Scrape Google Play Store App Data with Python (Ratings, Reviews, and Install Counts)

python -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml

from __future__ import annotations

import os
import time
import random
import requests

TIMEOUT = (10, 40)

session = requests.Session()

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/123.0.0.0 Safari/537.36"
    ),
    "Accept-Language": "en-US,en;q=0.9",
}


def jitter_sleep(min_s: float = 0.8, max_s: float = 2.0) -> None:
    time.sleep(random.uniform(min_s, max_s))


def get(url: str, *, params: dict | None = None) -> requests.Response:
    """HTTP GET with headers/timeouts. Swap proxy settings here."""

    # Direct:
    r = session.get(url, params=params, headers=HEADERS, timeout=TIMEOUT)

    # ProxiesAPI (illustrative; use your account’s exact proxy / gateway method):
    # proxy = os.environ.get("PROXIESAPI_HTTP_PROXY")
    # proxies = {"http": proxy, "https": proxy} if proxy else None
    # r = session.get(url, params=params, headers=HEADERS, proxies=proxies, timeout=TIMEOUT)

    r.raise_for_status()
    return r

from bs4 import BeautifulSoup


def app_url(package: str, hl: str = "en", gl: str = "US") -> str:
    return "https://play.google.com/store/apps/details"


def fetch_app_html(package: str, hl: str = "en", gl: str = "US") -> str:
    url = "https://play.google.com/store/apps/details"
    r = get(url, params={"id": package, "hl": hl, "gl": gl})
    return r.text

import re


def first_text(soup: BeautifulSoup, selectors: list[str]) -> str | None:
    for sel in selectors:
        el = soup.select_one(sel)
        if el:
            t = el.get_text(" ", strip=True)
            if t:
                return t
    return None


def parse_installs(text: str) -> str | None:
    # Common patterns: "10M+" / "1,000,000+" etc.
    m = re.search(r"\b([0-9][0-9,\.]*\+?)\b", text or "")
    return m.group(1) if m else None


def parse_app_metadata(html: str) -> dict:
    soup = BeautifulSoup(html, "lxml")

    title = first_text(soup, ["h1", "h1 span"])

    developer = first_text(
        soup,
        [
            "a[href*='/store/apps/dev']",
            "a[href*='developer?id=']",
        ],
    )

    rating = None
    rating_el = soup.select_one("div[role='img'][aria-label*='Rated']")
    if rating_el:
        rating = rating_el.get("aria-label")

    category = first_text(soup, ["a[itemprop='genre']", "a[href*='category']"])

    # Install count is not always visible everywhere; use a regex scan as fallback.
    text = soup.get_text("\n", strip=True)
    installs = None
    for pat in [r"\b\d+[MK]?\+\b", r"\b\d{1,3}(?:,\d{3})*\+\b"]:
        m = re.search(pat, text)
        if m:
            installs = m.group(0)
            break

    return {
        "title": title,
        "developer": developer,
        "rating_label": rating,
        "category": category,
        "installs": installs,
    }

import json


def post(url: str, *, data=None, headers=None) -> requests.Response:
    h = dict(HEADERS)
    if headers:
        h.update(headers)
    r = session.post(url, data=data, headers=h, timeout=TIMEOUT)
    r.raise_for_status()
    return r


def fetch_reviews_template(package: str, hl: str = "en", gl: str = "US", pages: int = 3) -> list[dict]:
    """Template: replace ENDPOINT + payload once you capture it from DevTools."""

    ENDPOINT = "https://play.google.com/_/PlayStoreUi/data/batchexecute"

    # The payload below is intentionally not hard-coded to a specific internal RPC
    # because it changes. Capture the exact 'f.req' payload from your browser.
    #
    # Steps:
    # 1) Visit app URL
    # 2) Open reviews modal
    # 3) Network → find POST to batchexecute
    # 4) Copy request payload (f.req)
    # 5) Paste into this function

    reviews: list[dict] = []

    for page in range(1, pages + 1):
        jitter_sleep()

        # Placeholder: you must paste the captured value.
        f_req = os.environ.get("PLAY_F_REQ")
        if not f_req:
            raise RuntimeError(
                "Set PLAY_F_REQ env var with the captured f.req payload from DevTools. "
                "This keeps the tutorial honest and update-friendly."
            )

        r = post(
            ENDPOINT,
            data={"f.req": f_req, "hl": hl, "gl": gl},
            headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"},
        )

        # Response is not clean JSON; it’s a structured text wrapper.
        # You’ll need to locate the JSON chunk inside. A simple strategy:
        txt = r.text
        # Find the first JSON-looking block
        start = txt.find("[")
        end = txt.rfind("]")
        if start == -1 or end == -1:
            break

        blob = txt[start : end + 1]
        try:
            data = json.loads(blob)
        except Exception:
            break

        # TODO: map the data structure into review objects.
        # This depends on the RPC response shape you captured.
        reviews.append({"page": page, "raw": data})

    return reviews

def scrape_app(package: str) -> dict:
    html = fetch_app_html(package, hl="en", gl="US")
    meta = parse_app_metadata(html)

    return {
        "package": package,
        "metadata": meta,
        "fetched_at": time.time(),
    }


if __name__ == "__main__":
    pkg = "com.spotify.music"  # replace
    data = scrape_app(pkg)
    print(json.dumps(data, ensure_ascii=False, indent=2))