Scrape Netflix Catalogue Data with Python + ProxiesAPI (Titles, Genres, Availability)

{"title":"Stranger Things","url":"https://www.netflix.com/title/80057281","title_id":"80057281","maturity":"TV-14","genres":["Sci-Fi TV"],"availability_country":"US"}

python -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml

import os
import time
import random
import requests

TIMEOUT = (15, 60)
PROXIESAPI_KEY = os.getenv("PROXIESAPI_KEY")

session = requests.Session()


def fetch(url: str, *, country: str | None = None, max_retries: int = 5) -> str:
    """Fetch a URL with retries/backoff.

    ProxiesAPI is used as a network reliability layer.
    Replace parameter names with the exact ProxiesAPI interface you use.
    """
    last_err = None

    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/122.0 Safari/537.36"
        ),
        "Accept-Language": "en-US,en;q=0.9",
    }

    for attempt in range(1, max_retries + 1):
        try:
            if not PROXIESAPI_KEY:
                raise RuntimeError("Missing PROXIESAPI_KEY env var")

            params = {
                "auth_key": PROXIESAPI_KEY,
                "url": url,
            }
            if country:
                params["country"] = country

            r = session.get(
                "https://api.proxiesapi.com",
                params=params,
                timeout=TIMEOUT,
                headers=headers,
            )
            r.raise_for_status()
            return r.text

        except Exception as e:
            last_err = e
            sleep_s = min(45, (2 ** (attempt - 1)) + random.random())
            time.sleep(sleep_s)

    raise RuntimeError(f"Failed to fetch after {max_retries} retries: {url}") from last_err

import re
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE = "https://www.netflix.com"


def extract_title_id(href: str) -> str | None:
    m = re.search(r"/title/(\d+)", href)
    return m.group(1) if m else None


def parse_titles_from_html(html: str) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")

    out = []
    for a in soup.select('a[href*="/title/"]'):
        href = a.get("href")
        if not href:
            continue

        title_id = extract_title_id(href)
        if not title_id:
            continue

        url = href if href.startswith("http") else urljoin(BASE, href)

        # The visible title text isn't always present, but try.
        text = a.get_text(" ", strip=True) or None

        out.append({
            "title_id": title_id,
            "url": url,
            "title": text,
        })

    # Dedupe by title_id
    seen = set()
    uniq = []
    for row in out:
        tid = row["title_id"]
        if tid in seen:
            continue
        seen.add(tid)
        uniq.append(row)

    return uniq

def crawl_seeds(seed_urls: list[str], *, country: str = "US") -> list[dict]:
    all_titles = []
    seen = set()

    for i, url in enumerate(seed_urls, start=1):
        html = fetch(url, country=country)
        batch = parse_titles_from_html(html)

        added = 0
        for row in batch:
            tid = row["title_id"]
            if tid in seen:
                continue
            seen.add(tid)
            row["availability_country"] = country
            row["source_url"] = url
            all_titles.append(row)
            added += 1

        print(f"seed {i}/{len(seed_urls)} -> found {len(batch)} titles, added {added}, total {len(all_titles)}")

    return all_titles

SEEDS = [
    "https://www.netflix.com/browse",
    # "https://www.netflix.com/browse/genre/83",  # TV Shows (example)
    # "https://www.netflix.com/browse/genre/1365",  # Action & Adventure (example)
]

rows = crawl_seeds(SEEDS, country="US")
print("unique titles:", len(rows))

def enrich_title(row: dict, *, country: str = "US") -> dict:
    url = row["url"]
    html = fetch(url, country=country)
    soup = BeautifulSoup(html, "lxml")

    # These selectors may change; keep them optional.
    maturity = None
    synopsis = None
    genres = []

    # Meta description sometimes includes synopsis-like content
    meta_desc = soup.select_one('meta[name="description"]')
    if meta_desc and meta_desc.get("content"):
        synopsis = meta_desc.get("content").strip() or None

    # Some pages include maturity rating in aria-label or text
    rating_el = soup.find(attrs={"data-uia": re.compile(r"maturity-rating", re.I)})
    if rating_el:
        maturity = rating_el.get_text(" ", strip=True) or None

    # Genres: look for links containing /browse/genre/
    for a in soup.select('a[href*="/browse/genre/"]'):
        g = a.get_text(" ", strip=True)
        if g and g not in genres:
            genres.append(g)

    row = dict(row)
    row.update({
        "maturity": maturity,
        "synopsis": synopsis,
        "genres": genres,
    })
    return row

import json

def write_jsonl(path: str, rows: list[dict]):
    with open(path, "w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")


if __name__ == "__main__":
    seeds = ["https://www.netflix.com/browse"]
    base_rows = crawl_seeds(seeds, country="US")

    # Optional: enrich first N titles
    enriched = []
    for row in base_rows[:50]:
        try:
            enriched.append(enrich_title(row, country="US"))
        except Exception:
            enriched.append(row)

    write_jsonl("netflix_catalogue_us.jsonl", enriched)
    print("wrote", len(enriched))