Scrape Book Reviews and Ratings from Goodreads with Python (JSON-LD + Top Reviews)

python3 -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml

from __future__ import annotations

import os
import random
import time
import requests

TIMEOUT = (10, 30)  # connect, read
UA = "Mozilla/5.0 (compatible; ProxiesAPI-Guides/1.0; +https://proxiesapi.com)"

# Optional: route through an HTTP proxy (including ProxiesAPI upstream proxy mode, if you use it).
PROXY = os.environ.get("HTTP_PROXY") or os.environ.get("HTTPS_PROXY")

session = requests.Session()
session.headers.update({"User-Agent": UA})


def fetch(url: str, attempts: int = 3) -> str:
    proxies = {"http": PROXY, "https": PROXY} if PROXY else None
    last_exc = None
    for i in range(attempts):
        try:
            r = session.get(url, timeout=TIMEOUT, proxies=proxies)
            r.raise_for_status()
            return r.text
        except Exception as e:  # noqa: BLE001
            last_exc = e
            sleep_s = (2**i) + random.random()
            time.sleep(sleep_s)
    raise last_exc  # type: ignore[misc]

import json
from bs4 import BeautifulSoup


def extract_jsonld_book(html: str) -> dict | None:
    soup = BeautifulSoup(html, "lxml")
    script = soup.select_one('script[type="application/ld+json"]')
    if not script or not script.string:
        return None
    data = json.loads(script.string)
    return data if isinstance(data, dict) else None


def extract_ratings(book_schema: dict) -> dict:
    agg = book_schema.get("aggregateRating") or {}
    return {
        "title": book_schema.get("name"),
        "author": (book_schema.get("author") or [{}])[0].get("name"),
        "rating_value": agg.get("ratingValue"),
        "rating_count": agg.get("ratingCount"),
        "review_count": agg.get("reviewCount"),
        "image": book_schema.get("image"),
        "url": book_schema.get("url"),
    }

def extract_top_reviews(html: str, limit: int = 5) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")

    reviews: list[dict] = []

    # Goodreads markup evolves; treat this as a pattern, not a promise.
    # Start broad: look for blocks that look like review text.
    for node in soup.select("article, div"):
        text = node.get_text(" ", strip=True)
        if not text or len(text) < 120:
            continue
        if "likes" in text.lower() and "comment" in text.lower():
            # common review footer markers
            reviews.append({"snippet": text[:420] + ("…" if len(text) > 420 else "")})
        if len(reviews) >= limit:
            break

    return reviews

import json


def scrape_goodreads_book(url: str) -> dict:
    html = fetch(url)
    schema = extract_jsonld_book(html) or {}
    out = extract_ratings(schema) if schema else {"url": url}
    out["top_reviews"] = extract_top_reviews(html, limit=5)
    return out


if __name__ == "__main__":
    url = "https://www.goodreads.com/book/show/4671.The_Great_Gatsby"
    data = scrape_goodreads_book(url)
    print(json.dumps(data, indent=2, ensure_ascii=False))