Scrape Google Scholar Search Results with Python (Authors, Citations, and Year)

python -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml

import time
import random
from typing import Optional

import requests

TIMEOUT = (10, 30)

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/123.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

session = requests.Session()


def fetch(url: str, *, proxy_url: Optional[str] = None, max_retries: int = 5) -> str:
    proxies = None
    if proxy_url:
        proxies = {"http": proxy_url, "https": proxy_url}

    last_err = None
    for attempt in range(1, max_retries + 1):
        try:
            r = session.get(url, headers=HEADERS, timeout=TIMEOUT, proxies=proxies)

            # Scholar often responds with 429/503 when it dislikes automation.
            if r.status_code in (429, 503, 500, 502, 504):
                time.sleep(min(30, (2 ** attempt) + random.random()))
                continue

            r.raise_for_status()
            return r.text
        except Exception as e:
            last_err = e
            time.sleep(min(30, (2 ** attempt) + random.random()))

    raise RuntimeError(f"fetch failed: {last_err}")

import re
from bs4 import BeautifulSoup


def parse_int(text: str) -> int | None:
    m = re.search(r"(\d+)", text or "")
    return int(m.group(1)) if m else None


def extract_year(text: str) -> int | None:
    # Scholar snippets often contain a 4-digit year.
    m = re.search(r"\b(19\d{2}|20\d{2})\b", text or "")
    return int(m.group(1)) if m else None


def parse_scholar_page(html: str) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")

    out = []
    for r in soup.select("div.gs_r"):
        title_a = r.select_one("h3.gs_rt a")
        title = title_a.get_text(" ", strip=True) if title_a else None
        link = title_a.get("href") if title_a else None

        meta = r.select_one("div.gs_a")
        meta_text = meta.get_text(" ", strip=True) if meta else ""

        # gs_a usually looks like:
        # "A Author, B Author - Venue, 2021 - publisher.com"
        year = extract_year(meta_text)

        # Best-effort author split: authors are before the first '-' separator
        authors = None
        if meta_text and " - " in meta_text:
            authors = meta_text.split(" - ", 1)[0].strip()

        # Citations
        cited_by = 0
        for a in r.select("div.gs_fl a"):
            t = a.get_text(" ", strip=True)
            if t.lower().startswith("cited by"):
                cited_by = parse_int(t) or 0

        snippet = None
        snip = r.select_one("div.gs_rs")
        if snip:
            snippet = snip.get_text(" ", strip=True)

        out.append({
            "title": title,
            "link": link,
            "authors": authors,
            "meta": meta_text or None,
            "year": year,
            "cited_by": cited_by,
            "snippet": snippet,
        })

    return out

from urllib.parse import urlencode

BASE = "https://scholar.google.com/scholar"


def build_search_url(query: str, start: int = 0) -> str:
    qs = urlencode({"q": query, "start": start})
    return f"{BASE}?{qs}"


def crawl_scholar(query: str, *, pages: int = 3, proxy_url: str | None = None) -> list[dict]:
    all_rows = []

    for p in range(pages):
        start = p * 10
        url = build_search_url(query, start=start)

        html = fetch(url, proxy_url=proxy_url)
        rows = parse_scholar_page(html)
        print(f"page {p+1}: rows {len(rows)}")

        all_rows.extend(rows)

        # pacing matters on Scholar
        time.sleep(6 + random.random() * 3)

        if not rows:
            break

    return all_rows

import csv


def write_csv(rows: list[dict], path: str = "scholar_results.csv"):
    fields = ["title", "link", "authors", "year", "cited_by", "meta", "snippet"]
    with open(path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in rows:
            w.writerow({k: r.get(k) for k in fields})


if __name__ == "__main__":
    proxy_url = None  # set ProxiesAPI proxy endpoint if you have one

    rows = crawl_scholar("graph neural networks", pages=2, proxy_url=proxy_url)
    print("total rows:", len(rows))

    write_csv(rows)
    print("wrote scholar_results.csv")