How to Scrape Google Search Results with Python (Without Getting Blocked)

python -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml

import os
import time
import random
from urllib.parse import urlencode

import requests

TIMEOUT = (10, 30)
MAX_RETRIES = 5

session = requests.Session()

DEFAULT_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/122.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}


def build_proxiesapi_url(target_url: str) -> str:
    api_key = os.environ.get("PROXIESAPI_KEY")
    if not api_key:
        raise RuntimeError("Missing PROXIESAPI_KEY env var")

    base = os.environ.get("PROXIESAPI_URL", "https://api.proxiesapi.com")
    qs = urlencode({"api_key": api_key, "url": target_url})
    return f"{base}/?{qs}"


def looks_blocked(html: str) -> bool:
    t = (html or "").lower()
    # Common Google block/interstitial phrases
    return any(s in t for s in [
        "our systems have detected unusual traffic",
        "to continue, please verify",
        "sorry",
        "/sorry/index",
        "captcha",
    ])


def fetch(url: str, *, use_proxiesapi: bool = True) -> str:
    for attempt in range(1, MAX_RETRIES + 1):
        try:
            final = build_proxiesapi_url(url) if use_proxiesapi else url
            r = session.get(final, headers=DEFAULT_HEADERS, timeout=TIMEOUT)

            if r.status_code in (429, 503):
                raise RuntimeError(f"transient status {r.status_code}")

            r.raise_for_status()
            html = r.text or ""

            if looks_blocked(html):
                raise RuntimeError("blocked/interstitial detected")

            return html

        except Exception as e:
            if attempt == MAX_RETRIES:
                raise

            sleep_s = min(30, 2 ** (attempt - 1)) + random.uniform(0, 0.7)
            print(f"fetch failed ({attempt}/{MAX_RETRIES}): {e} — sleeping {sleep_s:.1f}s")
            time.sleep(sleep_s)

from urllib.parse import quote_plus


def google_search_url(q: str, *, hl: str = "en", gl: str = "us", num: int = 10, start: int = 0) -> str:
    return (
        "https://www.google.com/search?"
        f"q={quote_plus(q)}&num={num}&hl={hl}&gl={gl}&pws=0&start={start}"
    )


url = google_search_url("site:github.com proxies")
print(url)
html = fetch(url)
print(len(html))

import re
from bs4 import BeautifulSoup
from urllib.parse import urlparse


def is_google_internal(href: str) -> bool:
    if not href:
        return True
    if href.startswith("/"):
        return True
    host = urlparse(href).netloc.lower()
    return host.endswith("google.com") or host.endswith("googleusercontent.com")


def parse_serp(html: str) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")

    scope = soup.select_one("div#search") or soup

    results = []
    seen = set()

    # Candidate blocks
    for block in scope.select("div"):
        a = block.select_one("a[href]")
        if not a:
            continue

        href = a.get("href")
        if not href or is_google_internal(href):
            continue

        title_el = block.select_one("h3")
        title = title_el.get_text(" ", strip=True) if title_el else a.get_text(" ", strip=True)
        title = title.strip() if title else None

        # Snippet: best-effort
        snippet = None
        # common snippet container candidates
        sn = block.select_one("div.VwiC3b") or block.select_one("span.aCOpRe")
        if sn:
            snippet = sn.get_text(" ", strip=True)

        if not title:
            continue

        key = href
        if key in seen:
            continue
        seen.add(key)

        results.append({
            "title": title,
            "url": href,
            "snippet": snippet,
        })

    return results


items = parse_serp(html)
print("organic candidates:", len(items))
print(items[:3])

def crawl_serp(q: str, pages: int = 3) -> list[dict]:
    all_items = []
    seen = set()

    for p in range(pages):
        url = google_search_url(q, start=p * 10)
        html = fetch(url)
        batch = parse_serp(html)

        new_count = 0
        for it in batch:
            if it["url"] in seen:
                continue
            seen.add(it["url"])
            all_items.append(it)
            new_count += 1

        print(f"page {p+1}: batch={len(batch)} new={new_count} total={len(all_items)}")

        # gentle pacing
        time.sleep(2.0 + random.uniform(0, 0.8))

    return all_items


rows = crawl_serp("proxies api", pages=2)

import csv
import json


def write_csv(path: str, rows: list[dict]) -> None:
    fields = ["title", "url", "snippet"]
    with open(path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in rows:
            w.writerow({k: r.get(k) for k in fields})


def write_json(path: str, rows: list[dict]) -> None:
    with open(path, "w", encoding="utf-8") as f:
        json.dump(rows, f, ensure_ascii=False, indent=2)


write_csv("google_serp.csv", rows)
write_json("google_serp.json", rows)
print("wrote google_serp.csv + google_serp.json")