Scrape Sports Scores from ESPN with Python (via ProxiesAPI)

curl -sL "https://www.espn.com/nba/scoreboard" | head -n 15

python -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml

import random
import time
from urllib.parse import quote

import requests

TIMEOUT = (10, 30)  # connect, read

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/123.0.0.0 Safari/537.36"
    ),
    "Accept-Language": "en-US,en;q=0.9",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}

session = requests.Session()


def fetch(url: str, *, proxiesapi_key: str | None = None, retries: int = 4) -> str:
    last = None
    for attempt in range(1, retries + 1):
        try:
            if proxiesapi_key:
                # ProxiesAPI simple proxy-backed fetch URL.
                # Note: some accounts support extra params (rendering, country, etc.).
                proxied = (
                    "http://api.proxiesapi.com/?key="
                    + quote(proxiesapi_key)
                    + "&url="
                    + quote(url, safe="")
                )
                r = session.get(proxied, headers=HEADERS, timeout=TIMEOUT)
            else:
                r = session.get(url, headers=HEADERS, timeout=TIMEOUT)

            r.raise_for_status()
            return r.text
        except Exception as e:
            last = e
            sleep_s = (2 ** attempt) + random.random()
            time.sleep(sleep_s)

    raise RuntimeError(f"fetch failed after {retries} retries: {last}")

html_direct = fetch("https://www.espn.com/nba/scoreboard")
print("direct chars:", len(html_direct))

# html_proxy = fetch("https://www.espn.com/nba/scoreboard", proxiesapi_key="YOUR_KEY")
# print("proxied chars:", len(html_proxy))

html = fetch("https://www.espn.com/nba/scoreboard")
with open("espn_scoreboard.html", "w", encoding="utf-8") as f:
    f.write(html)
print("wrote espn_scoreboard.html")

import re
from urllib.parse import urljoin
from bs4 import BeautifulSoup

BASE = "https://www.espn.com"


def clean(text: str | None) -> str | None:
    if not text:
        return None
    t = re.sub(r"\s+", " ", text).strip()
    return t or None


def parse_int(text: str | None) -> int | None:
    if not text:
        return None
    m = re.search(r"\d+", text)
    return int(m.group(0)) if m else None


def parse_scoreboard(html: str, page_url: str) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")

    games: list[dict] = []

    # Heuristic: find links that look like game pages and walk up to a container.
    # ESPN game links often contain /game/_/gameId/
    game_links = soup.select('a[href*="/game/_/gameId/"]')

    seen = set()
    for a in game_links:
        href = a.get("href")
        if not href:
            continue
        game_url = urljoin(BASE, href)
        if game_url in seen:
            continue
        seen.add(game_url)

        # Find a reasonable container around the link
        card = a
        for _ in range(6):
            if not card:
                break
            # Stop when the container has enough text to plausibly be a game card
            if getattr(card, "get_text", None):
                txt = clean(card.get_text(" ", strip=True)) or ""
                if len(txt) > 40:
                    break
            card = card.parent

        container = card if card else a

        # Team names: look for two repeated name elements inside container.
        # These selectors are intentionally broad; you should tighten them based on a real snapshot.
        name_candidates = [
            el.get_text(" ", strip=True)
            for el in container.select('span, div')
            if el.get_text(strip=True)
        ]

        # Try to detect team-like names by excluding very short/very long tokens.
        # This is a fallback; ideally you target specific selectors once you inspect your snapshot.
        team_names = []
        for t in name_candidates:
            t = clean(t)
            if not t:
                continue
            if len(t) < 3 or len(t) > 40:
                continue
            # Skip obvious non-team tokens
            if t.lower() in {"final", "preview", "recap", "tickets"}:
                continue
            team_names.append(t)

        # De-dupe while preserving order
        uniq_names = []
        seen_name = set()
        for n in team_names:
            if n in seen_name:
                continue
            seen_name.add(n)
            uniq_names.append(n)

        # Scores: many cards show numeric scores; grab a few numbers.
        nums = [parse_int(clean(el.get_text(strip=True))) for el in container.select("span, div")]
        nums = [n for n in nums if n is not None]

        status = None
        # Status tends to include words like Final, Q1, Half, or a time.
        status_el = container.find(string=re.compile(r"Final|Q\d|Half|AM|PM", re.I))
        if status_el:
            status = clean(str(status_el))

        games.append({
            "page_url": page_url,
            "game_url": game_url,
            "teams_guess": uniq_names[:6],
            "scores_guess": nums[:6],
            "status_guess": status,
        })

    return games

def normalize_game(g: dict) -> dict:
    teams = g.get("teams_guess") or []
    scores = g.get("scores_guess") or []

    away_team = teams[0] if len(teams) > 0 else None
    home_team = teams[1] if len(teams) > 1 else None

    away_score = scores[0] if len(scores) > 0 else None
    home_score = scores[1] if len(scores) > 1 else None

    return {
        "away_team": away_team,
        "home_team": home_team,
        "away_score": away_score,
        "home_score": home_score,
        "status": g.get("status_guess"),
        "game_url": g.get("game_url"),
        "page_url": g.get("page_url"),
    }

import csv


def to_csv(rows: list[dict], path: str) -> None:
    if not rows:
        raise ValueError("no rows")

    fields = ["away_team", "home_team", "away_score", "home_score", "status", "game_url", "page_url"]
    with open(path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in rows:
            w.writerow({k: r.get(k) for k in fields})


if __name__ == "__main__":
    url = "https://www.espn.com/nba/scoreboard"
    html = fetch(url)  # or fetch(url, proxiesapi_key="YOUR_KEY")

    raw = parse_scoreboard(html, page_url=url)
    normalized = [normalize_game(g) for g in raw]

    to_csv(normalized, "espn_scores.csv")
    print("wrote espn_scores.csv", len(normalized))