Scrape Government Contract Data from SAM.gov with Python (Opportunities + Details)

python -m venv .venv
source .venv/bin/activate
pip install requests beautifulsoup4 lxml tenacity

import os
import time
import random
import requests
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type

PROXIESAPI_KEY = os.environ.get("PROXIESAPI_KEY")
TIMEOUT = (10, 50)

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
    "Accept-Language": "en-US,en;q=0.9",
})


class FetchError(Exception):
    pass


def proxiesapi_url(target_url: str) -> str:
    if not PROXIESAPI_KEY:
        raise RuntimeError("Set PROXIESAPI_KEY env var")
    return f"https://proxiesapi.com/api?auth_key={PROXIESAPI_KEY}&url={requests.utils.quote(target_url, safe='')}"


@retry(
    reraise=True,
    stop=stop_after_attempt(4),
    wait=wait_exponential(multiplier=1, min=1, max=15),
    retry=retry_if_exception_type((requests.RequestException, FetchError)),
)
def fetch(url: str) -> str:
    r = session.get(proxiesapi_url(url), timeout=TIMEOUT)
    if r.status_code >= 400:
        raise FetchError(f"HTTP {r.status_code}")
    text = r.text or ""
    if len(text) < 3000:
        raise FetchError("Response too small (possible block/interstitial)")
    return text


def jitter_sleep(min_s=0.5, max_s=1.3):
    time.sleep(random.uniform(min_s, max_s))

import re
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE = "https://sam.gov"


def parse_opportunity_cards(html: str) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")

    out = []

    # Many rows include a link to a detail route; we collect those.
    for a in soup.select('a[href*="/opp/"] , a[href*="opportunity"], a[href*="/opportunities/"]'):
        href = a.get("href")
        if not href:
            continue

        url = href if href.startswith("http") else urljoin(BASE, href)
        title = a.get_text(" ", strip=True)

        # Try to pull a nearby ID-like token
        parent = a.find_parent(["div", "li", "article"]) or a
        blob = parent.get_text(" ", strip=True)
        m = re.search(r"\b([A-Z0-9][A-Z0-9\-]{5,})\b", blob)
        notice_id = m.group(1) if m else None

        out.append({
            "title": title or None,
            "notice_id": notice_id,
            "detail_url": url,
        })

    # Dedupe by URL
    seen = set()
    deduped = []
    for r in out:
        if r["detail_url"] in seen:
            continue
        seen.add(r["detail_url"])
        deduped.append(r)

    return deduped

from urllib.parse import urlparse, parse_qs, urlencode, urlunparse


def set_query_param(url: str, key: str, value: str) -> str:
    parts = urlparse(url)
    q = parse_qs(parts.query)
    q[key] = [value]
    return urlunparse((parts.scheme, parts.netloc, parts.path, parts.params, urlencode(q, doseq=True), parts.fragment))


def crawl_search(search_url: str, pages: int = 3) -> list[dict]:
    all_rows = []
    seen_urls = set()

    for p in range(1, pages + 1):
        page_url = set_query_param(search_url, "page", str(p))
        html = fetch(page_url)
        batch = parse_opportunity_cards(html)

        for r in batch:
            u = r["detail_url"]
            if u in seen_urls:
                continue
            seen_urls.add(u)
            all_rows.append(r)

        print(f"page {p}/{pages}: {len(batch)} cards (total {len(all_rows)})")
        jitter_sleep()

    return all_rows

from dataclasses import dataclass, asdict


@dataclass
class SamOpportunity:
    notice_id: str | None
    title: str | None
    detail_url: str
    agency: str | None
    posted_date: str | None
    response_deadline: str | None
    naics: str | None
    set_aside: str | None
    place_of_performance: str | None


def text_or_none(el):
    return el.get_text(" ", strip=True) if el else None


def find_value_by_label(soup: BeautifulSoup, labels: list[str]) -> str | None:
    # Look for label text in dt/dd pairs, or in two-column rows.
    label_set = {l.strip().lower() for l in labels}

    # 1) definition lists
    for dt in soup.select("dt"):
        t = dt.get_text(" ", strip=True).lower()
        if t in label_set:
            dd = dt.find_next_sibling("dd")
            return text_or_none(dd)

    # 2) generic: find an element whose text is exactly a label
    for lab in soup.find_all(string=True):
        t = (lab or "").strip().lower()
        if t in label_set:
            el = lab.parent
            # try next sibling
            sib = el.find_next_sibling()
            if sib:
                val = text_or_none(sib)
                if val and val.lower() not in label_set:
                    return val

    return None


def parse_detail(detail_url: str, html: str, base_row: dict) -> SamOpportunity:
    soup = BeautifulSoup(html, "lxml")

    title = base_row.get("title")
    notice_id = base_row.get("notice_id")

    agency = find_value_by_label(soup, ["Agency", "Office", "Department"])
    posted_date = find_value_by_label(soup, ["Posted Date", "Publish Date", "Posted"])
    response_deadline = find_value_by_label(soup, ["Response Date", "Due Date", "Response Deadline"])
    naics = find_value_by_label(soup, ["NAICS", "NAICS Code"])
    set_aside = find_value_by_label(soup, ["Set-Aside", "Set Aside"])
    place = find_value_by_label(soup, ["Place of Performance", "Place"])

    # If the title wasn’t captured from list page, try h1
    if not title:
        h1 = soup.select_one("h1")
        title = text_or_none(h1)

    return SamOpportunity(
        notice_id=notice_id,
        title=title,
        detail_url=detail_url,
        agency=agency,
        posted_date=posted_date,
        response_deadline=response_deadline,
        naics=naics,
        set_aside=set_aside,
        place_of_performance=place,
    )

import csv


def build_dataset(search_url: str, pages: int = 2) -> list[SamOpportunity]:
    base_rows = crawl_search(search_url, pages=pages)
    out: list[SamOpportunity] = []

    for i, r in enumerate(base_rows, start=1):
        url = r["detail_url"]
        html = fetch(url)
        opp = parse_detail(url, html, r)
        out.append(opp)
        print(f"{i}/{len(base_rows)} {opp.notice_id} {opp.title}")
        jitter_sleep()

    return out


def export_csv(rows: list[SamOpportunity], path: str = "samgov_opportunities.csv"):
    if not rows:
        raise RuntimeError("No rows")
    with open(path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=list(asdict(rows[0]).keys()))
        w.writeheader()
        for r in rows:
            w.writerow(asdict(r))


if __name__ == "__main__":
    # Tip: run a search in your browser (e.g., keyword=software, set a date range), then copy the URL.
    SEARCH_URL = "https://sam.gov/search/?index=opp&sort=-modifiedDate&keywords=software"

    rows = build_dataset(SEARCH_URL, pages=2)
    export_csv(rows)
    print("exported", len(rows))

from pathlib import Path

def save_debug_html(name: str, html: str):
    Path("debug_html").mkdir(exist_ok=True)
    Path(f"debug_html/{name}.html").write_text(html, encoding="utf-8")