Scrape Live Stock Prices from Yahoo Finance (Python + ProxiesAPI)

curl -s "https://finance.yahoo.com/quote/AAPL" | head -n 15

python -m venv .venv
source .venv/bin/activate

pip install requests beautifulsoup4 lxml python-dotenv

export PROXIESAPI_PROXY_URL="http://YOUR_USERNAME:YOUR_PASSWORD@gw.proxiesapi.com:8080"

import os
import time
import random
import requests

PROXY_URL = os.getenv("PROXIESAPI_PROXY_URL")
BASE = "https://finance.yahoo.com"
TIMEOUT = (10, 30)  # connect, read

session = requests.Session()


def fetch(url: str) -> str:
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/123.0.0.0 Safari/537.36"
        ),
        "Accept-Language": "en-US,en;q=0.9",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    }

    proxies = None
    if PROXY_URL:
        proxies = {"http": PROXY_URL, "https": PROXY_URL}

    r = session.get(url, headers=headers, proxies=proxies, timeout=TIMEOUT)
    r.raise_for_status()

    # light jitter so you don’t look like a metronome
    time.sleep(random.uniform(0.6, 1.4))

    return r.text

import re
from bs4 import BeautifulSoup


def _to_float(s: str):
    if s is None:
        return None
    s = s.replace(",", "").strip()
    m = re.search(r"[-+]?\d+(?:\.\d+)?", s)
    return float(m.group(0)) if m else None


def parse_quote(html: str) -> dict:
    soup = BeautifulSoup(html, "lxml")

    # 1) Headline price by data-field
    price = None
    price_el = soup.select_one('[data-field="regularMarketPrice"]')
    if price_el:
        price = _to_float(price_el.get_text(" ", strip=True))

    # 2) Change and % change often sit nearby
    change = None
    change_pct = None
    change_el = soup.select_one('[data-field="regularMarketChange"]')
    if change_el:
        change = _to_float(change_el.get_text(" ", strip=True))

    change_pct_el = soup.select_one('[data-field="regularMarketChangePercent"]')
    if change_pct_el:
        change_pct = _to_float(change_pct_el.get_text(" ", strip=True))

    # 3) Fallback: meta tags (not always present)
    if price is None:
        meta = soup.select_one('meta[name="price"]')
        if meta and meta.get("content"):
            price = _to_float(meta["content"])

    return {
        "price": price,
        "change": change,
        "change_percent": change_pct,
    }

def parse_market_cap(html: str) -> str | None:
    soup = BeautifulSoup(html, "lxml")

    # Look for table-like rows containing the label.
    # Yahoo’s markup changes, so we’re flexible: find any element whose
    # text is exactly "Market Cap" then read the closest sibling cell.
    label = soup.find(string=re.compile(r"^Market\s*Cap$"))
    if not label:
        return None

    # Walk up to a row container, then find the next "td"/"span" with a value.
    row = label.find_parent(["tr", "li", "div"])
    if not row:
        return None

    # Common pattern: label in one cell, value in the next cell
    cells = row.find_all(["td", "span", "div"], recursive=True)
    texts = [c.get_text(" ", strip=True) for c in cells]

    # Heuristic: after "Market Cap" comes the value like "2.91T"
    for i, t in enumerate(texts):
        if re.fullmatch(r"Market\s*Cap", t):
            if i + 1 < len(texts):
                val = texts[i + 1]
                return val if val else None

    return None

import csv
from datetime import datetime


def scrape_ticker(ticker: str) -> dict:
    url = f"{BASE}/quote/{ticker}"
    html = fetch(url)

    q = parse_quote(html)
    market_cap = parse_market_cap(html)

    return {
        "ticker": ticker,
        "url": url,
        "scraped_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
        "price": q["price"],
        "change": q["change"],
        "change_percent": q["change_percent"],
        "market_cap": market_cap,
    }


def export_csv(rows: list[dict], path: str):
    fields = [
        "ticker",
        "url",
        "scraped_at",
        "price",
        "change",
        "change_percent",
        "market_cap",
    ]

    with open(path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in rows:
            w.writerow(r)


if __name__ == "__main__":
    tickers = ["AAPL", "MSFT", "GOOGL", "TSLA"]

    out = []
    for t in tickers:
        try:
            row = scrape_ticker(t)
            print("ok", t, row["price"], row["market_cap"])
            out.append(row)
        except Exception as e:
            print("err", t, repr(e))

    export_csv(out, "yahoo_quotes.csv")
    print("wrote yahoo_quotes.csv", len(out))

ok AAPL 173.24 2.67T
ok MSFT 412.10 3.08T
...
wrote yahoo_quotes.csv 4