How to Find All URLs on Any Website: 5 Methods

curl -s https://example.com/sitemap.xml | head

import requests
import xml.etree.ElementTree as ET

TIMEOUT = (10, 30)

def fetch_text(url: str) -> str:
    r = requests.get(url, timeout=TIMEOUT, headers={"User-Agent": "Mozilla/5.0 (compatible; ProxiesAPI-Guides/1.0)"})
    r.raise_for_status()
    return r.text


def parse_sitemap_urls(xml_text: str) -> list[str]:
    root = ET.fromstring(xml_text)

    # The XML uses namespaces; easiest is to ignore the namespace by stripping it.
    def strip(tag: str) -> str:
        return tag.split('}', 1)[-1]

    urls = []
    for el in root.iter():
        if strip(el.tag) == "loc" and el.text:
            urls.append(el.text.strip())
    return urls

xml_text = fetch_text("https://example.com/sitemap.xml")
urls = parse_sitemap_urls(xml_text)
print("found", len(urls))
print(urls[:5])

curl -s https://example.com/robots.txt | grep -i sitemap

Sitemap: https://example.com/sitemap.xml
Sitemap: https://example.com/sitemap-1.xml

from collections import deque
from urllib.parse import urljoin, urlparse, urldefrag
import requests
from bs4 import BeautifulSoup

TIMEOUT = (10, 30)

def is_html_response(resp: requests.Response) -> bool:
    ctype = resp.headers.get("Content-Type", "")
    return "text/html" in ctype


def normalize_url(base: str, href: str) -> str | None:
    if not href:
        return None

    url = urljoin(base, href)
    url, _frag = urldefrag(url)

    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https"):
        return None

    return url


def crawl_site(seed: str, max_pages: int = 200) -> list[str]:
    seed_parsed = urlparse(seed)
    host = seed_parsed.netloc

    q = deque([seed])
    seen = set([seed])
    out = []

    s = requests.Session()
    s.headers.update({"User-Agent": "Mozilla/5.0 (compatible; ProxiesAPI-Guides/1.0)"})

    while q and len(out) < max_pages:
        url = q.popleft()

        try:
            resp = s.get(url, timeout=TIMEOUT, allow_redirects=True)
            resp.raise_for_status()
        except Exception as e:
            print("fetch failed", url, e)
            continue

        if not is_html_response(resp):
            continue

        out.append(url)

        soup = BeautifulSoup(resp.text, "lxml")
        for a in soup.select("a[href]"):
            nxt = normalize_url(url, a.get("href"))
            if not nxt:
                continue

            p = urlparse(nxt)
            if p.netloc != host:
                continue

            # avoid obvious non-content file types
            if any(p.path.lower().endswith(ext) for ext in (".jpg", ".png", ".gif", ".pdf", ".zip")):
                continue

            if nxt not in seen:
                seen.add(nxt)
                q.append(nxt)

    return out

urls = crawl_site("https://example.com/", max_pages=100)
print("discovered", len(urls))
print(urls[:10])

curl "http://api.proxiesapi.com/?key=API_KEY&url=https://example.com/" | head