crawler-intra-mart/crawler/http_client.py

import httpx

from utils.logger import logger
from crawler.playwright_client import is_binary_file


def _http2_available() -> bool:
    """
    httpx requires the optional 'h2' package for HTTP/2. Detect
    its presence so we transparently fall back to HTTP/1.1 +
    keep-alive when h2 isn't installed (no hard dependency).
    """
    try:
        import h2  # noqa: F401
        return True
    except ImportError:
        return False


class HTTPClient:
    """
    Lightweight HTTP fetcher used as the fast-path for
    static documentation pages. Returns a dict shaped like
    PlaywrightClient.fetch_page so process_page can consume
    either source.
    """

    USER_AGENT = (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/124.0.0.0 Safari/537.36"
    )

    def __init__(
        self,
        timeout=20,
        max_keepalive=20,
        max_connections=40,
    ):
        self.timeout = timeout

        self.client = httpx.AsyncClient(
            timeout=timeout,
            follow_redirects=True,
            http2=_http2_available(),
            headers={"User-Agent": self.USER_AGENT},
            limits=httpx.Limits(
                max_connections=max_connections,
                max_keepalive_connections=max_keepalive,
            ),
        )

    async def aclose(self):
        await self.client.aclose()

    async def fetch_page(
        self,
        url: str,
        etag: str | None = None,
        last_modified: str | None = None,
    ):
        if is_binary_file(url):
            return {
                "url": url,
                "title": url.split("/")[-1],
                "html": "",
                "status_code": None,
                "headers": {},
                "is_binary": True,
                "content_type": "",
            }

        # Build conditional-GET headers when we have a prior
        # ETag / Last-Modified for this URL. A 304 response
        # lets the caller skip re-fetching, re-parsing, and
        # re-writing markdown — pure savings on resumed runs.
        headers = {}
        if etag:
            headers["If-None-Match"] = etag
        if last_modified:
            headers["If-Modified-Since"] = last_modified

        try:
            response = await self.client.get(url, headers=headers)
        except Exception as e:
            logger.warning(f"HTTP fetch failed {url}: {e}")
            return None

        content_type = response.headers.get("content-type", "")

        if response.status_code == 304:
            return {
                "url": str(response.url),
                "title": "",
                "html": "",
                "status_code": 304,
                "headers": dict(response.headers),
                "is_binary": False,
                "content_type": content_type,
                "not_modified": True,
            }

        if response.status_code >= 400:
            return {
                "url": str(response.url),
                "title": "",
                "html": "",
                "status_code": response.status_code,
                "headers": dict(response.headers),
                "is_binary": False,
                "content_type": content_type,
                "ok": False,
            }

        if "html" not in content_type.lower():
            # Treat non-HTML responses (e.g. application/pdf,
            # application/javascript) as binary so the caller
            # skips parsing them as documentation pages.
            return {
                "url": str(response.url),
                "title": url.split("/")[-1],
                "html": "",
                "status_code": response.status_code,
                "headers": dict(response.headers),
                "is_binary": True,
                "content_type": content_type,
            }

        html = response.text
        title = self._extract_title(html)

        return {
            "url": str(response.url),
            "title": title,
            "html": html,
            "status_code": response.status_code,
            "headers": dict(response.headers),
            "is_binary": False,
            "content_type": content_type,
            "ok": True,
            "etag": response.headers.get("etag"),
            "last_modified": response.headers.get("last-modified"),
        }

    @staticmethod
    def _extract_title(html: str) -> str:
        # Avoid full BS4 parse for just a title — the next
        # stage will parse the HTML anyway.
        lower = html.lower()
        start = lower.find("<title")
        if start < 0:
            return ""
        gt = lower.find(">", start)
        end = lower.find("</title", gt)
        if gt < 0 or end < 0:
            return ""
        return html[gt + 1 : end].strip()