crawler-intra-mart/crawler/sphinx_discovery.py

import json
import re
from pathlib import Path
from urllib.parse import urljoin

import httpx
from bs4 import BeautifulSoup

from utils.logger import logger


def _ensure_soup(html_or_soup):
    if hasattr(html_or_soup, "select_one") and hasattr(
        html_or_soup, "find_all"
    ):
        return html_or_soup
    return BeautifulSoup(html_or_soup, "lxml")


class SphinxDiscovery:
    """
    Discover hidden / structured pages from Sphinx-like docs.

    Sources:
    - searchindex.js
    - genindex.html
    - global index pages
    - toctree/sidebar links
    """

    def __init__(
        self,
        url_manager,
        timeout=30
    ):
        self.url_manager = url_manager
        self.timeout = timeout

    async def discover_from_base(
        self,
        base_url: str
    ):
        """
        Main discovery entry point.
        """

        discovered = set()

        candidates = self.common_sphinx_urls(
            base_url
        )

        for url in candidates:
            urls = await self.discover_from_url(url)
            discovered.update(urls)

        logger.info(
            f"Sphinx discovery found {len(discovered)} URLs from {base_url}"
        )

        return sorted(discovered)

    def common_sphinx_urls(
        self,
        base_url: str
    ):
        """
        Common Sphinx-generated files.
        """

        base = base_url.rstrip("/") + "/"

        return [
            urljoin(base, "searchindex.js"),
            urljoin(base, "genindex.html"),
            urljoin(base, "py-modindex.html"),
            urljoin(base, "search.html"),
            urljoin(base, "contents.html"),
            urljoin(base, "index.html"),
        ]

    async def discover_from_url(
        self,
        url: str
    ):
        """
        Dispatch discovery by file type.
        """

        if url.endswith("searchindex.js"):
            return await self.parse_searchindex(url)

        if url.endswith(".html"):
            return await self.parse_html_index(url)

        return []

    async def fetch_text(
        self,
        url: str
    ):
        try:
            async with httpx.AsyncClient(
                timeout=self.timeout,
                follow_redirects=True
            ) as client:
                response = await client.get(url)

                if response.status_code >= 400:
                    return None

                return response.text

        except Exception as e:
            logger.debug(
                f"Sphinx discovery fetch failed {url}: {e}"
            )
            return None

    async def parse_searchindex(
        self,
        url: str
    ):
        """
        Parse Sphinx searchindex.js.

        Typical format:
        Search.setIndex({...})
        """

        text = await self.fetch_text(url)

        if not text:
            return []

        try:
            payload = self.extract_searchindex_json(text)

            if not payload:
                return []

            docnames = payload.get("docnames", [])

            base_url = url.rsplit("/", 1)[0] + "/"

            discovered = set()

            for docname in docnames:
                page_url = urljoin(
                    base_url,
                    docname + ".html"
                )

                normalized = self.url_manager.normalize(page_url)

                if (
                    normalized
                    and self.url_manager.is_allowed(normalized)
                ):
                    discovered.add(normalized)

            logger.info(
                f"Parsed {len(discovered)} URLs from searchindex: {url}"
            )

            return sorted(discovered)

        except Exception as e:
            logger.warning(
                f"Failed parsing searchindex {url}: {e}"
            )
            return []

    def extract_searchindex_json(
        self,
        text: str
    ):
        """
        Extract JSON-like object from Sphinx searchindex.js.
        """

        match = re.search(
            r"Search\.setIndex\((.*)\)\s*;?\s*$",
            text,
            re.DOTALL
        )

        if not match:
            return None

        raw = match.group(1)

        try:
            return json.loads(raw)
        except Exception:
            pass

        # Fallback for slightly non-standard JS object
        raw = raw.replace("'", '"')

        try:
            return json.loads(raw)
        except Exception:
            logger.debug(
                "searchindex.js is not valid JSON after normalization."
            )
            return None

    async def parse_html_index(
        self,
        url: str
    ):
        """
        Parse Sphinx index-like HTML files.
        """

        html = await self.fetch_text(url)

        if not html:
            return []

        soup = BeautifulSoup(html, "lxml")

        discovered = set()

        selectors = [
            "a.reference.internal",
            ".toctree-wrapper a",
            ".wy-menu a",
            ".sphinxsidebar a",
            ".globaltoc a",
            ".localtoc a",
            "nav a",
            "a[href]"
        ]

        for selector in selectors:
            for a in soup.select(selector):
                href = a.get("href")

                resolved = self.url_manager.resolve(
                    url,
                    href
                )

                if not resolved:
                    continue

                if not self.url_manager.is_allowed(resolved):
                    continue

                discovered.add(resolved)

        logger.info(
            f"Parsed {len(discovered)} URLs from HTML index: {url}"
        )

        return sorted(discovered)

    async def discover_from_rendered_html(
        self,
        base_url: str,
        html
    ):
        """
        Discover Sphinx links from already-rendered page HTML.
        Accepts either a string or a parsed soup.
        """

        soup = _ensure_soup(html)

        discovered = set()

        selectors = [
            "a.reference.internal",
            ".toctree-wrapper a.reference.internal",
            ".section a.reference.internal",
            ".body a.reference.internal",
            ".document a.reference.internal",
            ".wy-menu a",
            ".sphinxsidebar a",
            ".globaltoc a",
            ".localtoc a",
            "link[rel='next']",
            "link[rel='prev']",
            "link[rel='up']"
        ]

        for selector in selectors:
            for node in soup.select(selector):
                href = node.get("href")

                if not href:
                    continue

                resolved = self.url_manager.resolve(
                    base_url,
                    href
                )

                if not resolved:
                    continue

                if not self.url_manager.is_allowed(resolved):
                    continue

                discovered.add(resolved)

        return sorted(discovered)

    def build_site_graph_edges(
        self,
        source_url: str,
        target_urls: list[str]
    ):
        """
        Prepare graph edges.
        """

        edges = []

        for target in target_urls:
            edges.append({
                "source": source_url,
                "target": target,
                "type": "sphinx_internal_link"
            })

        return edges