crawler-intra-mart/crawler/url_manager.py

from urllib.parse import (
    urlparse,
    urljoin,
    parse_qs,
    urlencode
)

from pathlib import Path

from utils.logger import logger


class URLManager:
    """
    Production-ready URL manager.

    Handles:
    - normalization
    - deduplication
    - filtering
    - crawl scope
    - asset filtering
    """

    def __init__(
        self,
        allowed_domains,
        allowed_schemes=None,
        blocked_extensions=None,
        remove_query_params=True
    ):
        self.allowed_domains = set(
            allowed_domains
        )

        self.allowed_schemes = (
            allowed_schemes
            or {"http", "https"}
        )

        self.remove_query_params = (
            remove_query_params
        )

        self.blocked_extensions = (
            blocked_extensions
            or {
                ".jpg",
                ".jpeg",
                ".png",
                ".gif",
                ".svg",
                ".webp",
                ".pdf",
                ".xls",
                ".xlsx",
                ".doc",
                ".docx",
                ".ppt",
                ".pptx",
                ".zip",
                ".tar",
                ".gz",
                ".mp4",
                ".mp3",
                ".avi",
                ".woff",
                ".woff2",
                ".ttf",
                ".eot"
            }
        )

    def normalize(
        self,
        url: str
    ):
        """
        Normalize URL consistently.
        """

        if not url:
            return None

        parsed = urlparse(url)

        # Remove fragments
        parsed = parsed._replace(
            fragment=""
        )

        # Normalize scheme/domain
        scheme = parsed.scheme.lower()

        netloc = parsed.netloc.lower()

        # Remove default ports
        netloc = netloc.replace(
            ":80",
            ""
        ).replace(
            ":443",
            ""
        )

        # Normalize path
        path = parsed.path or "/"

        if path != "/" and path.endswith("/"):

            path = path.rstrip("/")

        # Remove tracking query params
        query = ""

        if (
            not self.remove_query_params
            and parsed.query
        ):
            query = parsed.query

        normalized = parsed._replace(
            scheme=scheme,
            netloc=netloc,
            path=path,
            query=query
        )

        return normalized.geturl()

    def resolve(
        self,
        base_url: str,
        href: str
    ):
        """
        Resolve relative URL.
        """

        if not href:
            return None

        href = href.strip()

        # Ignore anchors
        if href.startswith("#"):
            return None

        # Ignore JS/mail links
        ignored = [
            "javascript:",
            "mailto:",
            "tel:"
        ]

        if any(
            href.startswith(prefix)
            for prefix in ignored
        ):
            return None

        try:

            resolved = urljoin(
                base_url,
                href
            )

            return self.normalize(
                resolved
            )

        except Exception as e:

            logger.warning(
                f"URL resolve failed "
                f"{href}: {e}"
            )

            return None

    def is_allowed(
        self,
        url: str
    ):
        """
        Check crawl scope.
        """

        if not url:
            return False

        parsed = urlparse(url)

        # Scheme check
        if (
            parsed.scheme
            not in self.allowed_schemes
        ):

            return False

        domain = parsed.netloc.lower()

        # Domain check
        if not any(
            domain == allowed or domain.endswith("." + allowed)
            for allowed in self.allowed_domains
        ):
            return False

        # Extension filter
        extension = Path(
            parsed.path
        ).suffix.lower()

        if extension in self.blocked_extensions:

            return False

        return True

    def should_enqueue(
        self,
        url: str
    ):
        """
        Determine whether URL
        should be crawled.
        """

        normalized = self.normalize(url)

        if not normalized:

            return False

        if not self.is_allowed(normalized):

            return False

        return True

    def extract_domain(
        self,
        url: str
    ):
        """
        Extract domain.
        """

        parsed = urlparse(url)

        return parsed.netloc.lower()

    def extract_path(
        self,
        url: str
    ):
        """
        Extract path.
        """

        parsed = urlparse(url)

        return parsed.path

    def remove_query_tracking(
        self,
        url: str
    ):
        """
        Remove tracking params.
        """

        parsed = urlparse(url)

        query = parse_qs(
            parsed.query
        )

        blocked_params = {
            "utm_source",
            "utm_medium",
            "utm_campaign",
            "fbclid",
            "gclid"
        }

        clean_query = {
            k: v
            for k, v in query.items()
            if k not in blocked_params
        }

        encoded_query = urlencode(
            clean_query,
            doseq=True
        )

        parsed = parsed._replace(
            query=encoded_query
        )

        return parsed.geturl()

    def is_binary_url(self, url: str):
        parsed = urlparse(url)
        extension = Path(parsed.path).suffix.lower()

        binary_extensions = {
            ".pdf",
            ".xls",
            ".xlsx",
            ".doc",
            ".docx",
            ".ppt",
            ".pptx",
            ".zip"
        }

        return extension in binary_extensions
    
    def stats(self):
        return {
            "allowed_domains": list(self.allowed_domains),
            "blocked_extensions": list(self.blocked_extensions),
        }