crawler-intra-mart/storage/path_mapper.py

import hashlib

from pathlib import Path
from urllib.parse import urlparse

from utils.logger import logger


class PathMapper:
    """
    Production-ready URL -> filesystem mapper.

    Handles:
    - structure preservation
    - collision prevention
    - filename sanitization
    - cross-platform safety
    """

    def __init__(
        self,
        output_dir,
        include_domain=True,
        max_filename_length=180
    ):
        self.output_dir = Path(
            output_dir
        )

        self.include_domain = (
            include_domain
        )

        self.max_filename_length = (
            max_filename_length
        )

    def url_to_path(
        self,
        url: str,
        extension=".md"
    ):
        """
        Convert URL -> local file path.
        """

        parsed = urlparse(url)

        domain = parsed.netloc

        path = parsed.path.strip("/")

        # Homepage
        if not path:

            path = "index"

        # Remove trailing slash
        path = path.rstrip("/")

        # Remove html extension
        if path.endswith(".html"):

            path = path[:-5]

        # Handle query params
        if parsed.query:

            query_hash = self.hash_string(
                parsed.query
            )[:10]

            path += f"__{query_hash}"

        # Sanitize each segment
        segments = []

        for segment in path.split("/"):

            segment = self.sanitize_filename(
                segment
            )

            if not segment:
                segment = "untitled"

            segments.append(segment)

        path = "/".join(segments)

        # Append extension
        path += extension

        # Include domain
        if self.include_domain:

            output_path = (
                self.output_dir /
                domain /
                path
            )

        else:

            output_path = (
                self.output_dir /
                path
            )

        # Prevent very long filenames
        output_path = self.shorten_if_needed(
            output_path
        )

        output_path.parent.mkdir(
            parents=True,
            exist_ok=True
        )

        logger.debug(
            f"Mapped URL -> path: "
            f"{url} => {output_path}"
        )

        return output_path

    def asset_url_to_path(
        self,
        url: str
    ):
        """
        Map asset URL to asset path.
        """

        return self.url_to_path(
            url,
            extension=""
        )

    def sanitize_filename(
        self,
        filename: str
    ):
        """
        Remove filesystem-invalid chars.
        """

        invalid_chars = [
            "<",
            ">",
            ":",
            "\"",
            "/",
            "\\",
            "|",
            "?",
            "*",
            "\n",
            "\r",
            "\t"
        ]

        for char in invalid_chars:

            filename = filename.replace(
                char,
                "_"
            )

        filename = filename.strip()

        # Windows reserved names -- the OS blocks these
        # regardless of extension, so e.g. "CON.txt" is
        # also unusable on Windows. Check the stem (the
        # portion before the first dot) against the full
        # reserved set: CON, PRN, AUX, NUL, COM1..COM9,
        # LPT1..LPT9.
        reserved = {
            "CON",
            "PRN",
            "AUX",
            "NUL",
        }
        for i in range(1, 10):
            reserved.add(f"COM{i}")
            reserved.add(f"LPT{i}")

        stem = filename.split(".", 1)[0]

        if stem.upper() in reserved:

            filename = "_" + filename

        return filename

    def shorten_if_needed(
        self,
        path: Path
    ):
        """
        Prevent excessively long filenames.
        """

        filename = path.name

        if len(filename) <= self.max_filename_length:

            return path

        suffix = path.suffix

        stem = path.stem

        hash_part = self.hash_string(
            filename
        )[:12]

        shortened = (
            stem[:100]
            + "__"
            + hash_part
            + suffix
        )

        return path.with_name(
            shortened
        )

    def hash_string(
        self,
        value: str
    ):
        """
        Stable hash helper.
        """

        return hashlib.md5(
            value.encode("utf-8")
        ).hexdigest()

    def relative_path(
        self,
        path: Path
    ):
        """
        Get relative path from output dir.
        """

        try:

            return path.relative_to(
                self.output_dir
            )

        except Exception:

            return path

    def markdown_path_to_url(
        self,
        markdown_path: Path
    ):
        """
        Reverse mapping helper.
        """

        relative = markdown_path.relative_to(
            self.output_dir
        )

        return str(relative)

    def ensure_unique_path(
        self,
        path: Path
    ):
        """
        Prevent overwrite collisions.
        """

        if not path.exists():

            return path

        counter = 1

        while True:

            candidate = path.with_name(
                f"{path.stem}_{counter}"
                f"{path.suffix}"
            )

            if not candidate.exists():

                return candidate

            counter += 1

    def stats(
        self
    ):
        """
        Path mapper statistics.
        """

        return {
            "output_dir": str(
                self.output_dir
            ),
            "include_domain": (
                self.include_domain
            )
        }