crawler-intra-mart/models/crawl_result.py

from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any
from datetime import datetime, timezone


def _utcnow_naive() -> datetime:
    """
    Replacement for the deprecated datetime.utcnow().

    Returns a naive UTC datetime (no tzinfo) to preserve
    the historical wire format.
    """
    return datetime.now(timezone.utc).replace(tzinfo=None)


@dataclass
class CrawlResult:
    """
    Result object for a crawled page.
    """

    # Core
    url: str
    title: str

    # Content
    raw_html: Optional[str] = None
    markdown: Optional[str] = None

    markdown_path: Optional[str] = None

    # Status
    status_code: Optional[int] = None
    success: bool = True
    error_message: Optional[str] = None

    # Links (ONLY ONE SOURCE OF TRUTH)
    links: List[str] = field(default_factory=list)

    # Timing
    crawled_at: datetime = field(default_factory=_utcnow_naive)

    # Assets
    assets: List[Dict[str, Any]] = field(default_factory=list)

    # Navigation
    navigation_tree: Optional[list] = None

    # Stats
    content_length: int = 0
    markdown_length: int = 0
    word_count: int = 0

    # Metadata
    tags: List[str] = field(default_factory=list)
    metadata: Dict[str, Any] = field(default_factory=dict)

    def compute_statistics(self):
        """
        Compute markdown statistics.
        """

        if not self.markdown:
            return

        self.markdown_length = len(
            self.markdown
        )

        self.word_count = len(
            self.markdown.split()
        )

        self.content_length = len(
            self.markdown or ""
        )

    def add_asset(
        self,
        asset_url: str,
        local_path: str
    ):
        """
        Add downloaded asset.
        """

        self.assets.append({
            "url": asset_url,
            "local_path": local_path
        })

    def add_discovered_link(
        self,
        url: str
    ):
        """
        Add discovered URL.
        """

        if url not in self.links:
            self.links.append(url)

    def mark_failed(
        self,
        error_message: str
    ):
        """
        Mark crawl as failed.
        """

        self.success = False

        self.error_message = error_message

    def to_dict(self):
        """
        Convert object to serializable dict.
        """

        return {
            "url": self.url,
            "title": self.title,
            "markdown_path": self.markdown_path,
            "success": self.success,
            "error_message": self.error_message,
            "status_code": self.status_code,
            "crawled_at": self.crawled_at.isoformat(),
            "assets": self.assets,
            "links": self.links,
            "content_length": self.content_length,
            "markdown_length": self.markdown_length,
            "word_count": self.word_count,
            "tags": self.tags,
            "metadata": self.metadata
        }

    @classmethod
    def from_dict(
        cls,
        data: dict
    ):
        """
        Restore CrawlResult from dict.
        """

        obj = cls(
            url=data.get("url", ""),
            title=data.get("title", "")
        )

        obj.markdown_path = data.get(
            "markdown_path"
        )

        obj.success = data.get(
            "success",
            True
        )

        obj.error_message = data.get(
            "error_message"
        )

        obj.status_code = data.get(
            "status_code"
        )

        obj.assets = data.get(
            "assets",
            []
        )

        obj.links = data.get(
            "links",
            []
        )

        obj.content_length = data.get(
            "content_length",
            0
        )

        obj.markdown_length = data.get(
            "markdown_length",
            0
        )

        obj.word_count = data.get(
            "word_count",
            0
        )

        obj.tags = data.get(
            "tags",
            []
        )

        obj.metadata = data.get(
            "metadata",
            {}
        )

        crawled_at = data.get(
            "crawled_at"
        )

        if crawled_at:

            obj.crawled_at = datetime.fromisoformat(
                crawled_at
            )

        return obj

    def summary(self):
        """
        Human-readable summary.
        """

        return (
            f"CrawlResult("
            f"url={self.url}, "
            f"success={self.success}, "
            f"markdown_length={self.markdown_length}, "
            f"assets={len(self.assets)}, "
            f"links={len(self.links)}"
            f")"
        )