from dataclasses import dataclass, field from typing import Optional, List, Dict, Any from datetime import datetime, timezone def _utcnow_naive() -> datetime: """ Replacement for the deprecated datetime.utcnow(). Returns a naive UTC datetime (no tzinfo) to preserve the historical wire format. """ return datetime.now(timezone.utc).replace(tzinfo=None) @dataclass class CrawlResult: """ Result object for a crawled page. """ # Core url: str title: str # Content raw_html: Optional[str] = None markdown: Optional[str] = None markdown_path: Optional[str] = None # Status status_code: Optional[int] = None success: bool = True error_message: Optional[str] = None # Links (ONLY ONE SOURCE OF TRUTH) links: List[str] = field(default_factory=list) # Timing crawled_at: datetime = field(default_factory=_utcnow_naive) # Assets assets: List[Dict[str, Any]] = field(default_factory=list) # Navigation navigation_tree: Optional[list] = None # Stats content_length: int = 0 markdown_length: int = 0 word_count: int = 0 # Metadata tags: List[str] = field(default_factory=list) metadata: Dict[str, Any] = field(default_factory=dict) def compute_statistics(self): """ Compute markdown statistics. """ if not self.markdown: return self.markdown_length = len( self.markdown ) self.word_count = len( self.markdown.split() ) self.content_length = len( self.markdown or "" ) def add_asset( self, asset_url: str, local_path: str ): """ Add downloaded asset. """ self.assets.append({ "url": asset_url, "local_path": local_path }) def add_discovered_link( self, url: str ): """ Add discovered URL. """ if url not in self.links: self.links.append(url) def mark_failed( self, error_message: str ): """ Mark crawl as failed. """ self.success = False self.error_message = error_message def to_dict(self): """ Convert object to serializable dict. """ return { "url": self.url, "title": self.title, "markdown_path": self.markdown_path, "success": self.success, "error_message": self.error_message, "status_code": self.status_code, "crawled_at": self.crawled_at.isoformat(), "assets": self.assets, "links": self.links, "content_length": self.content_length, "markdown_length": self.markdown_length, "word_count": self.word_count, "tags": self.tags, "metadata": self.metadata } @classmethod def from_dict( cls, data: dict ): """ Restore CrawlResult from dict. """ obj = cls( url=data.get("url", ""), title=data.get("title", "") ) obj.markdown_path = data.get( "markdown_path" ) obj.success = data.get( "success", True ) obj.error_message = data.get( "error_message" ) obj.status_code = data.get( "status_code" ) obj.assets = data.get( "assets", [] ) obj.links = data.get( "links", [] ) obj.content_length = data.get( "content_length", 0 ) obj.markdown_length = data.get( "markdown_length", 0 ) obj.word_count = data.get( "word_count", 0 ) obj.tags = data.get( "tags", [] ) obj.metadata = data.get( "metadata", {} ) crawled_at = data.get( "crawled_at" ) if crawled_at: obj.crawled_at = datetime.fromisoformat( crawled_at ) return obj def summary(self): """ Human-readable summary. """ return ( f"CrawlResult(" f"url={self.url}, " f"success={self.success}, " f"markdown_length={self.markdown_length}, " f"assets={len(self.assets)}, " f"links={len(self.links)}" f")" )