import hashlib from pathlib import Path from urllib.parse import urlparse from utils.logger import logger class PathMapper: """ Production-ready URL -> filesystem mapper. Handles: - structure preservation - collision prevention - filename sanitization - cross-platform safety """ def __init__( self, output_dir, include_domain=True, max_filename_length=180 ): self.output_dir = Path( output_dir ) self.include_domain = ( include_domain ) self.max_filename_length = ( max_filename_length ) def url_to_path( self, url: str, extension=".md" ): """ Convert URL -> local file path. """ parsed = urlparse(url) domain = parsed.netloc path = parsed.path.strip("/") # Homepage if not path: path = "index" # Remove trailing slash path = path.rstrip("/") # Remove html extension if path.endswith(".html"): path = path[:-5] # Handle query params if parsed.query: query_hash = self.hash_string( parsed.query )[:10] path += f"__{query_hash}" # Sanitize each segment segments = [] for segment in path.split("/"): segment = self.sanitize_filename( segment ) if not segment: segment = "untitled" segments.append(segment) path = "/".join(segments) # Append extension path += extension # Include domain if self.include_domain: output_path = ( self.output_dir / domain / path ) else: output_path = ( self.output_dir / path ) # Prevent very long filenames output_path = self.shorten_if_needed( output_path ) output_path.parent.mkdir( parents=True, exist_ok=True ) logger.debug( f"Mapped URL -> path: " f"{url} => {output_path}" ) return output_path def asset_url_to_path( self, url: str ): """ Map asset URL to asset path. """ return self.url_to_path( url, extension="" ) def sanitize_filename( self, filename: str ): """ Remove filesystem-invalid chars. """ invalid_chars = [ "<", ">", ":", "\"", "/", "\\", "|", "?", "*", "\n", "\r", "\t" ] for char in invalid_chars: filename = filename.replace( char, "_" ) filename = filename.strip() # Windows reserved names -- the OS blocks these # regardless of extension, so e.g. "CON.txt" is # also unusable on Windows. Check the stem (the # portion before the first dot) against the full # reserved set: CON, PRN, AUX, NUL, COM1..COM9, # LPT1..LPT9. reserved = { "CON", "PRN", "AUX", "NUL", } for i in range(1, 10): reserved.add(f"COM{i}") reserved.add(f"LPT{i}") stem = filename.split(".", 1)[0] if stem.upper() in reserved: filename = "_" + filename return filename def shorten_if_needed( self, path: Path ): """ Prevent excessively long filenames. """ filename = path.name if len(filename) <= self.max_filename_length: return path suffix = path.suffix stem = path.stem hash_part = self.hash_string( filename )[:12] shortened = ( stem[:100] + "__" + hash_part + suffix ) return path.with_name( shortened ) def hash_string( self, value: str ): """ Stable hash helper. """ return hashlib.md5( value.encode("utf-8") ).hexdigest() def relative_path( self, path: Path ): """ Get relative path from output dir. """ try: return path.relative_to( self.output_dir ) except Exception: return path def markdown_path_to_url( self, markdown_path: Path ): """ Reverse mapping helper. """ relative = markdown_path.relative_to( self.output_dir ) return str(relative) def ensure_unique_path( self, path: Path ): """ Prevent overwrite collisions. """ if not path.exists(): return path counter = 1 while True: candidate = path.with_name( f"{path.stem}_{counter}" f"{path.suffix}" ) if not candidate.exists(): return candidate counter += 1 def stats( self ): """ Path mapper statistics. """ return { "output_dir": str( self.output_dir ), "include_domain": ( self.include_domain ) }