import os import sys from pathlib import Path from loguru import logger LOG_DIR = Path("logs") LOG_DIR.mkdir( parents=True, exist_ok=True ) def setup_logger(): """ Configure production logger. """ logger.remove() # Console logger logger.add( sys.stdout, level="INFO", colorize=True, enqueue=True, backtrace=True, diagnose=True, format=( "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:" "{function}:" "{line} - " "{message}" ) ) # Main crawler log logger.add( LOG_DIR / "crawler.log", level="INFO", rotation="50 MB", retention="30 days", compression="zip", enqueue=True, encoding="utf-8", backtrace=True, diagnose=True, format=( "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:{function}:{line} - " "{message}" ) ) # Error log logger.add( LOG_DIR / "errors.log", level="ERROR", rotation="20 MB", retention="60 days", compression="zip", enqueue=True, encoding="utf-8", backtrace=True, diagnose=True, format=( "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:{function}:{line} - " "{message}" ) ) # Debug log: very chatty (every DISCOVERED/ENQUEUED, every # save_*), so only enable when explicitly asked for. # Writing+rotating 100 MB of debug log per run silently # bottlenecks the crawler. if os.environ.get("CRAWLER_DEBUG"): logger.add( LOG_DIR / "debug.log", level="DEBUG", rotation="100 MB", retention="14 days", compression="zip", enqueue=True, encoding="utf-8", ) logger.info("Logger initialized.") # Initialize automatically setup_logger()