import os
import sys
from pathlib import Path
from loguru import logger
LOG_DIR = Path("logs")
LOG_DIR.mkdir(
parents=True,
exist_ok=True
)
def setup_logger():
"""
Configure production logger.
"""
logger.remove()
# Console logger
logger.add(
sys.stdout,
level="INFO",
colorize=True,
enqueue=True,
backtrace=True,
diagnose=True,
format=(
"{time:YYYY-MM-DD HH:mm:ss} | "
"{level: <8} | "
"{name}:"
"{function}:"
"{line} - "
"{message}"
)
)
# Main crawler log
logger.add(
LOG_DIR / "crawler.log",
level="INFO",
rotation="50 MB",
retention="30 days",
compression="zip",
enqueue=True,
encoding="utf-8",
backtrace=True,
diagnose=True,
format=(
"{time:YYYY-MM-DD HH:mm:ss} | "
"{level: <8} | "
"{name}:{function}:{line} - "
"{message}"
)
)
# Error log
logger.add(
LOG_DIR / "errors.log",
level="ERROR",
rotation="20 MB",
retention="60 days",
compression="zip",
enqueue=True,
encoding="utf-8",
backtrace=True,
diagnose=True,
format=(
"{time:YYYY-MM-DD HH:mm:ss} | "
"{level: <8} | "
"{name}:{function}:{line} - "
"{message}"
)
)
# Debug log: very chatty (every DISCOVERED/ENQUEUED, every
# save_*), so only enable when explicitly asked for.
# Writing+rotating 100 MB of debug log per run silently
# bottlenecks the crawler.
if os.environ.get("CRAWLER_DEBUG"):
logger.add(
LOG_DIR / "debug.log",
level="DEBUG",
rotation="100 MB",
retention="14 days",
compression="zip",
enqueue=True,
encoding="utf-8",
)
logger.info("Logger initialized.")
# Initialize automatically
setup_logger()