eb46e739e7
Playwright-based crawler that renders pages from api.intra-mart.jp and document.intra-mart.jp, extracts main content, and converts to Markdown via Pandoc. SQLite-backed queue drives a resumable sequential pipeline across crawler, extractor, converter, and storage modules. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
98 lines
2.1 KiB
Python
98 lines
2.1 KiB
Python
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from loguru import logger
|
|
|
|
|
|
LOG_DIR = Path("logs")
|
|
|
|
LOG_DIR.mkdir(
|
|
parents=True,
|
|
exist_ok=True
|
|
)
|
|
|
|
|
|
def setup_logger():
|
|
"""
|
|
Configure production logger.
|
|
"""
|
|
|
|
logger.remove()
|
|
|
|
# Console logger
|
|
logger.add(
|
|
sys.stdout,
|
|
level="INFO",
|
|
colorize=True,
|
|
enqueue=True,
|
|
backtrace=True,
|
|
diagnose=True,
|
|
format=(
|
|
"<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
|
|
"<level>{level: <8}</level> | "
|
|
"<cyan>{name}</cyan>:"
|
|
"<cyan>{function}</cyan>:"
|
|
"<cyan>{line}</cyan> - "
|
|
"<level>{message}</level>"
|
|
)
|
|
)
|
|
|
|
# Main crawler log
|
|
logger.add(
|
|
LOG_DIR / "crawler.log",
|
|
level="INFO",
|
|
rotation="50 MB",
|
|
retention="30 days",
|
|
compression="zip",
|
|
enqueue=True,
|
|
encoding="utf-8",
|
|
backtrace=True,
|
|
diagnose=True,
|
|
format=(
|
|
"{time:YYYY-MM-DD HH:mm:ss} | "
|
|
"{level: <8} | "
|
|
"{name}:{function}:{line} - "
|
|
"{message}"
|
|
)
|
|
)
|
|
|
|
# Error log
|
|
logger.add(
|
|
LOG_DIR / "errors.log",
|
|
level="ERROR",
|
|
rotation="20 MB",
|
|
retention="60 days",
|
|
compression="zip",
|
|
enqueue=True,
|
|
encoding="utf-8",
|
|
backtrace=True,
|
|
diagnose=True,
|
|
format=(
|
|
"{time:YYYY-MM-DD HH:mm:ss} | "
|
|
"{level: <8} | "
|
|
"{name}:{function}:{line} - "
|
|
"{message}"
|
|
)
|
|
)
|
|
|
|
# Debug log: very chatty (every DISCOVERED/ENQUEUED, every
|
|
# save_*), so only enable when explicitly asked for.
|
|
# Writing+rotating 100 MB of debug log per run silently
|
|
# bottlenecks the crawler.
|
|
if os.environ.get("CRAWLER_DEBUG"):
|
|
logger.add(
|
|
LOG_DIR / "debug.log",
|
|
level="DEBUG",
|
|
rotation="100 MB",
|
|
retention="14 days",
|
|
compression="zip",
|
|
enqueue=True,
|
|
encoding="utf-8",
|
|
)
|
|
|
|
logger.info("Logger initialized.")
|
|
|
|
|
|
# Initialize automatically
|
|
setup_logger() |