eb46e739e7
Playwright-based crawler that renders pages from api.intra-mart.jp and document.intra-mart.jp, extracts main content, and converts to Markdown via Pandoc. SQLite-backed queue drives a resumable sequential pipeline across crawler, extractor, converter, and storage modules. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
481 lines
8.8 KiB
Python
481 lines
8.8 KiB
Python
import json
|
|
import sqlite3
|
|
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
|
|
from utils.logger import logger
|
|
|
|
|
|
class MetadataDB:
|
|
"""
|
|
Metadata storage layer
|
|
for crawler state tracking.
|
|
|
|
Stores:
|
|
- crawled pages
|
|
- crawl status
|
|
- markdown outputs
|
|
- assets
|
|
- hashes
|
|
- timestamps
|
|
- incremental update info
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
db_path="data/metadata.db"
|
|
):
|
|
self.db_path = Path(db_path)
|
|
|
|
self.db_path.parent.mkdir(
|
|
parents=True,
|
|
exist_ok=True
|
|
)
|
|
|
|
self.conn = sqlite3.connect(
|
|
self.db_path
|
|
)
|
|
|
|
self.conn.row_factory = sqlite3.Row
|
|
|
|
self.initialize()
|
|
|
|
def initialize(self):
|
|
"""
|
|
Create required tables.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
# Pages table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS pages (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
url TEXT UNIQUE,
|
|
title TEXT,
|
|
|
|
markdown_path TEXT,
|
|
|
|
status TEXT,
|
|
status_code INTEGER,
|
|
|
|
content_hash TEXT,
|
|
|
|
word_count INTEGER,
|
|
markdown_length INTEGER,
|
|
|
|
crawled_at TEXT,
|
|
updated_at TEXT,
|
|
|
|
parent_url TEXT,
|
|
depth INTEGER,
|
|
|
|
metadata_json TEXT
|
|
)
|
|
""")
|
|
|
|
# Assets table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS assets (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
page_url TEXT,
|
|
|
|
asset_url TEXT,
|
|
local_path TEXT,
|
|
|
|
downloaded_at TEXT
|
|
)
|
|
""")
|
|
|
|
# Crawl queue state
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS crawl_queue (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
url TEXT UNIQUE,
|
|
|
|
status TEXT,
|
|
|
|
discovered_at TEXT
|
|
)
|
|
""")
|
|
|
|
# Failed pages
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS failures (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
url TEXT,
|
|
|
|
error_message TEXT,
|
|
|
|
failed_at TEXT
|
|
)
|
|
""")
|
|
|
|
self.conn.commit()
|
|
|
|
logger.info(
|
|
"Metadata database initialized."
|
|
)
|
|
|
|
def save_page(
|
|
self,
|
|
url,
|
|
title="",
|
|
markdown_path="",
|
|
status="success",
|
|
status_code=200,
|
|
content_hash="",
|
|
word_count=0,
|
|
markdown_length=0,
|
|
parent_url=None,
|
|
depth=0,
|
|
metadata=None
|
|
):
|
|
"""
|
|
Insert or update page metadata.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
now = datetime.utcnow().isoformat()
|
|
|
|
metadata_json = json.dumps(
|
|
metadata or {},
|
|
ensure_ascii=False
|
|
)
|
|
|
|
cursor.execute("""
|
|
INSERT INTO pages (
|
|
url,
|
|
title,
|
|
markdown_path,
|
|
status,
|
|
status_code,
|
|
content_hash,
|
|
word_count,
|
|
markdown_length,
|
|
crawled_at,
|
|
updated_at,
|
|
parent_url,
|
|
depth,
|
|
metadata_json
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
|
|
ON CONFLICT(url)
|
|
DO UPDATE SET
|
|
title=excluded.title,
|
|
markdown_path=excluded.markdown_path,
|
|
status=excluded.status,
|
|
status_code=excluded.status_code,
|
|
content_hash=excluded.content_hash,
|
|
word_count=excluded.word_count,
|
|
markdown_length=excluded.markdown_length,
|
|
updated_at=excluded.updated_at,
|
|
metadata_json=excluded.metadata_json
|
|
""", (
|
|
url,
|
|
title,
|
|
markdown_path,
|
|
status,
|
|
status_code,
|
|
content_hash,
|
|
word_count,
|
|
markdown_length,
|
|
now,
|
|
now,
|
|
parent_url,
|
|
depth,
|
|
metadata_json
|
|
))
|
|
|
|
self.conn.commit()
|
|
|
|
logger.info(
|
|
f"Saved page metadata: {url}"
|
|
)
|
|
|
|
def save_asset(
|
|
self,
|
|
page_url,
|
|
asset_url,
|
|
local_path
|
|
):
|
|
"""
|
|
Save asset metadata.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
INSERT INTO assets (
|
|
page_url,
|
|
asset_url,
|
|
local_path,
|
|
downloaded_at
|
|
)
|
|
VALUES (?, ?, ?, ?)
|
|
""", (
|
|
page_url,
|
|
asset_url,
|
|
local_path,
|
|
datetime.utcnow().isoformat()
|
|
))
|
|
|
|
self.conn.commit()
|
|
|
|
def save_failure(
|
|
self,
|
|
url,
|
|
error_message
|
|
):
|
|
"""
|
|
Save failed crawl.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
INSERT INTO failures (
|
|
url,
|
|
error_message,
|
|
failed_at
|
|
)
|
|
VALUES (?, ?, ?)
|
|
""", (
|
|
url,
|
|
error_message,
|
|
datetime.utcnow().isoformat()
|
|
))
|
|
|
|
self.conn.commit()
|
|
|
|
logger.warning(
|
|
f"Saved failure: {url}"
|
|
)
|
|
|
|
def enqueue_url(
|
|
self,
|
|
url,
|
|
status="pending"
|
|
):
|
|
"""
|
|
Save crawl queue item.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
INSERT OR IGNORE INTO crawl_queue (
|
|
url,
|
|
status,
|
|
discovered_at
|
|
)
|
|
VALUES (?, ?, ?)
|
|
""", (
|
|
url,
|
|
status,
|
|
datetime.utcnow().isoformat()
|
|
))
|
|
|
|
self.conn.commit()
|
|
|
|
def update_queue_status(
|
|
self,
|
|
url,
|
|
status
|
|
):
|
|
"""
|
|
Update queue item status.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
UPDATE crawl_queue
|
|
SET status=?
|
|
WHERE url=?
|
|
""", (
|
|
status,
|
|
url
|
|
))
|
|
|
|
self.conn.commit()
|
|
|
|
def page_exists(
|
|
self,
|
|
url
|
|
):
|
|
"""
|
|
Check if page already crawled.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT 1
|
|
FROM pages
|
|
WHERE url=?
|
|
LIMIT 1
|
|
""", (url,))
|
|
|
|
row = cursor.fetchone()
|
|
|
|
return row is not None
|
|
|
|
def get_page(
|
|
self,
|
|
url
|
|
):
|
|
"""
|
|
Retrieve page metadata.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT *
|
|
FROM pages
|
|
WHERE url=?
|
|
""", (url,))
|
|
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
return None
|
|
|
|
return dict(row)
|
|
|
|
def get_all_pages(self):
|
|
"""
|
|
Get all crawled pages.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT *
|
|
FROM pages
|
|
ORDER BY crawled_at DESC
|
|
""")
|
|
|
|
rows = cursor.fetchall()
|
|
|
|
return [
|
|
dict(row)
|
|
for row in rows
|
|
]
|
|
|
|
def get_failed_pages(self):
|
|
"""
|
|
Get all failures.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT *
|
|
FROM failures
|
|
ORDER BY failed_at DESC
|
|
""")
|
|
|
|
rows = cursor.fetchall()
|
|
|
|
return [
|
|
dict(row)
|
|
for row in rows
|
|
]
|
|
|
|
def get_pending_queue(self):
|
|
"""
|
|
Get pending URLs.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT *
|
|
FROM crawl_queue
|
|
WHERE status='pending'
|
|
""")
|
|
|
|
rows = cursor.fetchall()
|
|
|
|
return [
|
|
dict(row)
|
|
for row in rows
|
|
]
|
|
|
|
def delete_page(
|
|
self,
|
|
url
|
|
):
|
|
"""
|
|
Remove page metadata.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
DELETE FROM pages
|
|
WHERE url=?
|
|
""", (url,))
|
|
|
|
self.conn.commit()
|
|
|
|
def clear_failures(self):
|
|
"""
|
|
Clear failure table.
|
|
"""
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute("""
|
|
DELETE FROM failures
|
|
""")
|
|
|
|
self.conn.commit()
|
|
|
|
def export_pages_json(
|
|
self,
|
|
output_path
|
|
):
|
|
"""
|
|
Export all pages metadata.
|
|
"""
|
|
|
|
pages = self.get_all_pages()
|
|
|
|
with open(
|
|
output_path,
|
|
"w",
|
|
encoding="utf-8"
|
|
) as f:
|
|
|
|
json.dump(
|
|
pages,
|
|
f,
|
|
ensure_ascii=False,
|
|
indent=2
|
|
)
|
|
|
|
logger.info(
|
|
f"Exported metadata JSON: "
|
|
f"{output_path}"
|
|
)
|
|
|
|
def close(self):
|
|
"""
|
|
Close database connection.
|
|
"""
|
|
|
|
self.conn.close()
|
|
|
|
logger.info(
|
|
"Metadata database closed."
|
|
) |