eb46e739e7
Playwright-based crawler that renders pages from api.intra-mart.jp and document.intra-mart.jp, extracts main content, and converts to Markdown via Pandoc. SQLite-backed queue drives a resumable sequential pipeline across crawler, extractor, converter, and storage modules. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
201 lines
3.9 KiB
Python
201 lines
3.9 KiB
Python
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
class LinkRewriter:
|
|
"""
|
|
Rewrite internal intra-mart links
|
|
from online URLs -> local markdown paths
|
|
|
|
Example:
|
|
|
|
https://api.intra-mart.jp/iap/auth/login.html
|
|
|
|
=>
|
|
../auth/login.md
|
|
"""
|
|
|
|
def __init__(self, output_dir: str):
|
|
self.output_dir = Path(output_dir)
|
|
|
|
def rewrite(
|
|
self,
|
|
markdown: str,
|
|
current_url: str,
|
|
current_output_path: Path
|
|
):
|
|
"""
|
|
Rewrite markdown links
|
|
|
|
Parameters
|
|
----------
|
|
markdown : str
|
|
markdown content
|
|
|
|
current_url : str
|
|
original page url
|
|
|
|
current_output_path : Path
|
|
local markdown file path
|
|
"""
|
|
|
|
lines = markdown.splitlines()
|
|
|
|
rewritten_lines = []
|
|
|
|
for line in lines:
|
|
|
|
rewritten = self.rewrite_markdown_links(
|
|
line,
|
|
current_output_path
|
|
)
|
|
|
|
rewritten_lines.append(rewritten)
|
|
|
|
return "\n".join(rewritten_lines)
|
|
|
|
def rewrite_markdown_links(
|
|
self,
|
|
line: str,
|
|
current_output_path: Path
|
|
):
|
|
"""
|
|
Rewrite markdown inline links
|
|
|
|
Example:
|
|
[API](https://api.intra-mart.jp/iap/auth.html)
|
|
|
|
->
|
|
[API](../auth.md)
|
|
"""
|
|
|
|
import re
|
|
|
|
pattern = r"\[([^\]]+)\]\(([^)]+)\)"
|
|
|
|
matches = re.findall(pattern, line)
|
|
|
|
if not matches:
|
|
return line
|
|
|
|
rewritten_line = line
|
|
|
|
for text, url in matches:
|
|
|
|
if not self.is_internal_url(url):
|
|
continue
|
|
|
|
local_path = self.url_to_markdown_path(url)
|
|
|
|
relative_path = self.make_relative_path(
|
|
current_output_path,
|
|
local_path
|
|
)
|
|
|
|
old = f"[{text}]({url})"
|
|
|
|
new = f"[{text}]({relative_path})"
|
|
|
|
rewritten_line = rewritten_line.replace(
|
|
old,
|
|
new
|
|
)
|
|
|
|
return rewritten_line
|
|
|
|
def is_internal_url(self, url: str):
|
|
"""
|
|
Check if URL belongs to intra-mart docs
|
|
"""
|
|
|
|
parsed = urlparse(url)
|
|
|
|
allowed_domains = [
|
|
"api.intra-mart.jp",
|
|
"document.intra-mart.jp"
|
|
]
|
|
|
|
return any(
|
|
domain in parsed.netloc
|
|
for domain in allowed_domains
|
|
)
|
|
|
|
def url_to_markdown_path(self, url: str):
|
|
"""
|
|
Convert URL -> local markdown path
|
|
|
|
Example:
|
|
https://api.intra-mart.jp/iap/auth/login.html
|
|
|
|
=>
|
|
output/markdown/iap/auth/login.md
|
|
"""
|
|
|
|
parsed = urlparse(url)
|
|
|
|
path = parsed.path.strip("/")
|
|
|
|
if not path:
|
|
path = "index"
|
|
|
|
if path.endswith(".html"):
|
|
path = path[:-5]
|
|
|
|
local_path = (
|
|
self.output_dir /
|
|
(path + ".md")
|
|
)
|
|
|
|
return local_path
|
|
|
|
def make_relative_path(
|
|
self,
|
|
current_file: Path,
|
|
target_file: Path
|
|
):
|
|
"""
|
|
Generate relative markdown path
|
|
|
|
Example:
|
|
|
|
current:
|
|
output/markdown/iap/index.md
|
|
|
|
target:
|
|
output/markdown/iap/auth/login.md
|
|
|
|
=>
|
|
auth/login.md
|
|
"""
|
|
|
|
current_dir = current_file.parent
|
|
|
|
relative = target_file.relative_to(
|
|
self.output_dir
|
|
)
|
|
|
|
target_absolute = self.output_dir / relative
|
|
|
|
relative_path = Path(
|
|
target_absolute.relative_to(current_dir)
|
|
)
|
|
|
|
try:
|
|
relative_path = target_absolute.relative_to(
|
|
current_dir
|
|
)
|
|
|
|
return str(relative_path).replace("\\", "/")
|
|
|
|
except Exception:
|
|
|
|
import os
|
|
|
|
relative_path = os.path.relpath(
|
|
target_absolute,
|
|
current_dir
|
|
)
|
|
|
|
return relative_path.replace("\\", "/") |