eb46e739e7
Playwright-based crawler that renders pages from api.intra-mart.jp and document.intra-mart.jp, extracts main content, and converts to Markdown via Pandoc. SQLite-backed queue drives a resumable sequential pipeline across crawler, extractor, converter, and storage modules. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
64 lines
1.1 KiB
TOML
64 lines
1.1 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "intra-mart-doc-crawler"
|
|
version = "1.0.0"
|
|
description = "Production-grade documentation crawler and markdown exporter"
|
|
authors = [
|
|
{ name = "Iric Do" }
|
|
]
|
|
readme = "README.md"
|
|
requires-python = ">=3.11"
|
|
|
|
dependencies = [
|
|
"playwright>=1.53.0",
|
|
"beautifulsoup4>=4.12.3",
|
|
"lxml>=5.2.2",
|
|
"httpx>=0.28.1",
|
|
"aiofiles>=24.1.0",
|
|
"loguru>=0.7.2",
|
|
"tenacity>=9.0.0",
|
|
"markdownify>=0.13.1",
|
|
"PyYAML>=6.0.2"
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
|
|
dev = [
|
|
"black>=24.4.2",
|
|
"ruff>=0.5.0",
|
|
"pytest>=8.2.2",
|
|
"pytest-asyncio>=0.23.7",
|
|
"mypy>=1.10.1"
|
|
]
|
|
|
|
[tool.setuptools]
|
|
include-package-data = true
|
|
|
|
[tool.setuptools.packages.find]
|
|
include = [
|
|
"crawler*",
|
|
"converter*",
|
|
"extractor*",
|
|
"models*",
|
|
"storage*",
|
|
"utils*"
|
|
]
|
|
|
|
exclude = [
|
|
"output*",
|
|
"data*",
|
|
"logs*"
|
|
]
|
|
|
|
[tool.black]
|
|
line-length = 79
|
|
|
|
[tool.ruff]
|
|
line-length = 79
|
|
target-version = "py311"
|
|
|
|
[tool.pytest.ini_options]
|
|
asyncio_mode = "auto" |