. Use a # body-scoped selector so we don't match `.main` # elements nested inside navigation widgets. "body > div.main", # document.intra-mart.jp/library/... — most user-guide # pages put their content in a single top-level #

. "body > div.container", ] MAX_PAGES_PER_RUN = 100000 MAX_ATTEMPTS = 3 BROWSER_RECYCLE_EVERY = 500 class _PermanentFetchFailure(Exception): """Raised when re-fetching the URL cannot succeed (4xx).""" # Number of concurrent page workers. Each worker shares the # single PlaywrightContext + single httpx client; only DB # writes serialize through self.db_lock. The bottleneck is # network I/O, so this is the multiplier on throughput. CONCURRENCY = 8 class DocumentationCrawler: """ Main crawler orchestrator. """ def __init__(self): self.client = PlaywrightClient( headless=True, content_selectors=CONTENT_SELECTORS, ) self.http_client = HTTPClient() self.url_manager = URLManager( allowed_domains=ALLOWED_DOMAINS ) self.sphinx_discovery = SphinxDiscovery( url_manager=self.url_manager ) self.navigation_parser = NavigationParser( url_manager=self.url_manager ) self.navigation_trees = [] self.cleaner = HTMLCleaner( REMOVE_SELECTORS ) self.extractor = HTMLExtractor( CONTENT_SELECTORS ) self.asset_downloader = AssetDownloader( output_dir=str(Path(OUTPUT_DIR) / "assets"), http_client=self.http_client.client, ) self.converter = MarkdownConverter() self.writer = FileWriter() self.db = MetadataDB() self.path_mapper = PathMapper( Path(OUTPUT_DIR) / "markdown" ) self.processed_this_run = 0 # Single lock serializing DB write blocks across # concurrent workers. SQLite + Python's sqlite3 module # already mutex on the connection, but a transaction() # context spans many statements — without this lock, # two workers could interleave statements mid-tx. self.db_lock = asyncio.Lock() # In-flight worker count for clean shutdown: a worker # that sees an empty queue must wait until all peers # finish (their process_page may enqueue new URLs). self._in_flight = 0 self._stop = False # Track domains we've already navigation-parsed. # Sidebars are site-wide on Sphinx/Javadoc, so the same # tree is rediscovered on every page — running the # heavy parser repeatedly wastes CPU. Any per-section # URLs we miss here will still be picked up by # extract_links (it iterates all ). self._nav_parsed_domains = set() async def initialize(self): """ Initialize crawler resources. """ logger.info( "Initializing crawler..." ) self.db.reset_stuck_processing() await self.client.start() for url in START_URLS: normalized = self.url_manager.normalize(url) if normalized and self.url_manager.is_allowed(normalized): self.db.enqueue_url( normalized, parent_url=None, depth=0, priority=100 ) for url in START_URLS: discovered = await self.sphinx_discovery.discover_from_base(url) for discovered_url in discovered: if not self.url_manager.should_enqueue(discovered_url): continue if self.db.is_done(discovered_url): continue if self.db.is_known(discovered_url): continue self.db.enqueue_url( discovered_url, parent_url=url, depth=1, priority=50 ) self.db.save_discovery_source( url=discovered_url, discovery_type="sphinx_base_discovery", source_url=url ) logger.info(f"Initial queue stats: {self.db.queue_stats()}") async def shutdown(self): """ Cleanup resources. """ logger.info( "Shutting down crawler..." ) self.export_summary() await self.client.stop() await self.http_client.aclose() self.db.close() logger.info( "Crawler shutdown complete." ) async def crawl(self): """ Main crawl entry point. Spawns CONCURRENCY workers that share the single browser context + httpx client and serialize through self.db_lock for DB writes. """ await self.initialize() try: workers = [ asyncio.create_task(self._worker(i)) for i in range(CONCURRENCY) ] await asyncio.gather(*workers) finally: await self.shutdown() async def _claim_next_url(self): """ Atomically pick the next pending URL and mark it processing. Returns None when the queue is empty. """ async with self.db_lock: item = self.db.get_next_pending_url() if not item: return None url = item["url"] if self.db.is_done(url): # Stale row — skip without claiming. return "skip" self.db.mark_processing(url) self._in_flight += 1 return item async def _worker(self, worker_id: int): """ One concurrent worker. Loops until the queue stays empty *and* no peer worker is in-flight (peers can enqueue new URLs as they discover links). """ while not self._stop: if self.processed_this_run >= MAX_PAGES_PER_RUN: return item = await self._claim_next_url() if item == "skip": continue if item is None: # No work available — but a peer might enqueue # more. Exit only if everyone else is idle too. if self._in_flight == 0: return await asyncio.sleep(0.3) continue url = item["url"] depth = item["depth"] try: result = await self.process_page( url=url, depth=depth, ) async with self.db_lock: self.db.mark_done(url) if result and result.url != url: self.db.mark_done(result.url) self.processed_this_run += 1 n = self.processed_this_run if n % 20 == 0: logger.info( f"Progress: {n} pages this run. " f"Queue: {self.db.queue_stats()}" ) if n % 100 == 0: self.export_summary() logger.info( "Intermediate summary exported." ) except asyncio.CancelledError: raise except _PermanentFetchFailure as e: # Skip the retry ramp — re-fetching can't # succeed. logger.warning(f"Permanent failure on {url}: {e}") async with self.db_lock: self.db.mark_permanently_failed(url, str(e)) except Exception as e: logger.exception( f"Failed processing {url}: {e}" ) async with self.db_lock: self.db.mark_failed( url, str(e), max_attempts=MAX_ATTEMPTS, ) finally: async with self.db_lock: self._in_flight -= 1 async def _fetch_and_extract(self, url: str): """ Try a cheap HTTP fetch first. If a configured content selector matches we're done. Otherwise fall back to a full Playwright render (handles JS-only pages). Returns (response_dict, extraction_dict) or (None, None) when neither path produces usable content. """ # Conditional GET: ask the server whether the page # has changed since we last cached it. A 304 lets us # skip all downstream work. etag, last_modified = self.db.get_cached_headers(url) http_response = await self.http_client.fetch_page( url, etag=etag, last_modified=last_modified, ) if http_response is not None and http_response.get("is_binary"): return http_response, None if http_response is not None and http_response.get("not_modified"): # Signal "nothing changed" to process_page. return http_response, None # Hard 4xx (404, 410, 451, ...): there is nothing # Playwright can do that httpx cannot. Surface this so # the worker can mark the URL permanently failed and # stop wasting retries. if ( http_response is not None and http_response.get("status_code") is not None and 400 <= http_response["status_code"] < 500 ): return http_response, None if http_response is not None and http_response.get("ok"): # Parse the HTML once here. The cleaner mutates # the soup in place; the extractor uses it; assets # later use the matched node. No re-parses. cleaning_soup = BeautifulSoup( http_response["html"], "lxml" ) self.cleaner.clean(cleaning_soup) extraction = self.extractor.extract_with_status( cleaning_soup ) if extraction["matched"]: logger.info(f"HTTP fast-path success: {url}") return http_response, extraction logger.info( f"HTTP fetch ok but no content selector matched " f"— falling back to Playwright: {url}" ) # Fallback: full browser render pw_response = await self.client.fetch_page(url) if pw_response.get("is_binary"): return pw_response, None cleaning_soup = BeautifulSoup(pw_response["html"], "lxml") self.cleaner.clean(cleaning_soup) extraction = self.extractor.extract_with_status( cleaning_soup ) return pw_response, extraction async def process_page(self, url: str, depth: int): """ Process single page. """ logger.info( f"Crawling: {url}" ) if self.url_manager.is_binary_url(url): logger.info(f"Skipping binary URL as page: {url}") return None response, extraction = await self._fetch_and_extract(url) if response is None: return None if response.get("is_binary"): logger.info(f"Skipping binary response: {url}") return None # 304 Not Modified: skip the entire pipeline. Worker # will mark_done as usual. if response.get("not_modified"): logger.info(f"Not modified, skipping: {url}") return CrawlResult( status_code=304, url=url, title="", markdown_path="", links=[], success=True, ) # _fetch_and_extract returns extraction=None with a # 4xx response when the URL is a permanent failure # (404 etc.). Raise a typed error so the worker maps # it to mark_permanently_failed instead of retrying. status_code = response.get("status_code") if extraction is None and status_code and 400 <= status_code < 500: raise _PermanentFetchFailure( f"HTTP {status_code} on {url}" ) final_url = response.get("url", url) html = response["html"] title = response["title"] # The extracted node lives inside the cleaning soup # already produced in _fetch_and_extract. Use it # directly to avoid one BS4 re-parse per page. extracted_node = extraction["node"] # Resolve output path first output_path = self.path_mapper.url_to_path( final_url ) # Download assets — pass the already-parsed node # rather than its serialized HTML string. downloaded_assets = await self.asset_downloader.download_assets_from_html( base_url=final_url, html=extracted_node, ) async with self.db_lock: with self.db.transaction(): for asset in downloaded_assets: self.db.save_asset( page_url=final_url, asset_url=asset["url"], local_path=asset["local_path"] ) # Rewrite asset links in place on the same parsed # node — no re-parse needed. self.asset_downloader.rewrite_asset_links( html=extracted_node, base_url=final_url, current_output_path=output_path, ) # Convert markdown markdown = self.converter.html_to_markdown( str(extracted_node), title=title, source_url=final_url, media_dir=Path(OUTPUT_DIR) / "assets" / "pandoc", ) self.writer.write_text( output_path, markdown ) # Parse the ORIGINAL html ONCE for all three # downstream consumers. Previously each one re-parsed # the same string independently. original_soup = BeautifulSoup(html, "lxml") discovered_links = self.extract_links( final_url, original_soup, ) sphinx_links = await self.sphinx_discovery.discover_from_rendered_html( base_url=final_url, html=original_soup, ) # Only parse navigation the first time we visit a # given domain. The sidebar tree is site-wide and # repeating the parse on every page is the single # most expensive bit of per-page CPU we still have. nav_domain = self.url_manager.extract_domain(final_url) if nav_domain in self._nav_parsed_domains: navigation_result = {"urls": [], "tree": []} else: navigation_result = self.navigation_parser.parse_navigation( base_url=final_url, html=original_soup, ) self._nav_parsed_domains.add(nav_domain) self.navigation_trees.extend( navigation_result["tree"] ) for nav_url in navigation_result["urls"]: if nav_url not in discovered_links: discovered_links.append(nav_url) logger.info( f"SPHINX_LINKS: {len(sphinx_links)} from {final_url}" ) for sphinx_link in sphinx_links: if sphinx_link not in discovered_links: discovered_links.append(sphinx_link) # Batch every per-link write + the page row into one # transaction per page — turns hundreds of fsyncs into # one, and one acquisition of db_lock per worker turn. async with self.db_lock: with self.db.transaction(): for link in discovered_links: logger.debug(f"DISCOVERED: {link}") self.db.save_page_link( source_url=final_url, target_url=link, link_type="sphinx_or_html_internal" ) if not self.url_manager.should_enqueue(link): continue if self.db.is_done(link): continue if self.db.is_known(link): continue logger.debug(f"ENQUEUED: {link}") self.db.enqueue_url( url=link, parent_url=final_url, depth=depth + 1, priority=0 ) self.db.save_discovery_source( url=link, discovery_type="page_internal_link", source_url=final_url ) self.db.save_page( url=final_url, title=title, markdown_path=str(output_path), status="success", status_code=response.get( "status_code", 200 ), markdown_length=len(markdown), word_count=len( markdown.split() ), metadata={ "depth": depth, "links_found": len(discovered_links), "assets_found": len(downloaded_assets) }, etag=response.get("etag"), last_modified=response.get("last_modified"), ) logger.debug( f"Saved markdown: " f"{output_path}" ) return CrawlResult( status_code=response.get("status_code"), url=final_url, title=title, markdown_path=str(output_path), links=discovered_links, success=True ) def extract_links( self, base_url, html ): """ Extract crawlable links. Accepts an HTML string or a parsed BS4 soup/Tag. """ if hasattr(html, "select") and hasattr(html, "find_all"): soup = html else: soup = BeautifulSoup(html, "lxml") discovered = set() for a in soup.find_all( "a", href=True ): href = a.get("href") resolved = self.url_manager.resolve( base_url, href ) if not resolved: continue if not self.url_manager.is_allowed( resolved ): continue discovered.add( resolved ) selectors = [ "a.reference.internal", ".toctree-wrapper a", ".wy-menu a", ".sphinxsidebar a", ".globaltoc a", ".localtoc a", "link[rel='next']", "link[rel='prev']", "link[rel='up']" ] for selector in selectors: for node in soup.select(selector): href = node.get("href") resolved = self.url_manager.resolve( base_url, href ) if not resolved: continue if not self.url_manager.is_allowed(resolved): continue discovered.add(resolved) return list(discovered) def export_summary(self): """ Export crawl summary. """ summary_path = ( Path(OUTPUT_DIR) / "crawl_summary.json" ) data = { "processed_this_run": self.processed_this_run, "queue_stats": self.db.queue_stats() } self.writer.write_json( summary_path, data ) graph_path = ( Path(OUTPUT_DIR) / "site_graph.json" ) self.db.export_site_graph( graph_path ) navigation_tree = self.navigation_parser.dedupe_tree( self.navigation_trees ) navigation_path = ( Path(OUTPUT_DIR) / "navigation_tree.json" ) self.navigation_parser.export_tree_json( navigation_tree, navigation_path ) summary_md = self.navigation_parser.generate_summary_markdown( navigation_tree ) summary_md_path = ( Path(OUTPUT_DIR) / "SUMMARY.md" ) self.writer.write_text( summary_md_path, summary_md ) logger.info( f"Summary exported: " f"{summary_path}" ) async def recycle_browser(self): """ Restart Playwright browser/context periodically to avoid long-running memory/socket leaks. """ logger.info("Recycling browser context...") await self.client.stop() await asyncio.sleep(2) await self.client.start() logger.info("Browser context recycled.") async def async_main(): crawler = DocumentationCrawler() await crawler.crawl() def main(): asyncio.run( async_main() ) if __name__ == "__main__": main()