crawler-intra-mart/crawler/crawler.py

import asyncio
from typing import Set

from bs4 import BeautifulSoup

from crawler.playwright_client import PlaywrightClient
from crawler.url_manager import URLManager

from extractor.html_extractor import HTMLExtractor
from extractor.html_cleaner import HTMLCleaner

from converter.markdown_converter import MarkdownConverter
from converter.path_mapper import PathMapper

from storage.file_writer import FileWriter

from utils.logger import logger

from converter.link_rewriter import LinkRewriter
from extractor.asset_downloader import AssetDownloader

class DocumentationCrawler:
    def __init__(self, config):
        self.config = config

        self.client = PlaywrightClient(
            timeout=config["crawl"]["timeout"],
            headless=config["playwright"]["headless"]
        )

        self.url_manager = URLManager(
            config["allowed_domains"]
        )

        self.extractor = HTMLExtractor(
            config["selectors"]["main_content"]
        )

        self.cleaner = HTMLCleaner(
            config["selectors"]["remove"]
        )

        self.converter = MarkdownConverter()

        self.mapper = PathMapper(
            config["output_dir"] + "/markdown"
        )

        self.writer = FileWriter()

        self.link_rewriter = LinkRewriter(
            config["output_dir"] + "/markdown"
        )
        
        self.asset_downloader = AssetDownloader(
            output_dir=config["output_dir"] + "/assets"
        )
        self.delay_seconds = config["crawl"]["delay_seconds"]
        self.max_retries = config["crawl"]["max_retries"]

    async def crawl(self):
        """
        Main crawling entry point
        """

        logger.info("Starting crawler...")

        await self.client.start()

        queue = asyncio.Queue()

        for url in self.config["seed_urls"]:
            await queue.put(url)

        try:
            while not queue.empty():

                current_url = await queue.get()

                current_url = self.url_manager.normalize(current_url)

                if not self.url_manager.should_visit(current_url):
                    continue

                self.url_manager.mark_visited(current_url)

                logger.info(f"Crawling: {current_url}")

                try:
                    await self.process_page(current_url, queue)

                except Exception as e:
                    logger.exception(
                        f"Failed processing {current_url}: {e}"
                    )

                await asyncio.sleep(self.delay_seconds)

        finally:
            await self.client.stop()

            logger.info("Crawler stopped.")

    async def process_page(
        self,
        url: str,
        queue: asyncio.Queue
    ):
        """
        Process single page:
        - render html
        - extract content
        - clean html
        - convert markdown
        - save markdown
        - discover links
        """

        page_data = await self.fetch_with_retry(url)

        if not page_data:
            return

        html = page_data["html"]

        await self.asset_downloader.download_assets_from_html(
            base_url=url,
            html=html
        )
        
        logger.info(f"Extracting content: {url}")

        main_html = self.extractor.extract_main_content(html)

        clean_html = self.cleaner.clean(main_html)

        logger.info(f"Converting markdown: {url}")

        markdown = self.converter.html_to_markdown(
            clean_html
        )

        output_path = self.mapper.url_to_path(url)
        
        markdown = self.link_rewriter.rewrite(
            markdown=markdown,
            current_url=url,
            current_output_path=output_path
        )
        
        self.writer.write_text(
            output_path,
            markdown
        )

        logger.info(f"Saved markdown: {output_path}")

        await self.discover_links(
            base_url=url,
            html=html,
            queue=queue
        )

    async def fetch_with_retry(
        self,
        url: str
    ):
        """
        Retry wrapper for page fetching
        """

        for attempt in range(1, self.max_retries + 1):

            try:
                logger.info(
                    f"Fetching ({attempt}/{self.max_retries}): {url}"
                )

                return await self.client.fetch_page(url)

            except Exception as e:

                logger.warning(
                    f"Retry {attempt} failed for {url}: {e}"
                )

                await asyncio.sleep(2)

        logger.error(f"Max retries exceeded: {url}")

        return None

    async def discover_links(
        self,
        base_url: str,
        html: str,
        queue: asyncio.Queue
    ):
        """
        Extract all links from page
        and push valid links into queue
        """

        soup = BeautifulSoup(html, "lxml")

        links_found = 0

        for a in soup.find_all("a", href=True):

            href = a["href"].strip()

            if not href:
                continue

            if href.startswith("#"):
                continue

            if href.startswith("mailto:"):
                continue

            if href.startswith("javascript:"):
                continue

            try:
                next_url = self.url_manager.resolve(
                    base_url,
                    href
                )

                next_url = self.url_manager.normalize(
                    next_url
                )

                if self.url_manager.should_visit(next_url):

                    await queue.put(next_url)

                    links_found += 1

            except Exception as e:

                logger.warning(
                    f"Invalid URL {href} from {base_url}: {e}"
                )

        logger.info(
            f"Discovered {links_found} new links from {base_url}"
        )