breakpilot-lehrer/klausur-service/backend/github_crawler_core.py

"""
GitHub Crawler - Core Crawler and Downloader

GitHubCrawler for API-based repository crawling and RepositoryDownloader
for ZIP-based local extraction.

Extracted from github_crawler.py to keep files under 500 LOC.
"""

import asyncio
import logging
import os
import shutil
import tempfile
import zipfile
from fnmatch import fnmatch
from pathlib import Path
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
from urllib.parse import urlparse

import httpx

from template_sources import SourceConfig
from github_crawler_parsers import (
    ExtractedDocument,
    MarkdownParser,
    HTMLParser,
    JSONParser,
)

logger = logging.getLogger(__name__)

# Configuration
GITHUB_API_URL = "https://api.github.com"
GITLAB_API_URL = "https://gitlab.com/api/v4"
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")
MAX_FILE_SIZE = 1024 * 1024  # 1 MB max file size
REQUEST_TIMEOUT = 60.0
RATE_LIMIT_DELAY = 1.0


class GitHubCrawler:
    """Crawl GitHub repositories for legal templates."""

    def __init__(self, token: Optional[str] = None):
        self.token = token or GITHUB_TOKEN
        self.headers = {
            "Accept": "application/vnd.github.v3+json",
            "User-Agent": "LegalTemplatesCrawler/1.0",
        }
        if self.token:
            self.headers["Authorization"] = f"token {self.token}"

        self.http_client: Optional[httpx.AsyncClient] = None

    async def __aenter__(self):
        self.http_client = httpx.AsyncClient(
            timeout=REQUEST_TIMEOUT,
            headers=self.headers,
            follow_redirects=True,
        )
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        if self.http_client:
            await self.http_client.aclose()

    def _parse_repo_url(self, url: str) -> Tuple[str, str, str]:
        """Parse repository URL into owner, repo, and host."""
        parsed = urlparse(url)
        path_parts = parsed.path.strip('/').split('/')

        if len(path_parts) < 2:
            raise ValueError(f"Invalid repository URL: {url}")

        owner = path_parts[0]
        repo = path_parts[1].replace('.git', '')

        if 'gitlab' in parsed.netloc:
            host = 'gitlab'
        else:
            host = 'github'

        return owner, repo, host

    async def get_default_branch(self, owner: str, repo: str) -> str:
        """Get the default branch of a repository."""
        if not self.http_client:
            raise RuntimeError("Crawler not initialized. Use 'async with' context.")

        url = f"{GITHUB_API_URL}/repos/{owner}/{repo}"
        response = await self.http_client.get(url)
        response.raise_for_status()
        data = response.json()
        return data.get("default_branch", "main")

    async def get_latest_commit(self, owner: str, repo: str, branch: str = "main") -> str:
        """Get the latest commit SHA for a branch."""
        if not self.http_client:
            raise RuntimeError("Crawler not initialized. Use 'async with' context.")

        url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/commits/{branch}"
        response = await self.http_client.get(url)
        response.raise_for_status()
        data = response.json()
        return data.get("sha", "")

    async def list_files(
        self,
        owner: str,
        repo: str,
        path: str = "",
        branch: str = "main",
        patterns: List[str] = None,
        exclude_patterns: List[str] = None,
    ) -> List[Dict[str, Any]]:
        """List files in a repository matching the given patterns."""
        if not self.http_client:
            raise RuntimeError("Crawler not initialized. Use 'async with' context.")

        patterns = patterns or ["*.md", "*.txt", "*.html"]
        exclude_patterns = exclude_patterns or []

        url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1"
        response = await self.http_client.get(url)
        response.raise_for_status()
        data = response.json()

        files = []
        for item in data.get("tree", []):
            if item["type"] != "blob":
                continue

            file_path = item["path"]

            excluded = any(fnmatch(file_path, pattern) for pattern in exclude_patterns)
            if excluded:
                continue

            matched = any(fnmatch(file_path, pattern) for pattern in patterns)
            if not matched:
                continue

            if item.get("size", 0) > MAX_FILE_SIZE:
                logger.warning(f"Skipping large file: {file_path} ({item['size']} bytes)")
                continue

            files.append({
                "path": file_path,
                "sha": item["sha"],
                "size": item.get("size", 0),
                "url": item.get("url", ""),
            })

        return files

    async def get_file_content(self, owner: str, repo: str, path: str, branch: str = "main") -> str:
        """Get the content of a file from a repository."""
        if not self.http_client:
            raise RuntimeError("Crawler not initialized. Use 'async with' context.")

        url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}"
        response = await self.http_client.get(url)
        response.raise_for_status()
        return response.text

    async def crawl_repository(
        self,
        source: SourceConfig,
    ) -> AsyncGenerator[ExtractedDocument, None]:
        """Crawl a repository and yield extracted documents."""
        if not source.repo_url:
            logger.warning(f"No repo URL for source: {source.name}")
            return

        try:
            owner, repo, host = self._parse_repo_url(source.repo_url)
        except ValueError as e:
            logger.error(f"Failed to parse repo URL for {source.name}: {e}")
            return

        if host == "gitlab":
            logger.info(f"GitLab repos not yet supported: {source.name}")
            return

        logger.info(f"Crawling repository: {owner}/{repo}")

        try:
            branch = await self.get_default_branch(owner, repo)
            commit_sha = await self.get_latest_commit(owner, repo, branch)

            await asyncio.sleep(RATE_LIMIT_DELAY)

            files = await self.list_files(
                owner, repo,
                branch=branch,
                patterns=source.file_patterns,
                exclude_patterns=source.exclude_patterns,
            )

            logger.info(f"Found {len(files)} matching files in {source.name}")

            for file_info in files:
                await asyncio.sleep(RATE_LIMIT_DELAY)

                try:
                    content = await self.get_file_content(
                        owner, repo, file_info["path"], branch
                    )

                    file_path = file_info["path"]
                    source_url = f"https://github.com/{owner}/{repo}/blob/{branch}/{file_path}"

                    if file_path.endswith('.md'):
                        doc = MarkdownParser.parse(content, file_path)
                        doc.source_url = source_url
                        doc.source_commit = commit_sha
                        yield doc

                    elif file_path.endswith('.html') or file_path.endswith('.htm'):
                        doc = HTMLParser.parse(content, file_path)
                        doc.source_url = source_url
                        doc.source_commit = commit_sha
                        yield doc

                    elif file_path.endswith('.json'):
                        docs = JSONParser.parse(content, file_path)
                        for doc in docs:
                            doc.source_url = source_url
                            doc.source_commit = commit_sha
                            yield doc

                    elif file_path.endswith('.txt'):
                        yield ExtractedDocument(
                            text=content,
                            title=Path(file_path).stem,
                            file_path=file_path,
                            file_type="text",
                            source_url=source_url,
                            source_commit=commit_sha,
                            language=MarkdownParser._detect_language(content),
                            placeholders=MarkdownParser._find_placeholders(content),
                        )

                except httpx.HTTPError as e:
                    logger.warning(f"Failed to fetch {file_path}: {e}")
                    continue
                except Exception as e:
                    logger.error(f"Error processing {file_path}: {e}")
                    continue

        except httpx.HTTPError as e:
            logger.error(f"HTTP error crawling {source.name}: {e}")
        except Exception as e:
            logger.error(f"Error crawling {source.name}: {e}")


class RepositoryDownloader:
    """Download and extract repository archives."""

    def __init__(self):
        self.http_client: Optional[httpx.AsyncClient] = None

    async def __aenter__(self):
        self.http_client = httpx.AsyncClient(
            timeout=120.0,
            follow_redirects=True,
        )
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        if self.http_client:
            await self.http_client.aclose()

    async def download_zip(self, repo_url: str, branch: str = "main") -> Path:
        """Download repository as ZIP and extract to temp directory."""
        if not self.http_client:
            raise RuntimeError("Downloader not initialized. Use 'async with' context.")

        parsed = urlparse(repo_url)
        path_parts = parsed.path.strip('/').split('/')
        owner = path_parts[0]
        repo = path_parts[1].replace('.git', '')

        zip_url = f"https://github.com/{owner}/{repo}/archive/refs/heads/{branch}.zip"

        logger.info(f"Downloading ZIP from {zip_url}")

        response = await self.http_client.get(zip_url)
        response.raise_for_status()

        temp_dir = Path(tempfile.mkdtemp())
        zip_path = temp_dir / f"{repo}.zip"

        with open(zip_path, 'wb') as f:
            f.write(response.content)

        extract_dir = temp_dir / repo
        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
            zip_ref.extractall(temp_dir)

        extracted_dirs = list(temp_dir.glob(f"{repo}-*"))
        if extracted_dirs:
            return extracted_dirs[0]

        return extract_dir

    async def crawl_local_directory(
        self,
        directory: Path,
        source: SourceConfig,
        base_url: str,
    ) -> AsyncGenerator[ExtractedDocument, None]:
        """Crawl a local directory for documents."""
        patterns = source.file_patterns or ["*.md", "*.txt", "*.html"]
        exclude_patterns = source.exclude_patterns or []

        for pattern in patterns:
            for file_path in directory.rglob(pattern.replace("**/", "")):
                if not file_path.is_file():
                    continue

                rel_path = str(file_path.relative_to(directory))

                excluded = any(fnmatch(rel_path, ep) for ep in exclude_patterns)
                if excluded:
                    continue

                if file_path.stat().st_size > MAX_FILE_SIZE:
                    continue

                try:
                    content = file_path.read_text(encoding='utf-8')
                except UnicodeDecodeError:
                    try:
                        content = file_path.read_text(encoding='latin-1')
                    except Exception:
                        continue

                source_url = f"{base_url}/{rel_path}"

                if file_path.suffix == '.md':
                    doc = MarkdownParser.parse(content, rel_path)
                    doc.source_url = source_url
                    yield doc

                elif file_path.suffix in ['.html', '.htm']:
                    doc = HTMLParser.parse(content, rel_path)
                    doc.source_url = source_url
                    yield doc

                elif file_path.suffix == '.json':
                    docs = JSONParser.parse(content, rel_path)
                    for doc in docs:
                        doc.source_url = source_url
                        yield doc

                elif file_path.suffix == '.txt':
                    yield ExtractedDocument(
                        text=content,
                        title=file_path.stem,
                        file_path=rel_path,
                        file_type="text",
                        source_url=source_url,
                        language=MarkdownParser._detect_language(content),
                        placeholders=MarkdownParser._find_placeholders(content),
                    )

    def cleanup(self, directory: Path):
        """Clean up temporary directory."""
        if directory.exists():
            shutil.rmtree(directory, ignore_errors=True)


async def crawl_source(source: SourceConfig) -> List[ExtractedDocument]:
    """Crawl a source configuration and return all extracted documents."""
    documents = []

    if source.repo_url:
        async with GitHubCrawler() as crawler:
            async for doc in crawler.crawl_repository(source):
                documents.append(doc)

    return documents


# CLI for testing
async def main():
    """Test crawler with a sample source."""
    from template_sources import TEMPLATE_SOURCES

    source = next(s for s in TEMPLATE_SOURCES if s.name == "github-site-policy")

    async with GitHubCrawler() as crawler:
        count = 0
        async for doc in crawler.crawl_repository(source):
            count += 1
            print(f"\n{'='*60}")
            print(f"Title: {doc.title}")
            print(f"Path: {doc.file_path}")
            print(f"Type: {doc.file_type}")
            print(f"Language: {doc.language}")
            print(f"URL: {doc.source_url}")
            print(f"Placeholders: {doc.placeholders[:5] if doc.placeholders else 'None'}")
            print(f"Text preview: {doc.text[:200]}...")

        print(f"\n\nTotal documents: {count}")


if __name__ == "__main__":
    asyncio.run(main())