diff --git a/consent-tester/services/dsi_discovery.py b/consent-tester/services/dsi_discovery.py index 9f71b62..f1d34f3 100644 --- a/consent-tester/services/dsi_discovery.py +++ b/consent-tester/services/dsi_discovery.py @@ -217,7 +217,7 @@ async def discover_dsi_documents( try: # Step 1: Load the page - await page.goto(url, wait_until="networkidle", timeout=30000) + await page.goto(url, wait_until="networkidle", timeout=60000) await page.wait_for_timeout(2000) # Step 2: Find DSI links in current page @@ -290,11 +290,11 @@ async def discover_dsi_documents( continue # Navigate to page — wait for JS to load content - resp = await page.goto(href, wait_until="networkidle", timeout=20000) + resp = await page.goto(href, wait_until="networkidle", timeout=45000) if resp and resp.status < 400: - await page.wait_for_timeout(3000) # Extra wait for JS content loading + await page.wait_for_timeout(2000) await _expand_all_interactive(page) - await page.wait_for_timeout(1000) + await page.wait_for_timeout(500) # Extract text — try specific content areas, fall back to full body text = await page.evaluate(""" @@ -333,14 +333,14 @@ async def discover_dsi_documents( pending_links.append(nl) # Navigate back for next link - await page.goto(url, wait_until="networkidle", timeout=20000) + await page.goto(url, wait_until="networkidle", timeout=45000) await page.wait_for_timeout(500) await _expand_all_interactive(page) except Exception as e: result.errors.append(f"Failed to load {href}: {str(e)[:80]}") try: - await page.goto(url, wait_until="networkidle", timeout=20000) + await page.goto(url, wait_until="networkidle", timeout=45000) except Exception: pass diff --git a/consent-tester/services/playwright_scanner.py b/consent-tester/services/playwright_scanner.py index 49635d2..2aef6ef 100644 --- a/consent-tester/services/playwright_scanner.py +++ b/consent-tester/services/playwright_scanner.py @@ -157,7 +157,7 @@ async def _visit_page(page: Page, url: str, result: PlaywrightScanResult) -> Sca """Visit a page and capture its rendered HTML.""" sp = ScannedPage(url=url, status=0) try: - response = await page.goto(url, wait_until="networkidle", timeout=20000) + response = await page.goto(url, wait_until="networkidle", timeout=45000) sp.status = response.status if response else 0 await page.wait_for_timeout(2000)