Add opt-in crawl_timeout and shield page cleanup from cancellation

crawl4ai/async_configs.py

@@ -266,7 +266,7 @@ class UntrustedConfigError(ValueError):
         "no_cache_write", "check_cache_freshness", "cache_validation_timeout",
         "fetch_ssl_certificate",
         # timing / waiting
-        "wait_until", "page_timeout", "wait_for", "wait_for_timeout",
+        "wait_until", "page_timeout", "crawl_timeout", "wait_for", "wait_for_timeout",
         "body_visibility_timeout",
         "wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
         # scrolling / rendering
@@ -325,7 +325,7 @@ def _cap_timeout(v):
         return min(int(v), _MAX_TIMEOUT_MS)
 
     if type_name == "CrawlerRunConfig":
-        for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
+        for f in ("page_timeout", "crawl_timeout", "wait_for_timeout", "body_visibility_timeout"):
             if f in params:
                 params[f] = _cap_timeout(params[f])
         if isinstance(params.get("max_scroll_steps"), int):
@@ -1473,6 +1473,9 @@ class CrawlerRunConfig():
                           Default: "domcontentloaded".
         page_timeout (int): Timeout in ms for page operations like navigation.
                             Default: 60000 (60 seconds).
+        crawl_timeout (int or None): Timeout in ms for the whole page visit, from navigation to final HTML,
+                                     including js_code and hooks. None = no limit.
+                                     Default: None.
         wait_for (str or None): A CSS selector or JS condition to wait for before extracting content.
                                 Default: None.
         wait_for_timeout (int or None): Specific timeout in ms for the wait_for condition.
@@ -1666,6 +1669,7 @@ def __init__(
         # Page Navigation and Timing Parameters
         wait_until: str = "domcontentloaded",
         page_timeout: int = PAGE_TIMEOUT,
+        crawl_timeout: Optional[int] = None,
         wait_for: str = None,
         wait_for_timeout: int = None,
         wait_for_images: bool = False,
@@ -1796,6 +1800,7 @@ def __init__(
         # Page Navigation and Timing Parameters
         self.wait_until = wait_until
         self.page_timeout = page_timeout
+        self.crawl_timeout = crawl_timeout
         self.wait_for = wait_for
         self.wait_for_timeout = wait_for_timeout
         self.wait_for_images = wait_for_images
@@ -2173,6 +2178,7 @@ def to_dict(self):
             "shared_data": self.shared_data,
             "wait_until": self.wait_until,
             "page_timeout": self.page_timeout,
+            "crawl_timeout": self.crawl_timeout,
             "wait_for": self.wait_for,
             "wait_for_timeout": self.wait_for_timeout,
             "wait_for_images": self.wait_for_images,

crawl4ai/async_crawler_strategy.py

@@ -526,18 +526,9 @@ async def _crawl_web(
             AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data
         """
         config.url = url
-        response_headers = {}
-        execution_result = None
-        status_code = None
-        redirected_url = url
-        redirected_status_code = None
 
         # Reset downloaded files list for new crawl
         self._downloaded_files = []
-
-        # Initialize capture lists
-        captured_requests = []
-        captured_console = []
 
         # Handle user agent with magic mode.
         # For persistent contexts the UA is locked at browser launch time
@@ -574,6 +565,28 @@ async def _crawl_web(
             except Exception:
                 pass
 
+        if not config.crawl_timeout:
+            return await self._crawl_page(url, config, page, context, ua_changed)
+        try:
+            return await asyncio.wait_for(
+                asyncio.create_task(self._crawl_page(url, config, page, context, ua_changed)), config.crawl_timeout / 1000
+            )
+        except asyncio.TimeoutError:
+            await self._close_unresponsive_page(page, config)
+            raise RuntimeError(f"Crawl exceeded crawl_timeout of {config.crawl_timeout} ms")
+
+    async def _crawl_page(
+        self, url: str, config: CrawlerRunConfig, page: Page, context, ua_changed: bool
+    ) -> AsyncCrawlResponse:
+        """The page visit itself (navigation to final HTML plus cleanup); bounded by crawl_timeout in _crawl_web."""
+        response_headers = {}
+        execution_result = None
+        status_code = None
+        redirected_url = url
+        redirected_status_code = None
+        captured_requests = []
+        captured_console = []
+
         try:
             # Push updated UA + sec-ch-ua to the page so the server sees them
             if ua_changed:
@@ -1198,37 +1211,77 @@ async def get_delayed_content(delay: float = 5.0) -> str:
             raise e
 
         finally:
-            # Always clean up event listeners to prevent accumulation
-            # across reuses (even for session pages).
-            try:
-                if config.capture_network_requests:
-                    page.remove_listener("request", handle_request_capture)
-                    page.remove_listener("response", handle_response_capture)
-                    page.remove_listener("requestfailed", handle_request_failed_capture)
-                if config.capture_console_messages:
-                    if hasattr(self.adapter, 'retrieve_console_messages'):
-                        final_messages = await self.adapter.retrieve_console_messages(page)
-                        captured_console.extend(final_messages)
-                    await self.adapter.cleanup_console_capture(page, handle_console, handle_error)
-            except Exception:
-                pass
-
-            if not config.session_id:
-                # ALWAYS decrement refcount first — must succeed even if
-                # the browser crashed or the page is in a bad state.
+            async def _cleanup():
+                # Always clean up event listeners to prevent accumulation
+                # across reuses (even for session pages).
                 try:
-                    await self.browser_manager.release_page_with_context(page)
+                    if config.capture_network_requests:
+                        page.remove_listener("request", handle_request_capture)
+                        page.remove_listener("response", handle_response_capture)
+                        page.remove_listener("requestfailed", handle_request_failed_capture)
+                    if config.capture_console_messages:
+                        if hasattr(self.adapter, 'retrieve_console_messages'):
+                            final_messages = await asyncio.wait_for(self.adapter.retrieve_console_messages(page), 5)
+                            captured_console.extend(final_messages)
+                        await asyncio.wait_for(self.adapter.cleanup_console_capture(page, handle_console, handle_error), 5)
                 except Exception:
                     pass
 
-                # Close the page unless it's the last one in a headless/managed browser
-                try:
-                    all_contexts = page.context.browser.contexts
-                    total_pages = sum(len(context.pages) for context in all_contexts)
-                    if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
-                        await page.close()
-                except Exception:
-                    pass
+                if not config.session_id:
+                    # ALWAYS decrement refcount first — must succeed even if
+                    # the browser crashed or the page is in a bad state.
+                    try:
+                        await self.browser_manager.release_page_with_context(page)
+                    except Exception:
+                        pass
+
+                    # Close the page unless it's the last one in a headless/managed browser
+                    try:
+                        all_contexts = page.context.browser.contexts
+                        total_pages = sum(len(context.pages) for context in all_contexts)
+                        if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
+                            await page.close()
+                    except Exception:
+                        pass
+
+            # Shielded so a cancel landing mid-cleanup cannot skip page.close(); the cancel is re-raised once cleanup is done
+            cleanup = asyncio.create_task(_cleanup())
+            try:
+                await asyncio.shield(cleanup)
+            except asyncio.CancelledError:
+                await asyncio.shield(cleanup)
+                raise
+
+    async def _close_unresponsive_page(self, page: Page, config: CrawlerRunConfig) -> None:
+        """
+        Close (or drop the session of) a page whose crawl hit crawl_timeout.
+
+        Args:
+            page (Page): The Playwright page instance
+            config (CrawlerRunConfig): Crawler Config to check for session_id
+        """
+        async def _close():
+            browser = page.context.browser
+            if not page.is_closed() and browser and sum(len(c.pages) for c in browser.contexts) <= 1:
+                await page.context.new_page()  # a headed managed Chrome exits when its last tab closes
+            if config.session_id:
+                self.logger.warning(
+                    message="Dropping session {session_id}: crawl exceeded crawl_timeout",
+                    tag="TIMEOUT",
+                    params={"session_id": config.session_id},
+                )
+                await self.browser_manager.kill_session(config.session_id)
+            elif not page.is_closed():
+                await page.close()
+
+        try:
+            await asyncio.wait_for(_close(), 5)
+        except Exception as e:
+            self.logger.warning(
+                message="Could not close unresponsive page: {error}",
+                tag="TIMEOUT",
+                params={"error": str(e)},
+            )
 
     # async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1):
     async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, max_scroll_steps: Optional[int] = None):

deploy/docker/config.yml

@@ -74,6 +74,7 @@ security:
 crawler:
   base_config:
     simulate_user: true
+    crawl_timeout: 180000   # ms; bounds the whole page visit so a hung page cannot pin a renderer
   memory_threshold_percent: 95.0
   rate_limiter:
     enabled: true

docs/md_v2/api/parameters.md

@@ -140,6 +140,7 @@ Use these for controlling whether you read or write from a local content cache.
 |----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
 | **`wait_until`**           | `str` (domcontentloaded)| Condition for navigation to "complete". Often `"networkidle"` or `"domcontentloaded"`.                               |
 | **`page_timeout`**         | `int` (60000 ms)        | Timeout for page navigation or JS steps. Increase for slow sites.                                                    |
+| **`crawl_timeout`**        | `int or None` (None)    | Timeout in ms for the whole page visit, from navigation to final HTML, including `js_code` and hooks. On expiry the page is closed and the crawl fails. None = no limit. |
 | **`wait_for`**             | `str or None`           | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction.                     |
 | **`wait_for_timeout`**     | `int or None` (None)    | Specific timeout in ms for the `wait_for` condition. If None, uses `page_timeout`.                                   |
 | **`wait_for_images`**      | `bool` (False)          | Wait for images to load before finishing. Slows down if you only want text.                                          |

docs/md_v2/core/browser-crawler-config.md

@@ -298,6 +298,7 @@ class CrawlerRunConfig:
     - **`scan_full_page`**: If `True`, scroll through the entire page to load all content
     - **`wait_until`**: Condition to wait for when navigating (e.g., "domcontentloaded", "networkidle")
     - **`page_timeout`**: Timeout in milliseconds for page operations (default: 60000)
+    - **`crawl_timeout`**: Timeout in milliseconds for the whole page visit, navigation to final HTML, including `js_code` and hooks (default: None, no limit)
     - **`delay_before_return_html`**: Delay in seconds before retrieving final HTML.
 
 13.⠀**`url_matcher`** & **`match_mode`**:  

docs/md_v2/core/page-interaction.md

@@ -218,8 +218,9 @@ result = await crawler.arun(url="https://github.com/search", config=config)
 ## 4. Timing Control
 
 1. **`page_timeout`** (ms): Overall page load or script execution time limit.  
-2. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.  
-3. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
+2. **`crawl_timeout`** (ms): Limit for the whole page visit, navigation to final HTML, including `js_code` and hooks. None = no limit.  
+3. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.  
+4. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
 
 **Example**:
 

tests/test_crawl_timeout.py

@@ -0,0 +1,157 @@
+"""
+crawl_timeout must bound the whole page visit, not just navigation.
+
+A page whose JS thread goes busy right after load navigates fine, then every
+post-navigation call (evaluate / content) would wait forever. With crawl_timeout
+set the crawl must fail within it and leave no page or refcount behind.
+"""
+import asyncio
+import http.server
+import socketserver
+import threading
+import time
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
+
+TRAP_HTML = b"""<!doctype html><html><head><title>trap</title></head><body><p>hi</p>
+<script>document.addEventListener('DOMContentLoaded',function(){setTimeout(function(){while(true){}},0);});</script>
+</body></html>"""
+OK_HTML = b"<html><body><p>ok</p>" + b"<p>filler text so the anti-bot check does not flag a near-empty page</p>" * 20 + b"</body></html>"
+
+
+class _Handler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html")
+        self.end_headers()
+        self.wfile.write(TRAP_HTML if self.path == "/trap" else OK_HTML)
+
+    def log_message(self, *args):
+        pass
+
+
+@pytest.fixture
+def base_url():
+    srv = socketserver.TCPServer(("127.0.0.1", 0), _Handler)
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    yield f"http://127.0.0.1:{srv.server_address[1]}"
+    srv.shutdown()
+
+
+def _cfg(**kw):
+    return CrawlerRunConfig(crawl_timeout=5000, cache_mode=CacheMode.BYPASS, **kw)
+
+
+@pytest.mark.asyncio
+async def test_trap_page_fails_within_crawl_timeout_and_leaks_nothing(base_url):
+    cfg = _cfg()
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        t0 = time.perf_counter()
+        result = await asyncio.wait_for(crawler.arun(base_url + "/trap", config=cfg), 20)
+        elapsed = time.perf_counter() - t0
+
+        assert result.success is False
+        assert "exceeded crawl_timeout" in result.error_message
+        assert elapsed < 10, f"took {elapsed:.1f}s, crawl_timeout was 5s"
+
+        bm = crawler.crawler_strategy.browser_manager
+        open_urls = [p.url for c in bm.browser.contexts for p in c.pages]
+        assert base_url + "/trap" not in open_urls, open_urls
+        assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+
+
+@pytest.mark.asyncio
+async def test_trap_page_with_overlay_removal_does_not_hang(base_url):
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        result = await asyncio.wait_for(
+            crawler.arun(base_url + "/trap", config=_cfg(remove_overlay_elements=True)), 20
+        )
+        assert result.success is False
+        assert "exceeded crawl_timeout" in result.error_message
+
+
+@pytest.mark.asyncio
+async def test_trap_session_is_dropped_and_next_crawl_works(base_url):
+    cfg = _cfg(session_id="trap-session")
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        bm = crawler.crawler_strategy.browser_manager
+        result = await asyncio.wait_for(crawler.arun(base_url + "/trap", config=cfg), 20)
+        assert result.success is False
+        assert "trap-session" not in bm.sessions
+
+        result = await asyncio.wait_for(crawler.arun(base_url + "/ok", config=cfg), 20)
+        assert result.success is True
+        assert "ok" in result.html
+        assert "trap-session" in bm.sessions  # normal crawl keeps the session page
+
+
+@pytest.mark.asyncio
+async def test_no_crawl_timeout_means_no_limit(base_url):
+    """Default None must not cut a crawl; the trap page is bounded only by the test's own guard."""
+    cfg = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+    assert cfg.crawl_timeout is None
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        task = asyncio.create_task(crawler.arun(base_url + "/trap", config=cfg))
+        done, _ = await asyncio.wait({task}, timeout=8)
+        assert not done, "crawl finished without a crawl_timeout; expected it to still be running"
+        task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+        bm = crawler.crawler_strategy.browser_manager
+        assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+        assert sum(len(c.pages) for c in bm.browser.contexts) <= 1
+
+
+@pytest.mark.asyncio
+async def test_keep_last_page_rule_unchanged(base_url):
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        bm = crawler.crawler_strategy.browser_manager
+        for _ in range(3):
+            assert (await crawler.arun(base_url + "/ok", config=_cfg())).success
+        assert sum(len(c.pages) for c in bm.browser.contexts) <= 1
+
+
+@pytest.mark.asyncio
+async def test_cancel_during_cleanup_still_closes_page(base_url):
+    """A cancel landing inside the finally's console cleanup must not skip page close."""
+    cfg = _cfg(capture_console_messages=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        bm = crawler.crawler_strategy.browser_manager
+        in_cleanup = asyncio.Event()
+
+        async def slow_cleanup(page, *args):
+            in_cleanup.set()
+            await asyncio.sleep(2)
+
+        crawler.crawler_strategy.adapter.cleanup_console_capture = slow_cleanup
+
+        task = asyncio.create_task(crawler.arun(base_url + "/ok", config=cfg))
+        await asyncio.wait_for(in_cleanup.wait(), 20)
+        task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+        assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+        assert sum(len(c.pages) for c in bm.browser.contexts) <= 1
+
+
+@pytest.mark.asyncio
+async def test_hung_cleanup_does_not_block_crawl_timeout(base_url):
+    """An evaluate inside cleanup (UndetectedAdapter does this) hangs on a trap page; the 5 s cleanup bound must let the crawl fail."""
+    cfg = _cfg(capture_console_messages=True)
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        async def evaluate_in_cleanup(page, *args):
+            return await page.evaluate("1")
+
+        crawler.crawler_strategy.adapter.cleanup_console_capture = evaluate_in_cleanup
+
+        t0 = time.perf_counter()
+        result = await asyncio.wait_for(crawler.arun(base_url + "/trap", config=cfg), 30)
+        assert not result.success
+        assert "exceeded crawl_timeout" in result.error_message
+        assert time.perf_counter() - t0 < 20
+        bm = crawler.crawler_strategy.browser_manager
+        assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+        assert all(p.url != base_url + "/trap" for c in bm.browser.contexts for p in c.pages)