diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 383079fef..510e7b8ea 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -266,7 +266,7 @@ class UntrustedConfigError(ValueError):
"no_cache_write", "check_cache_freshness", "cache_validation_timeout",
"fetch_ssl_certificate",
# timing / waiting
- "wait_until", "page_timeout", "wait_for", "wait_for_timeout",
+ "wait_until", "page_timeout", "crawl_timeout", "wait_for", "wait_for_timeout",
"body_visibility_timeout",
"wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
# scrolling / rendering
@@ -325,7 +325,7 @@ def _cap_timeout(v):
return min(int(v), _MAX_TIMEOUT_MS)
if type_name == "CrawlerRunConfig":
- for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
+ for f in ("page_timeout", "crawl_timeout", "wait_for_timeout", "body_visibility_timeout"):
if f in params:
params[f] = _cap_timeout(params[f])
if isinstance(params.get("max_scroll_steps"), int):
@@ -1473,6 +1473,9 @@ class CrawlerRunConfig():
Default: "domcontentloaded".
page_timeout (int): Timeout in ms for page operations like navigation.
Default: 60000 (60 seconds).
+ crawl_timeout (int or None): Timeout in ms for the whole page visit, from navigation to final HTML,
+ including js_code and hooks. None = no limit.
+ Default: None.
wait_for (str or None): A CSS selector or JS condition to wait for before extracting content.
Default: None.
wait_for_timeout (int or None): Specific timeout in ms for the wait_for condition.
@@ -1666,6 +1669,7 @@ def __init__(
# Page Navigation and Timing Parameters
wait_until: str = "domcontentloaded",
page_timeout: int = PAGE_TIMEOUT,
+ crawl_timeout: Optional[int] = None,
wait_for: str = None,
wait_for_timeout: int = None,
wait_for_images: bool = False,
@@ -1796,6 +1800,7 @@ def __init__(
# Page Navigation and Timing Parameters
self.wait_until = wait_until
self.page_timeout = page_timeout
+ self.crawl_timeout = crawl_timeout
self.wait_for = wait_for
self.wait_for_timeout = wait_for_timeout
self.wait_for_images = wait_for_images
@@ -2173,6 +2178,7 @@ def to_dict(self):
"shared_data": self.shared_data,
"wait_until": self.wait_until,
"page_timeout": self.page_timeout,
+ "crawl_timeout": self.crawl_timeout,
"wait_for": self.wait_for,
"wait_for_timeout": self.wait_for_timeout,
"wait_for_images": self.wait_for_images,
diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py
index 6d0fb4769..1b7cef481 100644
--- a/crawl4ai/async_crawler_strategy.py
+++ b/crawl4ai/async_crawler_strategy.py
@@ -526,18 +526,9 @@ async def _crawl_web(
AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data
"""
config.url = url
- response_headers = {}
- execution_result = None
- status_code = None
- redirected_url = url
- redirected_status_code = None
# Reset downloaded files list for new crawl
self._downloaded_files = []
-
- # Initialize capture lists
- captured_requests = []
- captured_console = []
# Handle user agent with magic mode.
# For persistent contexts the UA is locked at browser launch time
@@ -574,6 +565,28 @@ async def _crawl_web(
except Exception:
pass
+ if not config.crawl_timeout:
+ return await self._crawl_page(url, config, page, context, ua_changed)
+ try:
+ return await asyncio.wait_for(
+ asyncio.create_task(self._crawl_page(url, config, page, context, ua_changed)), config.crawl_timeout / 1000
+ )
+ except asyncio.TimeoutError:
+ await self._close_unresponsive_page(page, config)
+ raise RuntimeError(f"Crawl exceeded crawl_timeout of {config.crawl_timeout} ms")
+
+ async def _crawl_page(
+ self, url: str, config: CrawlerRunConfig, page: Page, context, ua_changed: bool
+ ) -> AsyncCrawlResponse:
+ """The page visit itself (navigation to final HTML plus cleanup); bounded by crawl_timeout in _crawl_web."""
+ response_headers = {}
+ execution_result = None
+ status_code = None
+ redirected_url = url
+ redirected_status_code = None
+ captured_requests = []
+ captured_console = []
+
try:
# Push updated UA + sec-ch-ua to the page so the server sees them
if ua_changed:
@@ -1198,37 +1211,77 @@ async def get_delayed_content(delay: float = 5.0) -> str:
raise e
finally:
- # Always clean up event listeners to prevent accumulation
- # across reuses (even for session pages).
- try:
- if config.capture_network_requests:
- page.remove_listener("request", handle_request_capture)
- page.remove_listener("response", handle_response_capture)
- page.remove_listener("requestfailed", handle_request_failed_capture)
- if config.capture_console_messages:
- if hasattr(self.adapter, 'retrieve_console_messages'):
- final_messages = await self.adapter.retrieve_console_messages(page)
- captured_console.extend(final_messages)
- await self.adapter.cleanup_console_capture(page, handle_console, handle_error)
- except Exception:
- pass
-
- if not config.session_id:
- # ALWAYS decrement refcount first — must succeed even if
- # the browser crashed or the page is in a bad state.
+ async def _cleanup():
+ # Always clean up event listeners to prevent accumulation
+ # across reuses (even for session pages).
try:
- await self.browser_manager.release_page_with_context(page)
+ if config.capture_network_requests:
+ page.remove_listener("request", handle_request_capture)
+ page.remove_listener("response", handle_response_capture)
+ page.remove_listener("requestfailed", handle_request_failed_capture)
+ if config.capture_console_messages:
+ if hasattr(self.adapter, 'retrieve_console_messages'):
+ final_messages = await asyncio.wait_for(self.adapter.retrieve_console_messages(page), 5)
+ captured_console.extend(final_messages)
+ await asyncio.wait_for(self.adapter.cleanup_console_capture(page, handle_console, handle_error), 5)
except Exception:
pass
- # Close the page unless it's the last one in a headless/managed browser
- try:
- all_contexts = page.context.browser.contexts
- total_pages = sum(len(context.pages) for context in all_contexts)
- if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
- await page.close()
- except Exception:
- pass
+ if not config.session_id:
+ # ALWAYS decrement refcount first — must succeed even if
+ # the browser crashed or the page is in a bad state.
+ try:
+ await self.browser_manager.release_page_with_context(page)
+ except Exception:
+ pass
+
+ # Close the page unless it's the last one in a headless/managed browser
+ try:
+ all_contexts = page.context.browser.contexts
+ total_pages = sum(len(context.pages) for context in all_contexts)
+ if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
+ await page.close()
+ except Exception:
+ pass
+
+ # Shielded so a cancel landing mid-cleanup cannot skip page.close(); the cancel is re-raised once cleanup is done
+ cleanup = asyncio.create_task(_cleanup())
+ try:
+ await asyncio.shield(cleanup)
+ except asyncio.CancelledError:
+ await asyncio.shield(cleanup)
+ raise
+
+ async def _close_unresponsive_page(self, page: Page, config: CrawlerRunConfig) -> None:
+ """
+ Close (or drop the session of) a page whose crawl hit crawl_timeout.
+
+ Args:
+ page (Page): The Playwright page instance
+ config (CrawlerRunConfig): Crawler Config to check for session_id
+ """
+ async def _close():
+ browser = page.context.browser
+ if not page.is_closed() and browser and sum(len(c.pages) for c in browser.contexts) <= 1:
+ await page.context.new_page() # a headed managed Chrome exits when its last tab closes
+ if config.session_id:
+ self.logger.warning(
+ message="Dropping session {session_id}: crawl exceeded crawl_timeout",
+ tag="TIMEOUT",
+ params={"session_id": config.session_id},
+ )
+ await self.browser_manager.kill_session(config.session_id)
+ elif not page.is_closed():
+ await page.close()
+
+ try:
+ await asyncio.wait_for(_close(), 5)
+ except Exception as e:
+ self.logger.warning(
+ message="Could not close unresponsive page: {error}",
+ tag="TIMEOUT",
+ params={"error": str(e)},
+ )
# async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1):
async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1, max_scroll_steps: Optional[int] = None):
diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml
index 7aabc6814..1d6429ae7 100644
--- a/deploy/docker/config.yml
+++ b/deploy/docker/config.yml
@@ -74,6 +74,7 @@ security:
crawler:
base_config:
simulate_user: true
+ crawl_timeout: 180000 # ms; bounds the whole page visit so a hung page cannot pin a renderer
memory_threshold_percent: 95.0
rate_limiter:
enabled: true
diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md
index f9f759a58..59167d250 100644
--- a/docs/md_v2/api/parameters.md
+++ b/docs/md_v2/api/parameters.md
@@ -140,6 +140,7 @@ Use these for controlling whether you read or write from a local content cache.
|----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
| **`wait_until`** | `str` (domcontentloaded)| Condition for navigation to "complete". Often `"networkidle"` or `"domcontentloaded"`. |
| **`page_timeout`** | `int` (60000 ms) | Timeout for page navigation or JS steps. Increase for slow sites. |
+| **`crawl_timeout`** | `int or None` (None) | Timeout in ms for the whole page visit, from navigation to final HTML, including `js_code` and hooks. On expiry the page is closed and the crawl fails. None = no limit. |
| **`wait_for`** | `str or None` | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction. |
| **`wait_for_timeout`** | `int or None` (None) | Specific timeout in ms for the `wait_for` condition. If None, uses `page_timeout`. |
| **`wait_for_images`** | `bool` (False) | Wait for images to load before finishing. Slows down if you only want text. |
diff --git a/docs/md_v2/core/browser-crawler-config.md b/docs/md_v2/core/browser-crawler-config.md
index d9946c689..0b3c31fe6 100644
--- a/docs/md_v2/core/browser-crawler-config.md
+++ b/docs/md_v2/core/browser-crawler-config.md
@@ -298,6 +298,7 @@ class CrawlerRunConfig:
- **`scan_full_page`**: If `True`, scroll through the entire page to load all content
- **`wait_until`**: Condition to wait for when navigating (e.g., "domcontentloaded", "networkidle")
- **`page_timeout`**: Timeout in milliseconds for page operations (default: 60000)
+ - **`crawl_timeout`**: Timeout in milliseconds for the whole page visit, navigation to final HTML, including `js_code` and hooks (default: None, no limit)
- **`delay_before_return_html`**: Delay in seconds before retrieving final HTML.
13.⠀**`url_matcher`** & **`match_mode`**:
diff --git a/docs/md_v2/core/page-interaction.md b/docs/md_v2/core/page-interaction.md
index 4b28cac2b..b29028001 100644
--- a/docs/md_v2/core/page-interaction.md
+++ b/docs/md_v2/core/page-interaction.md
@@ -218,8 +218,9 @@ result = await crawler.arun(url="https://github.com/search", config=config)
## 4. Timing Control
1. **`page_timeout`** (ms): Overall page load or script execution time limit.
-2. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.
-3. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
+2. **`crawl_timeout`** (ms): Limit for the whole page visit, navigation to final HTML, including `js_code` and hooks. None = no limit.
+3. **`delay_before_return_html`** (seconds): Wait an extra moment before capturing the final HTML.
+4. **`mean_delay`** & **`max_range`**: If you call `arun_many()` with multiple URLs, these add a random pause between each request.
**Example**:
diff --git a/tests/test_crawl_timeout.py b/tests/test_crawl_timeout.py
new file mode 100644
index 000000000..d81ebec46
--- /dev/null
+++ b/tests/test_crawl_timeout.py
@@ -0,0 +1,157 @@
+"""
+crawl_timeout must bound the whole page visit, not just navigation.
+
+A page whose JS thread goes busy right after load navigates fine, then every
+post-navigation call (evaluate / content) would wait forever. With crawl_timeout
+set the crawl must fail within it and leave no page or refcount behind.
+"""
+import asyncio
+import http.server
+import socketserver
+import threading
+import time
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
+
+TRAP_HTML = b"""traphi
+
+"""
+OK_HTML = b"ok
" + b"filler text so the anti-bot check does not flag a near-empty page
" * 20 + b""
+
+
+class _Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(200)
+ self.send_header("Content-Type", "text/html")
+ self.end_headers()
+ self.wfile.write(TRAP_HTML if self.path == "/trap" else OK_HTML)
+
+ def log_message(self, *args):
+ pass
+
+
+@pytest.fixture
+def base_url():
+ srv = socketserver.TCPServer(("127.0.0.1", 0), _Handler)
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ yield f"http://127.0.0.1:{srv.server_address[1]}"
+ srv.shutdown()
+
+
+def _cfg(**kw):
+ return CrawlerRunConfig(crawl_timeout=5000, cache_mode=CacheMode.BYPASS, **kw)
+
+
+@pytest.mark.asyncio
+async def test_trap_page_fails_within_crawl_timeout_and_leaks_nothing(base_url):
+ cfg = _cfg()
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ t0 = time.perf_counter()
+ result = await asyncio.wait_for(crawler.arun(base_url + "/trap", config=cfg), 20)
+ elapsed = time.perf_counter() - t0
+
+ assert result.success is False
+ assert "exceeded crawl_timeout" in result.error_message
+ assert elapsed < 10, f"took {elapsed:.1f}s, crawl_timeout was 5s"
+
+ bm = crawler.crawler_strategy.browser_manager
+ open_urls = [p.url for c in bm.browser.contexts for p in c.pages]
+ assert base_url + "/trap" not in open_urls, open_urls
+ assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+
+
+@pytest.mark.asyncio
+async def test_trap_page_with_overlay_removal_does_not_hang(base_url):
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ result = await asyncio.wait_for(
+ crawler.arun(base_url + "/trap", config=_cfg(remove_overlay_elements=True)), 20
+ )
+ assert result.success is False
+ assert "exceeded crawl_timeout" in result.error_message
+
+
+@pytest.mark.asyncio
+async def test_trap_session_is_dropped_and_next_crawl_works(base_url):
+ cfg = _cfg(session_id="trap-session")
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ bm = crawler.crawler_strategy.browser_manager
+ result = await asyncio.wait_for(crawler.arun(base_url + "/trap", config=cfg), 20)
+ assert result.success is False
+ assert "trap-session" not in bm.sessions
+
+ result = await asyncio.wait_for(crawler.arun(base_url + "/ok", config=cfg), 20)
+ assert result.success is True
+ assert "ok" in result.html
+ assert "trap-session" in bm.sessions # normal crawl keeps the session page
+
+
+@pytest.mark.asyncio
+async def test_no_crawl_timeout_means_no_limit(base_url):
+ """Default None must not cut a crawl; the trap page is bounded only by the test's own guard."""
+ cfg = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+ assert cfg.crawl_timeout is None
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ task = asyncio.create_task(crawler.arun(base_url + "/trap", config=cfg))
+ done, _ = await asyncio.wait({task}, timeout=8)
+ assert not done, "crawl finished without a crawl_timeout; expected it to still be running"
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ bm = crawler.crawler_strategy.browser_manager
+ assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+ assert sum(len(c.pages) for c in bm.browser.contexts) <= 1
+
+
+@pytest.mark.asyncio
+async def test_keep_last_page_rule_unchanged(base_url):
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ bm = crawler.crawler_strategy.browser_manager
+ for _ in range(3):
+ assert (await crawler.arun(base_url + "/ok", config=_cfg())).success
+ assert sum(len(c.pages) for c in bm.browser.contexts) <= 1
+
+
+@pytest.mark.asyncio
+async def test_cancel_during_cleanup_still_closes_page(base_url):
+ """A cancel landing inside the finally's console cleanup must not skip page close."""
+ cfg = _cfg(capture_console_messages=True)
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ bm = crawler.crawler_strategy.browser_manager
+ in_cleanup = asyncio.Event()
+
+ async def slow_cleanup(page, *args):
+ in_cleanup.set()
+ await asyncio.sleep(2)
+
+ crawler.crawler_strategy.adapter.cleanup_console_capture = slow_cleanup
+
+ task = asyncio.create_task(crawler.arun(base_url + "/ok", config=cfg))
+ await asyncio.wait_for(in_cleanup.wait(), 20)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+ assert sum(len(c.pages) for c in bm.browser.contexts) <= 1
+
+
+@pytest.mark.asyncio
+async def test_hung_cleanup_does_not_block_crawl_timeout(base_url):
+ """An evaluate inside cleanup (UndetectedAdapter does this) hangs on a trap page; the 5 s cleanup bound must let the crawl fail."""
+ cfg = _cfg(capture_console_messages=True)
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ async def evaluate_in_cleanup(page, *args):
+ return await page.evaluate("1")
+
+ crawler.crawler_strategy.adapter.cleanup_console_capture = evaluate_in_cleanup
+
+ t0 = time.perf_counter()
+ result = await asyncio.wait_for(crawler.arun(base_url + "/trap", config=cfg), 30)
+ assert not result.success
+ assert "exceeded crawl_timeout" in result.error_message
+ assert time.perf_counter() - t0 < 20
+ bm = crawler.crawler_strategy.browser_manager
+ assert bm._context_refcounts.get(bm._make_config_signature(cfg), 0) == 0
+ assert all(p.url != base_url + "/trap" for c in bm.browser.contexts for p in c.pages)