From 788b04502ce48043922577a4daafac82ae989733 Mon Sep 17 00:00:00 2001 From: Battleplus <3559424769@qq.com> Date: Fri, 21 Aug 2026 11:30:46 +0800 Subject: [PATCH 1/2] fix: allow single-label hostnames in deep crawl URL validation The previous implementation rejected URLs with single-label hostnames (e.g. 'https://name/xyz/') by checking for a dot in netloc. This is overly strict and prevents crawling internal DNS hosts or local development servers. Fixes #2079 Signed-off-by: Battleplus <3559424769@qq.com> --- crawl4ai/deep_crawling/bff_strategy.py | 4 ++-- crawl4ai/deep_crawling/bfs_strategy.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 511fde692..fde34ce93 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -83,8 +83,8 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") + # Single-label hostnames (e.g. "name" without a dot) are valid + # when the URL has a scheme and non-empty netloc (e.g. internal DNS). except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index dfb759272..9025b185d 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -70,8 +70,8 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") + # Single-label hostnames (e.g. "name" without a dot) are valid + # when the URL has a scheme and non-empty netloc (e.g. internal DNS). except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False From a515dc14d2ebc26ca4e190acfc2ced3eafa71899 Mon Sep 17 00:00:00 2001 From: Battleplus <3559424769@qq.com> Date: Sat, 22 Aug 2026 18:54:37 +0800 Subject: [PATCH 2/2] test: cover deep-crawl hostname validation Signed-off-by: Battleplus <3559424769@qq.com> --- crawl4ai/deep_crawling/bff_strategy.py | 4 +- crawl4ai/deep_crawling/bfs_strategy.py | 4 +- .../test_deep_crawl_url_validation.py | 37 +++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 tests/deep_crawling/test_deep_crawl_url_validation.py diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index fde34ce93..ad1a54d74 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -79,8 +79,8 @@ async def can_process_url(self, url: str, depth: int) -> bool: """ try: parsed = urlparse(url) - if not parsed.scheme or not parsed.netloc: - raise ValueError("Missing scheme or netloc") + if not parsed.scheme or not parsed.hostname: + raise ValueError("Missing scheme or hostname") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") # Single-label hostnames (e.g. "name" without a dot) are valid diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index 9025b185d..b0dc3b72d 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -66,8 +66,8 @@ async def can_process_url(self, url: str, depth: int) -> bool: """ try: parsed = urlparse(url) - if not parsed.scheme or not parsed.netloc: - raise ValueError("Missing scheme or netloc") + if not parsed.scheme or not parsed.hostname: + raise ValueError("Missing scheme or hostname") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") # Single-label hostnames (e.g. "name" without a dot) are valid diff --git a/tests/deep_crawling/test_deep_crawl_url_validation.py b/tests/deep_crawling/test_deep_crawl_url_validation.py new file mode 100644 index 000000000..833716915 --- /dev/null +++ b/tests/deep_crawling/test_deep_crawl_url_validation.py @@ -0,0 +1,37 @@ +import pytest + +from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strategy_cls", [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy]) +@pytest.mark.parametrize( + "url", + [ + "http://intranet/path", + "https://localhost:8443/path", + "http://127.0.0.1/path", + "http://[::1]/path", + ], +) +async def test_can_process_url_accepts_valid_single_label_and_ip_hosts(strategy_cls, url): + strategy = strategy_cls(max_depth=1) + + assert await strategy.can_process_url(url, depth=0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strategy_cls", [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy]) +@pytest.mark.parametrize( + "url", + [ + "intranet/path", + "ftp://intranet/path", + "http:///path", + "http://:8080/path", + ], +) +async def test_can_process_url_rejects_missing_or_invalid_hosts(strategy_cls, url): + strategy = strategy_cls(max_depth=1) + + assert not await strategy.can_process_url(url, depth=0)