fix: allow single-label hostnames in deep crawl URL validation

crawl4ai/deep_crawling/bff_strategy.py

@@ -79,12 +79,12 @@ async def can_process_url(self, url: str, depth: int) -> bool:
         """
         try:
             parsed = urlparse(url)
-            if not parsed.scheme or not parsed.netloc:
-                raise ValueError("Missing scheme or netloc")
+            if not parsed.scheme or not parsed.hostname:
+                raise ValueError("Missing scheme or hostname")
             if parsed.scheme not in ("http", "https"):
                 raise ValueError("Invalid scheme")
-            if "." not in parsed.netloc:
-                raise ValueError("Invalid domain")
+            # Single-label hostnames (e.g. "name" without a dot) are valid
+            # when the URL has a scheme and non-empty netloc (e.g. internal DNS).
         except Exception as e:
             self.logger.warning(f"Invalid URL: {url}, error: {e}")
             return False

crawl4ai/deep_crawling/bfs_strategy.py

@@ -66,12 +66,12 @@ async def can_process_url(self, url: str, depth: int) -> bool:
         """
         try:
             parsed = urlparse(url)
-            if not parsed.scheme or not parsed.netloc:
-                raise ValueError("Missing scheme or netloc")
+            if not parsed.scheme or not parsed.hostname:
+                raise ValueError("Missing scheme or hostname")
             if parsed.scheme not in ("http", "https"):
                 raise ValueError("Invalid scheme")
-            if "." not in parsed.netloc:
-                raise ValueError("Invalid domain")
+            # Single-label hostnames (e.g. "name" without a dot) are valid
+            # when the URL has a scheme and non-empty netloc (e.g. internal DNS).
         except Exception as e:
             self.logger.warning(f"Invalid URL: {url}, error: {e}")
             return False

tests/deep_crawling/test_deep_crawl_url_validation.py

@@ -0,0 +1,37 @@
+import pytest
+
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("strategy_cls", [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy])
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://intranet/path",
+        "https://localhost:8443/path",
+        "http://127.0.0.1/path",
+        "http://[::1]/path",
+    ],
+)
+async def test_can_process_url_accepts_valid_single_label_and_ip_hosts(strategy_cls, url):
+    strategy = strategy_cls(max_depth=1)
+
+    assert await strategy.can_process_url(url, depth=0)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("strategy_cls", [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy])
+@pytest.mark.parametrize(
+    "url",
+    [
+        "intranet/path",
+        "ftp://intranet/path",
+        "http:///path",
+        "http://:8080/path",
+    ],
+)
+async def test_can_process_url_rejects_missing_or_invalid_hosts(strategy_cls, url):
+    strategy = strategy_cls(max_depth=1)
+
+    assert not await strategy.can_process_url(url, depth=0)