Security/codeql research findings

crawl4ai/utils.py

@@ -1,3 +1,5 @@
+import ipaddress
+import socket
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from bs4 import BeautifulSoup, Comment, element, Tag, NavigableString
@@ -50,6 +52,46 @@
 import inspect
 
 
+def _validate_public_http_url(url: str) -> tuple[str, int | None]:
+    """Return hostname/port only when the URL resolves to a public address."""
+    parsed = urlparse(url)
+    if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
+        raise ValueError("robots URL must use http(s) and include a hostname")
+    if parsed.username is not None or parsed.password is not None:
+        raise ValueError("robots URL must not contain user information")
+
+    try:
+        port = parsed.port
+    except ValueError as exc:
+        raise ValueError("robots URL contains an invalid port") from exc
+
+    try:
+        addresses = [ipaddress.ip_address(parsed.hostname)]
+    except ValueError:
+        try:
+            addresses = [
+                ipaddress.ip_address(info[4][0])
+                for info in socket.getaddrinfo(
+                    parsed.hostname, port, type=socket.SOCK_STREAM
+                )
+            ]
+        except (OSError, ValueError) as exc:
+            raise ValueError("robots hostname could not be resolved") from exc
+
+    if not addresses or any(
+        address.is_private
+        or address.is_loopback
+        or address.is_link_local
+        or address.is_reserved
+        or address.is_multicast
+        or address.is_unspecified
+        for address in addresses
+    ):
+        raise ValueError("robots URL resolves to a private or reserved address")
+
+    return parsed.hostname, port
+
+
 # Monkey patch to fix wildcard handling in urllib.robotparser
 from urllib.robotparser import RuleLine
 import re
@@ -327,7 +369,8 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool:
             domain = parsed.netloc
             if not domain:
                 return True
-        except Exception as _ex:
+            hostname, port = _validate_public_http_url(url)
+        except (TypeError, ValueError):
             return True
 
         # Fast path - check cache first
@@ -336,12 +379,18 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool:
         # If rules not found or stale, fetch new ones
         if not is_fresh:
             try:
-                # Ensure we use the same scheme as the input URL
-                scheme = parsed.scheme or 'http'
-                robots_url = f"{scheme}://{domain}/robots.txt"
-
+                # Use the validated hostname/port and never follow a
+                # redirect to an address that was not checked above.
+                scheme = parsed.scheme.lower()
+                netloc = f"[{hostname}]" if ":" in hostname else hostname
+                if port:
+                    netloc = f"{netloc}:{port}"
+                robots_url = f"{scheme}://{netloc}/robots.txt"
+
                 async with aiohttp.ClientSession() as session:
-                    async with session.get(robots_url, timeout=2, ssl=False) as response:
+                    async with session.get(
+                        robots_url, timeout=2, allow_redirects=False
+                    ) as response:
                         if response.status == 200:
                             rules = await response.text()
                             self._cache_rules(domain, rules)

docs/md_v2/marketplace/backend/database.py

@@ -77,14 +77,44 @@ def _create_or_update_table(self, table_name: str, columns: Dict):
 
         self.conn.commit()
 
-    def get_all(self, table: str, limit: int = 100, offset: int = 0, where: str = None) -> List[Dict]:
-        cursor = self.conn.cursor()
-        query = f"SELECT * FROM {table}"
-        if where:
-            query += f" WHERE {where}"
-        query += f" LIMIT {limit} OFFSET {offset}"
+    def get_all(
+        self,
+        table: str,
+        limit: int = 100,
+        offset: int = 0,
+        where: Dict[str, Any] | None = None,
+    ) -> List[Dict]:
+        """Return rows using schema-validated equality filters.
+
+        SQL identifiers cannot be bound as parameters, so table and filter
+        column names are validated against the loaded schema before they are
+        interpolated. Filter values, limits, and offsets remain bound
+        parameters; callers must not pass raw SQL fragments.
+        """
+        if table not in self.schema["tables"]:
+            raise ValueError(f"Unknown table: {table}")
+
+        filters = where or {}
+        if not isinstance(filters, dict):
+            raise TypeError("where must be a mapping of column names to values")
+
+        allowed_columns = set(self.schema["tables"][table]["columns"])
+        clauses = []
+        params = []
+        for column, value in filters.items():
+            if column not in allowed_columns:
+                raise ValueError(f"Unknown filter column for {table}: {column}")
+            clauses.append(f'"{column}" = ?')
+            params.append(value)
+
+        query = f'SELECT * FROM "{table}"'
+        if clauses:
+            query += " WHERE " + " AND ".join(clauses)
+        query += " LIMIT ? OFFSET ?"
+        params.extend([max(0, int(limit)), max(0, int(offset))])
 
-        cursor.execute(query)
+        cursor = self.conn.cursor()
+        cursor.execute(query, params)
         rows = cursor.fetchall()
         return [dict(row) for row in rows]
 
@@ -114,4 +144,4 @@ def search(self, query: str, tables: List[str] = None) -> Dict[str, List[Dict]]:
 
     def close(self):
         if self.conn:
-            self.conn.close()
+            self.conn.close()

docs/md_v2/marketplace/backend/server.py

@@ -95,18 +95,17 @@ async def get_apps(
     offset: int = Query(default=0)
 ):
     """Get apps with optional filters"""
-    where_clauses = []
+    filters = {}
     if category:
-        where_clauses.append(f"category = '{category}'")
+        filters["category"] = category
     if type:
-        where_clauses.append(f"type = '{type}'")
+        filters["type"] = type
     if featured is not None:
-        where_clauses.append(f"featured = {1 if featured else 0}")
+        filters["featured"] = int(featured)
     if sponsored is not None:
-        where_clauses.append(f"sponsored = {1 if sponsored else 0}")
+        filters["sponsored"] = int(sponsored)
 
-    where = " AND ".join(where_clauses) if where_clauses else None
-    apps = db.get_all('apps', limit=limit, offset=offset, where=where)
+    apps = db.get_all("apps", limit=limit, offset=offset, where=filters or None)
 
     # Parse JSON fields
     for app in apps:
@@ -118,7 +117,7 @@ async def get_apps(
 @router.get("/apps/{slug}")
 async def get_app(slug: str):
     """Get single app by slug"""
-    apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1)
+    apps = db.get_all("apps", where={"slug": slug}, limit=1)
     if not apps:
         raise HTTPException(status_code=404, detail="App not found")
 
@@ -135,8 +134,8 @@ async def get_articles(
     offset: int = Query(default=0)
 ):
     """Get articles with optional category filter"""
-    where = f"category = '{category}'" if category else None
-    articles = db.get_all('articles', limit=limit, offset=offset, where=where)
+    filters = {"category": category} if category else None
+    articles = db.get_all("articles", limit=limit, offset=offset, where=filters)
 
     # Parse JSON fields
     for article in articles:
@@ -150,7 +149,7 @@ async def get_articles(
 @router.get("/articles/{slug}")
 async def get_article(slug: str):
     """Get single article by slug"""
-    articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1)
+    articles = db.get_all("articles", where={"slug": slug}, limit=1)
     if not articles:
         raise HTTPException(status_code=404, detail="Article not found")
 
@@ -174,8 +173,8 @@ async def get_categories():
 @router.get("/sponsors")
 async def get_sponsors(active: Optional[bool] = True):
     """Get sponsors, default active only"""
-    where = f"active = {1 if active else 0}" if active is not None else None
-    sponsors = db.get_all('sponsors', where=where, limit=20)
+    filters = {"active": int(active)} if active is not None else None
+    sponsors = db.get_all("sponsors", where=filters, limit=20)
 
     # Filter by date if active
     if active:
@@ -211,10 +210,10 @@ async def search(q: str = Query(min_length=2)):
 async def get_stats():
     """Get marketplace statistics"""
     stats = {
-        "total_apps": len(db.get_all('apps', limit=10000)),
-        "total_articles": len(db.get_all('articles', limit=10000)),
-        "total_categories": len(db.get_all('categories', limit=1000)),
-        "active_sponsors": len(db.get_all('sponsors', where="active = 1", limit=1000))
+        "total_apps": len(db.get_all("apps", limit=10000)),
+        "total_articles": len(db.get_all("articles", limit=10000)),
+        "total_categories": len(db.get_all("categories", limit=1000)),
+        "active_sponsors": len(db.get_all("sponsors", where={"active": 1}, limit=1000)),
     }
     return json_response(stats, cache_time=1800)
 
@@ -279,13 +278,13 @@ async def get_admin_stats():
     stats = {
         "apps": {
             "total": len(db.get_all('apps', limit=10000)),
-            "featured": len(db.get_all('apps', where="featured = 1", limit=10000)),
-            "sponsored": len(db.get_all('apps', where="sponsored = 1", limit=10000))
+            "featured": len(db.get_all("apps", where={"featured": 1}, limit=10000)),
+            "sponsored": len(db.get_all("apps", where={"sponsored": 1}, limit=10000))
         },
         "articles": len(db.get_all('articles', limit=10000)),
         "categories": len(db.get_all('categories', limit=1000)),
         "sponsors": {
-            "active": len(db.get_all('sponsors', where="active = 1", limit=1000)),
+            "active": len(db.get_all("sponsors", where={"active": 1}, limit=1000)),
             "total": len(db.get_all('sponsors', limit=10000))
         },
         "total_views": sum(app.get('views', 0) for app in db.get_all('apps', limit=10000))
@@ -494,4 +493,4 @@ async def root():
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="127.0.0.1", port=8100)
+    uvicorn.run(app, host="127.0.0.1", port=8100)