fix: ignore negative Retry-After header instead of disabling 429 back-off

src/crawlee/_utils/http.py

@@ -24,11 +24,20 @@ def parse_retry_after_header(value: str | None) -> timedelta | None:
     if not value:
         return None
 
+    # Numeric form: `delay-seconds`, a non-negative integer per RFC 7231 §7.1.3.
     try:
-        return timedelta(seconds=int(value))
+        seconds = int(value)
     except ValueError:
-        pass
-
+        pass  # Not an integer, fall through to the HTTP-date form below.
+    else:
+        if seconds < 0:
+            # A negative delay is malformed. Reject it instead of returning a negative `timedelta`, which would
+            # push `throttled_until` into the past and silently disable the 429 back-off downstream.
+            logger.debug(f'Retry-After delay-seconds {value!r} is negative; ignoring.')
+            return None
+        return timedelta(seconds=seconds)
+
+    # HTTP-date form, e.g. "Wed, 21 Oct 2015 07:28:00 GMT".
     try:
         retry_date = parsedate_to_datetime(value)
         # `parsedate_to_datetime` may return a naive datetime when the input has no timezone info.

tests/unit/test_throttling_request_manager.py

@@ -543,6 +543,16 @@ def test_parse_retry_after_integer_seconds() -> None:
     assert result == timedelta(seconds=120)
 
 
+def test_parse_retry_after_zero_seconds() -> None:
+    """A delay of `0` ("retry immediately") is valid and must yield a zero delta, not None."""
+    assert parse_retry_after_header('0') == timedelta(0)
+
+
+def test_parse_retry_after_negative_seconds() -> None:
+    """`delay-seconds` is non-negative per RFC 7231; a malformed negative value must be ignored."""
+    assert parse_retry_after_header('-5') is None
+
+
 def test_parse_retry_after_invalid_value() -> None:
     assert parse_retry_after_header('not-a-date-or-number') is None