Make the CI unit test timeout fire before the job timeout does

.github/workflows/run-unit-tests.yml

@@ -129,11 +129,17 @@ on:  # yamllint disable-line rule:truthy
         description: "The default branch of the repository"
         required: true
         type: string
+      job-timeout-minutes:
+        description: "How long the job may run before GitHub cancels it"
+        required: false
+        default: 65
+        type: number
 permissions:
   contents: read
 jobs:
   tests:
-    timeout-minutes: 65
+    # run_unit_tests.sh derives the tests' own timeout from what is left of this budget.
+    timeout-minutes: ${{ fromJSON(inputs.job-timeout-minutes) }}
     # yamllint disable rule:line-length
     name: "\
       ${{ case(inputs.test-scope == 'Quarantined', 'Qrnt', inputs.test-scope == 'All', '', inputs.test-scope) }}\
@@ -176,9 +182,12 @@ jobs:
       AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}"
       VERBOSE: "true"
       DEFAULT_BRANCH: "${{ inputs.default-branch }}"
-      TOTAL_TEST_TIMEOUT: "3600"  # 60 minutes in seconds
+      JOB_TIMEOUT_MINUTES: "${{ inputs.job-timeout-minutes }}"
     if: inputs.test-group == 'core' || inputs.skip-providers-tests != 'true'
     steps:
+      - name: "Record job start time"
+        shell: bash
+        run: echo "JOB_START_EPOCH=$(date +%s)" >> "${GITHUB_ENV}"
       - name: "Cleanup repo"
         shell: bash
         run: sudo rm -rf ${GITHUB_WORKSPACE}/*

scripts/ci/testing/compute_remaining_test_timeout.py

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import argparse
+import time
+
+# Reserved so the test timeout handler can stop the containers and dump their logs, and the job
+# still has time to upload those logs as artifacts, before GitHub cancels the job.
+TEARDOWN_GRACE_SECONDS = 5 * 60
+
+# Floor applied when the steps before the tests already ate the budget - the job is doomed either
+# way, but the tests still get a chance to report something rather than none at all.
+MINIMUM_TEST_TIMEOUT_SECONDS = 10 * 60
+
+
+def compute_remaining_test_timeout(job_timeout_seconds: int, elapsed_seconds: int) -> int:
+    """Return how long tests may run so their own timeout fires before the job timeout does."""
+    remaining_seconds = job_timeout_seconds - elapsed_seconds - TEARDOWN_GRACE_SECONDS
+    return max(remaining_seconds, MINIMUM_TEST_TIMEOUT_SECONDS)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Print the number of seconds the tests may run within the remaining job budget."
+    )
+    parser.add_argument(
+        "--job-timeout-minutes", type=int, required=True, help="The job's `timeout-minutes` value"
+    )
+    parser.add_argument(
+        "--job-start-epoch", type=int, required=True, help="Unix timestamp stamped when the job started"
+    )
+    args = parser.parse_args()
+    # Wall clock rather than time.monotonic(): the start timestamp comes from an earlier step in a
+    # different process, so only an absolute clock is comparable across the two.
+    elapsed_seconds = int(time.time()) - args.job_start_epoch
+    print(compute_remaining_test_timeout(args.job_timeout_minutes * 60, elapsed_seconds))
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/testing/run_unit_tests.sh

@@ -30,6 +30,21 @@ fi
 TEST_GROUP=${1}
 TEST_SCOPE=${2}
 
+# The whole job - not only this step - has to fit in JOB_TIMEOUT_MINUTES, so the tests get whatever
+# is left of that budget rather than a fixed value.
+if [[ -n "${JOB_START_EPOCH:-}" && -n "${JOB_TIMEOUT_MINUTES:-}" ]]; then
+    TOTAL_TEST_TIMEOUT=$(python "$(dirname "${BASH_SOURCE[0]}")/compute_remaining_test_timeout.py" \
+        --job-timeout-minutes "${JOB_TIMEOUT_MINUTES}" --job-start-epoch "${JOB_START_EPOCH}") || exit 1
+    export TOTAL_TEST_TIMEOUT
+    TIMEOUT_DISPLAY="$((TOTAL_TEST_TIMEOUT / 60))m$((TOTAL_TEST_TIMEOUT % 60))s"
+    echo "${COLOR_BLUE}Tests get ${TIMEOUT_DISPLAY} of the ${JOB_TIMEOUT_MINUTES}m job budget${COLOR_RESET}"
+elif [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+    # Letting breeze fall back to its own default here would restore the timeout that cannot
+    # fire, and nothing would say so - the job would stay green until something hung.
+    echo "${COLOR_RED}JOB_START_EPOCH and JOB_TIMEOUT_MINUTES must both be set in CI${COLOR_RESET}"
+    exit 1
+fi
+
 function core_tests() {
     echo "${COLOR_BLUE}Running core tests${COLOR_RESET}"
     set +e

scripts/tests/ci/testing/test_compute_remaining_test_timeout.py

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import importlib.util
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pytest
+import time_machine
+
+MODULE_PATH = (
+    Path(__file__).resolve().parents[4] / "scripts" / "ci" / "testing" / "compute_remaining_test_timeout.py"
+)
+
+
+@pytest.fixture
+def timeout_module():
+    module_name = "test_compute_remaining_test_timeout_module"
+    sys.modules.pop(module_name, None)
+    spec = importlib.util.spec_from_file_location(module_name, MODULE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    ("job_timeout_seconds", "elapsed_seconds", "expected_seconds"),
+    [
+        pytest.param(65 * 60, 8 * 60, 3120, id="image-pull-only"),
+        pytest.param(65 * 60, 25 * 60, 2100, id="image-pull-and-slow-migration-tests"),
+        pytest.param(65 * 60, 55 * 60, 600, id="floor-applies-when-budget-nearly-gone"),
+        pytest.param(65 * 60, 70 * 60, 600, id="floor-applies-when-budget-overrun"),
+    ],
+)
+def test_compute_remaining_test_timeout(
+    timeout_module, job_timeout_seconds, elapsed_seconds, expected_seconds
+):
+    assert (
+        timeout_module.compute_remaining_test_timeout(job_timeout_seconds, elapsed_seconds)
+        == expected_seconds
+    )
+
+
+@time_machine.travel("2026-08-29 12:10:00+00:00", tick=False)
+def test_main_derives_the_budget_left_from_the_job_start_time(timeout_module, monkeypatch, capsys):
+    job_start_epoch = int(datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc).timestamp())
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "compute_remaining_test_timeout.py",
+            "--job-timeout-minutes",
+            "65",
+            "--job-start-epoch",
+            str(job_start_epoch),
+        ],
+    )
+
+    timeout_module.main()
+
+    assert int(capsys.readouterr().out) == (65 - 10 - 5) * 60

scripts/tests/ci/testing/test_run_unit_tests.py

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+SCRIPT_PATH = Path(__file__).resolve().parents[4] / "scripts" / "ci" / "testing" / "run_unit_tests.sh"
+
+# Deliberately excludes the directory holding breeze, so a regression that lets the script run on
+# past the guard cannot start a real test run - it fails to find breeze instead.
+MINIMAL_ENVIRONMENT = {"PATH": "/usr/bin:/bin"}
+
+
+def test_run_unit_tests_aborts_when_the_job_budget_is_missing_in_github_actions():
+    result = subprocess.run(
+        ["bash", str(SCRIPT_PATH), "core", "DB"],
+        env={**MINIMAL_ENVIRONMENT, "GITHUB_ACTIONS": "true"},
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 1
+    assert "JOB_START_EPOCH and JOB_TIMEOUT_MINUTES must both be set" in result.stdout