From bcaff122763b221945b08fb7dcf2548b62ce216b Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 30 May 2026 15:37:55 -0400 Subject: [PATCH 1/2] ci(test): enforce coverage-based test selection on PRs Flip the shadow --only-changes selection to --select-enforce on PR runs for the github (GNU/intel) jobs and the self-hosted Phoenix/Frontier jobs. PRs now run only the tests whose recorded coverage overlaps the changed files; the conservative ladder in coverage.py falls back to run-all for non-.fpp changes and .fpp files no test covers. Backstops retained: the NVHPC jobs still run --test-all on every PR (pre-merge full check), and pushes to master run the full suite (SELECT empty). --- .github/workflows/common/test.sh | 10 ++++++---- .github/workflows/test.yml | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/common/test.sh b/.github/workflows/common/test.sh index cb856f73ba..2420e89632 100644 --- a/.github/workflows/common/test.sh +++ b/.github/workflows/common/test.sh @@ -57,12 +57,14 @@ if [ -n "${job_shard:-}" ]; then shard_opts="--shard $job_shard" fi -# Coverage-based test selection in SHADOW mode on PRs: prints what it WOULD select but the -# full suite still runs (no --select-enforce). Changed files come from git detection -# (self-healing deepen) since the SLURM job doesn't receive the paths-filter list. +# Coverage-based test selection ENFORCED on PRs: runs only tests whose recorded coverage +# overlaps the PR's changed files (conservative ladder; non-.fpp and uncovered-.fpp changes +# fall back to run-all). Pushes to master run the full suite as a backstop. Changed files +# come from git detection (self-healing deepen) since the SLURM job doesn't receive the +# paths-filter list. select_opts="" if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then - select_opts="--only-changes" + select_opts="--only-changes --select-enforce" fi ./mfc.sh test -v --max-attempts 3 --no-build $select_opts -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c05d68c17..26a08a8461 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -264,11 +264,13 @@ jobs: - name: Test if: '!matrix.nvhpc' run: | - # Coverage-based test selection in SHADOW mode on PRs: the selector - # prints what it WOULD run, but the full suite still runs (no - # --select-enforce). Enforcement is a separate, later change. + # Coverage-based test selection ENFORCED on PRs: runs only the tests whose + # recorded coverage overlaps the PR's changed files (conservative ladder in + # coverage.py — non-.fpp changes and .fpp files no test covers fall back to + # run-all). Pushes to master run the full suite (SELECT empty) as a backstop, + # and the NVHPC jobs below still run --test-all as a pre-merge full check. SELECT=() - [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--only-changes --changed-files "$CHANGED_FILES") + [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--only-changes --select-enforce --changed-files "$CHANGED_FILES") /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION env: TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} From 3eaba1c7bc145082e39c450df2e600d7ab534594 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 30 May 2026 15:46:01 -0400 Subject: [PATCH 2/2] ci(test): make --select-enforce imply selection (drop redundant --only-changes) --select-enforce previously did nothing without --only-changes, so the enforce config had to pass both. Gate the selection block on either flag: --select-enforce alone now selects AND prunes; --only-changes alone stays shadow (print selection, run full suite). CI enforce paths simplify to just --select-enforce --changed-files. --- .github/workflows/common/test.sh | 2 +- .github/workflows/test.yml | 2 +- toolchain/mfc/cli/commands.py | 4 ++-- toolchain/mfc/test/test.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/common/test.sh b/.github/workflows/common/test.sh index 2420e89632..14927ee314 100644 --- a/.github/workflows/common/test.sh +++ b/.github/workflows/common/test.sh @@ -64,7 +64,7 @@ fi # paths-filter list. select_opts="" if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then - select_opts="--only-changes --select-enforce" + select_opts="--select-enforce" fi ./mfc.sh test -v --max-attempts 3 --no-build $select_opts -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26a08a8461..5fe8cda475 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -270,7 +270,7 @@ jobs: # run-all). Pushes to master run the full suite (SELECT empty) as a backstop, # and the NVHPC jobs below still run --test-all as a pre-merge full check. SELECT=() - [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--only-changes --select-enforce --changed-files "$CHANGED_FILES") + [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES") /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION env: TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} diff --git a/toolchain/mfc/cli/commands.py b/toolchain/mfc/cli/commands.py index 06c55cbdce..54bbff4641 100644 --- a/toolchain/mfc/cli/commands.py +++ b/toolchain/mfc/cli/commands.py @@ -472,14 +472,14 @@ dest="only_changes", action=ArgAction.STORE_TRUE, default=False, - help="Select only tests whose covered files overlap changed files (shadow mode unless --select-enforce).", + help="Shadow mode: compute and print the coverage-based test selection but still run the full suite. Use --select-enforce to actually prune.", ), Argument( name="select-enforce", dest="select_enforce", action=ArgAction.STORE_TRUE, default=False, - help="With --only-changes, actually skip unselected tests (otherwise shadow: print selection, run all).", + help="Run only the coverage-selected tests (implies selection; skips unselected tests). Without it, --only-changes is shadow-only.", ), Argument(name="changed-files", dest="changed_files", type=str, default=None, help="Changed-file list (newline-, space-, or comma-separated; from CI paths-filter). Overrides git detection."), Argument(name="changes-branch", dest="changes_branch", type=str, default="master", help="Branch to diff against for --only-changes."), diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index f312a7cf49..5f61416f8e 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -178,7 +178,7 @@ def is_uuid(term): if not cases: raise MFCException(f"--shard {ARG('shard')} matched zero test cases. Total cases before sharding may be less than shard count.") - if ARG("only_changes"): + if ARG("only_changes") or ARG("select_enforce"): import datetime from .. import common