From 64f51d4fb3efe1c4fa4c1e7ae216a6ffb199016c Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Wed, 22 Jul 2026 20:13:59 +0000 Subject: [PATCH 01/11] feat: Enable TensorRT-RTX build for linux-aarch64 (SBSA / DGX Spark) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TensorRT-RTX 1.5 ships a Linux aarch64 (SBSA) distribution that supports DGX Spark (GB10, compute capability 12.1, experimental), but the repo only wired up the x86_64 and Windows RTX archives. This adds the aarch64 RTX archive and threads a rtx_sbsa config_setting (aarch64 + linux + compute_libs=rtx) through the TensorRT selects, so USE_TRT_RTX=1 builds resolve @tensorrt_rtx_sbsa on aarch64. Build wiring: - MODULE.bazel: add tensorrt_rtx_sbsa http_archive (aarch64 tarball). - toolchains/ci_workspaces/MODULE.bazel.tmpl: same archive. CI REGENERATES MODULE.bazel from this template (pre_build_script.sh: envsubst > MODULE.bazel), so the real MODULE.bazel edit alone is not enough — the template must match or the RTX aarch64 build fails 'no repository visible as @tensorrt_rtx_sbsa'. - third_party/tensorrt_rtx/{archive,local}/BUILD: rtx_sbsa config_setting + lib/header/nvinfer selects (lib/libtensorrt_rtx.so, @cuda//:cudart). - ~20 repo BUILD files: rtx_sbsa config_setting + a :rtx_sbsa select entry mirroring :rtx_x86_64 (sourced from @tensorrt_rtx_sbsa). Basic CI: - .github/workflows/build-test-linux-aarch64_rtx.yml: mirrors the standard SBSA build with use-rtx: true. BUILD-ONLY (no aarch64 GPU test runners). Co-Authored-By: Claude Opus 4.8 --- .../build-test-linux-aarch64_rtx.yml | 91 +++++++++++++++++++ BUILD.bazel | 13 +++ MODULE.bazel | 10 ++ core/BUILD | 12 +++ core/conversion/BUILD | 12 +++ core/conversion/conversionctx/BUILD | 12 +++ core/conversion/converters/BUILD | 14 +++ core/conversion/evaluators/BUILD | 12 +++ core/conversion/tensorcontainer/BUILD | 12 +++ core/conversion/var/BUILD | 12 +++ core/ir/BUILD | 12 +++ core/lowering/BUILD | 12 +++ core/partitioning/BUILD | 12 +++ core/partitioning/partitioningctx/BUILD | 12 +++ core/partitioning/partitioninginfo/BUILD | 12 +++ core/partitioning/segmentedblock/BUILD | 12 +++ core/plugins/BUILD | 14 +++ core/runtime/BUILD | 13 +++ core/util/BUILD | 12 +++ core/util/logging/BUILD | 10 ++ cpp/BUILD | 11 +++ tests/util/BUILD | 12 +++ third_party/tensorrt_rtx/archive/BUILD | 11 +++ third_party/tensorrt_rtx/local/BUILD | 20 ++++ toolchains/ci_workspaces/MODULE.bazel.tmpl | 10 ++ 25 files changed, 385 insertions(+) create mode 100644 .github/workflows/build-test-linux-aarch64_rtx.yml diff --git a/.github/workflows/build-test-linux-aarch64_rtx.yml b/.github/workflows/build-test-linux-aarch64_rtx.yml new file mode 100644 index 0000000000..eb98862081 --- /dev/null +++ b/.github/workflows/build-test-linux-aarch64_rtx.yml @@ -0,0 +1,91 @@ +name: RTX - Build Linux aarch64 wheels + +# Basic RTX support for aarch64 (SBSA): mirrors build-test-linux-aarch64.yml but +# selects TensorRT-RTX (use-rtx: true). BUILD-ONLY — there are no aarch64 GPU test +# runners, so this just validates the RTX aarch64 wheel builds. The full +# manifest-based wiring lives in the CI redesign (test-dx) on top of this. + +on: + pull_request: + push: + branches: + - main + - nightly + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + filter-matrix: + needs: [generate-matrix] + outputs: + matrix: ${{ steps.filter.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/checkout@v6 + with: + repository: pytorch/tensorrt + - name: Filter matrix + id: filter + env: + LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} + MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + build: + needs: filter-matrix + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/tensorrt + pre-script: packaging/pre_build_script.sh + env-var-script: packaging/env_vars.txt + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/smoke_test_script.sh + package-name: torch_tensorrt + display-name: RTX - Build SBSA torch-tensorrt whl package + name: ${{ matrix.display-name }} + uses: ./.github/workflows/build_linux.yml + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + env-var-script: ${{ matrix.env-var-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + architecture: "aarch64" + use-rtx: true + pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} + cancel-in-progress: true diff --git a/BUILD.bazel b/BUILD.bazel index a0123ff3ee..4dbc88f847 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -46,6 +46,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "no_package_executorch", flag_values = { @@ -217,6 +228,7 @@ pkg_tar( ":no_package_executorch": [], ":rtx_win": [], ":rtx_x86_64": [], + ":rtx_sbsa": [], ":windows": [], "//conditions:default": [ "//examples/executorch_reference_runner:example_executorch_runner", @@ -260,6 +272,7 @@ pkg_tar( ":no_package_executorch": [], ":rtx_win": [], ":rtx_x86_64": [], + ":rtx_sbsa": [], ":windows": [], "//conditions:default": [ ":executorch_source_package", diff --git a/MODULE.bazel b/MODULE.bazel index 2f384e8066..93aca46848 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -195,6 +195,16 @@ http_archive( ], ) +http_archive( + name = "tensorrt_rtx_sbsa", + build_file = "@//third_party/tensorrt_rtx/archive:BUILD", + strip_prefix = "TensorRT-RTX-1.5.0.114", + type = "tar.zst", + urls = [ + "https://developer.nvidia.com/downloads/trt/rtx_sdk/secure/1.5/TensorRT-RTX-1.5.0.114-Linux-aarch64-cuda-13.2-Release-external.tar.zst", + ], +) + http_archive( name = "tensorrt_sbsa", build_file = "@//third_party/tensorrt/archive:BUILD", diff --git a/core/BUILD b/core/BUILD index 9c334ffcf6..51ddecb75b 100644 --- a/core/BUILD +++ b/core/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -83,6 +94,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/conversion/BUILD b/core/conversion/BUILD index 480481e6bd..4196f2eb1d 100644 --- a/core/conversion/BUILD +++ b/core/conversion/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -78,6 +89,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/conversion/conversionctx/BUILD b/core/conversion/conversionctx/BUILD index d0ad2e7bd1..b9f9b53d62 100644 --- a/core/conversion/conversionctx/BUILD +++ b/core/conversion/conversionctx/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -73,6 +84,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/conversion/converters/BUILD b/core/conversion/converters/BUILD index ee69320df5..6df31e1ff9 100644 --- a/core/conversion/converters/BUILD +++ b/core/conversion/converters/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -73,6 +84,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], @@ -102,6 +114,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], @@ -169,6 +182,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/conversion/evaluators/BUILD b/core/conversion/evaluators/BUILD index e9fc358582..f71df9f9e0 100644 --- a/core/conversion/evaluators/BUILD +++ b/core/conversion/evaluators/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -79,6 +90,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/conversion/tensorcontainer/BUILD b/core/conversion/tensorcontainer/BUILD index c6f56b70c8..629afce89f 100644 --- a/core/conversion/tensorcontainer/BUILD +++ b/core/conversion/tensorcontainer/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -72,6 +83,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/conversion/var/BUILD b/core/conversion/var/BUILD index ce58ca70f3..e6e66441c6 100644 --- a/core/conversion/var/BUILD +++ b/core/conversion/var/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -75,6 +86,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/ir/BUILD b/core/ir/BUILD index 5dfdeded90..7c3919bea9 100644 --- a/core/ir/BUILD +++ b/core/ir/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -75,6 +86,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/lowering/BUILD b/core/lowering/BUILD index 6084198c74..8f463cd40d 100644 --- a/core/lowering/BUILD +++ b/core/lowering/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -77,6 +88,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/partitioning/BUILD b/core/partitioning/BUILD index bbbb89af37..9e84dbff21 100644 --- a/core/partitioning/BUILD +++ b/core/partitioning/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -80,6 +91,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/partitioning/partitioningctx/BUILD b/core/partitioning/partitioningctx/BUILD index bae63241a6..9b2b9bcd5c 100644 --- a/core/partitioning/partitioningctx/BUILD +++ b/core/partitioning/partitioningctx/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -76,6 +87,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/partitioning/partitioninginfo/BUILD b/core/partitioning/partitioninginfo/BUILD index 04515abb10..f2b28e4f5a 100644 --- a/core/partitioning/partitioninginfo/BUILD +++ b/core/partitioning/partitioninginfo/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -75,6 +86,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/partitioning/segmentedblock/BUILD b/core/partitioning/segmentedblock/BUILD index 73916bb6bd..0aafc0b581 100644 --- a/core/partitioning/segmentedblock/BUILD +++ b/core/partitioning/segmentedblock/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -75,6 +86,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/plugins/BUILD b/core/plugins/BUILD index de425aeab4..1b9fc164bc 100644 --- a/core/plugins/BUILD +++ b/core/plugins/BUILD @@ -22,6 +22,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -69,6 +80,7 @@ cc_library( ], ":rtx_win": [], ":rtx_x86_64": [], + ":rtx_sbsa": [], "//conditions:default": [ "impl/interpolate_plugin.cpp", "impl/normalize_plugin.cpp", @@ -84,6 +96,7 @@ cc_library( ], ":rtx_win": [], ":rtx_x86_64": [], + ":rtx_sbsa": [], "//conditions:default": [ "impl/interpolate_plugin.h", "impl/normalize_plugin.h", @@ -115,6 +128,7 @@ cc_library( ], ":rtx_win": [], ":rtx_x86_64": [], + ":rtx_sbsa": [], ":sbsa": [ "@tensorrt_sbsa//:nvinfer", "@tensorrt_sbsa//:nvinferplugin", diff --git a/core/runtime/BUILD b/core/runtime/BUILD index e1eaaa08d3..51e77a8663 100644 --- a/core/runtime/BUILD +++ b/core/runtime/BUILD @@ -23,6 +23,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -74,6 +85,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], @@ -129,6 +141,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/util/BUILD b/core/util/BUILD index 0a8ed29005..22609d97ee 100644 --- a/core/util/BUILD +++ b/core/util/BUILD @@ -21,6 +21,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -140,6 +151,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/core/util/logging/BUILD b/core/util/logging/BUILD index c09df8139b..ff9c542d8a 100644 --- a/core/util/logging/BUILD +++ b/core/util/logging/BUILD @@ -19,6 +19,15 @@ config_setting( flag_values = {"//toolchains/dep_collection:compute_libs": "rtx"}, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = {"//toolchains/dep_collection:compute_libs": "rtx"}, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -67,6 +76,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/cpp/BUILD b/cpp/BUILD index 23509b9164..b54b2d24e5 100644 --- a/cpp/BUILD +++ b/cpp/BUILD @@ -24,6 +24,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "sbsa", constraint_values = [ diff --git a/tests/util/BUILD b/tests/util/BUILD index 1f2b66a711..6a4a18f51c 100644 --- a/tests/util/BUILD +++ b/tests/util/BUILD @@ -54,6 +54,17 @@ config_setting( }, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = { + "//toolchains/dep_collection:compute_libs": "rtx", + }, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -82,6 +93,7 @@ cc_library( ":jetpack": ["@tensorrt_l4t//:nvinfer"], ":rtx_win": ["@tensorrt_rtx_win//:nvinfer"], ":rtx_x86_64": ["@tensorrt_rtx//:nvinfer"], + ":rtx_sbsa": ["@tensorrt_rtx_sbsa//:nvinfer"], ":sbsa": ["@tensorrt_sbsa//:nvinfer"], ":windows": ["@tensorrt_win//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/third_party/tensorrt_rtx/archive/BUILD b/third_party/tensorrt_rtx/archive/BUILD index 008c2fbdf9..b1517bdc4f 100644 --- a/third_party/tensorrt_rtx/archive/BUILD +++ b/third_party/tensorrt_rtx/archive/BUILD @@ -11,6 +11,15 @@ config_setting( flag_values = {"@//toolchains/dep_collection:compute_libs": "rtx"}, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = {"@//toolchains/dep_collection:compute_libs": "rtx"}, +) + config_setting( name = "rtx_win", constraint_values = [ @@ -40,6 +49,7 @@ cc_import( shared_library = select({ ":rtx_win": "bin/tensorrt_rtx_1_5.dll", ":rtx_x86_64": "lib/libtensorrt_rtx.so", + ":rtx_sbsa": "lib/libtensorrt_rtx.so", }), visibility = ["//visibility:private"], ) @@ -64,5 +74,6 @@ cc_library( "@cuda_win//:cudart", ], ":rtx_x86_64": ["@cuda//:cudart"], + ":rtx_sbsa": ["@cuda//:cudart"], }), ) diff --git a/third_party/tensorrt_rtx/local/BUILD b/third_party/tensorrt_rtx/local/BUILD index 1b9e769dcf..bba3ece98e 100644 --- a/third_party/tensorrt_rtx/local/BUILD +++ b/third_party/tensorrt_rtx/local/BUILD @@ -19,6 +19,15 @@ config_setting( flag_values = {"@//toolchains/dep_collection:compute_libs": "rtx"}, ) +config_setting( + name = "rtx_sbsa", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + flag_values = {"@//toolchains/dep_collection:compute_libs": "rtx"}, +) + cc_library( name = "nvinfer_headers", hdrs = select({ @@ -42,6 +51,16 @@ cc_library( # "include/NvInferPluginUtils.h", # ], ), + ":rtx_sbsa": glob( + [ + "include/NvInfer*.h", + ], + allow_empty = True, + # exclude = [ + # "include/NvInferPlugin.h", + # "include/NvInferPluginUtils.h", + # ], + ), }), includes = ["include/"], visibility = ["//visibility:private"], @@ -60,6 +79,7 @@ cc_import( shared_library = select({ ":rtx_win": "bin/tensorrt_rtx_1_5.dll", ":rtx_x86_64": "lib/libtensorrt_rtx.so", + ":rtx_sbsa": "lib/libtensorrt_rtx.so", }), visibility = ["//visibility:private"], ) diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 46479dc114..fface1d10d 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -111,6 +111,16 @@ http_archive( ], ) +http_archive( + name = "tensorrt_rtx_sbsa", + build_file = "@//third_party/tensorrt_rtx/archive:BUILD", + strip_prefix = "TensorRT-RTX-1.5.0.114", + type = "tar.zst", + urls = [ + "https://developer.nvidia.com/downloads/trt/rtx_sdk/secure/1.5/TensorRT-RTX-1.5.0.114-Linux-aarch64-cuda-${CU_UPPERBOUND}-Release-external.tar.zst", + ], +) + http_archive( name = "tensorrt_sbsa", build_file = "@//third_party/tensorrt/archive:BUILD", From b635d1985a4591dab1eb2543abcb2eef0ef1ff34 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Tue, 30 Jun 2026 20:44:03 +0000 Subject: [PATCH 02/11] feat(test-dx): suite manifest + runner (Layer 0) and the testing/CI design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests/ci/{suites,runner,__main__}.py — a typed, declarative suite manifest plus one runner (`python -m tests.ci`) that replaces the trt_tier_* bash functions as the source of truth for what each test job runs. Faithfully reproduces today's tier selection (verified via --dry-run) while fixing the junit-path collisions, the double-run of hlo/, and giving every suite the uniform flake-rerun+repro wrapper. - pyproject: register smoke/flaky markers; norecursedirs += ci - justfile: doctor / suites / suite / lane / report recipes (thin callers) - TESTING_AND_CI_DESIGN.md: full local + CI design (manifest+runner, lanes without merge queue, GHA-cache caching, aggregated agent-friendly reports) --- TESTING_AND_CI_DESIGN.md | 391 +++++++++++++++++++++++++++++++++++++++ justfile | 40 ++++ pyproject.toml | 3 + tests/ci/__init__.py | 16 ++ tests/ci/__main__.py | 171 +++++++++++++++++ tests/ci/runner.py | 247 +++++++++++++++++++++++++ tests/ci/suites.py | 322 ++++++++++++++++++++++++++++++++ 7 files changed, 1190 insertions(+) create mode 100644 TESTING_AND_CI_DESIGN.md create mode 100644 tests/ci/__init__.py create mode 100644 tests/ci/__main__.py create mode 100644 tests/ci/runner.py create mode 100644 tests/ci/suites.py diff --git a/TESTING_AND_CI_DESIGN.md b/TESTING_AND_CI_DESIGN.md new file mode 100644 index 0000000000..c4abad2c60 --- /dev/null +++ b/TESTING_AND_CI_DESIGN.md @@ -0,0 +1,391 @@ +# Torch-TensorRT — Testing & CI Design + +> **Status:** proposal / north-star design. Some pieces already exist on `main` +> (marked **✓ exists**); the rest is the target end-state and a rollout plan. +> +> **One-line goal:** *a developer can run exactly what CI runs, locally, with one +> command — and when something fails, the failure tells them how to reproduce and +> fix it.* + +--- + +## 1. Goals & non-goals + +**Goals** +- **Full coverage** of torch-tensorrt: converters, runtime, lowering, dynamo, torch-compile, torchscript, plugins, models, distributed — across the platforms/CUDA/Python we ship. +- **Ergonomic for developers**: the local experience is the *primary* design target, not an afterthought bolted onto CI. +- **Zero local/CI drift**: the command that runs a tier in CI is the *same* command you run locally. +- **Fast feedback by default, full coverage on demand**: a PR gets a signal in ~15 min; the exhaustive matrix runs when it matters. +- **Failures are never hidden**: nothing gets clobbered, every failure prints its own reproduce command, and the aggregate is machine- and agent-readable. + +**Non-goals** +- Replacing the PyTorch test-infra build plumbing (we reuse it for wheels/runners). +- A bespoke remote-execution cluster (we explicitly assume **no owned cache/RBE infra**). +- 100% of the matrix on every push (that's what nightly + the full lane are for). + +--- + +## 2. Design principles — the "pleasant" contract + +These are the invariants. Every decision below serves one of them. + +1. **One command to run anything.** `just ` — never a 200-char `pytest` incantation. +2. **Local == CI.** Suites are a declarative manifest + one runner; `just` and CI both call `ci run `, so there's nothing to drift. +3. **No rebuild to test.** Running tests never triggers a Bazel rebuild. +4. **Reproduce in one line.** Every CI failure prints `uv run --no-sync pytest … -n0 `. +5. **Build once, test many.** One wheel per (platform, CUDA, Python); every tier reuses it. +6. **Fail loud, aggregate everything.** One consolidated, agent-friendly report; no silent truncation; no fragile third-party glue. +7. **Cheap to retrigger, easy to discover.** Re-run via a PR comment; a menu tells you how. +8. **Pay for the signal you need.** Tiers + lanes + path filters; don't run GPU jobs for a docs typo. +9. **Flakes never mask real failures.** Retries are scoped to known signatures; everything else is quarantined with a tracking issue. + +--- + +## 3. Mental model + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ LAYER 0 — Test logic (platform-agnostic, the single source) │ +│ tests/ci/suites.py SUITES = [Suite(...)] manifest (DATA)│ +│ tests/ci/__main__.py `ci run ` · `ci matrix` one engine │ +│ @pytest.mark.{smoke,critical} lane membership on the test │ +│ tests/py/utils/junit_summary.py aggregate + --agent report ✓ │ +│ tests/py/utils/ci_helpers.sh env + trt_pytest wrapper only │ +│ pyproject.toml markers, optional dep groups │ +└───────────────┬───────────────────────────────┬───────────────────┘ + │ calls "ci run " │ matrix from "ci matrix" + ┌───────▼────────┐ ┌───────▼─────────────────┐ + │ LAYER 1 — LOCAL │ │ LAYER 1 — CI │ + │ justfile ✓ │ │ _build.yml / _test.yml│ + │ uv, jj fix │ │ ci.yml (lanes) │ + └────────────────┘ └───────┬─────────────────┘ + │ + ┌────────▼─────────┐ + │ LAYER 2 — STATUS │ + │ rollup check │ + │ PR comment(--agent)│ + │ /rerun, menu ✓ │ + └──────────────────┘ +``` + +The point: **Layer 0 is the contract — and it is _data + one engine_, not a pile of +shell functions.** A declarative **suite manifest** (`tests/ci/suites.py`) says *what* +each suite is; a single **runner** (`tests/ci/__main__.py`) is the *only* place that knows +*how* to turn a suite into a `pytest` command. `just` and CI are both thin callers of +`ci run `, and CI's job matrix is *generated* by `ci matrix` — so they cannot drift, +and adding a job is a one-line data change (§5.3). + +--- + +## 4. Test taxonomy + +### 4.1 Two orthogonal axes (retiring L0/L1/L2-by-filename) + +A job is a cell in **(subsystem × lane × variant)** — three *independent* axes. The old +`L0/L1/L2` tier numbers fused "what subsystem" with "how deep," and worse, encoded depth +in **filenames** (`runtime/test_000_*` = L0, `runtime/test_001_*` = L1, the rest = L2 via +`-k "not test_000_…"`). That's retired. The axes are now: + +- **Subsystem** *(what — a directory)*: `converters`, `runtime`, `lowering`, `partitioning`, `dynamo-models`, `torch-compile`, `torchscript`, `plugins`, `kernels`, `quantization`, `llm`, `distributed`, `executorch`. +- **Lane** *(how deep — a marker)*: `fast` (= `-m smoke`, every push), `full` (default, the ready-signal lane), `nightly` (everything + perf). +- **Variant** *(where — a dimension)*: `standard` / `rtx` / platform — applied centrally by the runner, **not** as `if [ "$USE_TRT_RTX" ]` branches scattered through shell. + +| | fast (every push) | full (ready signal) | nightly | +|---|---|---|---| +| converters | smoke subset | all | + fuzz | +| runtime / lowering / partitioning | smoke subset | all | — | +| dynamo-models / torch-compile | — | `-m critical` | all + llm | +| torchscript | api smoke | models | integrations | +| plugins / kernels / quantization | — | — | all (+optional deps) | +| distributed | — | — | multi-GPU | + +Depth lives on the test (a marker), so **moving a test between lanes is editing a +decorator, never renaming a file.** Sharding a big subsystem is a manifest field +(`shards: N`), not a `test_000_*` filename convention. + +### 4.2 Lane membership = markers, uniformly (`pyproject.toml`) +Membership is expressed *one* way — a marker on the test — never filename prefixes or +`-k "not test_000_…"` strings (today three different mechanisms coexist for the same idea). +- `smoke` *(proposed)* — the fast-lane subset of any subsystem. +- `critical` ✓ — the `full`-lane core; nightly runs the complement (`-m "not critical"`). Clean partition, no overlap, no gap. +- `unit` ✓ — the bulk of the suite. +- `flaky` *(proposed)* — quarantined out of gating lanes, tracked by an issue, still run nightly for visibility. + +### 4.3 Optional dependency groups (`-ext`) +`test-ext`, `kernels`, `quantization` (uv groups) + `executorch` (plain package). Suites that need them **skip cleanly when absent** (e.g. `conftest.py` `skip_no_cuda_core`), so a base checkout never hard-fails. `just install-test-ext` pulls them all. + +### 4.4 Directory layout (`tests/py/`) +``` +tests/ + ci/ # ← NEW: the Layer-0 brain — suites.py (manifest) + __main__.py (runner) + py/ + dynamo/{conversion,lowering,runtime,partitioning,models,hlo,llm,executorch,…} + ts/ # torchscript frontend + kernels/ quantization/ # gated on optional deps + distributed/ # multi-GPU + utils/ # ci_helpers.sh (env + trt_pytest wrapper), junit_summary.py +``` + +--- + +## 5. Layer 0 — the suite manifest + one runner + +This replaces the bash `trt_tier_*` library as the source of truth. The bash version +*works*, but config-in-shell conflates data with mechanism and quietly hides errors. Today +it: tiers tests by **filename prefix**; mixes markers / `-k` strings / globs for the same +"depth" concept; drops the L2 suites out of the `trt_pytest` rerun+repro wrapper (so L2 +failures get no repro hint); and — really — points **four** pytest runs at one +`--junitxml` path so three suites' results vanish from the aggregate. Those are all +symptoms of the same root cause, fixed by separating data from engine. + +### 5.1 The manifest — `tests/ci/suites.py` *(data; typed, mypy-validated)* +```python +@dataclass(frozen=True) +class Suite: + name: str # "dynamo-runtime" + paths: tuple[str, ...] # ("dynamo/runtime",) under tests/py + lanes: tuple[Lane, ...] = ("full",) # fast | full | nightly + shards: int = 1 # split a big suite across N CI jobs + dist: str | None = None # "--dist=loadscope" + needs: tuple[str, ...] = () # optional dep groups: ("kernels", "cuda-core") + variants: tuple[Variant, ...] = ("standard", "rtx") + extra_args: tuple[str, ...] = () # ("--ir", "torch_compile"), ("--maxfail", "20") + +SUITES = [ + Suite("dynamo-converters", ("dynamo/conversion",), lanes=("fast", "full"), + dist="--dist=loadscope", shards=4), + Suite("dynamo-runtime", ("dynamo/runtime",), lanes=("fast", "full")), + Suite("dynamo-models", ("dynamo/models",), lanes=("full", "nightly")), + Suite("kernels", ("kernels",), lanes=("nightly",), + needs=("kernels", "cuda-core"), variants=("standard",)), + # … one line per subsystem +] +``` +A typo'd field or path is a **static error**, not a silently-empty glob. (YAML + a pydantic +schema is the friendlier-for-non-coders alternative; we pick Python for the static checking +and because the same module generates the CI matrix.) + +### 5.2 The runner — `tests/ci/__main__.py` *(the ONLY place pytest mechanics live)* +```bash +python -m tests.ci matrix --lane fast --variant standard # → JSON for the GH Actions matrix +python -m tests.ci run dynamo-runtime --shard 2/4 # builds + runs pytest +``` +The runner — and nothing else — knows how to: apply the lane's marker (`fast` → `-m smoke`), +derive a *unique* junit path from `name+shard` (no more collisions), wrap **every** suite in +the `trt_pytest` reruns+repro helper *uniformly*, apply `--dist`/`extra_args`, gate on `needs` +(skip cleanly if a dep group is absent), and resolve variants centrally. Its knobs: + +| Knob | Default | Meaning | +|---|---|---| +| `PYTHON` | `python` | CI: container python; local: `uv run --no-sync python` (no rebuild) | +| `TRT_JOBS` | per-suite | xdist `-n`; **GPU-memory-aware, not `-n auto`** (one GPU OOMs long before it runs out of cores) | +| `TRT_PYTEST_RERUNS` | `1` (CI) / `0` (local) | gated reruns for known flake signatures only | +| `RUNNER_TEST_RESULTS_DIR` | `$TMPDIR/trt_test_results` | where JUnit XMLs land | +| `TMPDIR` | per-user | engine/timing cache; per-user to dodge cross-user permission collisions | + +> **Lesson baked in:** rerun args are a Python list passed to `subprocess` (or, in the +> shrunken shell wrapper, a bash *array*) — **never** an unquoted string. A multi-word +> `--only-rerun "Stream capture invalidated"` expanded unquoted word-splits into phantom +> test paths and silently collects **0 tests** — this is exactly how a real CI run failed. + +### 5.3 Adding a job — one line +- **Before** (bash): write a `trt_tier_*` function in `ci_helpers.sh` + a `just` recipe + a matrix entry in `_test.yml` + a `tests-report` tier-list entry — *4 edits, 3 files, in bash.* +- **After**: add one `Suite(...)` to `SUITES`. `just` exposes it, CI's matrix includes it (it's generated), the report aggregates it — **automatically.** + +### 5.4 `tests/py/utils/ci_helpers.sh` — shrunk, not deleted ✓exists +Keeps only the genuinely-shell bits the runner shells out to: env setup and the +array-safe `trt_pytest` wrapper. No tier definitions, no selection logic. + +### 5.5 `tests/py/utils/junit_summary.py` ✓exists +Reads all JUnit XMLs and emits a **human** report (TTY colors, `NO_COLOR`/`FORCE_COLOR`) and +an **`--agent`** report: Markdown with node id, file, junit path, a copy-paste +`uv run --no-sync pytest -k -n0` repro, message, and capped detail — *paste it +to Claude and it can start fixing.* Exits non-zero on any failure **or empty result set** +(the XMLs are the source of truth for pass/fail). No third-party result actions that crash +on empty input. + +--- + +## 6. Local developer experience + +### 6.1 The golden path +```bash +just test tests/py/dynamo/conversion/test_foo.py # inner loop: one file, fast +just tests-l0 # the smoke tier, exactly as CI runs it +just tests-report l1-ext --agent # run a whole tier past failures → paste-ready report +just lint # all pre-commit hooks (== the linter CI job) +``` +Everything is `uv run --no-sync` underneath — **tests never rebuild torch-tensorrt.** + +### 6.2 Recipes (justfile — a thin caller of the runner) +- `test *args` — raw pytest in the uv env (honors `pyproject` addopts). +- `suite [-- pytest args]` — run one manifest suite, *exactly* as CI runs it (`ci run `). +- `lane ` — run every suite in a lane locally. +- `report [--agent]` — **run a lane past failures, then print one consolidated report** (run + report in one step). +- `summary [--agent]` — re-print the last run's report without re-running. +- `lint` / `lint-changed` — pre-commit over all / changed files. +- `build` *(proposed)* — wrap the clean-rebuild flow; auto-detect ABI staleness after a nightly bump. +- `jobs=N` knob — raise xdist parallelism when your GPU has headroom (`just jobs=8 lane full`). + +During migration the legacy `tests-l0/l1/l2` recipes become thin aliases for the matching lanes, then retire. + +### 6.3 Build ergonomics +- `uv pip install -e . --no-deps --no-build-isolation` for incremental; **clean rebuild after a torch-nightly bump** (libtorch ABI changes → `undefined symbol` otherwise). +- `PYTHON_ONLY=1` for pure-Python iteration (skips Bazel entirely). +- A `build` skill / recipe encodes the decision tree so nobody re-learns it. + +### 6.4 Formatting is automatic, not a chore +- `pre-commit` for the full gate (the checks: mypy, validate-pyproject, uv-lock, …). +- **`jj fix`** wired to the *formatters* (black, ruff, isort, clang-format, buildifier) as stdin→stdout filters, version-pinned via `uvx` to the exact pre-commit revs → format an entire stack in one shot, no per-commit `pre-commit` dance. + +### 6.5 The failure→fix loop +Any failing tier → `just test-summary --agent` → the `analyze-test-report` skill triages (real bug vs torch-API change vs OOM/skip vs flake) and drives each to a fix using the printed repro commands. + +--- + +## 7. CI design + +### 7.1 Lanes — *without a merge queue* + +We **cannot** use GitHub's merge queue, so the full suite is gated by an explicit +"ready" signal instead of a queue: + +| Lane | Trigger | What runs | Required? | +|---|---|---|---| +| **Fast** | every PR push | lint + **1 representative build** (py-latest + newest CUDA, standard) + **L0** | informational | +| **Full** | `ci: full` label · `/ci full` comment · **approval** (`pull_request_review`) | full matrix · L1/L2 · RTX · all platforms | **yes** (`CI / full` rollup) | +| **Main canary** | `push` to `main` | full lane | — (catches trunk breakage → fast revert) | +| **Nightly** | `schedule` | full + `-ext` model/kernels/quant + perf + exhaustive CUDA/Python | — | + +**Gating mechanics.** Branch protection requires `CI / full`. It only reports after +the full lane runs (on label/approval). Pair with **"dismiss stale approvals on new +commits"** so a post-approval push invalidates approval → re-approve → full re-runs on +the new HEAD. + +> **Honest gap.** Without a merge queue we lose the guarantee that the *actually-merged* +> tree (after other PRs land) was tested as a unit — two independently-green PRs can break +> `main` together. Mitigation: (a) require PRs rebased on fresh `main` before the full run, +> (b) the main canary + fast revert. This is the inherent cost of no queue; we accept it. + +### 7.2 Topology — build once, test many ✓(x86_64 today) + +``` +ci.yml (one entry: pull_request | pull_request_review | push | schedule | comment) + ├─ _build.yml matrix{platform, cuda, python, variant} → ONE wheel artifact each + └─ _test.yml download wheel → run tier(s) via ci_helpers.sh → JUnit artifact + └─ rollup job aggregate JUnit → single status/platform + PR comment +``` + +Platform / RTX / python-only stop being **separate workflow files** and become +**matrix inputs**. This collapses today's ~11 near-duplicate entry workflows (incl. the +461-line inline Windows files) into `_build.yml` + `_test.yml` + a thin `ci.yml`. + +### 7.3 Caching — *without owned infra* + +We assume **no remote cache/RBE server**. Use GitHub's own cache, two layers, both wired +into the **vendored** build workflow (it's a local copy, so we can edit its steps): + +1. **sccache with the GitHub Actions backend** (`SCCACHE_GHA_ENABLED=true`) — free, no infra; caches C++ object compiles. Biggest win for Bazel C++ recompiles. +2. **Bazel `--disk_cache=` persisted via `actions/cache@v4`** — key on hashes of `MODULE.bazel` / `.bazelversion` / **torch-nightly + CUDA version** + `restore-keys` for partial hits. + +**Container caveat:** the build runs inside a manylinux container; `actions/cache` runs on +the host. Either **bind-mount** a host dir as the Bazel disk-cache path (no tokens cross +the boundary — preferred), or forward `ACTIONS_CACHE_URL`/`ACTIONS_RUNTIME_TOKEN` into the +container for sccache. + +**Warming model (forks for free):** populate caches on `push` to `main` (trusted, can +write); PRs — **including forks** — restore **read-only**. Forks get no secrets and can't +write the base cache, but they *can* read main's warm cache, so a fork PR still builds +incrementally with zero secret/infra exposure. + +> Caveats: GHA cache is **10 GB/repo, LRU-evicted** — scope it (sccache's compact object +> cache fits better than a raw disk_cache). **Key on the nightly/CUDA version** so a bump +> busts it — a stale cache here is exactly the ABI-mismatch class of bug. Step-up option if +> limits bite: BuildBuddy's free *hosted* tier (just an API key; non-fork PRs + main). + +### 7.4 Status & reporting +- **One rollup check per platform** (`CI / Linux x86_64`, …) summarizes its child jobs → branch protection keys on a *stable* name; reviewers see one green/red + a Markdown table. ✓exists +- JUnit from every job is aggregated by `junit_summary.py` into **one artifact + an auto PR comment** (the `--agent` report). Nothing clobbered, nothing missed. +- `--fail-on-empty` so "0 tests collected" is a failure, not a silent green. + +### 7.5 Ergonomic CI commands +- **`/rerun`** / **`/rerun all`** — re-run failed/cancelled or everything, no new commit (write-access gated). ✓exists +- **PR command menu** — on PR open, a comment lists the commands + local-repro recipes (so contributors discover them). ✓built +- **`/ci full`** *(proposed)* — opt a PR into the full lane (the merge-queue substitute trigger). + +### 7.6 Flake handling +- **Gated reruns for known signatures only** (`--only-rerun cudaErrorStreamCaptureInvalidated`, "Stream capture invalidated"). Never a blanket retry — that masks real failures. +- **Quarantine** `@pytest.mark.flaky` out of gating lanes with a tracking issue; still run nightly for visibility. +- **Cache model weights** as an artifact → kills the corrupted-download flakes (e.g. the mobilenet `unexpected EOF` torchscript failure). + +### 7.7 Concurrency & path filters +- `concurrency: cancel-in-progress` keyed on PR ref — never burn runners on superseded commits. +- **Path filters**: docs-only PR → skip GPU/C++; `.py`-only → skip the C++ test tier. (The single biggest waste-cutter on trivial PRs.) +- Skip CI on draft PRs; fast lane on "ready for review". + +--- + +## 8. Target file layout + +``` +.github/workflows/ + ci.yml # the ONLY build+test entry: lanes by event + _build.yml # reusable: matrix → wheel artifact (all platforms/variants) + _test.yml # reusable: wheel → suite(s) → JUnit; matrix GENERATED by `ci matrix` + nightly.yml # exhaustive matrix + -ext + perf + retrigger-ci.yml # /rerun ✓exists + pr-command-menu.yml # discoverability comment on open ✓built + linter.yml # pre-commit gate + release-*.yml # publish on tags (unchanged) +tests/ci/ + suites.py # ← the manifest (DATA): SUITES = [Suite(...)] + __main__.py # ← the runner/engine: `ci run` · `ci matrix` +tests/py/utils/ + ci_helpers.sh # SHRUNK: env + array-safe trt_pytest wrapper only ✓exists + junit_summary.py # aggregation + --agent report ✓exists +justfile # local entry — thin caller of `ci run`/`ci matrix` ✓exists +pyproject.toml # markers + optional dep groups ✓exists +TESTING_AND_CI_DESIGN.md # this document +``` + +**Deletions this enables:** the ~11 per-platform entry workflows, the parallel +schedule/dispatch build-test set (the source of the old/new *double-run*), and the +inline 461/381-line Windows workflows — all collapse into `_build.yml` + `_test.yml`. + +--- + +## 9. Rollout plan (incremental, each step shippable + verified) + +Use a **byte-equivalence harness** at each step — diff `ci run `'s emitted `pytest` +command against the bash tier it replaces, and render the workflow's generated script — +before flipping it on. Prove we didn't change *what runs*, only *how it's wired*. + +0. **Kill the double-run.** Delete/disable the superseded schedule/dispatch build-test set so a PR runs *one* set of jobs. (Halves PR cost, removes the confusion that surfaced in #4352.) +1. **Stand up the manifest + runner.** Port the `trt_tier_*` functions into `tests/ci/suites.py` + `tests/ci/__main__.py` one suite at a time, each producing a byte-identical command to the bash tier it replaces. Add `smoke` markers to replace the `test_000_*`/`test_001_*` filename tiering; shrink `ci_helpers.sh` to env + wrapper. +2. **Generalize the core** → `_build.yml` + `_test.yml` with a `platform`/`variant` input; `_test.yml`'s matrix is generated by `ci matrix`. Fold RTX + python-only in as variants. +3. **Migrate aarch64 + Windows** onto the same reusables (deletes the inline Windows files). +4. **Lanes** — add `ci.yml` orchestrator + the fast/full split + approval/label/`/ci full` triggers + `concurrency`. +5. **Nightly** — `nightly.yml` for the exhaustive matrix + `-ext`; trim the PR lane to fast. +6. **Caching** — sccache + Bazel disk_cache via GHA cache, warmed on main. +7. **Polish** — path filters, weight-cache, flake quarantine, drop fragile result actions. + +--- + +## 10. Open decisions + +- **Runner availability** for GPU jobs (how many parallel full lanes can we afford?). +- **GHA cache budget** — is 10 GB enough, or do we want BuildBuddy's free hosted tier? +- **Required-check names** for branch protection (must stay stable across the migration). +- **Fork policy** — confirm fork PRs are read-only on cache and never see secrets. +- **`/ci full` vs approval-only** as the full-lane trigger (or both). + +--- + +## 11. What "pleasant" feels like, end-to-end + +> Open a PR → a comment greets you with the command menu → a **fast green check in ~15 min**. +> Iterate on the fast lane. When ready, a reviewer approves (or you comment `/ci full`) → the +> full suite runs once. A failure? **One PR comment** with the exact `uv run --no-sync pytest …` +> for each failing test — paste it to Claude, or run it locally via `just`. Flaky? `/rerun`. +> Formatting? `jj fix`. You never assembled a `pytest` command by hand, never waited on a job +> irrelevant to your change, and never had a real failure hidden behind a flaky one. diff --git a/justfile b/justfile index c0f31ed1f7..da35049f11 100644 --- a/justfile +++ b/justfile @@ -25,6 +25,11 @@ jobs := "4" # no rebuild) and TRT_JOBS feeds the parallel suites. _tier := 'mkdir -p "$TMPDIR" && source tests/py/utils/ci_helpers.sh && export PYTHON="uv run --no-sync python" TRT_JOBS="' + jobs + '" TRT_PYTEST_RERUNS=0 &&' +# Invocation for the new suite manifest/runner (tests/ci). Same env policy as +# _tier: the built venv via `uv --no-sync`, GPU-memory-aware jobs, reruns off +# locally (the rerunfailures plugin may be absent and you want to SEE flakes). +_ci := 'mkdir -p "$TMPDIR" && PYTHON="uv run --no-sync python" TRT_JOBS="' + jobs + '" TRT_PYTEST_RERUNS=0 uv run --no-sync python -m tests.ci' + # ── Testing ─────────────────────────────────────────────────────────────────── # Run pytest in the uv-managed env (honors pyproject addopts). Pass any args: @@ -34,6 +39,41 @@ test *args: @mkdir -p "$TMPDIR" uv run --no-sync pytest {{args}} +# ── Suite manifest (tests/ci) ────────────────────────────────────────────────── +# +# The suite manifest is the single source of truth for what each job runs; CI +# and these recipes both call `python -m tests.ci`. See TESTING_AND_CI_DESIGN.md. + +# Validate the suite manifest (unique names/junit paths, valid setup steps) +doctor: + uv run --no-sync python -m tests.ci doctor + +# List every suite with its tier, lanes, and variants +suites: + uv run --no-sync python -m tests.ci list + +# Run ONE suite exactly as CI runs it. Extra pytest args after `--`: +# just suite dynamo-runtime -- -k test_foo -x +suite name *args: + {{_ci}} run {{name}} {{args}} + +# Run every suite in a LANE (fast|full|nightly), continuing past failures: +# just lane fast +lane name *args: + {{_ci}} run-lane --lane {{name}} --variant standard {{args}} + +# Run a lane past failures, then print one consolidated report (--agent for Claude): +# just report fast --agent +report lane *summary_args: + #!/usr/bin/env bash + set -uo pipefail + mkdir -p "$TMPDIR" + results="${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" + rm -f "$results"/*.xml 2>/dev/null || true + export PYTHON="uv run --no-sync python" TRT_JOBS="{{jobs}}" TRT_PYTEST_RERUNS=0 + uv run --no-sync python -m tests.ci run-lane --lane {{lane}} --variant standard || true + uv run --no-sync python tests/py/utils/junit_summary.py "$results" {{summary_args}} + # ── CI tier reproduction ────────────────────────────────────────────────────── # # Run exactly what a CI tier runs, before pushing. Each recipe calls the tier diff --git a/pyproject.toml b/pyproject.toml index 88d9bbc743..b88fc82931 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,6 +143,8 @@ addopts = "-n auto --dist=loadfile" markers = [ "unit: a focused unit test (the bulk of the suite)", "critical: a smoke/critical-path case run in the L1 tier (pytest -m critical); the L2 tier runs the complement (-m 'not critical')", + "smoke: the fast-lane subset of a subsystem (pytest -m smoke); run on every push by the `fast` lane", + "flaky: a known-flaky test, quarantined out of the gating lanes and tracked by an issue; still run in nightly", ] norecursedirs = [ "bazel-*", @@ -156,6 +158,7 @@ norecursedirs = [ "tools", "modules", "utils", + "ci", ] [tool.uv] diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 0000000000..20cec47fd1 --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1,16 @@ +"""Torch-TensorRT test-suite manifest + runner (the Layer-0 "brain"). + +This package is the single source of truth for *what each CI/local test job runs*. +It replaces the ``trt_tier_*`` bash functions in ``tests/py/utils/ci_helpers.sh``. + + * ``suites.py`` — the manifest: ``SUITES = [Suite(...), ...]`` (pure data). + * ``runner.py`` — the one engine that turns a ``Suite`` into a ``pytest`` command. + * ``__main__.py`` — the CLI: ``python -m tests.ci {list,show,run,matrix}``. + +Both ``just`` (local) and CI call ``python -m tests.ci run ``; CI's job +matrix is generated by ``python -m tests.ci matrix``. There is nothing to drift. +""" + +from .suites import SUITES, Lane, Suite, Tier, Variant + +__all__ = ["SUITES", "Suite", "Lane", "Tier", "Variant"] diff --git a/tests/ci/__main__.py b/tests/ci/__main__.py new file mode 100644 index 0000000000..708ad3fcc0 --- /dev/null +++ b/tests/ci/__main__.py @@ -0,0 +1,171 @@ +"""CLI for the test-suite manifest: python -m tests.ci {list,show,run,matrix,doctor} + +list all suites, tiers, lanes, variants +show a suite's resolved command per variant +run [opts] [-- ...] run one suite (the call CI + just both make) +matrix [--lane|--tier] JSON matrix `include` for GitHub Actions +doctor validate the manifest (CI lints this) +""" + +from __future__ import annotations + +import argparse +import json +import sys + +from .runner import REPO_ROOT, describe, junit_path, matrix, run_suite, select +from .suites import SUITES, by_name + + +def _cmd_list(_: argparse.Namespace) -> int: + width = max(len(s.name) for s in SUITES) + print(f"{'SUITE'.ljust(width)} TIER LANES VARIANTS") + for s in SUITES: + print( + f"{s.name.ljust(width)} {s.tier:<4} " + f"{','.join(s.lanes):<21} {','.join(s.variants)}" + ) + print( + f"\n{len(SUITES)} suites. " + f"Run one: python -m tests.ci run (or `just suite `)" + ) + return 0 + + +def _cmd_show(args: argparse.Namespace) -> int: + s = by_name(args.name) + print(f"# {s.name} (tier={s.tier}, lanes={','.join(s.lanes)})") + for var in s.variants: + print(f"\n## variant: {var} junit: {junit_path(s).name}") + print(describe(s, var)) + return 0 + + +def _cmd_run(args: argparse.Namespace) -> int: + s = by_name(args.name) + variants = [args.variant] if args.variant else list(s.variants) + if args.variant and args.variant not in s.variants: + print( + f"::warning::{s.name} does not run on variant {args.variant!r}; " + f"it runs on {s.variants}", + file=sys.stderr, + ) + return 0 + rc = 0 + for var in variants: + rc = run_suite(s, var, dry_run=args.dry_run, extra=args.pytest_args) or rc + return rc + + +def _cmd_run_lane(args: argparse.Namespace) -> int: + """Run every suite in a lane/tier, continuing past failures (so one consolidated + report sees them all). Returns non-zero if any suite failed.""" + jobs = select(lane=args.lane, tier=args.tier, variant=args.variant) + if not jobs: + print("::warning::no suites match the given filters", file=sys.stderr) + return 0 + rc = 0 + for s, var in jobs: + rc = run_suite(s, var, dry_run=args.dry_run) or rc + return rc + + +def _cmd_matrix(args: argparse.Namespace) -> int: + include = matrix(lane=args.lane, tier=args.tier, variant=args.variant) + if not include: + print("::warning::matrix is empty for the given filters", file=sys.stderr) + print(json.dumps({"include": include})) + return 0 + + +def _cmd_doctor(_: argparse.Namespace) -> int: + """Static checks CI can gate on: unique names, unique junit paths, valid setup + steps, declared cwd dirs exist, every suite is reachable by some lane.""" + problems: list[str] = [] + names = [s.name for s in SUITES] + dupes = {n for n in names if names.count(n) > 1} + if dupes: + problems.append(f"duplicate suite names: {sorted(dupes)}") + + junits = [junit_path(s).name for s in SUITES] + jdupes = {j for j in junits if junits.count(j) > 1} + if jdupes: + problems.append(f"colliding junit paths: {sorted(jdupes)}") + + valid_setup = {"hub", "executorch", "cuda-core", "mpi"} + for s in SUITES: + for step in s.setup: + if step not in valid_setup: + problems.append(f"{s.name}: unknown setup step {step!r}") + if not s.lanes: + problems.append(f"{s.name}: belongs to no lane") + if not s.variants: + problems.append(f"{s.name}: runs on no variant") + cwd = REPO_ROOT / s.cwd + if not cwd.is_dir(): + problems.append(f"{s.name}: cwd {s.cwd} does not exist") + for var in s.variants: + if var not in (s.overrides.keys() | {"standard", "rtx"}): + problems.append(f"{s.name}: bad variant {var!r}") + + # Every suite should be exercised by some lane and some tier path. + if problems: + for p in problems: + print(f"✗ {p}", file=sys.stderr) + print(f"\n{len(problems)} manifest problem(s).", file=sys.stderr) + return 1 + print( + f"✓ manifest OK — {len(SUITES)} suites, " + f"{len(set(junits))} unique junit paths, no collisions." + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="python -m tests.ci", description=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + + sub.add_parser("list", help="list all suites").set_defaults(fn=_cmd_list) + + sp = sub.add_parser("show", help="show a suite's resolved command") + sp.add_argument("name") + sp.set_defaults(fn=_cmd_show) + + sp = sub.add_parser("run", help="run one suite") + sp.add_argument("name") + sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument( + "--dry-run", action="store_true", help="print the command, don't run" + ) + sp.add_argument( + "pytest_args", + nargs="*", + help="extra args forwarded to pytest " "(use `-- -k foo`)", + ) + sp.set_defaults(fn=_cmd_run) + + sp = sub.add_parser( + "run-lane", help="run every suite in a lane/tier, past failures" + ) + g = sp.add_mutually_exclusive_group() + g.add_argument("--lane", choices=("fast", "full", "nightly")) + g.add_argument("--tier", choices=("l0", "l1", "l2")) + sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument("--dry-run", action="store_true") + sp.set_defaults(fn=_cmd_run_lane) + + sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON") + g = sp.add_mutually_exclusive_group() + g.add_argument("--lane", choices=("fast", "full", "nightly")) + g.add_argument("--tier", choices=("l0", "l1", "l2")) + sp.add_argument("--variant", choices=("standard", "rtx")) + sp.set_defaults(fn=_cmd_matrix) + + sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor) + + args = p.parse_args(argv) + return args.fn(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ci/runner.py b/tests/ci/runner.py new file mode 100644 index 0000000000..4943234b66 --- /dev/null +++ b/tests/ci/runner.py @@ -0,0 +1,247 @@ +"""The one engine that turns a ``Suite`` into a ``pytest`` command and runs it. + +This is the *only* place that knows pytest mechanics (xdist workers, junit paths, +the flake-rerun wrapper, ``--dist``, optional-dep setup, variant resolution). The +manifest (``suites.py``) stays pure data. +""" + +from __future__ import annotations + +import glob +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from .suites import SUITES, Suite, Variant, by_name + +# Repo root: tests/ci/runner.py -> parents[2]. Honor TRT_REPO_ROOT like the bash. +REPO_ROOT = Path( + os.environ.get("TRT_REPO_ROOT", str(Path(__file__).resolve().parents[2])) +) + +# Known transient cudagraph/TRT-driver flake signatures. Expand ONLY with +# concrete evidence — a broad regex hides real bugs. +_RERUN_ARGS = [ + "--reruns", + "1", + "--reruns-delay", + "5", + "--only-rerun", + "cudaErrorStreamCaptureInvalidated", + "--only-rerun", + "Stream capture invalidated", +] + + +def _launcher() -> list[str]: + """The python/pytest launcher. CI leaves PYTHON unset (-> container python); + the justfile sets PYTHON='uv run --no-sync python' to use the built venv.""" + return shlex.split(os.environ.get("PYTHON", "python")) + + +def _results_dir() -> Path: + d = os.environ.get("RUNNER_TEST_RESULTS_DIR") + if not d: + tmp = os.environ.get("TMPDIR", "/tmp") + d = str(Path(tmp) / "trt_test_results") + Path(d).mkdir(parents=True, exist_ok=True) + return Path(d) + + +def junit_path(suite: Suite) -> Path: + """Unique per suite (no more four-runs-one-file collisions).""" + return _results_dir() / f"{suite.name}.xml" + + +def _nproc(jobs: str | None) -> list[str]: + """``-n`` token. TRT_JOBS overrides the suite default; jobs=None -> serial.""" + if jobs is None: + return [] + return ["-n", os.environ.get("TRT_JOBS") or jobs] + + +def _reruns_enabled(reruns: bool) -> bool: + return reruns and os.environ.get("TRT_PYTEST_RERUNS", "1") != "0" + + +def _expand_paths(cwd: Path, paths: tuple[str, ...]) -> list[str]: + """Shell-style glob expansion (the bash relied on the shell for this). + + A pattern with ``*`` is expanded relative to ``cwd`` and sorted; a pattern + that matches nothing is kept literal (matches bash nullglob-off behavior, so + pytest reports the missing path rather than silently collecting 0 tests). + """ + out: list[str] = [] + for p in paths: + if "*" in p: + matches = sorted( + str(Path(m).relative_to(cwd)) for m in glob.glob(str(cwd / p)) + ) + out.extend(matches or [p]) + else: + out.append(p) + return out + + +def build_pytest_args(suite: Suite, variant: Variant) -> list[str]: + """The pytest args (everything after ``-m pytest``) for this suite+variant.""" + v = suite.for_variant(variant) + cwd = REPO_ROOT / v["cwd"] + args: list[str] = [] + if _reruns_enabled(v["reruns"]): + args += _RERUN_ARGS + if v["markers"]: + args += ["-m", v["markers"]] + args += ["-ra"] + args += _nproc(v["jobs"]) + args += ["--junitxml", str(junit_path(suite))] + if v["dist"]: + args += [v["dist"]] + if v["maxfail"] is not None: + args += [f"--maxfail={v['maxfail']}"] + if v["ir"]: + args += ["--ir", v["ir"]] + if v["keyword"]: + args += ["-k", v["keyword"]] + if v["verbose"]: + args += ["-v"] + args += _expand_paths(cwd, tuple(v["paths"])) + return args + + +def _setup_commands(step: str) -> list[tuple[list[str], Path]]: + """(argv, cwd) pairs for a named setup step.""" + launcher = _launcher() + if step == "hub": + return [(launcher + ["hub.py"], REPO_ROOT / "tests/modules")] + if step == "executorch": + return [ + ( + launcher + ["-m", "pip", "install", "pyyaml", "executorch>=1.3.1"], + REPO_ROOT, + ) + ] + if step == "cuda-core": + return [ + (launcher + ["-m", "pip", "install", "cuda-python", "cuda-core"], REPO_ROOT) + ] + if step == "mpi": + return [ + ( + [ + "dnf", + "install", + "-y", + "mpich", + "mpich-devel", + "openmpi", + "openmpi-devel", + ], + REPO_ROOT, + ) + ] + raise KeyError(f"unknown setup step {step!r} in a suite definition") + + +def describe(suite: Suite, variant: Variant) -> str: + """The full command line, for --dry-run / show (quoting-safe display).""" + v = suite.for_variant(variant) + pre = [] + for step in v["setup"]: + for argv, cwd in _setup_commands(step): + pre.append(f" (cd {cwd.relative_to(REPO_ROOT)} && {shlex.join(argv)})") + cmd = shlex.join(_launcher() + ["-m", "pytest"] + build_pytest_args(suite, variant)) + lines = pre + [f" (cd {v['cwd']} && {cmd})"] + for f in v["follow"]: + lines.append(f" (cd {v['cwd']} && {shlex.join(_launcher() + list(f))})") + return "\n".join(lines) + + +def run_suite( + suite: Suite, + variant: Variant, + *, + dry_run: bool = False, + extra: list[str] | None = None, +) -> int: + """Run setup steps, the pytest command, then any follow commands. Returns the + process exit code (non-zero on first failure), mirroring the bash tiers.""" + v = suite.for_variant(variant) + extra = extra or [] + env = {**os.environ, **{k: str(val) for k, val in v["env"].items()}} + cwd = REPO_ROOT / v["cwd"] + pytest_cmd = ( + _launcher() + ["-m", "pytest"] + build_pytest_args(suite, variant) + extra + ) + + if dry_run: + print(describe(suite, variant)) + if extra: + print(f" # + extra pytest args: {shlex.join(extra)}") + return 0 + + for step in v["setup"]: + for argv, scwd in _setup_commands(step): + print(f"==> setup[{step}]: {shlex.join(argv)}", flush=True) + rc = subprocess.run(argv, cwd=scwd, env=env).returncode + if rc != 0: + print(f"::warning::setup step {step!r} exited {rc}", flush=True) + + print(f"==> {suite.name} [{variant}]: {shlex.join(pytest_cmd)}", flush=True) + rc = subprocess.run(pytest_cmd, cwd=cwd, env=env).returncode + if rc != 0: + repro = shlex.join( + ["uv", "run", "--no-sync", "pytest"] + + build_pytest_args(suite, variant) + + extra + ) + print( + f"::warning::{suite.name} failed. Reproduce: cd {v['cwd']} && {repro}", + flush=True, + ) + return rc + + for f in v["follow"]: + fcmd = _launcher() + list(f) + print(f"==> {suite.name} follow: {shlex.join(fcmd)}", flush=True) + frc = subprocess.run(fcmd, cwd=cwd, env=env).returncode + if frc != 0: + return frc + return 0 + + +def select( + *, + lane: str | None = None, + tier: str | None = None, + variant: str | None = None, + names: list[str] | None = None, +) -> list[tuple[Suite, Variant]]: + """All (suite, variant) jobs matching the filters. No filter on an axis = all.""" + jobs: list[tuple[Suite, Variant]] = [] + pool = [by_name(n) for n in names] if names else list(SUITES) + for s in pool: + if lane is not None and lane not in s.lanes: + continue + if tier is not None and s.tier != tier: + continue + for var in s.variants: + if variant is not None and var != variant: + continue + jobs.append((s, var)) + return jobs + + +def matrix(**filters: str | None) -> list[dict[str, str]]: + """GitHub-Actions matrix ``include`` entries for the selected jobs.""" + return [ + { + "suite": s.name, + "variant": var, + "tier": s.tier, + "cwd": s.for_variant(var)["cwd"], + } + for s, var in select(**filters) + ] diff --git a/tests/ci/suites.py b/tests/ci/suites.py new file mode 100644 index 0000000000..044413b8c8 --- /dev/null +++ b/tests/ci/suites.py @@ -0,0 +1,322 @@ +"""The test-suite manifest — pure data describing every test job. + +A *suite* is one ``pytest`` invocation against one subsystem. A CI/local *job* +is a (suite x variant) pair, optionally sharded across N runners. Suites are +grouped two ways: + + * ``tier`` — the legacy L0/L1/L2 grouping, kept so the migration can be + coverage-equivalent to today's ``ci_helpers.sh`` (``ci matrix --tier l0``). + * ``lanes`` — the target grouping (``fast`` / ``full`` / ``nightly``) the + redesign moves to (``ci matrix --lane fast``). Depth within a subsystem is + expressed by a marker on the test, not a filename prefix. + +Deliberate differences from the bash tiers (improvements, not regressions): + * ``hlo/`` ran in BOTH l0_core (standard) and l1_dynamo_core — wasteful. It is + one ``dynamo-hlo`` suite here, run once. + * l2_plugin re-ran the whole ``conversion/`` dir (already covered by + ``dynamo-converters``) and pointed four ``--junitxml`` runs at ONE path, so + three suites' results vanished from the aggregate. Each suite here owns a + unique junit name (derived from ``name`` + shard), and the redundant + ``conversion/`` re-run is dropped. + * L2 suites used raw ``pytest`` (no rerun wrapper / no repro hint); every suite + now runs through the same wrapper uniformly (gated by ``TRT_PYTEST_RERUNS``). + +Validate the manifest: ``python -m tests.ci doctor`` +Inspect a command: ``python -m tests.ci run --dry-run`` +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +Tier = Literal["l0", "l1", "l2"] +Lane = Literal["fast", "full", "nightly"] +Variant = Literal["standard", "rtx"] + +ALL_VARIANTS: tuple[Variant, ...] = ("standard", "rtx") + + +@dataclass(frozen=True) +class Suite: + """One ``pytest`` invocation against one subsystem. + + Fields map 1:1 onto the command the runner builds. ``overrides`` lets a + single suite differ per variant (e.g. RTX selects a different path set) + without splitting it into two entries with two names. + """ + + name: str + tier: Tier + lanes: tuple[Lane, ...] + cwd: str = "tests/py/dynamo" # relative to repo root + paths: tuple[str, ...] = () # pytest positionals (rel to cwd); globs ok + markers: str | None = None # -m EXPR + keyword: str | None = None # -k EXPR + dist: str | None = None # --dist=loadscope + maxfail: int | None = None # --maxfail=N + ir: str | None = None # --ir torch_compile + jobs: str | None = None # xdist default: None=serial, "8"/"auto"/"4" + reruns: bool = True # wrap in the flake-rerun helper + verbose: bool = False # -v + variants: tuple[Variant, ...] = ALL_VARIANTS + setup: tuple[str, ...] = () # named pre-steps: hub|executorch|cuda-core|mpi + follow: tuple[tuple[str, ...], ...] = () # extra argv to run AFTER pytest + env: dict[str, str] = field(default_factory=dict) + overrides: dict[str, dict[str, Any]] = field(default_factory=dict) # per-variant + + def for_variant(self, variant: Variant) -> dict[str, Any]: + """This suite's effective fields for ``variant`` (applies overrides).""" + base = { + f: getattr(self, f) + for f in ( + "cwd", + "paths", + "markers", + "keyword", + "dist", + "maxfail", + "ir", + "jobs", + "reruns", + "verbose", + "setup", + "follow", + "env", + ) + } + base.update(self.overrides.get(variant, {})) + return base + + +# ── L0 — smoke / fast lane ──────────────────────────────────────────────────── +_L0: list[Suite] = [ + Suite( + "dynamo-converters", + tier="l0", + lanes=("fast", "full"), + paths=("conversion/",), + dist="--dist=loadscope", + maxfail=20, + jobs="8", + # RTX does not shard converters with loadscope. + overrides={"rtx": {"dist": None}}, + ), + Suite( + "dynamo-runtime-smoke", + tier="l0", + lanes=("fast", "full"), + paths=("runtime/test_000_*",), + jobs="8", + ), + Suite( + "dynamo-partitioning-smoke", + tier="l0", + lanes=("fast", "full"), + paths=("partitioning/test_000_*",), + jobs="8", + # RTX runs the whole partitioning suite (no smoke subset split). + overrides={"rtx": {"paths": ("partitioning/",)}}, + ), + Suite( + "dynamo-lowering", + tier="l0", + lanes=("fast", "full"), + paths=("lowering/",), + jobs="8", + ), + Suite( + "py-core", + tier="l0", + lanes=("fast", "full"), + cwd="tests/py/core", + paths=(".",), + jobs="8", + ), + Suite( + "ts-api", + tier="l0", + lanes=("fast", "full"), + cwd="tests/py/ts", + paths=("api/",), + setup=("hub",), + variants=("standard",), + ), +] + +# ── L1 — critical-path / full lane ──────────────────────────────────────────── +_L1: list[Suite] = [ + Suite( + "dynamo-runtime", + tier="l1", + lanes=("full",), + paths=("runtime/test_001_*",), + jobs="8", + ), + Suite( + "dynamo-partitioning", + tier="l1", + lanes=("full",), + paths=("partitioning/test_001_*",), + jobs="8", + variants=("standard",), + ), + Suite( + # Was run in BOTH l0_core (std) and l1_dynamo_core (both) — deduped to once. + "dynamo-hlo", + tier="l1", + lanes=("full",), + paths=("hlo/",), + jobs="8", + ), + Suite( + "dynamo-models-critical", + tier="l1", + lanes=("full",), + paths=("models/",), + markers="critical", + ), + Suite( + "torch-compile-backend", + tier="l1", + lanes=("full",), + paths=("backend/",), + ), + Suite( + "torch-compile-models-critical", + tier="l1", + lanes=("full",), + paths=("models/test_models.py", "models/test_dyn_models.py"), + markers="critical", + ir="torch_compile", + ), + Suite( + "ts-models", + tier="l1", + lanes=("full",), + cwd="tests/py/ts", + paths=("models/",), + setup=("hub",), + variants=("standard",), + ), +] + +# ── L2 — exhaustive / full + nightly ────────────────────────────────────────── +_L2: list[Suite] = [ + Suite( + "torch-compile-models", + tier="l2", + lanes=("full", "nightly"), + paths=("models/test_models.py", "models/test_dyn_models.py"), + markers="not critical", + ir="torch_compile", + jobs="auto", + ), + Suite( + "dynamo-models", + tier="l2", + lanes=("full", "nightly"), + paths=("models/",), + markers="not critical", + jobs="auto", + ), + Suite( + "dynamo-llm", + tier="l2", + lanes=("nightly",), + paths=("llm/",), + jobs="auto", + ), + Suite( + "dynamo-runtime-full", + tier="l2", + lanes=("full", "nightly"), + paths=("runtime/",), + keyword="not test_000_ and not test_001_", + jobs="auto", + ), + Suite( + "executorch", + tier="l2", + lanes=("nightly",), + paths=("executorch/",), + setup=("executorch",), + jobs="auto", + variants=("standard",), + ), + Suite( + # Standard: the automatic-plugin trio. RTX: the whole automatic_plugin dir. + # (The redundant conversion/ re-run from the old l2_plugin is dropped.) + "plugins-automatic", + tier="l2", + lanes=("nightly",), + jobs="auto", + paths=( + "automatic_plugin/test_automatic_plugin.py", + "automatic_plugin/test_automatic_plugin_with_attrs.py", + "automatic_plugin/test_flashinfer_rmsnorm.py", + ), + overrides={"rtx": {"paths": ("automatic_plugin/",)}}, + ), + Suite( + "kernels", + tier="l2", + lanes=("nightly",), + cwd="tests/py/kernels", + paths=(".",), + setup=("cuda-core",), + jobs="auto", + variants=("standard",), + ), + Suite( + "ts-integrations", + tier="l2", + lanes=("nightly",), + cwd="tests/py/ts", + paths=("integrations/",), + setup=("hub",), + jobs="auto", + variants=("standard",), + ), + Suite( + "distributed", + tier="l2", + lanes=("nightly",), + paths=( + "distributed/test_nccl_ops.py", + "distributed/test_native_nccl.py", + "distributed/test_export_save_load.py", + ), + jobs="auto", + verbose=True, + reruns=False, + variants=("standard",), + setup=("mpi",), + env={"USE_HOST_DEPS": "1", "CI_BUILD": "1", "USE_TRTLLM_PLUGINS": "1"}, + follow=( + ( + "-m", + "torch_tensorrt.distributed.run", + "--nproc_per_node=2", + "distributed/test_native_nccl.py", + "--multirank", + ), + ( + "-m", + "torch_tensorrt.distributed.run", + "--nproc_per_node=2", + "distributed/test_export_save_load.py", + "--multirank", + ), + ), + ), +] + +SUITES: tuple[Suite, ...] = tuple(_L0 + _L1 + _L2) + + +def by_name(name: str) -> Suite: + for s in SUITES: + if s.name == name: + return s + raise KeyError(f"no suite named {name!r}; try `python -m tests.ci list`") From 735b174af6c2118e3a5f67a12b9dbf99ce2f70a1 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Tue, 30 Jun 2026 21:10:27 +0000 Subject: [PATCH 03/11] tests: Overhauling CI infra ci(test-dx): grant id-token: write at ci.yml top level The reusable build jobs (build_linux/build_windows) request 'id-token: write' for OIDC; a reusable workflow can't exceed its caller's permission ceiling, so ci.yml's top-level 'contents: read' made GitHub reject it (startup_failure: "nested job 'build' is requesting 'id-token: write', but is only allowed 'id-token: none'"). Grant id-token: write + contents: read, matching the original per-platform entry workflows. --- .github/workflows/_decide.yml | 62 ++ .github/workflows/_linux-x86_64-core.yml | 671 ------------------ .github/workflows/_test-linux.yml | 148 ++++ .github/workflows/_test-windows.yml | 139 ++++ .github/workflows/blossom-ci.yml | 104 --- .github/workflows/build-tensorrt-linux.yml | 222 ------ .github/workflows/build-tensorrt-windows.yml | 232 ------ .../build-test-linux-aarch64-jetpack.yml | 89 --- .../build-test-linux-aarch64-python-only.yml | 89 --- .../workflows/build-test-linux-aarch64.yml | 85 --- .../build-test-linux-x86_64-python-only.yml | 117 --- .github/workflows/build-test-linux-x86_64.yml | 123 ---- ...uild-test-linux-x86_64_rtx-python-only.yml | 118 --- .../workflows/build-test-linux-x86_64_rtx.yml | 109 --- .../workflows/build-test-tensorrt-linux.yml | 336 --------- .../workflows/build-test-tensorrt-windows.yml | 320 --------- .../build-test-windows-python-only.yml | 122 ---- .github/workflows/build-test-windows.yml | 461 ------------ .../build-test-windows_rtx-python-only.yml | 124 ---- .github/workflows/build-test-windows_rtx.yml | 381 ---------- .github/workflows/ci-linux-x86_64.yml | 135 ++++ .github/workflows/ci-sbsa.yml | 87 +++ .github/workflows/ci-windows.yml | 129 ++++ .github/workflows/linux-test.yml | 11 + .github/workflows/windows-test.yml | 9 + TESTING_AND_CI_DESIGN.md | 5 +- justfile | 189 +---- tests/ci/__main__.py | 22 +- tests/ci/runner.py | 3 + tests/ci/suites.py | 28 +- 30 files changed, 798 insertions(+), 3872 deletions(-) create mode 100644 .github/workflows/_decide.yml delete mode 100644 .github/workflows/_linux-x86_64-core.yml create mode 100644 .github/workflows/_test-linux.yml create mode 100644 .github/workflows/_test-windows.yml delete mode 100644 .github/workflows/blossom-ci.yml delete mode 100644 .github/workflows/build-tensorrt-linux.yml delete mode 100644 .github/workflows/build-tensorrt-windows.yml delete mode 100644 .github/workflows/build-test-linux-aarch64-jetpack.yml delete mode 100644 .github/workflows/build-test-linux-aarch64-python-only.yml delete mode 100644 .github/workflows/build-test-linux-aarch64.yml delete mode 100644 .github/workflows/build-test-linux-x86_64-python-only.yml delete mode 100644 .github/workflows/build-test-linux-x86_64.yml delete mode 100644 .github/workflows/build-test-linux-x86_64_rtx-python-only.yml delete mode 100644 .github/workflows/build-test-linux-x86_64_rtx.yml delete mode 100644 .github/workflows/build-test-tensorrt-linux.yml delete mode 100644 .github/workflows/build-test-tensorrt-windows.yml delete mode 100644 .github/workflows/build-test-windows-python-only.yml delete mode 100644 .github/workflows/build-test-windows.yml delete mode 100644 .github/workflows/build-test-windows_rtx-python-only.yml delete mode 100644 .github/workflows/build-test-windows_rtx.yml create mode 100644 .github/workflows/ci-linux-x86_64.yml create mode 100644 .github/workflows/ci-sbsa.yml create mode 100644 .github/workflows/ci-windows.yml diff --git a/.github/workflows/_decide.yml b/.github/workflows/_decide.yml new file mode 100644 index 0000000000..bcfc32289b --- /dev/null +++ b/.github/workflows/_decide.yml @@ -0,0 +1,62 @@ +name: _decide + +# Resolve the lane + backend from the triggering event. Shared by the +# per-platform entry workflows (ci-linux-x86_64 / ci-windows / ci-sbsa) so the +# rules live in exactly one place. `github.*` here refers to the caller's event. +# +# lane: fast (PR push) | full (approval / `ci: full` / main push) | +# nightly (schedule) | skip (non-approval review) +# backend: standard | rtx | both — from the `backend: TensorRT[-RTX]` labels +# (or the workflow_dispatch `backend` input) + +on: + workflow_call: + outputs: + lane: + description: "fast | full | nightly | skip" + value: ${{ jobs.decide.outputs.lane }} + backend: + description: "standard | rtx | both" + value: ${{ jobs.decide.outputs.backend }} + +jobs: + decide: + runs-on: ubuntu-latest + outputs: + lane: ${{ steps.pick.outputs.lane }} + backend: ${{ steps.pick.outputs.backend }} + steps: + - id: pick + env: + EVENT: ${{ github.event_name }} + REVIEW_STATE: ${{ github.event.review.state }} + HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') }}" + # Exact array-element match: 'backend: TensorRT' != 'backend: TensorRT-RTX'. + HAS_RTX_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT-RTX') }}" + HAS_STD_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT') }}" + DISPATCH_LANE: ${{ github.event.inputs.lane }} + DISPATCH_BACKEND: ${{ github.event.inputs.backend }} + run: | + set -euo pipefail + case "$EVENT" in + schedule) lane=nightly ;; + workflow_dispatch) lane="${DISPATCH_LANE:-full}" ;; + push) lane=full ;; # main canary + pull_request_review) + [ "$REVIEW_STATE" = "approved" ] && lane=full || lane=skip ;; + pull_request) + [ "$HAS_FULL_LABEL" = "true" ] && lane=full || lane=fast ;; + *) lane=fast ;; + esac + echo "lane=$lane" >> "$GITHUB_OUTPUT" + case "$EVENT" in + workflow_dispatch) backend="${DISPATCH_BACKEND:-both}" ;; + pull_request) + if [ "$HAS_RTX_LABEL" = "true" ] && [ "$HAS_STD_LABEL" = "true" ]; then backend=both + elif [ "$HAS_RTX_LABEL" = "true" ]; then backend=rtx + elif [ "$HAS_STD_LABEL" = "true" ]; then backend=standard + else backend=standard; fi ;; + *) backend=both ;; # push / approval / schedule + esac + echo "backend=$backend" >> "$GITHUB_OUTPUT" + echo "Resolved lane='$lane' backend='$backend' (event=$EVENT)." diff --git a/.github/workflows/_linux-x86_64-core.yml b/.github/workflows/_linux-x86_64-core.yml deleted file mode 100644 index c0011470b2..0000000000 --- a/.github/workflows/_linux-x86_64-core.yml +++ /dev/null @@ -1,671 +0,0 @@ -# Reusable core for the Linux x86_64 build+test pipeline. -# -# Both build-test-linux-x86_64.yml (standard TensorRT) and -# build-test-linux-x86_64_rtx.yml (TensorRT-RTX) call this with use-rtx -# false/true. Keep ALL build/test logic here so a change lands once. -# -# RTX differences are expressed two ways: -# * Jobs that only exist for standard TRT (torchscript, distributed, -# executorch) are gated with ``if: ${{ !inputs.use-rtx && ... }}``. -# * Jobs whose pytest scope differs branch inside the script on the -# ``$USE_TRT_RTX`` env var that linux-test.yml exports. -# -# The per-job pass/fail map is bubbled up via the ``results`` output so the -# thin caller workflows can render a single ``ci-rollup`` status check whose -# name (e.g. "CI / Linux x86_64") stays stable for branch protection. -name: _linux-x86_64-core - -on: - workflow_call: - inputs: - use-rtx: - description: "Build/test against TensorRT-RTX instead of standard TensorRT" - type: boolean - default: false - name-prefix: - description: "Display-name prefix for the build job (e.g. 'RTX - ')" - type: string - default: "" - outputs: - results: - description: "JSON map of {job-id: {result, outputs}} for every build/test job" - value: ${{ jobs.collect-results.outputs.results }} - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - LIMIT_PR=${{ github.event_name == 'pull_request' && 'true' || 'false' }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: ${{ inputs.name-prefix }}Build Linux x86_64 torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "x86_64" - use-rtx: ${{ inputs.use-rtx }} - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - - # Standard-TRT only: ExecuTorch static build is not part of the RTX matrix. - executorch-static-build: - needs: [filter-matrix, build] - if: ${{ !inputs.use-rtx }} - uses: ./.github/workflows/executorch-static-linux.yml - with: - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - - L0-dynamo-converter-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 dynamo converter tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-dynamo-converter-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_converter - L0-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 dynamo core tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-dynamo-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_core - L0-py-core-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 core python tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-py-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_py_core - # Standard-TRT only: TorchScript frontend is not exercised on RTX. - L0-torchscript-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ !inputs.use-rtx && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 torchscript tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-torchscript-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_torchscript - L1-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 dynamo core tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-dynamo-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_dynamo_core - L1-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 dynamo compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-dynamo-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_dynamo_compile - L1-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 torch compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-torch-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_torch_compile - # Standard-TRT only: TorchScript frontend is not exercised on RTX. - L1-torchscript-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-core-tests, - L0-dynamo-converter-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: "${{ !inputs.use-rtx && !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 torch script tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-torchscript-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_torchscript - L2-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-torch-compile-tests, - L1-dynamo-compile-tests, - L1-dynamo-core-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 torch compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-torch-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_torch_compile - L2-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-compile-tests, - L1-dynamo-core-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_dynamo_compile - L2-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo core tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_dynamo_core - L2-dynamo-plugin-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo plugin tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-plugin-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_plugin - # Standard-TRT only: TorchScript frontend is not exercised on RTX. - L2-torchscript-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !inputs.use-rtx && !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 torch script tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-torchscript-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_torchscript - # Standard-TRT only: distributed tests need a multi-GPU runner not in the RTX matrix. - L2-dynamo-distributed-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !inputs.use-rtx && !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo distributed tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-distributed-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - runner: linux.g4dn.12xlarge.nvidia.gpu - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_distributed - # Gather every build/test job's result into a single JSON map and expose it - # as the ``results`` workflow output. The thin caller workflows render this - # into one ``ci-rollup`` status check. ``if: always()`` so the map is - # produced even when upstream jobs failed / were skipped / were cancelled. - collect-results: - if: ${{ always() }} - permissions: {} - needs: - [ - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - L2-torch-compile-tests, - L2-dynamo-compile-tests, - L2-dynamo-core-tests, - L2-dynamo-plugin-tests, - L2-torchscript-tests, - L2-dynamo-distributed-tests, - ] - runs-on: ubuntu-latest - outputs: - results: ${{ steps.collect.outputs.results }} - steps: - - name: Collect job results - id: collect - env: - RESULTS: ${{ toJSON(needs) }} - run: | - set -euo pipefail - { - echo "results<> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_test-linux.yml b/.github/workflows/_test-linux.yml new file mode 100644 index 0000000000..9f4a2c56b9 --- /dev/null +++ b/.github/workflows/_test-linux.yml @@ -0,0 +1,148 @@ +name: _test-linux + +# Manifest-driven build (+ optional test) for ONE Linux channel. Covers both: +# * x86_64 — os=linux, architecture=x86_64, run-tests=true (build + suites) +# * SBSA — os=linux-aarch64, architecture=aarch64, run-tests=false (build-only; +# there are no aarch64 GPU test runners) +# +# One reusable serves both because the build (build_linux.yml) and test +# (linux-test.yml) reusables are the same — only inputs differ. (GitHub requires +# `uses:` to be a literal, so Windows, which needs build_windows/windows-test, +# has its own _test-windows.yml.) + +on: + workflow_call: + inputs: + lane: + description: "fast | full | nightly" + required: true + type: string + use-rtx: + description: "Test against TensorRT-RTX (x86_64 only)" + type: boolean + default: false + name-prefix: + description: "Display-name prefix (e.g. 'RTX - ', 'SBSA ')" + type: string + default: "" + raw-matrix: + description: "Raw generate_binary_build_matrix output. Generated ONCE by the + caller and shared across channels — generate_binary_build_matrix has a + concurrency group keyed on (workflow, PR, os) with cancel-in-progress, so + calling it per-channel makes the channels cancel each other's matrix job." + required: true + type: string + architecture: + description: "x86_64 | aarch64" + type: string + default: x86_64 + run-tests: + description: "Run the suite matrix after building (false = build-only, e.g. SBSA)" + type: boolean + default: true + python-only: + description: "Build the PYTHON_ONLY=1 wheel (no C++ runtime) via env_vars_python_only.txt" + type: boolean + default: false + +jobs: + # The build matrix is generated ONCE by the caller (see raw-matrix input) and + # passed in; here we only filter it per-variant (a plain script job — no + # test-infra concurrency group, so it's safe to run per-channel). + filter-matrix: + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/checkout@v6 + with: + repository: pytorch/tensorrt + - name: Generate matrix + id: generate + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(inputs.raw-matrix) }} + # Build one representative wheel for PR-triggered runs (any lane); + # the full python×cuda matrix only on push-to-main and nightly. + LIMIT_PR=${{ (github.event_name == 'push' || github.event_name == 'schedule') && 'false' || 'true' }} + MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + build: + needs: filter-matrix + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/tensorrt + pre-script: packaging/pre_build_script.sh + env-var-script: packaging/env_vars.txt + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/smoke_test_script.sh + package-name: torch_tensorrt + display-name: ${{ inputs.name-prefix }}Build Linux ${{ inputs.architecture }} torch-tensorrt whl package + name: ${{ matrix.display-name }} + uses: ./.github/workflows/build_linux.yml + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + # python-only swaps in the env_vars_python_only.txt (sets PYTHON_ONLY=1). + env-var-script: ${{ inputs.python-only && 'packaging/env_vars_python_only.txt' || matrix.env-var-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + architecture: ${{ inputs.architecture }} + use-rtx: ${{ inputs.use-rtx }} + pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" + + # Suite list from the manifest. Skipped for build-only channels (SBSA). + suite-matrix: + if: ${{ inputs.run-tests }} + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - id: gen + run: | + set -euo pipefail + variant=${{ inputs.use-rtx && 'rtx' || 'standard' }} + json="$(python -m tests.ci matrix --platform linux-x86_64 --lane '${{ inputs.lane }}' --variant "${variant}")" + echo "matrix=${json}" >> "$GITHUB_OUTPUT" + echo "Lane '${{ inputs.lane }}' (${variant}) suites:"; echo "${json}" | python -m json.tool + + # One test job per suite (auto-skips when suite-matrix is skipped, i.e. SBSA). + test: + needs: [filter-matrix, build, suite-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.suite-matrix.outputs.matrix) }} + uses: ./.github/workflows/linux-test.yml + with: + job-name: ${{ matrix.suite }}-${{ matrix.variant }} + repository: "pytorch/tensorrt" + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + use-rtx: ${{ inputs.use-rtx }} + fail-on-empty: true + script: | + set -euo pipefail + python -m tests.ci run "${{ matrix.suite }}" --variant "${{ matrix.variant }}" diff --git a/.github/workflows/_test-windows.yml b/.github/workflows/_test-windows.yml new file mode 100644 index 0000000000..77174ceeb2 --- /dev/null +++ b/.github/workflows/_test-windows.yml @@ -0,0 +1,139 @@ +name: _test-windows + +# Manifest-driven build + test for the Windows channel. Mirrors _test-linux.yml +# but uses build_windows.yml + windows-test.yml, and wraps the suite run in +# vc_env_helper.bat (the MSVC env). Suite SELECTION is identical to Linux — the +# manifest is platform-agnostic (`ci matrix --platform windows`); only the build +# reusable and the execution wrapper differ. + +on: + workflow_call: + inputs: + lane: + description: "fast | full | nightly" + required: true + type: string + use-rtx: + description: "Test against TensorRT-RTX" + type: boolean + default: false + name-prefix: + description: "Display-name prefix (e.g. 'RTX - ')" + type: string + default: "" + raw-matrix: + description: "Raw generate_binary_build_matrix output, generated ONCE by the + caller and shared across channels (generate_binary_build_matrix's concurrency + group would otherwise make per-channel calls cancel each other)." + required: true + type: string + +jobs: + # The build matrix is generated ONCE by the caller (raw-matrix); here we only + # filter it per-variant (a plain script job — no test-infra concurrency group). + filter-matrix: + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/checkout@v6 + with: + repository: pytorch/tensorrt + - name: Generate matrix + id: generate + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(inputs.raw-matrix) }} + # Build one representative wheel for PR-triggered runs (any lane); + # the full python×cuda matrix only on push-to-main and nightly. + LIMIT_PR=${{ (github.event_name == 'push' || github.event_name == 'schedule') && 'false' || 'true' }} + MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + # Swap the build runner label for a GPU test runner (same as the legacy windows flow). + substitute-runner: + needs: filter-matrix + outputs: + matrix: ${{ steps.substitute.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - name: Substitute runner + id: substitute + run: | + echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> "${GITHUB_OUTPUT}" + + build: + needs: substitute-runner + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/tensorrt + pre-script: packaging/pre_build_script_windows.sh + env-script: packaging/vc_env_helper.bat + smoke-test-script: packaging/smoke_test_windows.py + package-name: torch_tensorrt + display-name: ${{ inputs.name-prefix }}Build Windows torch-tensorrt whl package + name: ${{ matrix.display-name }} + uses: ./.github/workflows/build_windows.yml + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + env-script: ${{ matrix.env-script }} + smoke-test-script: ${{ matrix.smoke-test-script }} + package-name: ${{ matrix.package-name }} + trigger-event: ${{ github.event_name }} + # Build the RTX wheel when requested (matches the original RTX-windows + # entry); empty params + use-rtx=false for the standard build. + use-rtx: ${{ inputs.use-rtx }} + wheel-build-params: ${{ inputs.use-rtx && '--use-rtx' || '' }} + timeout: 120 + + suite-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - id: gen + run: | + set -euo pipefail + variant=${{ inputs.use-rtx && 'rtx' || 'standard' }} + json="$(python -m tests.ci matrix --platform windows --lane '${{ inputs.lane }}' --variant "${variant}")" + echo "matrix=${json}" >> "$GITHUB_OUTPUT" + echo "Lane '${{ inputs.lane }}' (${variant}) windows suites:"; echo "${json}" | python -m json.tool + + test: + needs: [substitute-runner, build, suite-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.suite-matrix.outputs.matrix) }} + uses: ./.github/workflows/windows-test.yml + with: + job-name: ${{ matrix.suite }}-${{ matrix.variant }} + repository: "pytorch/tensorrt" + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + pre-script: packaging/driver_upgrade.bat + use-rtx: ${{ inputs.use-rtx }} + # vc_env_helper.bat sets the MSVC env, then runs the manifest runner; the + # inner `python -m pytest` it spawns inherits that env. + script: | + set -euo pipefail + packaging/vc_env_helper.bat python -m tests.ci run "${{ matrix.suite }}" --variant "${{ matrix.variant }}" diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml deleted file mode 100644 index d5bdf0ed95..0000000000 --- a/.github/workflows/blossom-ci.yml +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A workflow to trigger ci on hybrid infra (github + self hosted runner) -name: Blossom-CI -on: - issue_comment: - types: [created] - workflow_dispatch: - inputs: - platform: - description: 'runs-on argument' - required: false - args: - description: 'argument' - required: false -jobs: - Authorization: - name: Authorization - runs-on: blossom - outputs: - args: ${{ env.args }} - - # This job only runs for pull request comments - if: | - contains( 'andi4191, narendasan, peri044, bowang007,', format('{0},', github.actor)) && - github.event.comment.body == '/blossom-ci' - steps: - - name: Check if comment is issued by authorized person - run: blossom-ci - env: - OPERATION: 'AUTH' - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }} - - Vulnerability-scan: - name: Vulnerability scan - needs: [Authorization] - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - repository: ${{ fromJson(needs.Authorization.outputs.args).repo }} - ref: ${{ fromJson(needs.Authorization.outputs.args).ref }} - lfs: 'true' - - # repo specific steps - #- name: Setup java - # uses: actions/setup-java@v5 - # with: - # java-version: 1.8 - - # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file - #- name: Setup blackduck properties - # run: | - # PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g') - # echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties - # echo detect.maven.included.scopes=compile >> application.properties - - - name: Run blossom action - uses: NVIDIA/blossom-action@main - env: - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }} - with: - args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }} - args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }} - args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }} - - Job-trigger: - name: Start ci job - needs: [Vulnerability-scan] - runs-on: blossom - steps: - - name: Start ci job - run: blossom-ci - env: - OPERATION: 'START-CI-JOB' - CI_SERVER: ${{ secrets.CI_SERVER }} - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - Upload-Log: - name: Upload log - runs-on: blossom - if : github.event_name == 'workflow_dispatch' - steps: - - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here) - run: blossom-ci - env: - OPERATION: 'POST-PROCESSING' - CI_SERVER: ${{ secrets.CI_SERVER }} - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-tensorrt-linux.yml b/.github/workflows/build-tensorrt-linux.yml deleted file mode 100644 index 11555e9f8f..0000000000 --- a/.github/workflows/build-tensorrt-linux.yml +++ /dev/null @@ -1,222 +0,0 @@ -name: Build Torch-TensorRT wheel on Linux with Future TensorRT Versions - -on: - workflow_call: - inputs: - repository: - description: 'Repository to checkout, defaults to ""' - default: "" - type: string - ref: - description: 'Reference to checkout, defaults to "nightly"' - default: "nightly" - type: string - test-infra-repository: - description: "Test infra repository to use" - default: "pytorch/test-infra" - type: string - test-infra-ref: - description: "Test infra reference to use" - default: "" - type: string - build-matrix: - description: "Build matrix to utilize" - default: "" - type: string - pre-script: - description: "Pre script to run prior to build" - default: "" - type: string - post-script: - description: "Post script to run prior to build" - default: "" - type: string - smoke-test-script: - description: "Script for Smoke Test for a specific domain" - default: "" - type: string - env-var-script: - description: "Script that sets Domain-Specific Environment Variables" - default: "" - type: string - package-name: - description: "Name of the actual python package that is imported" - default: "" - type: string - trigger-event: - description: "Trigger Event in caller that determines whether or not to upload" - default: "" - type: string - cache-path: - description: "The path(s) on the runner to cache or restore. The path is relative to repository." - default: "" - type: string - cache-key: - description: "The key created when saving a cache and the key used to search for a cache." - default: "" - type: string - architecture: - description: Architecture to build for x86_64 for default Linux, or aarch64 for Linux aarch64 builds - required: false - type: string - default: x86_64 - submodules: - description: Works as stated in actions/checkout, but the default value is recursive - required: false - type: string - default: recursive - setup-miniconda: - description: Set to true if setup-miniconda is needed - required: false - type: boolean - default: true - -permissions: - id-token: write - contents: read - -jobs: - build: - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.build-matrix) }} - env: - PYTHON_VERSION: ${{ matrix.python_version }} - PACKAGE_TYPE: wheel - REPOSITORY: ${{ inputs.repository }} - REF: ${{ inputs.ref }} - CU_VERSION: ${{ matrix.desired_cuda }} - UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} - ARCH: ${{ inputs.architecture }} - TENSORRT_STRIP_PREFIX: ${{ matrix.tensorrt.strip_prefix }} - TENSORRT_VERSION: ${{ matrix.tensorrt.version }} - TENSORRT_URLS: ${{ matrix.tensorrt.urls }} - TENSORRT_SHA256: ${{ matrix.tensorrt.sha256 }} - UPLOAD_ARTIFACT_NAME: pytorch_tensorrt_${{ matrix.tensorrt.version }}_${{ matrix.python_version }}_${{ matrix.desired_cuda }}_${{ inputs.architecture }} - name: build_tensorrt${{ matrix.tensorrt.version }}_py${{matrix.python_version}}_${{matrix.desired_cuda}} - runs-on: ${{ matrix.validation_runner }} - container: - image: ${{ matrix.container_image }} - options: ${{ matrix.gpu_arch_type == 'cuda' && '--gpus all' || ' ' }} - # If a build is taking longer than 120 minutes on these runners we need - # to have a conversation - timeout-minutes: 120 - - steps: - - name: Clean workspace - shell: bash -l {0} - run: | - set -x - echo "::group::Cleanup debug output" - rm -rf "${GITHUB_WORKSPACE}" - mkdir -p "${GITHUB_WORKSPACE}" - if [[ "${{ inputs.architecture }}" = "aarch64" ]]; then - rm -rf "${RUNNER_TEMP}/*" - fi - echo "::endgroup::" - - uses: actions/checkout@v6 - with: - # Support the use case where we need to checkout someone's fork - repository: ${{ inputs.test-infra-repository }} - ref: ${{ inputs.test-infra-ref }} - path: test-infra - - uses: actions/checkout@v6 - if: ${{ env.ARCH == 'aarch64' }} - with: - # Support the use case where we need to checkout someone's fork - repository: "pytorch/builder" - ref: "main" - path: builder - - name: Set linux aarch64 CI - if: ${{ inputs.architecture == 'aarch64' }} - shell: bash -l {0} - env: - DESIRED_PYTHON: ${{ matrix.python_version }} - run: | - set +e - # TODO: This is temporary aarch64 setup script, this should be integrated into aarch64 docker. - ${GITHUB_WORKSPACE}/builder/aarch64_linux/aarch64_ci_setup.sh - echo "/opt/conda/bin" >> $GITHUB_PATH - set -e - - uses: ./test-infra/.github/actions/set-channel - - name: Set PYTORCH_VERSION - if: ${{ env.CHANNEL == 'test' }} - run: | - # When building RC, set the version to be the current candidate version, - # otherwise, leave it alone so nightly will pick up the latest - echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}" - - uses: ./test-infra/.github/actions/setup-binary-builds - env: - PLATFORM: ${{ inputs.architecture == 'aarch64' && 'linux-aarch64' || ''}} - with: - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - submodules: ${{ inputs.submodules }} - setup-miniconda: ${{ inputs.setup-miniconda }} - python-version: ${{ env.PYTHON_VERSION }} - cuda-version: ${{ env.CU_VERSION }} - arch: ${{ env.ARCH }} - - name: Combine Env Var and Build Env Files - if: ${{ inputs.env-var-script != '' }} - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}" - - name: Install torch dependency - shell: bash -l {0} - run: | - set -x - # shellcheck disable=SC1090 - source "${BUILD_ENV_FILE}" - # shellcheck disable=SC2086 - ${CONDA_RUN} ${PIP_INSTALL_TORCH} - - name: Run Pre-Script with Caching - if: ${{ inputs.pre-script != '' }} - uses: ./test-infra/.github/actions/run-script-with-cache - with: - cache-path: ${{ inputs.cache-path }} - cache-key: ${{ inputs.cache-key }} - repository: ${{ inputs.repository }} - script: ${{ inputs.pre-script }} - - name: Build clean - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - set -x - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} python setup.py clean - - name: Build the wheel (bdist_wheel) - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - set -x - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} python setup.py bdist_wheel - - - name: Run Post-Script - if: ${{ inputs.post-script != '' }} - uses: ./test-infra/.github/actions/run-script-with-cache - with: - repository: ${{ inputs.repository }} - script: ${{ inputs.post-script }} - - name: Smoke Test - shell: bash -l {0} - env: - PACKAGE_NAME: ${{ inputs.package-name }} - SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }} - run: | - set -x - source "${BUILD_ENV_FILE}" - # TODO: add smoke test for the auditwheel tarball built - - # NB: Only upload to GitHub after passing smoke tests - - name: Upload wheel to GitHub - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: ${{ env.UPLOAD_ARTIFACT_NAME }} - path: ${{ inputs.repository }}/dist - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-tensorrt-windows.yml b/.github/workflows/build-tensorrt-windows.yml deleted file mode 100644 index 3c69427a22..0000000000 --- a/.github/workflows/build-tensorrt-windows.yml +++ /dev/null @@ -1,232 +0,0 @@ -name: Build Torch-TensorRT wheel on Windows with Future TensorRT Versions - -on: - workflow_call: - inputs: - repository: - description: 'Repository to checkout, defaults to ""' - default: "" - type: string - ref: - description: 'Reference to checkout, defaults to "nightly"' - default: "nightly" - type: string - test-infra-repository: - description: "Test infra repository to use" - default: "pytorch/test-infra" - type: string - test-infra-ref: - description: "Test infra reference to use" - default: "" - type: string - build-matrix: - description: "Build matrix to utilize" - default: "" - type: string - pre-script: - description: "Pre script to run prior to build" - default: "" - type: string - env-script: - description: "Script to setup environment variables for the build" - default: "" - type: string - wheel-build-params: - description: "Additional parameters for bdist_wheel" - default: "" - type: string - post-script: - description: "Post script to run prior to build" - default: "" - type: string - smoke-test-script: - description: "Script for Smoke Test for a specific domain" - default: "" - type: string - package-name: - description: "Name of the actual python package that is imported" - default: "" - type: string - trigger-event: - description: "Trigger Event in caller that determines whether or not to upload" - default: "" - type: string - cache-path: - description: "The path(s) on the runner to cache or restore. The path is relative to repository." - default: "" - type: string - cache-key: - description: "The key created when saving a cache and the key used to search for a cache." - default: "" - type: string - submodules: - description: "Works as stated in actions/checkout, but the default value is recursive" - required: false - type: string - default: recursive - timeout: - description: 'Timeout for the job (in minutes)' - default: 60 - type: number - -permissions: - id-token: write - contents: read - -jobs: - build: - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.build-matrix) }} - env: - PYTHON_VERSION: ${{ matrix.python_version }} - PACKAGE_TYPE: wheel - REPOSITORY: ${{ inputs.repository }} - REF: ${{ inputs.ref }} - CU_VERSION: ${{ matrix.desired_cuda }} - UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} - ARCH: win_amd64 - TENSORRT_STRIP_PREFIX: ${{ matrix.tensorrt.strip_prefix }} - TENSORRT_VERSION: ${{ matrix.tensorrt.version }} - TENSORRT_URLS: ${{ matrix.tensorrt.urls }} - TENSORRT_SHA256: ${{ matrix.tensorrt.sha256 }} - UPLOAD_ARTIFACT_NAME: pytorch_tensorrt_${{ matrix.tensorrt.version }}_${{ matrix.python_version }}_${{ matrix.desired_cuda }}_win_amd64 - name: build_tensorrt${{ matrix.tensorrt.version }}_py${{matrix.python_version}}_${{matrix.desired_cuda}} - runs-on: ${{ matrix.validation_runner }} - defaults: - run: - shell: bash -l {0} - # If a build is taking longer than 60 minutes on these runners we need - # to have a conversation - timeout-minutes: 120 - steps: - - uses: actions/checkout@v6 - with: - # Support the use case where we need to checkout someone's fork - repository: ${{ inputs.test-infra-repository }} - ref: ${{ inputs.test-infra-ref }} - path: test-infra - - uses: ./test-infra/.github/actions/setup-ssh - name: Setup SSH - with: - github-secret: ${{ secrets.GITHUB_TOKEN }} - activate-with-label: false - instructions: "SSH with rdesktop using ssh -L 3389:localhost:3389 %%username%%@%%hostname%%" - - name: Add Conda scripts to GitHub path - run: | - echo "C:/Jenkins/Miniconda3/Scripts" >> $GITHUB_PATH - - uses: ./test-infra/.github/actions/set-channel - - name: Set PYTORCH_VERSION - if: ${{ env.CHANNEL == 'test' }} - run: | - # When building RC, set the version to be the current candidate version, - # otherwise, leave it alone so nightly will pick up the latest - echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}" - - uses: ./test-infra/.github/actions/setup-binary-builds - with: - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - submodules: ${{ inputs.submodules }} - setup-miniconda: false - python-version: ${{ env.PYTHON_VERSION }} - cuda-version: ${{ env.CU_VERSION }} - arch: ${{ env.ARCH }} - - name: Shorten Conda environment path - run: | - bash "${REPOSITORY}/.github/scripts/shorten-conda-env-windows.sh" - - name: Install XPU support package - if: ${{ matrix.gpu_arch_type == 'xpu' }} - run: | - cmd //c .\\test-infra\\.github\\scripts\\install_xpu.bat - - name: Install torch dependency - run: | - source "${BUILD_ENV_FILE}" - # shellcheck disable=SC2086 - ${CONDA_RUN} ${PIP_INSTALL_TORCH} - - name: Run Pre-Script with Caching - if: ${{ inputs.pre-script != '' }} - uses: ./test-infra/.github/actions/run-script-with-cache - with: - cache-path: ${{ inputs.cache-path }} - cache-key: ${{ inputs.cache-key }} - repository: ${{ inputs.repository }} - script: ${{ inputs.pre-script }} - is_windows: 'enabled' - - name: Build clean - working-directory: ${{ inputs.repository }} - env: - ENV_SCRIPT: ${{ inputs.env-script }} - run: | - source "${BUILD_ENV_FILE}" - if [[ -z "${ENV_SCRIPT}" ]]; then - ${CONDA_RUN} python setup.py clean - else - if [[ ! -f ${ENV_SCRIPT} ]]; then - echo "::error::Specified env-script file (${ENV_SCRIPT}) not found" - exit 1 - else - ${CONDA_RUN} ${ENV_SCRIPT} python setup.py clean - fi - fi - - name: Build the wheel (bdist_wheel) - working-directory: ${{ inputs.repository }} - env: - ENV_SCRIPT: ${{ inputs.env-script }} - BUILD_PARAMS: ${{ inputs.wheel-build-params }} - run: | - source "${BUILD_ENV_FILE}" - - if [[ "$CU_VERSION" == "cpu" ]]; then - # CUDA and CPU are ABI compatible on the CPU-only parts, so strip - # in this case - export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" - else - export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//')" - fi - - if [[ -z "${ENV_SCRIPT}" ]]; then - ${CONDA_RUN} python setup.py bdist_wheel - else - ${CONDA_RUN} ${ENV_SCRIPT} python setup.py bdist_wheel ${BUILD_PARAMS} - fi - - name: Run post-script - working-directory: ${{ inputs.repository }} - env: - POST_SCRIPT: ${{ inputs.post-script }} - ENV_SCRIPT: ${{ inputs.env-script }} - if: ${{ inputs.post-script != '' }} - run: | - set -euxo pipefail - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} ${ENV_SCRIPT} ${POST_SCRIPT} - - name: Smoke Test - env: - ENV_SCRIPT: ${{ inputs.env-script }} - PACKAGE_NAME: ${{ inputs.package-name }} - SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }} - run: | - source "${BUILD_ENV_FILE}" - WHEEL_NAME=$(ls "${{ inputs.repository }}/dist/") - echo "$WHEEL_NAME" - ${CONDA_RUN} pip install "${{ inputs.repository }}/dist/$WHEEL_NAME" - if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then - echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" - ${CONDA_RUN} "${{ inputs.repository }}/${ENV_SCRIPT}" python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" - else - echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - ${CONDA_RUN} "${{ inputs.repository }}/${ENV_SCRIPT}" python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" - fi - # NB: Only upload to GitHub after passing smoke tests - - name: Upload wheel to GitHub - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: ${{ env.UPLOAD_ARTIFACT_NAME }} - path: ${{ inputs.repository }}/dist/ - - uses: ./test-infra/.github/actions/teardown-windows - if: always() - name: Teardown Windows - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-test-linux-aarch64-jetpack.yml b/.github/workflows/build-test-linux-aarch64-jetpack.yml deleted file mode 100644 index 5cda6acec5..0000000000 --- a/.github/workflows/build-test-linux-aarch64-jetpack.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Build and test Linux aarch64 wheels for Jetpack - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - echo "LIMIT_PR_BUILDS=${LIMIT_PR_BUILDS}" - echo '${{ github.event_name }}' - echo '${{ github.event.ref}}' - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}" --jetpack true --limit-pr-builds "${LIMIT_PR_BUILDS}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Build Jetpack torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - is-jetpack: true - - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-test-linux-aarch64-python-only.yml b/.github/workflows/build-test-linux-aarch64-python-only.yml deleted file mode 100644 index 9900877d82..0000000000 --- a/.github/workflows/build-test-linux-aarch64-python-only.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Python-only build Linux aarch64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars_python_only.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Python-only build Linux aarch64 torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - use-rtx: false - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-aarch64.yml b/.github/workflows/build-test-linux-aarch64.yml deleted file mode 100644 index eea7ec01fa..0000000000 --- a/.github/workflows/build-test-linux-aarch64.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Build and test Linux aarch64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Build SBSA torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-test-linux-x86_64-python-only.yml b/.github/workflows/build-test-linux-x86_64-python-only.yml deleted file mode 100644 index c79d4157bb..0000000000 --- a/.github/workflows/build-test-linux-x86_64-python-only.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Python-only build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars_python_only.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Python-only build Linux x86_64 torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "x86_64" - use-rtx: false - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: Python-only dynamo runtime tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: python-only-dynamo-runtime-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-python-only-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml deleted file mode 100644 index 89b3ee3a32..0000000000 --- a/.github/workflows/build-test-linux-x86_64.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - # All build/test logic lives in the shared reusable core; this workflow - # just selects standard TensorRT (use-rtx: false). See - # .github/workflows/_linux-x86_64-core.yml. - core: - permissions: - id-token: write - contents: read - uses: ./.github/workflows/_linux-x86_64-core.yml - with: - use-rtx: false - - # Single rollup status that summarises every build/test job. Mark this one - # as the required check in branch protection — reviewers see a single - # ✅/❌ instead of 50 matrix entries. Click-through still surfaces the - # individual job logs. - # - # ``if: always()`` makes the rollup run even if the core jobs failed, - # were skipped, or were cancelled (so we always render a check). The body - # fails the rollup iff any build/test job ended in 'failure'; 'skipped' - # (label-gated) and 'success' both count as healthy. - ci-rollup: - name: CI / Linux x86_64 - if: ${{ always() }} - permissions: {} - needs: [core] - runs-on: ubuntu-latest - steps: - - name: Aggregate job results - env: - RESULTS: ${{ needs.core.outputs.results }} - # Safety net: if the core reusable call concluded - # failure/cancelled but its results output came back empty - # (a known edge case), still fail the rollup rather than - # silently reporting green. - CORE_RESULT: ${{ needs.core.result }} - WORKFLOW_LABEL: "Linux x86_64" - run: | - set -euo pipefail - # Emit two surfaces: - # * stdout / job exit code → drives the green/red rollup - # status that branch protection keys on. - # * $GITHUB_STEP_SUMMARY → the markdown that renders - # on the workflow run page, with a per-job result table. - python3 - <<'PY' - import json, os, sys - raw = os.environ.get("RESULTS") or "{}" - core_result = os.environ.get("CORE_RESULT", "") - try: - needs = json.loads(raw) - except json.JSONDecodeError: - needs = {} - label = os.environ.get("WORKFLOW_LABEL", "Linux x86_64") - by_result = {"success": [], "failure": [], "skipped": [], "cancelled": []} - for name, info in needs.items(): - by_result.setdefault(info.get("result") or "unknown", []).append(name) - failed = sorted(by_result["failure"]) - passed = sorted(by_result["success"]) - skipped = sorted(by_result["skipped"]) - cancelled = sorted(by_result["cancelled"]) - - # --- stdout: short pass/fail summary for the log tab --- - print(f"PASS: {len(passed)}") - print(f"FAIL: {len(failed)}") - print(f"SKIPPED: {len(skipped)} (label-gated or never started)") - print(f"CANCELLED: {len(cancelled)}") - if failed: - print() - print("Failed jobs:") - for name in failed: - print(f" - {name}") - - # --- step summary: markdown table for reviewers --- - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - icon = {"success": "✅", "failure": "❌", "skipped": "⏭️", "cancelled": "🚫"} - with open(summary_path, "a", encoding="utf-8") as f: - f.write(f"# CI / {label} — rollup\n\n") - f.write( - f"**{len(passed)}** passed · " - f"**{len(failed)}** failed · " - f"**{len(skipped)}** skipped · " - f"**{len(cancelled)}** cancelled\n\n" - ) - f.write("| Result | Job |\n|---|---|\n") - for status in ("failure", "cancelled", "skipped", "success"): - for name in sorted(by_result.get(status, [])): - f.write(f"| {icon.get(status, '?')} {status} | `{name}` |\n") - if failed: - f.write( - "\n> Click into a failed job above to see " - "the rendered test table (via `pytest-results-action`) " - "and the `::warning::Reproduce locally with: ...` hint " - "near the bottom of the log.\n" - ) - - # Fail if any job failed, OR the core call did not succeed - # (covers an empty/missing results payload). - if failed or core_result not in ("success", "skipped", ""): - if core_result not in ("success", "skipped", "") and not failed: - print(f"\nCore reusable workflow concluded '{core_result}' " - f"with no per-job results; failing rollup defensively.") - sys.exit(1) - PY - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-x86_64_rtx-python-only.yml b/.github/workflows/build-test-linux-x86_64_rtx-python-only.yml deleted file mode 100644 index 27e6a21f69..0000000000 --- a/.github/workflows/build-test-linux-x86_64_rtx-python-only.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: RTX - Python-only build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars_python_only.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: RTX - Python-only build Linux x86_64 torch-tensorrt-rtx whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "x86_64" - use-rtx: true - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: RTX - Python-only dynamo runtime tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: rtx-python-only-dynamo-runtime-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-x86_64_rtx.yml b/.github/workflows/build-test-linux-x86_64_rtx.yml deleted file mode 100644 index aed838a50b..0000000000 --- a/.github/workflows/build-test-linux-x86_64_rtx.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: RTX - Build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - # All build/test logic lives in the shared reusable core; this workflow - # just selects TensorRT-RTX (use-rtx: true). See - # .github/workflows/_linux-x86_64-core.yml. - core: - permissions: - id-token: write - contents: read - uses: ./.github/workflows/_linux-x86_64-core.yml - with: - use-rtx: true - name-prefix: "RTX - " - - # Single rollup status for the RTX matrix; mirror the non-RTX workflow's - # ci-rollup so branch protection can require one check per workflow. - ci-rollup: - name: CI / Linux x86_64 (RTX) - if: ${{ always() }} - permissions: {} - needs: [core] - runs-on: ubuntu-latest - steps: - - name: Aggregate job results - env: - RESULTS: ${{ needs.core.outputs.results }} - # Safety net: fail the rollup if the core call concluded - # failure/cancelled even when its results output came back empty. - CORE_RESULT: ${{ needs.core.result }} - # Surface a label so the markdown summary disambiguates RTX vs standard. - WORKFLOW_LABEL: "Linux x86_64 (RTX)" - run: | - set -euo pipefail - # Same logic as the non-RTX rollup: stdout for the rollup status, - # $GITHUB_STEP_SUMMARY for the reviewer-facing markdown table. - python3 - <<'PY' - import json, os, sys - raw = os.environ.get("RESULTS") or "{}" - core_result = os.environ.get("CORE_RESULT", "") - try: - needs = json.loads(raw) - except json.JSONDecodeError: - needs = {} - label = os.environ.get("WORKFLOW_LABEL", "Linux x86_64") - by_result = {"success": [], "failure": [], "skipped": [], "cancelled": []} - for name, info in needs.items(): - by_result.setdefault(info.get("result") or "unknown", []).append(name) - failed = sorted(by_result["failure"]) - passed = sorted(by_result["success"]) - skipped = sorted(by_result["skipped"]) - cancelled = sorted(by_result["cancelled"]) - - print(f"PASS: {len(passed)}") - print(f"FAIL: {len(failed)}") - print(f"SKIPPED: {len(skipped)} (label-gated or never started)") - print(f"CANCELLED: {len(cancelled)}") - if failed: - print() - print("Failed jobs:") - for name in failed: - print(f" - {name}") - - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - icon = {"success": "✅", "failure": "❌", "skipped": "⏭️", "cancelled": "🚫"} - with open(summary_path, "a", encoding="utf-8") as f: - f.write(f"# CI / {label} — rollup\n\n") - f.write( - f"**{len(passed)}** passed · " - f"**{len(failed)}** failed · " - f"**{len(skipped)}** skipped · " - f"**{len(cancelled)}** cancelled\n\n" - ) - f.write("| Result | Job |\n|---|---|\n") - for status in ("failure", "cancelled", "skipped", "success"): - for name in sorted(by_result.get(status, [])): - f.write(f"| {icon.get(status, '?')} {status} | `{name}` |\n") - if failed: - f.write( - "\n> Click into a failed job above to see the " - "rendered test table (via `pytest-results-action`) " - "and the `::warning::Reproduce locally with: ...` " - "hint near the bottom of the log.\n" - ) - - if failed or core_result not in ("success", "skipped", ""): - if core_result not in ("success", "skipped", "") and not failed: - print(f"\nCore reusable workflow concluded '{core_result}' " - f"with no per-job results; failing rollup defensively.") - sys.exit(1) - PY - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-tensorrt-linux.yml b/.github/workflows/build-test-tensorrt-linux.yml deleted file mode 100644 index 2b1a08c45a..0000000000 --- a/.github/workflows/build-test-tensorrt-linux.yml +++ /dev/null @@ -1,336 +0,0 @@ -name: Build and Test Torch-TensorRT on Linux with Future TensorRT Versions - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 0' # Runs at 00:00 UTC every Sunday (minute hour day-of-month month-of-year day-of-week) - -permissions: - id-token: write - contents: read - packages: write - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - python-versions: '["3.11"]' - - generate-tensorrt-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate tensorrt matrix - id: generate - run: | - set -eou pipefail - python -m pip install --upgrade pip - pip install requests - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/generate-tensorrt-test-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: [generate-tensorrt-matrix] - name: Build torch-tensorrt whl package - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - uses: ./.github/workflows/build-tensorrt-linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - - tests-py-torchscript-fe: - name: Test torchscript frontend [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-torchscript-fe - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd - - tests-py-dynamo-converters: - name: Test dynamo converters [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-converters - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - popd - - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-fe - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd - - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-serde - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - popd - - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-torch-compile-be - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd - - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd - - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - nvidia-smi - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py || true - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py || true - popd - - tests-py-core: - name: Test core [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-tensorrt-windows.yml b/.github/workflows/build-test-tensorrt-windows.yml deleted file mode 100644 index 6c66c8d7c6..0000000000 --- a/.github/workflows/build-test-tensorrt-windows.yml +++ /dev/null @@ -1,320 +0,0 @@ -name: Build and Test Torch-TensorRT on Windows with Future TensorRT Versions - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 0' # Runs at 00:00 UTC every Sunday (minute hour day-of-month month-of-year day-of-week) - -permissions: - id-token: write - contents: read - packages: write - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - python-versions: '["3.11"]' - - generate-tensorrt-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate tensorrt matrix - id: generate - run: | - set -eou pipefail - python -m pip install --upgrade pip - pip install requests - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/generate-tensorrt-test-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: generate-tensorrt-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.generate-tensorrt-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - name: Build torch-tensorrt whl package - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - uses: ./.github/workflows/build-tensorrt-windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - - tests-py-torchscript-fe: - name: Test torchscript frontend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-torchscript-fe - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd - - tests-py-dynamo-converters: - name: Test dynamo converters [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-converters - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - popd - - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-fe - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd - - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-serde - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - popd - - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-torch-compile-be - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd - - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd - - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py - popd - - tests-py-core: - name: Test core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows-python-only.yml b/.github/workflows/build-test-windows-python-only.yml deleted file mode 100644 index c48a054479..0000000000 --- a/.github/workflows/build-test-windows-python-only.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Python-only build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: Python-only build Windows torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - wheel-build-params: "--py-only" - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: Python-only dynamo runtime tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: python-only-dynamo-runtime-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - ../../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml deleted file mode 100644 index f092e5a7d0..0000000000 --- a/.github/workflows/build-test-windows.yml +++ /dev/null @@ -1,461 +0,0 @@ -name: Build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: Build Windows torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - - L0-dynamo-converter-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo converter tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-converter-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_converter_tests_results.xml --dist=loadscope --maxfail=20 conversion/ - popd - - L0-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_runtime_tests_results.xml runtime/test_000_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_partitioning_tests_results.xml partitioning/test_000_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_lowering_tests_results.xml lowering/ - popd - - L0-py-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 core python tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-core-python-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/core - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_py_core_tests_results.xml . - popd - - L0-torchscript-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 torchscript tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-torchscript-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_ts_api_tests_results.xml api/ - popd - - L1-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_core_tests_results.xml runtime/test_001_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_core_partitioning_tests_results.xml partitioning/test_001_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_hlo_tests_results.xml hlo/ - popd - - L1-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_compile_tests_results.xml models/ - popd - - L1-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_be_tests_results.xml backend/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L1-torchscript-tests: - name: L1 torchscript tests - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-torchscript-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_ts_models_tests_results.xml models/ - popd - - L2-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L2-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_tests_results.xml models/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_llm_tests_results.xml llm/ - popd - - L2-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_core_tests_results.xml -k "not test_000_ and not test_001_" runtime/* - popd - - L2-dynamo-plugin-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo plugin tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-plugin-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_plugins_tests_results.xml automatic_plugin/ - popd - - L2-torchscript-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 torchscript tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-torchscript-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_ts_integrations_tests_results.xml integrations/ - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows_rtx-python-only.yml b/.github/workflows/build-test-windows_rtx-python-only.yml deleted file mode 100644 index ab4678caa4..0000000000 --- a/.github/workflows/build-test-windows_rtx-python-only.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: RTX - Python-only build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: RTX - Python-only build Windows torch-tensorrt-rtx whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - wheel-build-params: "--py-only --use-rtx" - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - use-rtx: true - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: RTX - Python-only dynamo runtime tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: rtx-python-only-dynamo-runtime-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - ../../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows_rtx.yml b/.github/workflows/build-test-windows_rtx.yml deleted file mode 100644 index 53d2bd6a35..0000000000 --- a/.github/workflows/build-test-windows_rtx.yml +++ /dev/null @@ -1,381 +0,0 @@ -name: RTX - Build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: RTX - Build Windows torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - wheel-build-params: "--use-rtx" - use-rtx: true - timeout: 120 - - L0-dynamo-converter-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo converter tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-converter-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_converter_tests_results.xml --dist=loadscope --maxfail=20 conversion/ - popd - - L0-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_runtime_tests_results.xml runtime/test_000_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_partitioning_tests_results.xml partitioning/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_lowering_tests_results.xml lowering/ - popd - - L0-py-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 core python tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-core-python-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/core - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_py_core_tests_results.xml . - popd - - L1-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_core_tests_results.xml runtime/test_001_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_hlo_tests_results.xml hlo/ - popd - - L1-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_compile_tests_results.xml models/ - popd - - L1-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_be_tests_results.xml backend/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L2-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L2-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_tests_results.xml models/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_llm_tests_results.xml llm/ - popd - - L2-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_core_tests_results.xml -k "not test_000_ and not test_001_" runtime/* - popd - - L2-dynamo-plugin-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo plugin tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-plugin-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_plugins_tests_results.xml automatic_plugin/ - popd - - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/ci-linux-x86_64.yml b/.github/workflows/ci-linux-x86_64.yml new file mode 100644 index 0000000000..f3b0569f49 --- /dev/null +++ b/.github/workflows/ci-linux-x86_64.yml @@ -0,0 +1,135 @@ +name: CI Linux x86_64 + +# Per-platform entry (one small, independent run — not the old 8-channel mega-run). +# Runs the linux-x86_64 channels: standard, RTX, and python-only. Lane + backend +# come from the shared _decide reusable. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + pull_request_review: + types: [submitted] + push: + branches: [main] + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + lane: + description: "Which lane to run" + type: choice + options: [fast, full, nightly] + default: full + backend: + description: "Which backend(s) to test" + type: choice + options: [standard, rtx, both] + default: both + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + decide: + uses: ./.github/workflows/_decide.yml + + # Generate the build matrix ONCE. generate_binary_build_matrix's concurrency + # group is keyed on (workflow, PR, os) with cancel-in-progress, so calling it + # per-channel makes the channels cancel each other's matrix job. All channels + # share this one output and filter it per-variant. + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + # Standard — the only channel on the fast lane (every PR push). + standard: + needs: [decide, generate-matrix] + if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: false + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + rtx: + needs: [decide, generate-matrix] + if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: true + name-prefix: "RTX - " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + name-prefix: "Python-only " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + # python-only against TensorRT-RTX (so backend=both runs BOTH python-only variants). + python-only-rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + use-rtx: true + name-prefix: "Python-only RTX " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + # Required check. Fails ONLY on a genuine failure; skipped/cancelled don't block. + gate: + needs: [decide, standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - name: Gate + run: | + set -euo pipefail + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then + echo "::error::a Linux x86_64 channel failed — see the channel jobs above" + exit 1 + fi + echo "Linux x86_64 gate OK (cancelled/skipped channels ignored)." + + # Consolidated, agent-friendly report over this run's suites (informational). + report: + needs: [standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/download-artifact@v7 + with: + pattern: junit-* + path: all-results + merge-multiple: true + - name: Consolidated report + run: | + set -uo pipefail + mkdir -p all-results + if [ -z "$(find all-results -name '*.xml' 2>/dev/null)" ]; then + echo "No JUnit results uploaded." >> "$GITHUB_STEP_SUMMARY"; exit 0 + fi + python tests/py/utils/junit_summary.py all-results --agent >> "$GITHUB_STEP_SUMMARY" || true + python tests/py/utils/junit_summary.py all-results || true diff --git a/.github/workflows/ci-sbsa.yml b/.github/workflows/ci-sbsa.yml new file mode 100644 index 0000000000..3f7cdaae58 --- /dev/null +++ b/.github/workflows/ci-sbsa.yml @@ -0,0 +1,87 @@ +name: CI SBSA + +# Per-platform entry — SBSA (linux-aarch64) is BUILD-ONLY (no aarch64 GPU test +# runners), so this just validates the wheel builds (standard + python-only) on +# the full/nightly lanes. No test/report jobs since nothing runs. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + pull_request_review: + types: [submitted] + push: + branches: [main] + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + lane: + description: "Which lane to run" + type: choice + options: [fast, full, nightly] + default: full + backend: + description: "Which backend(s) to test" + type: choice + options: [standard, rtx, both] + default: both + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + decide: + uses: ./.github/workflows/_decide.yml + + # Generate the aarch64 build matrix once (shared by both build channels). + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + build: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + architecture: aarch64 + run-tests: false + name-prefix: "SBSA " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + architecture: aarch64 + run-tests: false + name-prefix: "Python-only SBSA " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + gate: + needs: [decide, build, python-only] + if: always() + runs-on: ubuntu-latest + steps: + - name: Gate + run: | + set -euo pipefail + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then + echo "::error::an SBSA build failed — see the build jobs above" + exit 1 + fi + echo "SBSA gate OK (cancelled/skipped channels ignored)." diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml new file mode 100644 index 0000000000..5b0d4f13d0 --- /dev/null +++ b/.github/workflows/ci-windows.yml @@ -0,0 +1,129 @@ +name: CI Windows + +# Per-platform entry — runs the Windows channels (standard, RTX, python-only) as +# one small independent run. Windows is heavier, so it runs on the full/nightly +# lanes only (never the fast PR-push lane). + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + pull_request_review: + types: [submitted] + push: + branches: [main] + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + lane: + description: "Which lane to run" + type: choice + options: [fast, full, nightly] + default: full + backend: + description: "Which backend(s) to test" + type: choice + options: [standard, rtx, both] + default: both + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + decide: + uses: ./.github/workflows/_decide.yml + + # Generate the windows build matrix once (shared across channels; + # generate_binary_build_matrix's cancel-in-progress concurrency would otherwise + # make per-channel calls cancel each other). + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + standard: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-windows.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: false + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-windows.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: true + name-prefix: "RTX - " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-windows.yml + with: + lane: python-only + name-prefix: "Python-only " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + # python-only against TensorRT-RTX (so backend=both runs BOTH python-only variants). + python-only-rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-windows.yml + with: + lane: python-only + use-rtx: true + name-prefix: "Python-only RTX " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + gate: + needs: [decide, standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - name: Gate + run: | + set -euo pipefail + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then + echo "::error::a Windows channel failed — see the channel jobs above" + exit 1 + fi + echo "Windows gate OK (cancelled/skipped channels ignored)." + + report: + needs: [standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/download-artifact@v7 + with: + pattern: junit-* + path: all-results + merge-multiple: true + - name: Consolidated report + run: | + set -uo pipefail + mkdir -p all-results + if [ -z "$(find all-results -name '*.xml' 2>/dev/null)" ]; then + echo "No JUnit results uploaded." >> "$GITHUB_STEP_SUMMARY"; exit 0 + fi + python tests/py/utils/junit_summary.py all-results --agent >> "$GITHUB_STEP_SUMMARY" || true + python tests/py/utils/junit_summary.py all-results || true diff --git a/.github/workflows/linux-test.yml b/.github/workflows/linux-test.yml index d055276143..4537d4d010 100644 --- a/.github/workflows/linux-test.yml +++ b/.github/workflows/linux-test.yml @@ -174,6 +174,17 @@ jobs: display-options: fEs fail-on-empty: ${{ inputs.fail-on-empty }} + # Upload the raw JUnit so a downstream job can aggregate every suite's + # results into one consolidated report (junit_summary.py). Additive: the + # per-job pmeier summary above is unchanged. + - name: Upload JUnit results + if: always() + uses: actions/upload-artifact@v6 + with: + name: junit-${{ inputs.job-name }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }} + path: ${{ env.RUNNER_TEST_RESULTS_DIR }}/**/*.xml + if-no-files-found: ignore + - name: Prepare artifacts for upload working-directory: ${{ inputs.repository }} id: check-artifacts diff --git a/.github/workflows/windows-test.yml b/.github/workflows/windows-test.yml index 0d919ca3c3..89c2e83aab 100644 --- a/.github/workflows/windows-test.yml +++ b/.github/workflows/windows-test.yml @@ -153,6 +153,15 @@ jobs: summary: true display-options: fEs fail-on-empty: true + # Upload raw JUnit so the consolidated report (junit_summary.py) can + # aggregate Windows results alongside Linux. Additive. + - name: Upload JUnit results + if: always() + uses: actions/upload-artifact@v6 + with: + name: junit-${{ inputs.job-name }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }} + path: ${{ env.RUNNER_TEST_RESULTS_DIR }}/*.xml + if-no-files-found: ignore - name: Teardown Windows if: ${{ always() }} uses: ./test-infra/.github/actions/teardown-windows diff --git a/TESTING_AND_CI_DESIGN.md b/TESTING_AND_CI_DESIGN.md index c4abad2c60..a73eafaad1 100644 --- a/TESTING_AND_CI_DESIGN.md +++ b/TESTING_AND_CI_DESIGN.md @@ -87,8 +87,9 @@ in **filenames** (`runtime/test_000_*` = L0, `runtime/test_001_*` = L1, the rest `-k "not test_000_…"`). That's retired. The axes are now: - **Subsystem** *(what — a directory)*: `converters`, `runtime`, `lowering`, `partitioning`, `dynamo-models`, `torch-compile`, `torchscript`, `plugins`, `kernels`, `quantization`, `llm`, `distributed`, `executorch`. -- **Lane** *(how deep — a marker)*: `fast` (= `-m smoke`, every push), `full` (default, the ready-signal lane), `nightly` (everything + perf). -- **Variant** *(where — a dimension)*: `standard` / `rtx` / platform — applied centrally by the runner, **not** as `if [ "$USE_TRT_RTX" ]` branches scattered through shell. +- **Lane** *(how deep — a marker)*: `fast` (= `-m smoke`, every push), `full` (default, the ready-signal lane), `nightly` (everything + perf), plus `python-only` (a build-mode lane: the `PYTHON_ONLY=1` wheel validated against the runtime suite). +- **Variant** *(where — a dimension)*: `standard` / `rtx` — applied centrally by the runner, **not** as `if [ "$USE_TRT_RTX" ]` branches scattered through shell. +- **Platform / channel** *(another dimension)*: `linux-x86_64` / `windows` for tests (SBSA/aarch64 is build-only — no GPU test runners). One reusable per OS family (`uses:` must be literal); the suite *selection* is platform-agnostic. | | fast (every push) | full (ready signal) | nightly | |---|---|---|---| diff --git a/justfile b/justfile index da35049f11..25a433674b 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,4 @@ -# Recipes source tests/py/utils/ci_helpers.sh (a bash library), so run them in bash. +# Recipes use bash (shebang recipes + the _ci launcher), so run them in bash. set shell := ["bash", "-cu"] # List all available recipes @@ -11,22 +11,24 @@ default: # PermissionError. Giving each user their own TMPDIR sidesteps that entirely. export TMPDIR := env_var_or_default("TMPDIR", "/tmp/torch_tensorrt_" + env_var_or_default("USER", "local")) -# pytest xdist parallelism for the parallel suites. Defaults to 2: these tiers +# pytest xdist parallelism for the parallel suites. Defaults to 4: these suites # build TensorRT engines, which are GPU-memory-heavy, and a single local GPU # OOMs (CUDA out-of-memory + segfaulting workers) well before it runs out of -# CPU cores — so `-n auto` is the wrong default here. CI runs 8 on dedicated -# GPU runners. Raise it if your GPU has headroom: just jobs=8 l0 +# CPU cores — so `-n auto` is the wrong default here. CI runs more on dedicated +# GPU runners. Raise it if your GPU has headroom: just jobs=8 lane full jobs := "4" -# Launcher + env shared by every tier recipe. The tier definitions themselves -# live in tests/py/utils/ci_helpers.sh — the SAME functions CI calls — so there is a -# single source of truth for what each tier runs. We only set environment -# policy here: PYTHON runs pytest against the already-built .venv (uv --no-sync, -# no rebuild) and TRT_JOBS feeds the parallel suites. -_tier := 'mkdir -p "$TMPDIR" && source tests/py/utils/ci_helpers.sh && export PYTHON="uv run --no-sync python" TRT_JOBS="' + jobs + '" TRT_PYTEST_RERUNS=0 &&' - -# Invocation for the new suite manifest/runner (tests/ci). Same env policy as -# _tier: the built venv via `uv --no-sync`, GPU-memory-aware jobs, reruns off +# Which backend variant to run: standard | rtx. This selects the test SELECTION +# (RTX drops --dist on converters, runs the whole partitioning dir, etc.) — it +# does NOT switch the installed build. To actually run RTX locally you must have +# a torch-tensorrt built with USE_TRT_RTX=1 installed in the .venv. +# just variant=rtx lane fast +variant := "standard" + +# Launcher + env for the suite recipes. Suite definitions live in the manifest +# (tests/ci/suites.py) — the SAME data CI uses — so local and CI cannot drift. +# We only set environment policy here: PYTHON runs pytest against the already-built +# .venv (uv --no-sync, no rebuild), TRT_JOBS feeds xdist, and reruns are off # locally (the rerunfailures plugin may be absent and you want to SEE flakes). _ci := 'mkdir -p "$TMPDIR" && PYTHON="uv run --no-sync python" TRT_JOBS="' + jobs + '" TRT_PYTEST_RERUNS=0 uv run --no-sync python -m tests.ci' @@ -39,10 +41,10 @@ test *args: @mkdir -p "$TMPDIR" uv run --no-sync pytest {{args}} -# ── Suite manifest (tests/ci) ────────────────────────────────────────────────── +# ── Suite manifest (tests/ci) — the single source of truth ────────────────────── # -# The suite manifest is the single source of truth for what each job runs; CI -# and these recipes both call `python -m tests.ci`. See TESTING_AND_CI_DESIGN.md. +# CI and these recipes both call `python -m tests.ci`, so what runs locally is +# exactly what runs in CI. See TESTING_AND_CI_DESIGN.md. # Validate the suite manifest (unique names/junit paths, valid setup steps) doctor: @@ -52,15 +54,15 @@ doctor: suites: uv run --no-sync python -m tests.ci list -# Run ONE suite exactly as CI runs it. Extra pytest args after `--`: -# just suite dynamo-runtime -- -k test_foo -x +# Run ONE suite exactly as CI runs it (uses the {{variant}} backend). Args after `--`: +# just suite dynamo-runtime -- -k test_foo -x just variant=rtx suite dynamo-converters suite name *args: - {{_ci}} run {{name}} {{args}} + {{_ci}} run {{name}} --variant {{variant}} {{args}} # Run every suite in a LANE (fast|full|nightly), continuing past failures: -# just lane fast +# just lane fast just variant=rtx lane fast lane name *args: - {{_ci}} run-lane --lane {{name}} --variant standard {{args}} + {{_ci}} run-lane --lane {{name}} --variant {{variant}} {{args}} # Run a lane past failures, then print one consolidated report (--agent for Claude): # just report fast --agent @@ -71,151 +73,20 @@ report lane *summary_args: results="${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" rm -f "$results"/*.xml 2>/dev/null || true export PYTHON="uv run --no-sync python" TRT_JOBS="{{jobs}}" TRT_PYTEST_RERUNS=0 - uv run --no-sync python -m tests.ci run-lane --lane {{lane}} --variant standard || true + uv run --no-sync python -m tests.ci run-lane --lane {{lane}} --variant {{variant}} || true uv run --no-sync python tests/py/utils/junit_summary.py "$results" {{summary_args}} -# ── CI tier reproduction ────────────────────────────────────────────────────── -# -# Run exactly what a CI tier runs, before pushing. Each recipe calls the tier -# function from tests/py/utils/ci_helpers.sh — the same one .github/workflows/ -# _linux-x86_64-core.yml invokes — so local and CI cannot drift. Extra args are -# forwarded to pytest, e.g. `just tests-l0-core -x -k test_foo`. -# (Standard-TensorRT scope; export USE_TRT_RTX=true before running for RTX.) - -# Full L0 smoke tier -tests-l0: tests-l0-converter tests-l0-core tests-l0-py-core tests-l0-torchscript - -tests-l0-converter *args: - {{_tier}} trt_tier_l0_converter {{args}} - -tests-l0-core *args: - {{_tier}} trt_tier_l0_core {{args}} - -tests-l0-py-core *args: - {{_tier}} trt_tier_l0_py_core {{args}} - -tests-l0-torchscript *args: - {{_tier}} trt_tier_l0_torchscript {{args}} - -# Full L1 tier -tests-l1: tests-l1-dynamo-core tests-l1-dynamo-compile tests-l1-torch-compile tests-l1-torchscript +# Re-print the LAST run's consolidated report without re-running (--agent for Claude) +summary *args: + uv run --no-sync python tests/py/utils/junit_summary.py "${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" {{args}} -tests-l1-dynamo-core *args: - {{_tier}} trt_tier_l1_dynamo_core {{args}} - -tests-l1-dynamo-compile *args: - {{_tier}} trt_tier_l1_dynamo_compile {{args}} - -tests-l1-torch-compile *args: - {{_tier}} trt_tier_l1_torch_compile {{args}} - -tests-l1-torchscript *args: - {{_tier}} trt_tier_l1_torchscript {{args}} - -# Pulls every optional test-dep group so the model / kernels / quantization / -# executorch suites run instead of skipping. Uses `uv pip install` (not `uv -# sync`) so it ADDS the deps without rebuilding torch-tensorrt or tearing down -# the env — this also sidesteps the test_ext↔quantization `uv sync` conflict, -# which only applies to lockfile resolution. executorch has no group, so it is -# installed as a plain package. - -# Install all optional test deps (test-ext + kernels + quantization + executorch) +# Added without a rebuild via `uv pip install --group` (sidesteps the +# test-ext↔quantization `uv sync` lockfile conflict). Run before `just lane full`. +# Install optional test deps so model/kernels/quantization/executorch suites run install-test-ext: uv pip install --group test-ext --group kernels --group quantization uv pip install pyyaml "executorch>=1.3.1" -# Full L1 tier + test-ext deps, so model-level cases run instead of skipping -tests-l1-ext: install-test-ext tests-l1-dynamo-core tests-l1-dynamo-compile tests-l1-torch-compile tests-l1-torchscript - -# Excludes tests-l2-distributed (needs 2+ GPUs and system MPI). Most L2 suites -# are model-level, so run tests-l2-ext (or `just install-test-ext` first) so -# they don't skip. - -# Full L2 tier (locally runnable suites) -tests-l2: tests-l2-torch-compile tests-l2-dynamo-compile tests-l2-dynamo-core tests-l2-plugin tests-l2-torchscript - -tests-l2-torch-compile *args: - {{_tier}} trt_tier_l2_torch_compile {{args}} - -tests-l2-dynamo-compile *args: - {{_tier}} trt_tier_l2_dynamo_compile {{args}} - -# Also installs executorch (additively) for the executorch/ suite. -tests-l2-dynamo-core *args: - {{_tier}} trt_tier_l2_dynamo_core {{args}} - -# Also installs cuda-python/cuda-core (additively) for the kernels/ QDP suite. -tests-l2-plugin *args: - {{_tier}} trt_tier_l2_plugin {{args}} - -tests-l2-torchscript *args: - {{_tier}} trt_tier_l2_torchscript {{args}} - -# Installs mpich/openmpi via dnf (root-capable container) and runs -# --nproc_per_node=2. Not part of the `tests-l2` aggregate. - -# CI-only: needs 2+ GPUs and system MPI -tests-l2-distributed *args: - {{_tier}} trt_tier_l2_distributed {{args}} - -# Full L2 tier + test-ext deps, so model-level cases run instead of skipping -tests-l2-ext: install-test-ext tests-l2-torch-compile tests-l2-dynamo-compile tests-l2-dynamo-core tests-l2-plugin tests-l2-torchscript - -# ── Run-all + consolidated failure report ───────────────────────────────────── - -# Unlike `just tests-l` (which aborts on the first failing suite), this runs -# every suite and aggregates the JUnit XMLs, so a single consolidated report -# shows all failures — nothing gets clobbered or missed. Append `-ext` to also -# install the test-ext deps first so model-level cases run instead of skipping. -# Pass `--agent` to print the paste-to-Claude Markdown report instead of the -# terminal one — i.e. run + report in one step (e.g. `just tests-report l1-ext -# --agent`). Exits non-zero if anything failed/errored. - -# Run a whole tier (l0|l1|l2[-ext]) past failures, then print one report [--agent] -tests-report level *report_args: - #!/usr/bin/env bash - set -uo pipefail # deliberately no -e: keep going past failures - mkdir -p "$TMPDIR" - # Accept an optional `-ext` suffix: install the test-ext group first. - lvl="{{level}}" - ext=0 - if [[ "$lvl" == *-ext ]]; then ext=1; lvl="${lvl%-ext}"; fi - case "$lvl" in - l0) tiers=(l0_converter l0_core l0_py_core l0_torchscript) ;; - l1) tiers=(l1_dynamo_core l1_dynamo_compile l1_torch_compile l1_torchscript) ;; - l2) tiers=(l2_torch_compile l2_dynamo_compile l2_dynamo_core l2_plugin l2_torchscript) ;; - *) echo "unknown level '{{level}}' (use l0|l1|l2, optionally with -ext)" >&2; exit 2 ;; - esac - if [[ "$ext" == 1 ]]; then - # Same set as `just install-test-ext`: pull every optional test-dep group - # so the model / kernels / quantization / executorch suites run instead of - # skipping. cuda-core resolves via uv here (the plugin tier's vanilla - # `pip install cuda-core` cannot fetch it on all hosts). - echo "==> installing test-ext + kernels + quantization + executorch deps" - uv pip install --group test-ext --group kernels --group quantization \ - || { echo "test-ext group install failed" >&2; exit 1; } - uv pip install pyyaml "executorch>=1.3.1" || true - fi - results="${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" - rm -f "$results"/*.xml 2>/dev/null || true # drop stale results from prior runs - source tests/py/utils/ci_helpers.sh - export PYTHON="uv run --no-sync python" TRT_JOBS="{{jobs}}" TRT_PYTEST_RERUNS=0 - for t in "${tiers[@]}"; do - echo "==> trt_tier_$t" - "trt_tier_$t" || echo "::: trt_tier_$t exited non-zero (continuing) :::" - done - # Source of truth is the JUnit XMLs, not exit codes; this sets the final - # status. Extra args (e.g. --agent) are forwarded to the summary. - python3 tests/py/utils/junit_summary.py "$results" {{report_args}} - -# Reads the JUnit XMLs from the previous run (does not re-run). Pass --agent for -# a plain Markdown report to hand to Claude (`just test-summary --agent`). Exits -# non-zero if that run had failures. - -# Print the consolidated failure summary from the last run's JUnit results -test-summary *args: - python3 tests/py/utils/junit_summary.py {{args}} - # ── Linting ─────────────────────────────────────────────────────────────────── # Run all pre-commit hooks across the repo (matches the linter CI job) diff --git a/tests/ci/__main__.py b/tests/ci/__main__.py index 708ad3fcc0..fc59e2ecbc 100644 --- a/tests/ci/__main__.py +++ b/tests/ci/__main__.py @@ -19,11 +19,13 @@ def _cmd_list(_: argparse.Namespace) -> int: width = max(len(s.name) for s in SUITES) - print(f"{'SUITE'.ljust(width)} TIER LANES VARIANTS") + print( + f"{'SUITE'.ljust(width)} TIER LANES VARIANTS PLATFORMS" + ) for s in SUITES: print( f"{s.name.ljust(width)} {s.tier:<4} " - f"{','.join(s.lanes):<21} {','.join(s.variants)}" + f"{','.join(s.lanes):<21} {','.join(s.variants):<15} {','.join(s.platforms)}" ) print( f"\n{len(SUITES)} suites. " @@ -60,7 +62,9 @@ def _cmd_run(args: argparse.Namespace) -> int: def _cmd_run_lane(args: argparse.Namespace) -> int: """Run every suite in a lane/tier, continuing past failures (so one consolidated report sees them all). Returns non-zero if any suite failed.""" - jobs = select(lane=args.lane, tier=args.tier, variant=args.variant) + jobs = select( + lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform + ) if not jobs: print("::warning::no suites match the given filters", file=sys.stderr) return 0 @@ -71,7 +75,9 @@ def _cmd_run_lane(args: argparse.Namespace) -> int: def _cmd_matrix(args: argparse.Namespace) -> int: - include = matrix(lane=args.lane, tier=args.tier, variant=args.variant) + include = matrix( + lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform + ) if not include: print("::warning::matrix is empty for the given filters", file=sys.stderr) print(json.dumps({"include": include})) @@ -101,6 +107,8 @@ def _cmd_doctor(_: argparse.Namespace) -> int: problems.append(f"{s.name}: belongs to no lane") if not s.variants: problems.append(f"{s.name}: runs on no variant") + if not s.platforms: + problems.append(f"{s.name}: runs on no platform") cwd = REPO_ROOT / s.cwd if not cwd.is_dir(): problems.append(f"{s.name}: cwd {s.cwd} does not exist") @@ -148,17 +156,19 @@ def main(argv: list[str] | None = None) -> int: "run-lane", help="run every suite in a lane/tier, past failures" ) g = sp.add_mutually_exclusive_group() - g.add_argument("--lane", choices=("fast", "full", "nightly")) + g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only")) g.add_argument("--tier", choices=("l0", "l1", "l2")) sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument("--platform", choices=("linux-x86_64", "windows")) sp.add_argument("--dry-run", action="store_true") sp.set_defaults(fn=_cmd_run_lane) sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON") g = sp.add_mutually_exclusive_group() - g.add_argument("--lane", choices=("fast", "full", "nightly")) + g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only")) g.add_argument("--tier", choices=("l0", "l1", "l2")) sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument("--platform", choices=("linux-x86_64", "windows")) sp.set_defaults(fn=_cmd_matrix) sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor) diff --git a/tests/ci/runner.py b/tests/ci/runner.py index 4943234b66..96fa3ddd69 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -217,6 +217,7 @@ def select( lane: str | None = None, tier: str | None = None, variant: str | None = None, + platform: str | None = None, names: list[str] | None = None, ) -> list[tuple[Suite, Variant]]: """All (suite, variant) jobs matching the filters. No filter on an axis = all.""" @@ -227,6 +228,8 @@ def select( continue if tier is not None and s.tier != tier: continue + if platform is not None and platform not in s.platforms: + continue for var in s.variants: if variant is not None and var != variant: continue diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 044413b8c8..4e87e53898 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -31,10 +31,17 @@ from typing import Any, Literal Tier = Literal["l0", "l1", "l2"] -Lane = Literal["fast", "full", "nightly"] +# python-only validates the PYTHON_ONLY=1 wheel (no C++ runtime) against the +# dynamo runtime suite. It's its own lane because it pairs a distinct BUILD mode +# with a focused test set, orthogonal to fast/full/nightly depth. +Lane = Literal["fast", "full", "nightly", "python-only"] Variant = Literal["standard", "rtx"] +# Test channels. SBSA (linux-aarch64) is build-only — no GPU test runners — so it +# is a build channel handled at the workflow level, not a suite platform here. +Platform = Literal["linux-x86_64", "windows"] ALL_VARIANTS: tuple[Variant, ...] = ("standard", "rtx") +ALL_PLATFORMS: tuple[Platform, ...] = ("linux-x86_64", "windows") @dataclass(frozen=True) @@ -60,6 +67,7 @@ class Suite: reruns: bool = True # wrap in the flake-rerun helper verbose: bool = False # -v variants: tuple[Variant, ...] = ALL_VARIANTS + platforms: tuple[Platform, ...] = ALL_PLATFORMS # channels this suite runs on setup: tuple[str, ...] = () # named pre-steps: hub|executorch|cuda-core|mpi follow: tuple[tuple[str, ...], ...] = () # extra argv to run AFTER pytest env: dict[str, str] = field(default_factory=dict) @@ -243,6 +251,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: setup=("executorch",), jobs="auto", variants=("standard",), + platforms=("linux-x86_64",), ), Suite( # Standard: the automatic-plugin trio. RTX: the whole automatic_plugin dir. @@ -267,6 +276,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: setup=("cuda-core",), jobs="auto", variants=("standard",), + platforms=("linux-x86_64",), ), Suite( "ts-integrations", @@ -291,6 +301,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: verbose=True, reruns=False, variants=("standard",), + platforms=("linux-x86_64",), setup=("mpi",), env={"USE_HOST_DEPS": "1", "CI_BUILD": "1", "USE_TRTLLM_PLUGINS": "1"}, follow=( @@ -312,7 +323,20 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: ), ] -SUITES: tuple[Suite, ...] = tuple(_L0 + _L1 + _L2) +# ── python-only — validates the PYTHON_ONLY=1 wheel against the runtime suite ── +_PYTHON_ONLY: list[Suite] = [ + Suite( + "python-only-runtime", + tier="l1", + lanes=("python-only",), + paths=("runtime/",), + jobs="8", + # Runs for BOTH backends: the PYTHON_ONLY=1 wheel is validated against + # standard TensorRT and TensorRT-RTX (variants default to both). + ), +] + +SUITES: tuple[Suite, ...] = tuple(_L0 + _L1 + _L2 + _PYTHON_ONLY) def by_name(name: str) -> Suite: From c44b116f0793d767960dde300f4c8ba748138c80 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Wed, 8 Jul 2026 20:25:12 +0000 Subject: [PATCH 04/11] tool: A UI to look at CI results that doesnt suck --- justfile | 15 + tools/ci-dashboard/.gitignore | 2 + tools/ci-dashboard/README.md | 91 ++ tools/ci-dashboard/ci_dashboard.py | 1100 +++++++++++++++++++++++++ tools/ci-dashboard/static/htmx.min.js | 1 + tools/ci-dashboard/static/index.html | 79 ++ tools/ci-dashboard/static/style.css | 222 +++++ 7 files changed, 1510 insertions(+) create mode 100644 tools/ci-dashboard/.gitignore create mode 100644 tools/ci-dashboard/README.md create mode 100755 tools/ci-dashboard/ci_dashboard.py create mode 100644 tools/ci-dashboard/static/htmx.min.js create mode 100644 tools/ci-dashboard/static/index.html create mode 100644 tools/ci-dashboard/static/style.css diff --git a/justfile b/justfile index 25a433674b..54df4ff6e1 100644 --- a/justfile +++ b/justfile @@ -114,3 +114,18 @@ docs-serve port="3000": # Build docs then serve them docs-build-serve port="3000": docs python3 -m http.server {{port}} --directory docs + +# ── CI dashboard ────────────────────────────────────────────────────────────── + +# Local web UI for GitHub Actions CI. Per-platform pass/fail, a python×cuda×tier +# grid, failing tests mapped back to source (file:line), a copy-paste local +# repro, and a cross-platform failure rollup. Needs an authenticated `gh` +# (check with `gh auth status`). Defaults to the current branch; override with +# branch= and port=, e.g. just ci branch=nightly | just ci branch=main port=9000 +# +# Uses `uv run --no-sync` on purpose: the dashboard is stdlib-only and has no +# torch-tensorrt dependency, so --no-sync skips the project build entirely and +# just reuses the existing .venv interpreter. Binds 0.0.0.0 so it's reachable +# over Tailscale. +ci branch="" port="8712": + uv run --no-sync tools/ci-dashboard/ci_dashboard.py -b "{{branch}}" -p {{port}} diff --git a/tools/ci-dashboard/.gitignore b/tools/ci-dashboard/.gitignore new file mode 100644 index 0000000000..7a60b85e14 --- /dev/null +++ b/tools/ci-dashboard/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/tools/ci-dashboard/README.md b/tools/ci-dashboard/README.md new file mode 100644 index 0000000000..6381d56753 --- /dev/null +++ b/tools/ci-dashboard/README.md @@ -0,0 +1,91 @@ +# CI dashboard + +A local web UI for Torch-TensorRT's GitHub Actions CI. It answers the questions +the raw Actions tab makes painful: **which platforms are green, which are red, +which test broke, and where that test lives in the tree.** + +![what it shows](#) + +## Run it + +```bash +just ci # current branch +just ci branch=nightly # a specific branch +just ci branch=main port=9000 # pick a port + +# or directly (no build — stdlib only): +uv run --no-sync tools/ci-dashboard/ci_dashboard.py -b nightly +python3 tools/ci-dashboard/ci_dashboard.py -b nightly # also fine +``` + +It opens `http://127.0.0.1:8712/` locally. It also **binds `0.0.0.0`**, so it's +reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN URLs +(e.g. `http://100.x.y.z:8712/`). To keep it local-only, pass `--host 127.0.0.1`. + +### Requirements + +- An **authenticated `gh`** — the only data source. Check with `gh auth status`; + log in with `gh auth login`. +- Python 3.9+ (**stdlib only** — no `pip install`, and **no torch-tensorrt build**). + `just ci` uses `uv run --no-sync`, which reuses the existing `.venv` interpreter + and skips the project build; plain `python3` works too since nothing outside the + stdlib is imported. `htmx` is vendored under `static/`, so it works offline. + +## What you get + +- **Platform board** — one card per workflow (Linux x86_64, aarch64, Windows, + the RTX and python-only variants, jetpack…), sorted worst-first, with a live + status badge. A summary strip up top counts failing / running / queued / + passing at a glance. +- **Drill in** — open a platform to see its jobs as a **python × cuda × tier** + grid. Green/red cells; L0/L1/L2 tiers are labelled with the pytest paths they + run (pulled from `tests/py/utils/ci_helpers.sh`). +- **Click a red cell** — a drawer shows every failing test with its error, the + **source file:line it maps to** (local path + a GitHub link pinned to the run's + commit), and a **copy-paste command to reproduce it locally** via the same + tier function CI used. +- **Relevant logs, captured** — the drawer then pulls that job's log and extracts + just the **pytest FAILURES block per failing test** (traceback + captured + stdout/stderr), so you read the actual failure without scrolling a 1 MB log. + A **raw log ↗** link opens the full plain-text log (grep/save it); a + build/env failure with no test annotations falls back to the end of the log. +- **Failures across platforms** — a rollup that dedupes a failing test across + every platform it breaks on ("fails on 3 platforms → likely this code"), so a + systemic break stands out from a one-off. +- **"Didn't run" ≠ "failed"** — skipped/gated-off tiers (common on PR branches), + jobs blocked by a failed dependency, and cancelled/never-started jobs are shown + as a distinct, dashed/muted **didn't run** state — never colored red and never + counted as failing. The summary strip tallies them separately, and the + cross-platform rollup only ever counts jobs that actually *ran and failed*. +- **Live** — status badges refresh in the background (~every 25s) without + collapsing whatever you've expanded. `↻ Refresh` (or `r`) forces a full reload; + `/` focuses the branch box; the *failing only* toggle hides the green cards. + +## How it works + +The server (`ci_dashboard.py`) is a thin, caching proxy over `gh`; all rendering +is server-side HTML fragments and `htmx` wires up the interactions (lazy-load a +platform on open, out-of-band badge polling, the failure drawer). + +- **Runs / jobs**: `gh run list` + `gh api …/actions/runs/{id}/jobs`. Job names + encode the matrix — `core / L2 dynamo distributed tests / + L2-dynamo-distributed-tests--3.12-cu130` — which we parse into + `{tier, python, cuda, kind}`. +- **Why a red cell → a test → a file**: the test jobs surface each pytest + failure as a GitHub **check-run annotation** (via `pytest-results-action`), so + the failing test name + traceback is one cheap API call + (`…/check-runs/{id}/annotations`) — no wheel or junit download. We then + `git grep` the test symbol in `tests/` to resolve it to `file:line`. +- **Tier → paths** live in `TIER_MAP`, a mirror of the `trt_tier_*` selectors in + `tests/py/utils/ci_helpers.sh`. If a tier is added/renamed there, add it here. + +## Limitations + +- Results are only as granular as the CI annotations. A **build/env/setup** + failure (not a test assertion) has no pytest annotation — the drawer says so + and links you to the raw job log. +- Expanded platform grids don't auto-refresh (only the top-level badges do). Open + a stale platform again, or hit `↻ Refresh`, to re-pull its jobs. +- The cross-platform rollup fans out `gh` calls for every failing test job; on a + branch that's failing everywhere the first scan takes a few seconds (results + are cached; `↻ rescan` forces a refresh). diff --git a/tools/ci-dashboard/ci_dashboard.py b/tools/ci-dashboard/ci_dashboard.py new file mode 100755 index 0000000000..e8c4923118 --- /dev/null +++ b/tools/ci-dashboard/ci_dashboard.py @@ -0,0 +1,1100 @@ +#!/usr/bin/env python3 +"""Torch-TensorRT CI dashboard — a local, pleasant view of GitHub Actions. + +Run: uv run tools/ci-dashboard/ci_dashboard.py # current branch + uv run tools/ci-dashboard/ci_dashboard.py -b nightly # a branch + python3 tools/ci-dashboard/ci_dashboard.py --port 8712 + +No pip dependencies. Data comes from the `gh` CLI (must be authenticated: +`gh auth status`). The server renders HTML fragments; htmx (vendored under +./static) drives lazy-loading, live status polling, and the failure drawer. + +Why annotations, not artifacts: the test jobs surface every pytest failure as a +GitHub check-run *annotation* (via pytest-results-action), so the failing test +name + traceback is one cheap API call away — no wheel/junit download needed. + +Job names encode the whole matrix, e.g. + core / L2 dynamo distributed tests / L2-dynamo-distributed-tests--3.12-cu130 +We parse that into {group/tier, python, cuda, kind} and map the tier back to the +pytest paths in tests/py/utils/ci_helpers.sh so a red cell points at code. +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import socket +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from functools import lru_cache +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, quote, urlparse + +HERE = os.path.dirname(os.path.abspath(__file__)) +STATIC = os.path.join(HERE, "static") +REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) + +# ── Tier → source mapping ──────────────────────────────────────────────────── +# Mirrors the trt_tier_* selectors in tests/py/utils/ci_helpers.sh (the single +# source of truth for "what does each CI tier run"). Keys are normalized job +# group names (see _norm()); `paths` are repo-relative; `fn` is the shell tier +# function you'd invoke locally to reproduce. +TIER_MAP = { + "l0 dynamo converter tests": dict( + fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"] + ), + "l0 dynamo core tests": dict( + fn="trt_tier_l0_core", + paths=[ + "tests/py/dynamo/runtime/test_000_*", + "tests/py/dynamo/partitioning/test_000_*", + "tests/py/dynamo/lowering/", + "tests/py/dynamo/hlo/", + ], + ), + "l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]), + "l0 torchscript tests": dict( + fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"] + ), + "l1 dynamo core tests": dict( + fn="trt_tier_l1_dynamo_core", + paths=[ + "tests/py/dynamo/runtime/test_001_*", + "tests/py/dynamo/partitioning/test_001_*", + "tests/py/dynamo/hlo/", + ], + ), + "l1 dynamo compile tests": dict( + fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"] + ), + "l1 torch compile tests": dict( + fn="trt_tier_l1_torch_compile", + paths=[ + "tests/py/dynamo/backend/", + "tests/py/dynamo/models/test_models.py", + "tests/py/dynamo/models/test_dyn_models.py", + ], + ), + "l1 torch script tests": dict( + fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"] + ), + "l2 torch compile tests": dict( + fn="trt_tier_l2_torch_compile", + paths=[ + "tests/py/dynamo/models/test_models.py", + "tests/py/dynamo/models/test_dyn_models.py", + ], + ), + "l2 dynamo compile tests": dict( + fn="trt_tier_l2_dynamo_compile", + paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"], + ), + "l2 dynamo core tests": dict( + fn="trt_tier_l2_dynamo_core", + paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"], + ), + "l2 dynamo plugin tests": dict( + fn="trt_tier_l2_plugin", + paths=[ + "tests/py/dynamo/conversion/", + "tests/py/dynamo/automatic_plugin/", + "tests/py/kernels/", + ], + ), + "l2 torch script tests": dict( + fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"] + ), + "l2 dynamo distributed tests": dict( + fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"] + ), +} + + +def _norm(s: str) -> str: + """Lowercase and collapse separators so 'L2-dynamo-core-tests' and + 'L2 dynamo core tests' hash to the same tier key.""" + return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip() + + +def tier_for(group: str): + return TIER_MAP.get(_norm(group)) + + +# ── gh plumbing (cached) ───────────────────────────────────────────────────── +class Cache: + """Tiny TTL cache. Completed runs/jobs are immutable so we cache them long; + anything still in flight gets a short TTL so the UI stays live.""" + + def __init__(self): + self._d: dict[str, tuple[float, object]] = {} + self._lock = threading.Lock() + + def get(self, key): + with self._lock: + hit = self._d.get(key) + if not hit: + return None + expires, val = hit + return None if time.time() > expires else val + + def put(self, key, val, ttl): + with self._lock: + self._d[key] = (time.time() + ttl, val) + + def clear(self): + with self._lock: + self._d.clear() + + +CACHE = Cache() + + +def gh(args, parse=True, timeout=90): + proc = subprocess.run( + ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT + ) + if proc.returncode != 0: + raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip()) + return json.loads(proc.stdout) if parse else proc.stdout + + +@lru_cache(maxsize=1) +def repo_slug(): + try: + return gh( + ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"], + parse=False, + ).strip() + except Exception: + return "pytorch/TensorRT" + + +_DEFAULT_BRANCH = None + + +def default_branch(): + if _DEFAULT_BRANCH: + return _DEFAULT_BRANCH + try: + b = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ).stdout.strip() + return b if b and b != "HEAD" else "main" + except Exception: + return "main" + + +# ── data layer ─────────────────────────────────────────────────────────────── +def get_runs(branch, limit=40, refresh=False): + key = f"runs:{branch}:{limit}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + fields = ( + "databaseId,workflowName,displayTitle,headBranch,headSha,status," + "conclusion,event,createdAt,updatedAt,url,number" + ) + runs = gh( + ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields] + ) + newest = {} + for r in runs: + r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId) + wf = r.get("workflowName") or "?" + if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]: + newest[wf] = r + result = list(newest.values()) + live = any(r["status"] != "completed" for r in result) + CACHE.put(key, result, ttl=15 if live else 120) + return result + + +def _parse_job(j): + raw = j.get("name", "") + parts = [p.strip() for p in raw.split(" / ")] + last = parts[-1] + py = re.search(r"(? 1 and bool(re.search(r"--|build-wheel|3\.\d+", last)) + group_parts = parts[:-1] if is_matrix else parts[:] + if group_parts and group_parts[0] in ("core", "build"): + rest = group_parts[1:] + group_parts = rest if rest else (parts[:-1] if is_matrix else parts) + group = " / ".join(group_parts) if group_parts else raw + gl = _norm(group) + if "build-wheel" in last or gl.startswith("build "): + kind = "build" + elif re.match(r"l[012]\b", gl): + kind = "test" + elif "matrix" in gl or "generate" in gl or group == "filter-matrix": + kind = "setup" + elif ( + raw.startswith("CI /") + or "aggregate" in gl + or "collect results" in gl + or "rollup" in gl + ): + kind = "rollup" + else: + kind = "other" + return dict( + id=j.get("databaseId") or j.get("id"), + raw=raw, + group=group, + kind=kind, + python=py.group(0) if py else None, + cuda=cu.group(0) if cu else None, + status=j.get("status"), + conclusion=j.get("conclusion"), + url=j.get("html_url") or j.get("url"), + failedSteps=[ + s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure" + ], + ) + + +def get_jobs(run_id, refresh=False): + key = f"jobs:{run_id}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + data = gh( + [ + "api", + f"repos/{repo_slug()}/actions/runs/{run_id}/jobs", + "--paginate", + "-q", + ".jobs[]", + ], + parse=False, + ) + jobs = [json.loads(line) for line in data.splitlines() if line.strip()] + parsed = [_parse_job(j) for j in jobs] + live = any(j["status"] != "completed" for j in parsed) + CACHE.put(key, parsed, ttl=15 if live else 300) + return parsed + + +NOISE = re.compile( + r"Process completed with exit code|Node\.js \d+ is deprecated|" + r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|" + r"might have been added implicitly", + re.I, +) +TESTLINE = re.compile(r"^(?P[\w./-]+(?:\.[\w\[\]-]+)+)\s*$") + + +@lru_cache(maxsize=4096) +def grep_test(symbol): + """Resolve a failing-test symbol (Class.method / module.func) to (file, line) + via `git grep`. Returns None if not found.""" + leaf = re.split(r"[.:]", symbol.strip())[-1] + leaf = re.sub(r"\[.*$", "", leaf) + if not re.match(r"^[A-Za-z_]\w+$", leaf): + return None + pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b" + try: + out = subprocess.run( + ["git", "grep", "-n", "-E", pat, "--", "tests/"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + timeout=15, + ).stdout + except Exception: + return None + for line in out.splitlines(): + path, lineno, _ = line.split(":", 2) + return (path, int(lineno)) + return None + + +def get_failures(job_id, refresh=False): + key = f"fail:{job_id}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + try: + anns = gh(["api", f"repos/{repo_slug()}/check-runs/{job_id}/annotations"]) + except Exception as e: + return dict(error=str(e), failures=[]) + seen, failures = set(), [] + for a in anns: + if a.get("annotation_level") != "failure": + continue + msg = (a.get("message") or "").strip() + if not msg or NOISE.search(msg.splitlines()[0]): + continue + head = msg.splitlines()[0].strip() + m = TESTLINE.match(head) + test = m.group("test") if m else None + dedupe = test or head + if dedupe in seen: + continue + seen.add(dedupe) + loc = grep_test(test) if test else None + failures.append( + dict( + test=test, + message=msg[:1400], + file=loc[0] if loc else None, + line=loc[1] if loc else None, + ) + ) + result = dict(failures=failures) + CACHE.put(key, result, ttl=600) + return result + + +# ── job logs (fetch + per-test extraction) ────────────────────────────────── +_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix +_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep +_SECT = re.compile(r"^={4,}") # ==== section ==== + + +def get_job_log(job_id, refresh=False): + """Raw plain-text log for one job (immutable once complete → cached 1h).""" + key = f"log:{job_id}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + try: + raw = gh( + ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"], + parse=False, + timeout=60, + ) + except Exception: + raw = None + CACHE.put(key, raw, ttl=3600 if raw else 20) + return raw + + +def _clean_lines(raw): + return [_TS.sub("", ln) for ln in raw.splitlines()] + + +def extract_test_log(lines, test, max_lines=160): + """Pull the pytest FAILURES block for `test`. Falls back to a context window + around the test's last mention (covers custom harnesses w/o a FAILURES sep). + Returns the excerpt string, or None.""" + leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None + seps = [ + (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln)) + ] + start = None + if test: + # Prefer an exact title match — the leaf method name (e.g. test_trt_compile) + # is shared across many classes, so fuzzy matching would grab the wrong block. + for i, title in seps: + if title == test or title.split("[")[0] == test: + start = i + break + if start is None and leaf: # fall back when the annotation gave only a leaf + for i, title in seps: + base = title.split("[")[0] + if base == leaf or base.endswith("." + leaf): + start = i + break + if start is not None: + nexts = [i for i, _ in seps if i > start] + [len(lines)] + end = min(nexts) + for j in range(start + 1, end): + if _SECT.match(lines[j]): + end = j + break + block = lines[start:end] + else: + needle = leaf or test + hit = next( + (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), + None, + ) + if hit is None: + return None + block = lines[max(0, hit - 4) : hit + 44] + if len(block) > max_lines: + block = block[:max_lines] + [ + f"… (+{len(block) - max_lines} more lines — open the raw log)" + ] + return "\n".join(block).strip() + + +def summary_reasons(lines, test): + """The concise `FAILED …/ERROR … - ` lines from pytest's summary.""" + leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None + return [ + ln + for ln in lines + if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln) + ] + + +def render_joblog(job_id, url): + log = get_job_log(job_id) + ghlink = ( + f'job on GitHub ↗' + if url + else "" + ) + rawlink = f'raw log ↗' + head = f'
relevant logs{rawlink} · {ghlink}
' + if not log: + return head + ( + '
Log not available yet — the job may still be ' + f"running, or GitHub expired it. {ghlink}
" + ) + lines = _clean_lines(log) + fails = get_failures(job_id).get("failures", []) + blocks = [] + for f in fails: + excerpt = extract_test_log(lines, f["test"]) + reasons = summary_reasons(lines, f["test"]) + title = e(f["test"] or "failure") + reason_html = ( + f'
{e(reasons[0][:300])}
' if reasons else "" + ) + if excerpt: + blocks.append( + f'
{title}' + f"{reason_html}
{e(excerpt)}
" + ) + else: + blocks.append( + f'
{title}' + f'{reason_html}
No matching block in the log — ' + f"see the raw log.
" + ) + if not fails: # build/env/setup failure: no test annotations → show the tail + tail = "\n".join(lines[-180:]).strip() + blocks.append( + '
end of job log' + f"
{e(tail)}
" + ) + return head + "".join(blocks) + + +# ── status helpers ─────────────────────────────────────────────────────────── +def classify(status, conclusion): + """→ (css class, label). "Didn't run" states (skipped/cancelled/never-started) + are kept DISTINCT from "fail" — a gated-off or blocked tier is not a failure, + and a cancelled job didn't produce a verdict. Only `failure`/`timed_out` + (actually executed and failed) are `fail`.""" + if status and status != "completed": + if status in ("in_progress", "running"): + return "run", "running" + return "queue", status.replace( + "_", " " + ) # queued / waiting / pending / requested + c = conclusion or "" + return { + "success": ("pass", "passed"), + "failure": ("fail", "failed"), + "timed_out": ("fail", "timed out"), + # ── didn't run (by design / gating / blocked by a failed dependency) ── + "skipped": ("skip", "didn’t run"), + "neutral": ("skip", "neutral"), + "stale": ("skip", "stale"), + # ── didn't finish / never started (distinct again from a test failure) ── + "cancelled": ("cancel", "cancelled"), + "startup_failure": ("cancel", "didn’t start"), + "action_required": ("queue", "action req"), + }.get(c, ("skip", c or "no result")) + + +# Sort worst-first: real failures, then in-flight, then didn't-finish, then green, +# then the didn't-run noise last. "skip" (gated-off) is deliberately below "pass". +RANK = {"fail": 0, "run": 1, "queue": 2, "cancel": 3, "pass": 4, "skip": 5} +DIDNT_RUN = ("skip", "cancel") + + +def rel_time(iso): + if not iso: + return "" + try: + t = datetime.fromisoformat(iso.replace("Z", "+00:00")) + except Exception: + return iso + s = (datetime.now(timezone.utc) - t).total_seconds() + if s < 60: + return "just now" + for unit, n in (("d", 86400), ("h", 3600), ("m", 60)): + if s >= n: + return f"{int(s // n)}{unit} ago" + return "just now" + + +def pretty_platform(name): + n = ( + name.replace("Python-only build and test ", "py-only ") + .replace("RTX - Build and test ", "RTX ") + .replace("RTX - Python-only build and test ", "RTX py-only ") + .replace("Build and test ", "") + .replace(" wheels", "") + .replace(" for Jetpack", " · jetpack") + ) + return n.strip() + + +def e(s): + return html.escape(str(s if s is not None else "")) + + +def qp(**kw): + """Build a URL-encoded, HTML-attribute-safe query string from kwargs, so + values with spaces/&/ (platform + tier names, job URLs) survive intact.""" + return e( + "&".join( + f"{k}={quote(str(v if v is not None else ''), safe='')}" + for k, v in kw.items() + ) + ) + + +# ── HTML fragment renderers ────────────────────────────────────────────────── +def _summary_html(runs, oob=False): + """The top counts strip. Shared by the board and the live poller so the two + never drift. 'didn't run' folds skipped + cancelled together and is counted + separately from 'failing'.""" + counts = {"pass": 0, "fail": 0, "run": 0, "queue": 0, "skip": 0, "cancel": 0} + for r in runs: + cls, _ = classify(r["status"], r["conclusion"]) + counts[cls] = counts.get(cls, 0) + 1 + didnt_run = counts["skip"] + counts["cancel"] + head = runs[0] if runs else {} + sha = (head.get("headSha") or "")[:9] + commit = ( + f'
{e(sha)} · {e(head.get("displayTitle", ""))[:80]}' + f' · {e(rel_time(head.get("createdAt")))}
' + ) + + def stat(cls, n, word): + return ( + ( + f'' + f'{n} {word}' + ) + if n + else "" + ) + + oob_attr = ' hx-swap-oob="true"' if oob else "" + return ( + f'
' + f'{stat("fail", counts["fail"], "failing")}' + f'{stat("run", counts["run"], "running")}' + f'{stat("queue", counts["queue"], "queued")}' + f'{stat("pass", counts["pass"], "passing")}' + f'{stat("skip", didnt_run, "didn’t run")}' + f"{commit}
" + ) + + +def render_board(branch, refresh=False): + try: + runs = get_runs(branch, refresh=refresh) + except Exception as ex: + return f'
Could not load {e(branch)}:

{e(ex)}
' + if not runs: + return f'
No CI runs found for {e(branch)}.
' + + for r in runs: + r["_cls"], r["_label"] = classify(r["status"], r["conclusion"]) + runs.sort( + key=lambda r: ( + RANK.get(r["_cls"], 9), + pretty_platform(r["workflowName"]).lower(), + ) + ) + summary = _summary_html(runs) + + cards = [] + for r in runs: + rid, cls, label = r["id"], r["_cls"], r["_label"] + plat = pretty_platform(r["workflowName"]) + sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}' + q = qp( + run=rid, + sha=r.get("headSha", ""), + platform=plat, + refresh="1" if refresh else "0", + ) + cards.append(f""" +
+ + + {e(plat)}
{sub}
 + {e(label)} + ▸ + +
loading jobs…
+
""") + + poller = ( + f'
' + ) + agg = ( + f'

Failures across platforms

' + f'
' + f'
scanning failing tests…
' + ) + board = f'

Platforms

{"".join(cards)}
' + return summary + agg + board + poller + + +def render_status(branch): + """OOB-only fragment: refresh the summary counts + every card badge in place + without disturbing expanded platform grids.""" + try: + runs = get_runs(branch, refresh=True) + except Exception: + return "" + spans = [] + for r in runs: + cls, label = classify(r["status"], r["conclusion"]) + spans.append( + f'{e(label)}' + ) + return _summary_html(runs, oob=True) + "".join(spans) + + +KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4} + + +def render_platform(run_id, sha, platform, refresh=False): + try: + jobs = get_jobs(run_id, refresh=refresh) + except Exception as ex: + return f'
Could not load jobs: {e(ex)}
' + if not jobs: + return '
No jobs.
' + + # group by (kind, group) + groups = {} + for j in jobs: + groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j) + + out = [] + for (_, gname), gjobs in sorted( + groups.items(), key=lambda kv: (kv[0][0], kv[0][1]) + ): + tier = tier_for(gname) + tierpath = ( + f'{e(", ".join(tier["paths"]))}' + if tier + else "" + ) + cells, rows = [], [] + for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")): + cls, label = classify(j["status"], j["conclusion"]) + failable = cls == "fail" and j["kind"] in ("test", "build") + if j["python"] and j["cuda"]: + inner = ( + f'{e(j["python"])}·{e(j["cuda"])}' + f'{e(label)}' + ) + if failable: + q = qp( + job=j["id"], + sha=sha, + platform=platform, + tier=gname, + py=j["python"], + cuda=j["cuda"], + url=j["url"], + ) + cells.append( + f'
{inner}
' + ) + else: + cells.append(f'
{inner}
') + else: + name = j["raw"].split(" / ")[-1] or j["group"] + link = ( + f' log ↗' + if j["url"] + else "" + ) + extra = "" + if failable: + q = qp( + job=j["id"], + sha=sha, + platform=platform, + tier=gname, + url=j["url"], + ) + extra = ( + f' · details' + ) + rows.append( + f'
' + f'{e(name)}' + f'{e(label)}{link}{extra}
' + ) + body = "" + if cells: + body += f'
{"".join(cells)}
' + if rows: + body += f'
{"".join(rows)}
' + out.append(f'

{e(gname)} {tierpath}

{body}
') + return "".join(out) + + +def render_failures(job_id, sha, platform, tier, py, cuda, url): + data = get_failures(job_id) + joblink = ( + (f' · job log ↗') + if url + else "" + ) + oob = ( + f'{e(platform)} — {e(tier)}' + f'{e(py)} {e(cuda)}{joblink}' + ) + if data.get("error"): + return ( + oob + + f'
Could not load annotations: {e(data["error"])}
' + ) + fails = data["failures"] + tinfo = tier_for(tier) + body = [] + if not fails: + loglink = ( + f'Open the job log ↗' + if url + else "" + ) + body.append( + '
No pytest failure annotations — this is likely a ' + f"build / environment / setup failure rather than a test assertion.

{loglink}
" + ) + for f in fails: + loc = "" + if f["file"]: + gh_url = ( + f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}' + ) + loc = ( + f'
↳ ' + f'{e(f["file"])}:{f["line"]} ↗
' + ) + body.append( + f'
{e(f["test"] or "failure")}
' + f'{loc}
{e(f["message"])}
' + ) + if tinfo: + leaves = [] + for f in fails: + if f["test"]: + leaf = re.split(r"[.:]", f["test"])[-1] + leaf = re.sub(r"\[.*$", "", leaf) + if leaf and leaf not in leaves: + leaves.append(leaf) + k = f' -k "{" or ".join(leaves[:6])}"' if leaves else "" + cmd = ( + f"source tests/py/utils/ci_helpers.sh && " + f'PYTHON="uv run --no-sync python" TRT_PYTEST_RERUNS=0 {tinfo["fn"]}{k}' + ) + body.append( + f'
' + f'
reproduce locally
{e(cmd)}
' + ) + # Lazy: fetch the job log + extract the relevant block(s) once the drawer is in. + body.append( + f'
' + f'
' + f"fetching relevant logs…
" + ) + return oob + "".join(body) + + +def render_aggregate(branch, refresh=False): + """Cross-platform failure rollup: dedupe a failing test across every platform + it breaks on, so 'fails on N platforms' points straight at the likely code.""" + try: + runs = get_runs(branch, refresh=refresh) + except Exception as ex: + return f'
Could not scan: {e(ex)}
' + + def jobs_of(r): + try: + return [(r, j) for j in get_jobs(r["id"], refresh=refresh)] + except Exception: + return [] + + with ThreadPoolExecutor(max_workers=8) as ex: + alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub] + failed = [ + (r, j) + for r, j in alljobs + if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test" + ] + if not failed: + healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs) + msg = ( + "No failing test jobs 🎉" + if healthy + else "No test-level failures found (failures are build/setup — see the platform grids)." + ) + return f'
{msg}
' + + def fails_of(rj): + r, j = rj + return (r, j, get_failures(j["id"], refresh=refresh).get("failures", [])) + + with ThreadPoolExecutor(max_workers=8) as ex: + results = list(ex.map(fails_of, failed)) + + agg = {} + for r, j, fails in results: + plat = pretty_platform(r["workflowName"]) + if not fails: # failed job but no parseable test → bucket under the tier + key = f"[{j['group']}]" + a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[])) + a["occ"].append( + dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="") + ) + continue + for f in fails: + key = f["test"] or f["message"].splitlines()[0] + a = agg.setdefault( + key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]) + ) + a["file"] = a["file"] or f["file"] + a["line"] = a["line"] or f["line"] + a["occ"].append( + dict( + plat=plat, + py=j["python"], + cuda=j["cuda"], + url=j["url"], + msg=f["message"], + ) + ) + + rows = sorted( + agg.values(), + key=lambda a: (-len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]), + ) + out = [] + for a in rows: + plats = sorted({o["plat"] for o in a["occ"]}) + loc = "" + if a["file"]: + gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}' + loc = f'
↳ {e(a["file"])}:{a["line"]} ↗
' + chips = "".join(f'{e(p)}' for p in plats) + sample = next((o["msg"] for o in a["occ"] if o["msg"]), "") + out.append(f"""
+ +
{e(a["test"])}
{loc}
+ {len(plats)} platform{"s" if len(plats)!=1 else ""} + +
{chips}
{f"
{e(sample)}
" if sample else ""}
+
""") + header = ( + f'
{len(rows)} distinct failing test' + f'{"s" if len(rows)!=1 else ""} across {len({o["plat"] for a in rows for o in a["occ"]})} platforms' + f'
' + ) + return header + f'
{"".join(out)}
' + + +# ── HTTP server ────────────────────────────────────────────────────────────── +class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self, code, body, ctype="text/html; charset=utf-8"): + if isinstance(body, (dict, list)): + body, ctype = json.dumps(body), "application/json" + body = body.encode() if isinstance(body, str) else body + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + u = urlparse(self.path) + q = {k: v[0] for k, v in parse_qs(u.query).items()} + refresh = q.get("refresh") == "1" + try: + p = u.path + if p in ("/", "/index.html"): + return self._index() + if p.startswith("/static/"): + return self._static(p[len("/static/") :]) + if p == "/ui/board": + return self._send( + 200, render_board(q.get("branch") or default_branch(), refresh) + ) + if p == "/ui/status": + return self._send( + 200, render_status(q.get("branch") or default_branch()) + ) + if p == "/ui/platform": + return self._send( + 200, + render_platform( + q["run"], q.get("sha", ""), q.get("platform", ""), refresh + ), + ) + if p == "/ui/failures": + return self._send( + 200, + render_failures( + q["job"], + q.get("sha", ""), + q.get("platform", ""), + q.get("tier", ""), + q.get("py", ""), + q.get("cuda", ""), + q.get("url", ""), + ), + ) + if p == "/ui/aggregate": + return self._send( + 200, render_aggregate(q.get("branch") or default_branch(), refresh) + ) + if p == "/ui/joblog": + if q.get("raw") == "1": + log = get_job_log(q["job"], refresh) + return self._send( + 200 if log else 404, + log or "log not available", + "text/plain; charset=utf-8", + ) + return self._send(200, render_joblog(q["job"], q.get("url", ""))) + return self._send(404, "not found", "text/plain") + except KeyError as ex: + self._send(400, f"missing param {ex}", "text/plain") + except Exception as ex: + self._send(500, f'
error: {e(ex)}
') + + def _index(self): + with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f: + html_s = f.read() + html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace( + "{{REPO}}", e(repo_slug()) + ) + self._send(200, html_s) + + def _static(self, rel): + rel = rel.split("?")[0].lstrip("/") + path = os.path.normpath(os.path.join(STATIC, rel)) + if not path.startswith(STATIC) or not os.path.isfile(path): + return self._send(404, "not found", "text/plain") + ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get( + rel.rsplit(".", 1)[-1], "application/octet-stream" + ) + with open(path, "rb") as f: + self._send(200, f.read(), ctype + "; charset=utf-8") + + +def preflight(): + try: + subprocess.run(["gh", "auth", "status"], capture_output=True, check=True) + except Exception: + sys.exit("error: `gh` is not authenticated. Run `gh auth login` first.") + + +def tailscale_ip(): + """Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present, + otherwise sniffs local interfaces. Returns None if not on a tailnet.""" + try: + out = ( + subprocess.run( + ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3 + ) + .stdout.strip() + .splitlines() + ) + if out and out[0]: + return out[0].strip() + except Exception: + pass + try: + for res in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET): + ip = res[4][0] + if ip.startswith("100.") and 64 <= int(ip.split(".")[1]) <= 127: + return ip + except Exception: + pass + return None + + +def lan_ip(): + """Primary outbound-facing LAN IP (no packets sent).""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + return ip + except Exception: + return None + + +def main(): + global _DEFAULT_BRANCH + ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard") + ap.add_argument("-b", "--branch", default=None, help="default branch to show") + ap.add_argument("-p", "--port", type=int, default=8712) + ap.add_argument( + "--host", + default="0.0.0.0", + help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; " + "use 127.0.0.1 to keep it local-only)", + ) + ap.add_argument("--no-open", action="store_true") + args = ap.parse_args() + preflight() + if args.branch: + _DEFAULT_BRANCH = args.branch + srv = ThreadingHTTPServer((args.host, args.port), Handler) + local = f"http://127.0.0.1:{args.port}/" + print(f"CI dashboard (repo {repo_slug()}, branch {default_branch()})") + print(f" local {local}") + if args.host in ("0.0.0.0", "::"): + ts = tailscale_ip() + if ts: + print(f" tailscale http://{ts}:{args.port}/") + lan = lan_ip() + if lan and lan != ts: + print(f" lan http://{lan}:{args.port}/") + print( + " (bound to all interfaces — anyone who can reach this host can view CI status)" + ) + print("Ctrl-C to stop.", flush=True) + if not args.no_open: + try: + import webbrowser + + webbrowser.open(local) # always open the loopback URL locally + except Exception: + pass + try: + srv.serve_forever() + except KeyboardInterrupt: + print("\nbye") + + +if __name__ == "__main__": + main() diff --git a/tools/ci-dashboard/static/htmx.min.js b/tools/ci-dashboard/static/htmx.min.js new file mode 100644 index 0000000000..59937d712d --- /dev/null +++ b/tools/ci-dashboard/static/htmx.min.js @@ -0,0 +1 @@ +var htmx=function(){"use strict";const Q={onLoad:null,process:null,on:null,off:null,trigger:null,ajax:null,find:null,findAll:null,closest:null,values:function(e,t){const n=cn(e,t||"post");return n.values},remove:null,addClass:null,removeClass:null,toggleClass:null,takeClass:null,swap:null,defineExtension:null,removeExtension:null,logAll:null,logNone:null,logger:null,config:{historyEnabled:true,historyCacheSize:10,refreshOnHistoryMiss:false,defaultSwapStyle:"innerHTML",defaultSwapDelay:0,defaultSettleDelay:20,includeIndicatorStyles:true,indicatorClass:"htmx-indicator",requestClass:"htmx-request",addedClass:"htmx-added",settlingClass:"htmx-settling",swappingClass:"htmx-swapping",allowEval:true,allowScriptTags:true,inlineScriptNonce:"",inlineStyleNonce:"",attributesToSettle:["class","style","width","height"],withCredentials:false,timeout:0,wsReconnectDelay:"full-jitter",wsBinaryType:"blob",disableSelector:"[hx-disable], [data-hx-disable]",scrollBehavior:"instant",defaultFocusScroll:false,getCacheBusterParam:false,globalViewTransitions:false,methodsThatUseUrlParams:["get","delete"],selfRequestsOnly:true,ignoreTitle:false,scrollIntoViewOnBoost:true,triggerSpecsCache:null,disableInheritance:false,responseHandling:[{code:"204",swap:false},{code:"[23]..",swap:true},{code:"[45]..",swap:false,error:true}],allowNestedOobSwaps:true},parseInterval:null,_:null,version:"2.0.4"};Q.onLoad=j;Q.process=kt;Q.on=ye;Q.off=be;Q.trigger=he;Q.ajax=Rn;Q.find=u;Q.findAll=x;Q.closest=g;Q.remove=z;Q.addClass=K;Q.removeClass=G;Q.toggleClass=W;Q.takeClass=Z;Q.swap=$e;Q.defineExtension=Fn;Q.removeExtension=Bn;Q.logAll=V;Q.logNone=_;Q.parseInterval=d;Q._=e;const n={addTriggerHandler:St,bodyContains:le,canAccessLocalStorage:B,findThisElement:Se,filterValues:hn,swap:$e,hasAttribute:s,getAttributeValue:te,getClosestAttributeValue:re,getClosestMatch:o,getExpressionVars:En,getHeaders:fn,getInputValues:cn,getInternalData:ie,getSwapSpecification:gn,getTriggerSpecs:st,getTarget:Ee,makeFragment:P,mergeObjects:ce,makeSettleInfo:xn,oobSwap:He,querySelectorExt:ae,settleImmediately:Kt,shouldCancel:ht,triggerEvent:he,triggerErrorEvent:fe,withExtensions:Ft};const r=["get","post","put","delete","patch"];const H=r.map(function(e){return"[hx-"+e+"], [data-hx-"+e+"]"}).join(", ");function d(e){if(e==undefined){return undefined}let t=NaN;if(e.slice(-2)=="ms"){t=parseFloat(e.slice(0,-2))}else if(e.slice(-1)=="s"){t=parseFloat(e.slice(0,-1))*1e3}else if(e.slice(-1)=="m"){t=parseFloat(e.slice(0,-1))*1e3*60}else{t=parseFloat(e)}return isNaN(t)?undefined:t}function ee(e,t){return e instanceof Element&&e.getAttribute(t)}function s(e,t){return!!e.hasAttribute&&(e.hasAttribute(t)||e.hasAttribute("data-"+t))}function te(e,t){return ee(e,t)||ee(e,"data-"+t)}function c(e){const t=e.parentElement;if(!t&&e.parentNode instanceof ShadowRoot)return e.parentNode;return t}function ne(){return document}function m(e,t){return e.getRootNode?e.getRootNode({composed:t}):ne()}function o(e,t){while(e&&!t(e)){e=c(e)}return e||null}function i(e,t,n){const r=te(t,n);const o=te(t,"hx-disinherit");var i=te(t,"hx-inherit");if(e!==t){if(Q.config.disableInheritance){if(i&&(i==="*"||i.split(" ").indexOf(n)>=0)){return r}else{return null}}if(o&&(o==="*"||o.split(" ").indexOf(n)>=0)){return"unset"}}return r}function re(t,n){let r=null;o(t,function(e){return!!(r=i(t,ue(e),n))});if(r!=="unset"){return r}}function h(e,t){const n=e instanceof Element&&(e.matches||e.matchesSelector||e.msMatchesSelector||e.mozMatchesSelector||e.webkitMatchesSelector||e.oMatchesSelector);return!!n&&n.call(e,t)}function T(e){const t=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i;const n=t.exec(e);if(n){return n[1].toLowerCase()}else{return""}}function q(e){const t=new DOMParser;return t.parseFromString(e,"text/html")}function L(e,t){while(t.childNodes.length>0){e.append(t.childNodes[0])}}function A(e){const t=ne().createElement("script");se(e.attributes,function(e){t.setAttribute(e.name,e.value)});t.textContent=e.textContent;t.async=false;if(Q.config.inlineScriptNonce){t.nonce=Q.config.inlineScriptNonce}return t}function N(e){return e.matches("script")&&(e.type==="text/javascript"||e.type==="module"||e.type==="")}function I(e){Array.from(e.querySelectorAll("script")).forEach(e=>{if(N(e)){const t=A(e);const n=e.parentNode;try{n.insertBefore(t,e)}catch(e){O(e)}finally{e.remove()}}})}function P(e){const t=e.replace(/]*)?>[\s\S]*?<\/head>/i,"");const n=T(t);let r;if(n==="html"){r=new DocumentFragment;const i=q(e);L(r,i.body);r.title=i.title}else if(n==="body"){r=new DocumentFragment;const i=q(t);L(r,i.body);r.title=i.title}else{const i=q('");r=i.querySelector("template").content;r.title=i.title;var o=r.querySelector("title");if(o&&o.parentNode===r){o.remove();r.title=o.innerText}}if(r){if(Q.config.allowScriptTags){I(r)}else{r.querySelectorAll("script").forEach(e=>e.remove())}}return r}function oe(e){if(e){e()}}function t(e,t){return Object.prototype.toString.call(e)==="[object "+t+"]"}function k(e){return typeof e==="function"}function D(e){return t(e,"Object")}function ie(e){const t="htmx-internal-data";let n=e[t];if(!n){n=e[t]={}}return n}function M(t){const n=[];if(t){for(let e=0;e=0}function le(e){return e.getRootNode({composed:true})===document}function F(e){return e.trim().split(/\s+/)}function ce(e,t){for(const n in t){if(t.hasOwnProperty(n)){e[n]=t[n]}}return e}function S(e){try{return JSON.parse(e)}catch(e){O(e);return null}}function B(){const e="htmx:localStorageTest";try{localStorage.setItem(e,e);localStorage.removeItem(e);return true}catch(e){return false}}function U(t){try{const e=new URL(t);if(e){t=e.pathname+e.search}if(!/^\/$/.test(t)){t=t.replace(/\/+$/,"")}return t}catch(e){return t}}function e(e){return vn(ne().body,function(){return eval(e)})}function j(t){const e=Q.on("htmx:load",function(e){t(e.detail.elt)});return e}function V(){Q.logger=function(e,t,n){if(console){console.log(t,e,n)}}}function _(){Q.logger=null}function u(e,t){if(typeof e!=="string"){return e.querySelector(t)}else{return u(ne(),e)}}function x(e,t){if(typeof e!=="string"){return e.querySelectorAll(t)}else{return x(ne(),e)}}function E(){return window}function z(e,t){e=y(e);if(t){E().setTimeout(function(){z(e);e=null},t)}else{c(e).removeChild(e)}}function ue(e){return e instanceof Element?e:null}function $(e){return e instanceof HTMLElement?e:null}function J(e){return typeof e==="string"?e:null}function f(e){return e instanceof Element||e instanceof Document||e instanceof DocumentFragment?e:null}function K(e,t,n){e=ue(y(e));if(!e){return}if(n){E().setTimeout(function(){K(e,t);e=null},n)}else{e.classList&&e.classList.add(t)}}function G(e,t,n){let r=ue(y(e));if(!r){return}if(n){E().setTimeout(function(){G(r,t);r=null},n)}else{if(r.classList){r.classList.remove(t);if(r.classList.length===0){r.removeAttribute("class")}}}}function W(e,t){e=y(e);e.classList.toggle(t)}function Z(e,t){e=y(e);se(e.parentElement.children,function(e){G(e,t)});K(ue(e),t)}function g(e,t){e=ue(y(e));if(e&&e.closest){return e.closest(t)}else{do{if(e==null||h(e,t)){return e}}while(e=e&&ue(c(e)));return null}}function l(e,t){return e.substring(0,t.length)===t}function Y(e,t){return e.substring(e.length-t.length)===t}function ge(e){const t=e.trim();if(l(t,"<")&&Y(t,"/>")){return t.substring(1,t.length-2)}else{return t}}function p(t,r,n){if(r.indexOf("global ")===0){return p(t,r.slice(7),true)}t=y(t);const o=[];{let t=0;let n=0;for(let e=0;e"){t--}}if(n0){const r=ge(o.shift());let e;if(r.indexOf("closest ")===0){e=g(ue(t),ge(r.substr(8)))}else if(r.indexOf("find ")===0){e=u(f(t),ge(r.substr(5)))}else if(r==="next"||r==="nextElementSibling"){e=ue(t).nextElementSibling}else if(r.indexOf("next ")===0){e=pe(t,ge(r.substr(5)),!!n)}else if(r==="previous"||r==="previousElementSibling"){e=ue(t).previousElementSibling}else if(r.indexOf("previous ")===0){e=me(t,ge(r.substr(9)),!!n)}else if(r==="document"){e=document}else if(r==="window"){e=window}else if(r==="body"){e=document.body}else if(r==="root"){e=m(t,!!n)}else if(r==="host"){e=t.getRootNode().host}else{s.push(r)}if(e){i.push(e)}}if(s.length>0){const e=s.join(",");const c=f(m(t,!!n));i.push(...M(c.querySelectorAll(e)))}return i}var pe=function(t,e,n){const r=f(m(t,n)).querySelectorAll(e);for(let e=0;e=0;e--){const o=r[e];if(o.compareDocumentPosition(t)===Node.DOCUMENT_POSITION_FOLLOWING){return o}}};function ae(e,t){if(typeof e!=="string"){return p(e,t)[0]}else{return p(ne().body,e)[0]}}function y(e,t){if(typeof e==="string"){return u(f(t)||document,e)}else{return e}}function xe(e,t,n,r){if(k(t)){return{target:ne().body,event:J(e),listener:t,options:n}}else{return{target:y(e),event:J(t),listener:n,options:r}}}function ye(t,n,r,o){Vn(function(){const e=xe(t,n,r,o);e.target.addEventListener(e.event,e.listener,e.options)});const e=k(n);return e?n:r}function be(t,n,r){Vn(function(){const e=xe(t,n,r);e.target.removeEventListener(e.event,e.listener)});return k(n)?n:r}const ve=ne().createElement("output");function we(e,t){const n=re(e,t);if(n){if(n==="this"){return[Se(e,t)]}else{const r=p(e,n);if(r.length===0){O('The selector "'+n+'" on '+t+" returned no matches!");return[ve]}else{return r}}}}function Se(e,t){return ue(o(e,function(e){return te(ue(e),t)!=null}))}function Ee(e){const t=re(e,"hx-target");if(t){if(t==="this"){return Se(e,"hx-target")}else{return ae(e,t)}}else{const n=ie(e);if(n.boosted){return ne().body}else{return e}}}function Ce(t){const n=Q.config.attributesToSettle;for(let e=0;e0){s=e.substring(0,e.indexOf(":"));n=e.substring(e.indexOf(":")+1)}else{s=e}o.removeAttribute("hx-swap-oob");o.removeAttribute("data-hx-swap-oob");const r=p(t,n,false);if(r){se(r,function(e){let t;const n=o.cloneNode(true);t=ne().createDocumentFragment();t.appendChild(n);if(!Re(s,e)){t=f(n)}const r={shouldSwap:true,target:e,fragment:t};if(!he(e,"htmx:oobBeforeSwap",r))return;e=r.target;if(r.shouldSwap){qe(t);_e(s,e,e,t,i);Te()}se(i.elts,function(e){he(e,"htmx:oobAfterSwap",r)})});o.parentNode.removeChild(o)}else{o.parentNode.removeChild(o);fe(ne().body,"htmx:oobErrorNoTarget",{content:o})}return e}function Te(){const e=u("#--htmx-preserve-pantry--");if(e){for(const t of[...e.children]){const n=u("#"+t.id);n.parentNode.moveBefore(t,n);n.remove()}e.remove()}}function qe(e){se(x(e,"[hx-preserve], [data-hx-preserve]"),function(e){const t=te(e,"id");const n=ne().getElementById(t);if(n!=null){if(e.moveBefore){let e=u("#--htmx-preserve-pantry--");if(e==null){ne().body.insertAdjacentHTML("afterend","
");e=u("#--htmx-preserve-pantry--")}e.moveBefore(n,null)}else{e.parentNode.replaceChild(n,e)}}})}function Le(l,e,c){se(e.querySelectorAll("[id]"),function(t){const n=ee(t,"id");if(n&&n.length>0){const r=n.replace("'","\\'");const o=t.tagName.replace(":","\\:");const e=f(l);const i=e&&e.querySelector(o+"[id='"+r+"']");if(i&&i!==e){const s=t.cloneNode();Oe(t,i);c.tasks.push(function(){Oe(t,s)})}}})}function Ae(e){return function(){G(e,Q.config.addedClass);kt(ue(e));Ne(f(e));he(e,"htmx:load")}}function Ne(e){const t="[autofocus]";const n=$(h(e,t)?e:e.querySelector(t));if(n!=null){n.focus()}}function a(e,t,n,r){Le(e,n,r);while(n.childNodes.length>0){const o=n.firstChild;K(ue(o),Q.config.addedClass);e.insertBefore(o,t);if(o.nodeType!==Node.TEXT_NODE&&o.nodeType!==Node.COMMENT_NODE){r.tasks.push(Ae(o))}}}function Ie(e,t){let n=0;while(n0}function $e(e,t,r,o){if(!o){o={}}e=y(e);const i=o.contextElement?m(o.contextElement,false):ne();const n=document.activeElement;let s={};try{s={elt:n,start:n?n.selectionStart:null,end:n?n.selectionEnd:null}}catch(e){}const l=xn(e);if(r.swapStyle==="textContent"){e.textContent=t}else{let n=P(t);l.title=n.title;if(o.selectOOB){const u=o.selectOOB.split(",");for(let t=0;t0){E().setTimeout(c,r.settleDelay)}else{c()}}function Je(e,t,n){const r=e.getResponseHeader(t);if(r.indexOf("{")===0){const o=S(r);for(const i in o){if(o.hasOwnProperty(i)){let e=o[i];if(D(e)){n=e.target!==undefined?e.target:n}else{e={value:e}}he(n,i,e)}}}else{const s=r.split(",");for(let e=0;e0){const s=o[0];if(s==="]"){e--;if(e===0){if(n===null){t=t+"true"}o.shift();t+=")})";try{const l=vn(r,function(){return Function(t)()},function(){return true});l.source=t;return l}catch(e){fe(ne().body,"htmx:syntax:error",{error:e,source:t});return null}}}else if(s==="["){e++}if(tt(s,n,i)){t+="(("+i+"."+s+") ? ("+i+"."+s+") : (window."+s+"))"}else{t=t+s}n=o.shift()}}}function C(e,t){let n="";while(e.length>0&&!t.test(e[0])){n+=e.shift()}return n}function rt(e){let t;if(e.length>0&&Ye.test(e[0])){e.shift();t=C(e,Qe).trim();e.shift()}else{t=C(e,v)}return t}const ot="input, textarea, select";function it(e,t,n){const r=[];const o=et(t);do{C(o,w);const l=o.length;const c=C(o,/[,\[\s]/);if(c!==""){if(c==="every"){const u={trigger:"every"};C(o,w);u.pollInterval=d(C(o,/[,\[\s]/));C(o,w);var i=nt(e,o,"event");if(i){u.eventFilter=i}r.push(u)}else{const a={trigger:c};var i=nt(e,o,"event");if(i){a.eventFilter=i}C(o,w);while(o.length>0&&o[0]!==","){const f=o.shift();if(f==="changed"){a.changed=true}else if(f==="once"){a.once=true}else if(f==="consume"){a.consume=true}else if(f==="delay"&&o[0]===":"){o.shift();a.delay=d(C(o,v))}else if(f==="from"&&o[0]===":"){o.shift();if(Ye.test(o[0])){var s=rt(o)}else{var s=C(o,v);if(s==="closest"||s==="find"||s==="next"||s==="previous"){o.shift();const h=rt(o);if(h.length>0){s+=" "+h}}}a.from=s}else if(f==="target"&&o[0]===":"){o.shift();a.target=rt(o)}else if(f==="throttle"&&o[0]===":"){o.shift();a.throttle=d(C(o,v))}else if(f==="queue"&&o[0]===":"){o.shift();a.queue=C(o,v)}else if(f==="root"&&o[0]===":"){o.shift();a[f]=rt(o)}else if(f==="threshold"&&o[0]===":"){o.shift();a[f]=C(o,v)}else{fe(e,"htmx:syntax:error",{token:o.shift()})}C(o,w)}r.push(a)}}if(o.length===l){fe(e,"htmx:syntax:error",{token:o.shift()})}C(o,w)}while(o[0]===","&&o.shift());if(n){n[t]=r}return r}function st(e){const t=te(e,"hx-trigger");let n=[];if(t){const r=Q.config.triggerSpecsCache;n=r&&r[t]||it(e,t,r)}if(n.length>0){return n}else if(h(e,"form")){return[{trigger:"submit"}]}else if(h(e,'input[type="button"], input[type="submit"]')){return[{trigger:"click"}]}else if(h(e,ot)){return[{trigger:"change"}]}else{return[{trigger:"click"}]}}function lt(e){ie(e).cancelled=true}function ct(e,t,n){const r=ie(e);r.timeout=E().setTimeout(function(){if(le(e)&&r.cancelled!==true){if(!gt(n,e,Mt("hx:poll:trigger",{triggerSpec:n,target:e}))){t(e)}ct(e,t,n)}},n.pollInterval)}function ut(e){return location.hostname===e.hostname&&ee(e,"href")&&ee(e,"href").indexOf("#")!==0}function at(e){return g(e,Q.config.disableSelector)}function ft(t,n,e){if(t instanceof HTMLAnchorElement&&ut(t)&&(t.target===""||t.target==="_self")||t.tagName==="FORM"&&String(ee(t,"method")).toLowerCase()!=="dialog"){n.boosted=true;let r,o;if(t.tagName==="A"){r="get";o=ee(t,"href")}else{const i=ee(t,"method");r=i?i.toLowerCase():"get";o=ee(t,"action");if(o==null||o===""){o=ne().location.href}if(r==="get"&&o.includes("?")){o=o.replace(/\?[^#]+/,"")}}e.forEach(function(e){pt(t,function(e,t){const n=ue(e);if(at(n)){b(n);return}de(r,o,n,t)},n,e,true)})}}function ht(e,t){const n=ue(t);if(!n){return false}if(e.type==="submit"||e.type==="click"){if(n.tagName==="FORM"){return true}if(h(n,'input[type="submit"], button')&&(h(n,"[form]")||g(n,"form")!==null)){return true}if(n instanceof HTMLAnchorElement&&n.href&&(n.getAttribute("href")==="#"||n.getAttribute("href").indexOf("#")!==0)){return true}}return false}function dt(e,t){return ie(e).boosted&&e instanceof HTMLAnchorElement&&t.type==="click"&&(t.ctrlKey||t.metaKey)}function gt(e,t,n){const r=e.eventFilter;if(r){try{return r.call(t,n)!==true}catch(e){const o=r.source;fe(ne().body,"htmx:eventFilter:error",{error:e,source:o});return true}}return false}function pt(l,c,e,u,a){const f=ie(l);let t;if(u.from){t=p(l,u.from)}else{t=[l]}if(u.changed){if(!("lastValue"in f)){f.lastValue=new WeakMap}t.forEach(function(e){if(!f.lastValue.has(u)){f.lastValue.set(u,new WeakMap)}f.lastValue.get(u).set(e,e.value)})}se(t,function(i){const s=function(e){if(!le(l)){i.removeEventListener(u.trigger,s);return}if(dt(l,e)){return}if(a||ht(e,l)){e.preventDefault()}if(gt(u,l,e)){return}const t=ie(e);t.triggerSpec=u;if(t.handledFor==null){t.handledFor=[]}if(t.handledFor.indexOf(l)<0){t.handledFor.push(l);if(u.consume){e.stopPropagation()}if(u.target&&e.target){if(!h(ue(e.target),u.target)){return}}if(u.once){if(f.triggeredOnce){return}else{f.triggeredOnce=true}}if(u.changed){const n=event.target;const r=n.value;const o=f.lastValue.get(u);if(o.has(n)&&o.get(n)===r){return}o.set(n,r)}if(f.delayed){clearTimeout(f.delayed)}if(f.throttle){return}if(u.throttle>0){if(!f.throttle){he(l,"htmx:trigger");c(l,e);f.throttle=E().setTimeout(function(){f.throttle=null},u.throttle)}}else if(u.delay>0){f.delayed=E().setTimeout(function(){he(l,"htmx:trigger");c(l,e)},u.delay)}else{he(l,"htmx:trigger");c(l,e)}}};if(e.listenerInfos==null){e.listenerInfos=[]}e.listenerInfos.push({trigger:u.trigger,listener:s,on:i});i.addEventListener(u.trigger,s)})}let mt=false;let xt=null;function yt(){if(!xt){xt=function(){mt=true};window.addEventListener("scroll",xt);window.addEventListener("resize",xt);setInterval(function(){if(mt){mt=false;se(ne().querySelectorAll("[hx-trigger*='revealed'],[data-hx-trigger*='revealed']"),function(e){bt(e)})}},200)}}function bt(e){if(!s(e,"data-hx-revealed")&&X(e)){e.setAttribute("data-hx-revealed","true");const t=ie(e);if(t.initHash){he(e,"revealed")}else{e.addEventListener("htmx:afterProcessNode",function(){he(e,"revealed")},{once:true})}}}function vt(e,t,n,r){const o=function(){if(!n.loaded){n.loaded=true;he(e,"htmx:trigger");t(e)}};if(r>0){E().setTimeout(o,r)}else{o()}}function wt(t,n,e){let i=false;se(r,function(r){if(s(t,"hx-"+r)){const o=te(t,"hx-"+r);i=true;n.path=o;n.verb=r;e.forEach(function(e){St(t,e,n,function(e,t){const n=ue(e);if(g(n,Q.config.disableSelector)){b(n);return}de(r,o,n,t)})})}});return i}function St(r,e,t,n){if(e.trigger==="revealed"){yt();pt(r,n,t,e);bt(ue(r))}else if(e.trigger==="intersect"){const o={};if(e.root){o.root=ae(r,e.root)}if(e.threshold){o.threshold=parseFloat(e.threshold)}const i=new IntersectionObserver(function(t){for(let e=0;e0){t.polling=true;ct(ue(r),n,e)}else{pt(r,n,t,e)}}function Et(e){const t=ue(e);if(!t){return false}const n=t.attributes;for(let e=0;e", "+e).join(""));return o}else{return[]}}function Tt(e){const t=g(ue(e.target),"button, input[type='submit']");const n=Lt(e);if(n){n.lastButtonClicked=t}}function qt(e){const t=Lt(e);if(t){t.lastButtonClicked=null}}function Lt(e){const t=g(ue(e.target),"button, input[type='submit']");if(!t){return}const n=y("#"+ee(t,"form"),t.getRootNode())||g(t,"form");if(!n){return}return ie(n)}function At(e){e.addEventListener("click",Tt);e.addEventListener("focusin",Tt);e.addEventListener("focusout",qt)}function Nt(t,e,n){const r=ie(t);if(!Array.isArray(r.onHandlers)){r.onHandlers=[]}let o;const i=function(e){vn(t,function(){if(at(t)){return}if(!o){o=new Function("event",n)}o.call(t,e)})};t.addEventListener(e,i);r.onHandlers.push({event:e,listener:i})}function It(t){ke(t);for(let e=0;eQ.config.historyCacheSize){i.shift()}while(i.length>0){try{localStorage.setItem("htmx-history-cache",JSON.stringify(i));break}catch(e){fe(ne().body,"htmx:historyCacheError",{cause:e,cache:i});i.shift()}}}function Vt(t){if(!B()){return null}t=U(t);const n=S(localStorage.getItem("htmx-history-cache"))||[];for(let e=0;e=200&&this.status<400){he(ne().body,"htmx:historyCacheMissLoad",i);const e=P(this.response);const t=e.querySelector("[hx-history-elt],[data-hx-history-elt]")||e;const n=Ut();const r=xn(n);kn(e.title);qe(e);Ve(n,t,r);Te();Kt(r.tasks);Bt=o;he(ne().body,"htmx:historyRestore",{path:o,cacheMiss:true,serverResponse:this.response})}else{fe(ne().body,"htmx:historyCacheMissLoadError",i)}};e.send()}function Wt(e){zt();e=e||location.pathname+location.search;const t=Vt(e);if(t){const n=P(t.content);const r=Ut();const o=xn(r);kn(t.title);qe(n);Ve(r,n,o);Te();Kt(o.tasks);E().setTimeout(function(){window.scrollTo(0,t.scroll)},0);Bt=e;he(ne().body,"htmx:historyRestore",{path:e,item:t})}else{if(Q.config.refreshOnHistoryMiss){window.location.reload(true)}else{Gt(e)}}}function Zt(e){let t=we(e,"hx-indicator");if(t==null){t=[e]}se(t,function(e){const t=ie(e);t.requestCount=(t.requestCount||0)+1;e.classList.add.call(e.classList,Q.config.requestClass)});return t}function Yt(e){let t=we(e,"hx-disabled-elt");if(t==null){t=[]}se(t,function(e){const t=ie(e);t.requestCount=(t.requestCount||0)+1;e.setAttribute("disabled","");e.setAttribute("data-disabled-by-htmx","")});return t}function Qt(e,t){se(e.concat(t),function(e){const t=ie(e);t.requestCount=(t.requestCount||1)-1});se(e,function(e){const t=ie(e);if(t.requestCount===0){e.classList.remove.call(e.classList,Q.config.requestClass)}});se(t,function(e){const t=ie(e);if(t.requestCount===0){e.removeAttribute("disabled");e.removeAttribute("data-disabled-by-htmx")}})}function en(t,n){for(let e=0;en.indexOf(e)<0)}else{e=e.filter(e=>e!==n)}r.delete(t);se(e,e=>r.append(t,e))}}function on(t,n,r,o,i){if(o==null||en(t,o)){return}else{t.push(o)}if(tn(o)){const s=ee(o,"name");let e=o.value;if(o instanceof HTMLSelectElement&&o.multiple){e=M(o.querySelectorAll("option:checked")).map(function(e){return e.value})}if(o instanceof HTMLInputElement&&o.files){e=M(o.files)}nn(s,e,n);if(i){sn(o,r)}}if(o instanceof HTMLFormElement){se(o.elements,function(e){if(t.indexOf(e)>=0){rn(e.name,e.value,n)}else{t.push(e)}if(i){sn(e,r)}});new FormData(o).forEach(function(e,t){if(e instanceof File&&e.name===""){return}nn(t,e,n)})}}function sn(e,t){const n=e;if(n.willValidate){he(n,"htmx:validation:validate");if(!n.checkValidity()){t.push({elt:n,message:n.validationMessage,validity:n.validity});he(n,"htmx:validation:failed",{message:n.validationMessage,validity:n.validity})}}}function ln(n,e){for(const t of e.keys()){n.delete(t)}e.forEach(function(e,t){n.append(t,e)});return n}function cn(e,t){const n=[];const r=new FormData;const o=new FormData;const i=[];const s=ie(e);if(s.lastButtonClicked&&!le(s.lastButtonClicked)){s.lastButtonClicked=null}let l=e instanceof HTMLFormElement&&e.noValidate!==true||te(e,"hx-validate")==="true";if(s.lastButtonClicked){l=l&&s.lastButtonClicked.formNoValidate!==true}if(t!=="get"){on(n,o,i,g(e,"form"),l)}on(n,r,i,e,l);if(s.lastButtonClicked||e.tagName==="BUTTON"||e.tagName==="INPUT"&&ee(e,"type")==="submit"){const u=s.lastButtonClicked||e;const a=ee(u,"name");nn(a,u.value,o)}const c=we(e,"hx-include");se(c,function(e){on(n,r,i,ue(e),l);if(!h(e,"form")){se(f(e).querySelectorAll(ot),function(e){on(n,r,i,e,l)})}});ln(r,o);return{errors:i,formData:r,values:An(r)}}function un(e,t,n){if(e!==""){e+="&"}if(String(n)==="[object Object]"){n=JSON.stringify(n)}const r=encodeURIComponent(n);e+=encodeURIComponent(t)+"="+r;return e}function an(e){e=qn(e);let n="";e.forEach(function(e,t){n=un(n,t,e)});return n}function fn(e,t,n){const r={"HX-Request":"true","HX-Trigger":ee(e,"id"),"HX-Trigger-Name":ee(e,"name"),"HX-Target":te(t,"id"),"HX-Current-URL":ne().location.href};bn(e,"hx-headers",false,r);if(n!==undefined){r["HX-Prompt"]=n}if(ie(e).boosted){r["HX-Boosted"]="true"}return r}function hn(n,e){const t=re(e,"hx-params");if(t){if(t==="none"){return new FormData}else if(t==="*"){return n}else if(t.indexOf("not ")===0){se(t.slice(4).split(","),function(e){e=e.trim();n.delete(e)});return n}else{const r=new FormData;se(t.split(","),function(t){t=t.trim();if(n.has(t)){n.getAll(t).forEach(function(e){r.append(t,e)})}});return r}}else{return n}}function dn(e){return!!ee(e,"href")&&ee(e,"href").indexOf("#")>=0}function gn(e,t){const n=t||re(e,"hx-swap");const r={swapStyle:ie(e).boosted?"innerHTML":Q.config.defaultSwapStyle,swapDelay:Q.config.defaultSwapDelay,settleDelay:Q.config.defaultSettleDelay};if(Q.config.scrollIntoViewOnBoost&&ie(e).boosted&&!dn(e)){r.show="top"}if(n){const s=F(n);if(s.length>0){for(let e=0;e0?o.join(":"):null;r.scroll=u;r.scrollTarget=i}else if(l.indexOf("show:")===0){const a=l.slice(5);var o=a.split(":");const f=o.pop();var i=o.length>0?o.join(":"):null;r.show=f;r.showTarget=i}else if(l.indexOf("focus-scroll:")===0){const h=l.slice("focus-scroll:".length);r.focusScroll=h=="true"}else if(e==0){r.swapStyle=l}else{O("Unknown modifier in hx-swap: "+l)}}}}return r}function pn(e){return re(e,"hx-encoding")==="multipart/form-data"||h(e,"form")&&ee(e,"enctype")==="multipart/form-data"}function mn(t,n,r){let o=null;Ft(n,function(e){if(o==null){o=e.encodeParameters(t,r,n)}});if(o!=null){return o}else{if(pn(n)){return ln(new FormData,qn(r))}else{return an(r)}}}function xn(e){return{tasks:[],elts:[e]}}function yn(e,t){const n=e[0];const r=e[e.length-1];if(t.scroll){var o=null;if(t.scrollTarget){o=ue(ae(n,t.scrollTarget))}if(t.scroll==="top"&&(n||o)){o=o||n;o.scrollTop=0}if(t.scroll==="bottom"&&(r||o)){o=o||r;o.scrollTop=o.scrollHeight}}if(t.show){var o=null;if(t.showTarget){let e=t.showTarget;if(t.showTarget==="window"){e="body"}o=ue(ae(n,e))}if(t.show==="top"&&(n||o)){o=o||n;o.scrollIntoView({block:"start",behavior:Q.config.scrollBehavior})}if(t.show==="bottom"&&(r||o)){o=o||r;o.scrollIntoView({block:"end",behavior:Q.config.scrollBehavior})}}}function bn(r,e,o,i){if(i==null){i={}}if(r==null){return i}const s=te(r,e);if(s){let e=s.trim();let t=o;if(e==="unset"){return null}if(e.indexOf("javascript:")===0){e=e.slice(11);t=true}else if(e.indexOf("js:")===0){e=e.slice(3);t=true}if(e.indexOf("{")!==0){e="{"+e+"}"}let n;if(t){n=vn(r,function(){return Function("return ("+e+")")()},{})}else{n=S(e)}for(const l in n){if(n.hasOwnProperty(l)){if(i[l]==null){i[l]=n[l]}}}}return bn(ue(c(r)),e,o,i)}function vn(e,t,n){if(Q.config.allowEval){return t()}else{fe(e,"htmx:evalDisallowedError");return n}}function wn(e,t){return bn(e,"hx-vars",true,t)}function Sn(e,t){return bn(e,"hx-vals",false,t)}function En(e){return ce(wn(e),Sn(e))}function Cn(t,n,r){if(r!==null){try{t.setRequestHeader(n,r)}catch(e){t.setRequestHeader(n,encodeURIComponent(r));t.setRequestHeader(n+"-URI-AutoEncoded","true")}}}function On(t){if(t.responseURL&&typeof URL!=="undefined"){try{const e=new URL(t.responseURL);return e.pathname+e.search}catch(e){fe(ne().body,"htmx:badResponseUrl",{url:t.responseURL})}}}function R(e,t){return t.test(e.getAllResponseHeaders())}function Rn(t,n,r){t=t.toLowerCase();if(r){if(r instanceof Element||typeof r==="string"){return de(t,n,null,null,{targetOverride:y(r)||ve,returnPromise:true})}else{let e=y(r.target);if(r.target&&!e||r.source&&!e&&!y(r.source)){e=ve}return de(t,n,y(r.source),r.event,{handler:r.handler,headers:r.headers,values:r.values,targetOverride:e,swapOverride:r.swap,select:r.select,returnPromise:true})}}else{return de(t,n,null,null,{returnPromise:true})}}function Hn(e){const t=[];while(e){t.push(e);e=e.parentElement}return t}function Tn(e,t,n){let r;let o;if(typeof URL==="function"){o=new URL(t,document.location.href);const i=document.location.origin;r=i===o.origin}else{o=t;r=l(t,document.location.origin)}if(Q.config.selfRequestsOnly){if(!r){return false}}return he(e,"htmx:validateUrl",ce({url:o,sameHost:r},n))}function qn(e){if(e instanceof FormData)return e;const t=new FormData;for(const n in e){if(e.hasOwnProperty(n)){if(e[n]&&typeof e[n].forEach==="function"){e[n].forEach(function(e){t.append(n,e)})}else if(typeof e[n]==="object"&&!(e[n]instanceof Blob)){t.append(n,JSON.stringify(e[n]))}else{t.append(n,e[n])}}}return t}function Ln(r,o,e){return new Proxy(e,{get:function(t,e){if(typeof e==="number")return t[e];if(e==="length")return t.length;if(e==="push"){return function(e){t.push(e);r.append(o,e)}}if(typeof t[e]==="function"){return function(){t[e].apply(t,arguments);r.delete(o);t.forEach(function(e){r.append(o,e)})}}if(t[e]&&t[e].length===1){return t[e][0]}else{return t[e]}},set:function(e,t,n){e[t]=n;r.delete(o);e.forEach(function(e){r.append(o,e)});return true}})}function An(o){return new Proxy(o,{get:function(e,t){if(typeof t==="symbol"){const r=Reflect.get(e,t);if(typeof r==="function"){return function(){return r.apply(o,arguments)}}else{return r}}if(t==="toJSON"){return()=>Object.fromEntries(o)}if(t in e){if(typeof e[t]==="function"){return function(){return o[t].apply(o,arguments)}}else{return e[t]}}const n=o.getAll(t);if(n.length===0){return undefined}else if(n.length===1){return n[0]}else{return Ln(e,t,n)}},set:function(t,n,e){if(typeof n!=="string"){return false}t.delete(n);if(e&&typeof e.forEach==="function"){e.forEach(function(e){t.append(n,e)})}else if(typeof e==="object"&&!(e instanceof Blob)){t.append(n,JSON.stringify(e))}else{t.append(n,e)}return true},deleteProperty:function(e,t){if(typeof t==="string"){e.delete(t)}return true},ownKeys:function(e){return Reflect.ownKeys(Object.fromEntries(e))},getOwnPropertyDescriptor:function(e,t){return Reflect.getOwnPropertyDescriptor(Object.fromEntries(e),t)}})}function de(t,n,r,o,i,D){let s=null;let l=null;i=i!=null?i:{};if(i.returnPromise&&typeof Promise!=="undefined"){var e=new Promise(function(e,t){s=e;l=t})}if(r==null){r=ne().body}const M=i.handler||Dn;const X=i.select||null;if(!le(r)){oe(s);return e}const c=i.targetOverride||ue(Ee(r));if(c==null||c==ve){fe(r,"htmx:targetError",{target:te(r,"hx-target")});oe(l);return e}let u=ie(r);const a=u.lastButtonClicked;if(a){const L=ee(a,"formaction");if(L!=null){n=L}const A=ee(a,"formmethod");if(A!=null){if(A.toLowerCase()!=="dialog"){t=A}}}const f=re(r,"hx-confirm");if(D===undefined){const K=function(e){return de(t,n,r,o,i,!!e)};const G={target:c,elt:r,path:n,verb:t,triggeringEvent:o,etc:i,issueRequest:K,question:f};if(he(r,"htmx:confirm",G)===false){oe(s);return e}}let h=r;let d=re(r,"hx-sync");let g=null;let F=false;if(d){const N=d.split(":");const I=N[0].trim();if(I==="this"){h=Se(r,"hx-sync")}else{h=ue(ae(r,I))}d=(N[1]||"drop").trim();u=ie(h);if(d==="drop"&&u.xhr&&u.abortable!==true){oe(s);return e}else if(d==="abort"){if(u.xhr){oe(s);return e}else{F=true}}else if(d==="replace"){he(h,"htmx:abort")}else if(d.indexOf("queue")===0){const W=d.split(" ");g=(W[1]||"last").trim()}}if(u.xhr){if(u.abortable){he(h,"htmx:abort")}else{if(g==null){if(o){const P=ie(o);if(P&&P.triggerSpec&&P.triggerSpec.queue){g=P.triggerSpec.queue}}if(g==null){g="last"}}if(u.queuedRequests==null){u.queuedRequests=[]}if(g==="first"&&u.queuedRequests.length===0){u.queuedRequests.push(function(){de(t,n,r,o,i)})}else if(g==="all"){u.queuedRequests.push(function(){de(t,n,r,o,i)})}else if(g==="last"){u.queuedRequests=[];u.queuedRequests.push(function(){de(t,n,r,o,i)})}oe(s);return e}}const p=new XMLHttpRequest;u.xhr=p;u.abortable=F;const m=function(){u.xhr=null;u.abortable=false;if(u.queuedRequests!=null&&u.queuedRequests.length>0){const e=u.queuedRequests.shift();e()}};const B=re(r,"hx-prompt");if(B){var x=prompt(B);if(x===null||!he(r,"htmx:prompt",{prompt:x,target:c})){oe(s);m();return e}}if(f&&!D){if(!confirm(f)){oe(s);m();return e}}let y=fn(r,c,x);if(t!=="get"&&!pn(r)){y["Content-Type"]="application/x-www-form-urlencoded"}if(i.headers){y=ce(y,i.headers)}const U=cn(r,t);let b=U.errors;const j=U.formData;if(i.values){ln(j,qn(i.values))}const V=qn(En(r));const v=ln(j,V);let w=hn(v,r);if(Q.config.getCacheBusterParam&&t==="get"){w.set("org.htmx.cache-buster",ee(c,"id")||"true")}if(n==null||n===""){n=ne().location.href}const S=bn(r,"hx-request");const _=ie(r).boosted;let E=Q.config.methodsThatUseUrlParams.indexOf(t)>=0;const C={boosted:_,useUrlParams:E,formData:w,parameters:An(w),unfilteredFormData:v,unfilteredParameters:An(v),headers:y,target:c,verb:t,errors:b,withCredentials:i.credentials||S.credentials||Q.config.withCredentials,timeout:i.timeout||S.timeout||Q.config.timeout,path:n,triggeringEvent:o};if(!he(r,"htmx:configRequest",C)){oe(s);m();return e}n=C.path;t=C.verb;y=C.headers;w=qn(C.parameters);b=C.errors;E=C.useUrlParams;if(b&&b.length>0){he(r,"htmx:validation:halted",C);oe(s);m();return e}const z=n.split("#");const $=z[0];const O=z[1];let R=n;if(E){R=$;const Z=!w.keys().next().done;if(Z){if(R.indexOf("?")<0){R+="?"}else{R+="&"}R+=an(w);if(O){R+="#"+O}}}if(!Tn(r,R,C)){fe(r,"htmx:invalidPath",C);oe(l);return e}p.open(t.toUpperCase(),R,true);p.overrideMimeType("text/html");p.withCredentials=C.withCredentials;p.timeout=C.timeout;if(S.noHeaders){}else{for(const k in y){if(y.hasOwnProperty(k)){const Y=y[k];Cn(p,k,Y)}}}const H={xhr:p,target:c,requestConfig:C,etc:i,boosted:_,select:X,pathInfo:{requestPath:n,finalRequestPath:R,responsePath:null,anchor:O}};p.onload=function(){try{const t=Hn(r);H.pathInfo.responsePath=On(p);M(r,H);if(H.keepIndicators!==true){Qt(T,q)}he(r,"htmx:afterRequest",H);he(r,"htmx:afterOnLoad",H);if(!le(r)){let e=null;while(t.length>0&&e==null){const n=t.shift();if(le(n)){e=n}}if(e){he(e,"htmx:afterRequest",H);he(e,"htmx:afterOnLoad",H)}}oe(s);m()}catch(e){fe(r,"htmx:onLoadError",ce({error:e},H));throw e}};p.onerror=function(){Qt(T,q);fe(r,"htmx:afterRequest",H);fe(r,"htmx:sendError",H);oe(l);m()};p.onabort=function(){Qt(T,q);fe(r,"htmx:afterRequest",H);fe(r,"htmx:sendAbort",H);oe(l);m()};p.ontimeout=function(){Qt(T,q);fe(r,"htmx:afterRequest",H);fe(r,"htmx:timeout",H);oe(l);m()};if(!he(r,"htmx:beforeRequest",H)){oe(s);m();return e}var T=Zt(r);var q=Yt(r);se(["loadstart","loadend","progress","abort"],function(t){se([p,p.upload],function(e){e.addEventListener(t,function(e){he(r,"htmx:xhr:"+t,{lengthComputable:e.lengthComputable,loaded:e.loaded,total:e.total})})})});he(r,"htmx:beforeSend",H);const J=E?null:mn(p,r,w);p.send(J);return e}function Nn(e,t){const n=t.xhr;let r=null;let o=null;if(R(n,/HX-Push:/i)){r=n.getResponseHeader("HX-Push");o="push"}else if(R(n,/HX-Push-Url:/i)){r=n.getResponseHeader("HX-Push-Url");o="push"}else if(R(n,/HX-Replace-Url:/i)){r=n.getResponseHeader("HX-Replace-Url");o="replace"}if(r){if(r==="false"){return{}}else{return{type:o,path:r}}}const i=t.pathInfo.finalRequestPath;const s=t.pathInfo.responsePath;const l=re(e,"hx-push-url");const c=re(e,"hx-replace-url");const u=ie(e).boosted;let a=null;let f=null;if(l){a="push";f=l}else if(c){a="replace";f=c}else if(u){a="push";f=s||i}if(f){if(f==="false"){return{}}if(f==="true"){f=s||i}if(t.pathInfo.anchor&&f.indexOf("#")===-1){f=f+"#"+t.pathInfo.anchor}return{type:a,path:f}}else{return{}}}function In(e,t){var n=new RegExp(e.code);return n.test(t.toString(10))}function Pn(e){for(var t=0;t0){E().setTimeout(e,x.swapDelay)}else{e()}}if(f){fe(o,"htmx:responseError",ce({error:"Response Status Error Code "+s.status+" from "+i.pathInfo.requestPath},i))}}const Mn={};function Xn(){return{init:function(e){return null},getSelectors:function(){return null},onEvent:function(e,t){return true},transformResponse:function(e,t,n){return e},isInlineSwap:function(e){return false},handleSwap:function(e,t,n,r){return false},encodeParameters:function(e,t,n){return null}}}function Fn(e,t){if(t.init){t.init(n)}Mn[e]=ce(Xn(),t)}function Bn(e){delete Mn[e]}function Un(e,n,r){if(n==undefined){n=[]}if(e==undefined){return n}if(r==undefined){r=[]}const t=te(e,"hx-ext");if(t){se(t.split(","),function(e){e=e.replace(/ /g,"");if(e.slice(0,7)=="ignore:"){r.push(e.slice(7));return}if(r.indexOf(e)<0){const t=Mn[e];if(t&&n.indexOf(t)<0){n.push(t)}}})}return Un(ue(c(e)),n,r)}var jn=false;ne().addEventListener("DOMContentLoaded",function(){jn=true});function Vn(e){if(jn||ne().readyState==="complete"){e()}else{ne().addEventListener("DOMContentLoaded",e)}}function _n(){if(Q.config.includeIndicatorStyles!==false){const e=Q.config.inlineStyleNonce?` nonce="${Q.config.inlineStyleNonce}"`:"";ne().head.insertAdjacentHTML("beforeend"," ."+Q.config.indicatorClass+"{opacity:0} ."+Q.config.requestClass+" ."+Q.config.indicatorClass+"{opacity:1; transition: opacity 200ms ease-in;} ."+Q.config.requestClass+"."+Q.config.indicatorClass+"{opacity:1; transition: opacity 200ms ease-in;} ")}}function zn(){const e=ne().querySelector('meta[name="htmx-config"]');if(e){return S(e.content)}else{return null}}function $n(){const e=zn();if(e){Q.config=ce(Q.config,e)}}Vn(function(){$n();_n();let e=ne().body;kt(e);const t=ne().querySelectorAll("[hx-trigger='restored'],[data-hx-trigger='restored']");e.addEventListener("htmx:abort",function(e){const t=e.target;const n=ie(t);if(n&&n.xhr){n.xhr.abort()}});const n=window.onpopstate?window.onpopstate.bind(window):null;window.onpopstate=function(e){if(e.state&&e.state.htmx){Wt();se(t,function(e){he(e,"htmx:restored",{document:ne(),triggerEvent:he})})}else{if(n){n(e)}}};E().setTimeout(function(){he(e,"htmx:load",{});e=null},0)});return Q}(); \ No newline at end of file diff --git a/tools/ci-dashboard/static/index.html b/tools/ci-dashboard/static/index.html new file mode 100644 index 0000000000..23c0aa8475 --- /dev/null +++ b/tools/ci-dashboard/static/index.html @@ -0,0 +1,79 @@ + + + + + + Torch-TensorRT CI · {{BRANCH}} + + + + +
+

Torch-TensorRT CI

+
+ + + + +
+
+ +
+
+ passed + failed + running + queued + didn’t run + · open a platform for its python×cuda×tier grid · click a red cell for failing tests + code +
+
+
loading {{BRANCH}}…
+
+
+ +
+ + +
+ + + + diff --git a/tools/ci-dashboard/static/style.css b/tools/ci-dashboard/static/style.css new file mode 100644 index 0000000000..e8cb475723 --- /dev/null +++ b/tools/ci-dashboard/static/style.css @@ -0,0 +1,222 @@ +:root { + --bg: #f6f7f9; --surface: #fff; --surface-2: #f0f2f5; --border: #e2e6ea; + --text: #1c2024; --muted: #5b6570; --accent: #0969da; + --pass: #1a7f37; --pass-bg: #e6f4ea; --pass-line: #b7dcc0; + --fail: #cf222e; --fail-bg: #ffebe9; --fail-line: #f3c0c2; + --run: #9a6700; --run-bg: #fff8e6; --run-line: #e6d9a8; + --queue: #6e7781; --queue-bg: #eef0f2; --queue-line: #dbe0e4; + --skip: #7d8590; --skip-bg: #eef1f4; --skip-line: #d3d9df; + --cancel: #8250df; --cancel-bg: #f4f0fb; --cancel-line: #e1d5f5; + --radius: 10px; --shadow: 0 1px 2px rgba(0,0,0,.06), 0 3px 12px rgba(0,0,0,.04); + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #0d1117; --surface: #161b22; --surface-2: #1c2128; --border: #30363d; + --text: #e6edf3; --muted: #8b949e; --accent: #58a6ff; + --pass: #3fb950; --pass-bg: #12261a; --pass-line: #23552f; + --fail: #f85149; --fail-bg: #2c1518; --fail-line: #5c2a2a; + --run: #d29922; --run-bg: #2a230f; --run-line: #574a1e; + --queue: #8b949e; --queue-bg: #1c2128; --queue-line: #363c44; + --skip: #8b949e; --skip-bg: #1a1f26; --skip-line: #363c44; + --cancel: #a371f7; --cancel-bg: #201a2c; --cancel-line: #3d3357; + --shadow: 0 1px 2px rgba(0,0,0,.4), 0 3px 14px rgba(0,0,0,.3); + } +} +* { box-sizing: border-box; } +body { + margin: 0; background: var(--bg); color: var(--text); + font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; +} +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } +button, input { font: inherit; color: inherit; } + +/* header */ +header { + position: sticky; top: 0; z-index: 20; background: var(--surface); + border-bottom: 1px solid var(--border); padding: 10px 18px; + display: flex; align-items: center; gap: 14px; flex-wrap: wrap; +} +header h1 { font-size: 15px; margin: 0; font-weight: 650; letter-spacing: .2px; } +header h1 .dim { color: var(--muted); font-weight: 400; } +.controls { display: flex; align-items: center; gap: 8px; margin-left: auto; flex-wrap: wrap; } +.controls input[type=text] { + background: var(--bg); border: 1px solid var(--border); border-radius: 8px; + padding: 6px 10px; width: 190px; +} +.controls input[type=text]:focus { outline: 2px solid var(--accent); border-color: transparent; } +.btn { + background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; + padding: 6px 12px; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; +} +.btn:hover { border-color: var(--accent); } +.btn.primary { background: var(--accent); color: #fff; border-color: transparent; } +.btn.primary:hover { filter: brightness(1.08); } +.toggle { display: inline-flex; align-items: center; gap: 6px; color: var(--muted); font-size: 13px; cursor: pointer; } + +main { max-width: 1280px; margin: 0 auto; padding: 18px; } + +/* summary strip */ +.summary { display: flex; gap: 10px; flex-wrap: wrap; margin-bottom: 8px; align-items: center; } +.stat { display: inline-flex; align-items: center; gap: 7px; padding: 5px 12px; border-radius: 999px; + border: 1px solid var(--border); background: var(--surface); font-weight: 600; } +.stat .n { font-variant-numeric: tabular-nums; } +.stat.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } +.stat.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } +.stat.run { color: var(--run); border-color: var(--run-line); background: var(--run-bg); } +.stat.skip { color: var(--skip); border-color: var(--skip-line); background: var(--skip-bg); border-style: dashed; } +.commit { color: var(--muted); font-size: 13px; margin-left: auto; } +.commit code { font-family: var(--mono); color: var(--text); } + +/* dot / pill */ +.dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; flex: none; } +.dot.pass { background: var(--pass); } +.dot.fail { background: var(--fail); } +.dot.run { background: var(--run); animation: pulse 1.4s ease-in-out infinite; } +.dot.queue{ background: var(--queue); } +.dot.skip { background: transparent; box-shadow: inset 0 0 0 1.5px var(--skip); } +.dot.cancel { background: var(--cancel); } +@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .35; } } + +/* section headings */ +h2.sec { font-size: 12px; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); + margin: 22px 0 10px; font-weight: 700; } + +/* aggregate failures panel */ +.agg { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); + box-shadow: var(--shadow); overflow: hidden; margin-bottom: 8px; } +.agg-row-d { border-top: 1px solid var(--border); } +.agg-row-d:first-child { border-top: none; } +.agg-row-d > summary { list-style: none; } +.agg-row-d > summary::-webkit-details-marker { display: none; } +.agg-row { display: grid; grid-template-columns: 1fr auto; gap: 12px; align-items: center; + padding: 11px 14px; cursor: pointer; } +.agg-row:hover { background: var(--surface-2); } +.agg-head-row { display: flex; align-items: center; gap: 10px; color: var(--muted); + font-size: 13px; margin: 2px 0 8px; } +.agg-loading { color: var(--muted); padding: 12px 2px; } +.agg-ok { color: var(--pass); font-weight: 600; padding: 12px 2px; } +.btn.small { padding: 3px 9px; font-size: 12px; margin-left: auto; } +#content.filter-fail .card:not(.fail) { display: none; } +.agg-test { font-family: var(--mono); font-size: 13px; word-break: break-all; } +.agg-file { color: var(--muted); font-size: 12px; margin-top: 2px; } +.agg-file code { font-family: var(--mono); } +.agg-count { display: inline-flex; align-items: center; gap: 6px; white-space: nowrap; + color: var(--fail); font-weight: 650; } +.agg-detail { padding: 10px 14px 12px 14px; border-top: 1px dashed var(--border); + background: var(--surface-2); font-size: 13px; } +.agg-detail pre { background: var(--bg); border: 1px solid var(--border); border-radius: 8px; + padding: 10px; overflow: auto; font-family: var(--mono); font-size: 12px; margin: 10px 0; max-height: 240px; } +.chips { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 8px; } +.chip { font-size: 11px; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--border); + background: var(--surface); color: var(--muted); } + +/* platform board */ +.board { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 12px; } +.card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); + box-shadow: var(--shadow); overflow: hidden; } +.card.fail { border-color: var(--fail-line); } +.card-head { display: flex; align-items: center; gap: 10px; padding: 12px 14px; cursor: pointer; } +.card-head:hover { background: var(--surface-2); } +.card-title { font-weight: 620; flex: 1; min-width: 0; } +.card-title .sub { color: var(--muted); font-weight: 400; font-size: 12px; margin-top: 1px; } +.card-badge { font-size: 12px; font-weight: 650; padding: 3px 9px; border-radius: 999px; white-space: nowrap; } +.card-badge.pass { color: var(--pass); background: var(--pass-bg); border: 1px solid var(--pass-line); } +.card-badge.fail { color: var(--fail); background: var(--fail-bg); border: 1px solid var(--fail-line); } +.card-badge.run { color: var(--run); background: var(--run-bg); border: 1px solid var(--run-line); } +.card-badge.queue{ color: var(--queue);background: var(--queue-bg);border: 1px solid var(--queue-line); } +.card-badge.skip { color: var(--skip); background: var(--skip-bg); border: 1px dashed var(--skip-line); } +.card-badge.cancel { color: var(--cancel); background: var(--cancel-bg); border: 1px solid var(--cancel-line); } +.progress { height: 4px; background: var(--surface-2); display: flex; } +.progress > span { height: 100%; } +.progress .p-pass { background: var(--pass); } +.progress .p-fail { background: var(--fail); } +.progress .p-run { background: var(--run); } +.card-body { padding: 6px 14px 14px; border-top: 1px solid var(--border); } +.card > summary { list-style: none; } +.card > summary::-webkit-details-marker { display: none; } +.caret { color: var(--muted); transition: transform .15s; } +.card[open] .caret { transform: rotate(90deg); } + +/* job groups + matrix */ +.jobgroup { margin-top: 12px; } +.jobgroup h3 { font-size: 12px; font-weight: 650; margin: 0 0 6px; display: flex; align-items: center; gap: 8px; } +.jobgroup h3 .tierpath { color: var(--muted); font-weight: 400; font-family: var(--mono); font-size: 11px; } +.matrix { display: flex; gap: 6px; flex-wrap: wrap; } +.cell { min-width: 84px; border: 1px solid var(--border); border-radius: 8px; padding: 6px 9px; + display: flex; flex-direction: column; gap: 3px; cursor: default; background: var(--surface); } +.cell.pass { background: var(--pass-bg); border-color: var(--pass-line); } +.cell.fail { background: var(--fail-bg); border-color: var(--fail-line); cursor: pointer; } +.cell.fail:hover { outline: 2px solid var(--fail); } +.cell.run { background: var(--run-bg); border-color: var(--run-line); } +.cell.queue{ background: var(--queue-bg); } +/* "didn't run" — dashed + muted so it reads as inactive, never as pass or fail */ +.cell.skip { background: var(--surface); border-style: dashed; border-color: var(--skip-line); color: var(--muted); } +.cell.skip .k, .cell.cancel .k { color: var(--muted); font-weight: 500; } +.cell.cancel { background: var(--cancel-bg); border-style: dashed; border-color: var(--cancel-line); } +.cell .k { font-size: 11px; font-weight: 650; font-variant-numeric: tabular-nums; display: flex; gap: 5px; align-items: center; } +.cell .v { font-size: 10px; color: var(--muted); font-family: var(--mono); } +.rows { display: flex; flex-direction: column; gap: 4px; } +.jobrow { display: flex; align-items: center; gap: 8px; padding: 4px 6px; border-radius: 6px; font-size: 13px; } +.jobrow:hover { background: var(--surface-2); } +.jobrow .name { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.jobrow.fail .name { color: var(--fail); } +.jobstate { font-size: 11px; padding: 1px 8px; border-radius: 999px; white-space: nowrap; color: var(--muted); border: 1px solid var(--border); } +.jobstate.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } +.jobstate.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } +.jobstate.skip { border-style: dashed; } +.jobstate.cancel { color: var(--cancel); border-color: var(--cancel-line); border-style: dashed; } + +/* drawer */ +.drawer-bg { position: fixed; inset: 0; background: rgba(0,0,0,.35); z-index: 40; display: none; } +.drawer-bg.open { display: block; } +.drawer { position: fixed; top: 0; right: 0; height: 100%; width: min(560px, 94vw); z-index: 41; + background: var(--surface); border-left: 1px solid var(--border); box-shadow: -8px 0 30px rgba(0,0,0,.2); + transform: translateX(100%); transition: transform .2s ease; display: flex; flex-direction: column; } +.drawer.open { transform: translateX(0); } +.drawer-head { padding: 14px 16px; border-bottom: 1px solid var(--border); display: flex; align-items: flex-start; gap: 10px; } +.drawer-head .t { flex: 1; } +.drawer-head .t .title { font-weight: 650; } +.drawer-head .t .sub { color: var(--muted); font-size: 12px; margin-top: 2px; } +.drawer-head .x { cursor: pointer; border: none; background: none; color: var(--muted); font-size: 20px; line-height: 1; } +.drawer-body { padding: 14px 16px; overflow: auto; } +.fail-item { border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; margin-bottom: 10px; background: var(--surface); } +.fail-item .ftest { font-family: var(--mono); font-size: 13px; font-weight: 600; word-break: break-all; } +.fail-item .floc { font-size: 12px; margin-top: 4px; } +.fail-item .floc code { font-family: var(--mono); } +.fail-item pre { background: var(--bg); border: 1px solid var(--border); border-radius: 6px; + padding: 9px; font-family: var(--mono); font-size: 12px; overflow: auto; max-height: 260px; margin: 8px 0 0; white-space: pre-wrap; } +.repro { background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; margin-top: 10px; } +.repro .lbl { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); margin-bottom: 6px; } +.repro code { font-family: var(--mono); font-size: 12px; word-break: break-all; display: block; } +.copy { float: right; font-size: 11px; cursor: pointer; color: var(--accent); background: none; border: none; } + +/* relevant logs (in drawer) */ +.logsec { margin-top: 14px; border-top: 1px solid var(--border); padding-top: 12px; } +.logsec-head { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--muted); + margin-bottom: 8px; flex-wrap: wrap; } +.logsec-head .lbl { text-transform: uppercase; letter-spacing: .06em; font-weight: 600; + color: var(--text); margin-right: auto; } +.logblock { border: 1px solid var(--border); border-radius: 8px; margin-bottom: 8px; overflow: hidden; } +.logblock > summary { cursor: pointer; padding: 8px 12px; font-family: var(--mono); font-size: 12px; + font-weight: 600; background: var(--surface-2); word-break: break-all; list-style: none; } +.logblock > summary::-webkit-details-marker { display: none; } +.logblock > summary::before { content: "▸ "; color: var(--muted); } +.logblock[open] > summary::before { content: "▾ "; } +.logblock .logreason { padding: 8px 12px 0; font-family: var(--mono); font-size: 12px; color: var(--fail); word-break: break-word; } +.logblock pre { margin: 8px 12px 12px; background: var(--bg); border: 1px solid var(--border); + border-radius: 6px; padding: 10px; font-family: var(--mono); font-size: 12px; line-height: 1.45; + overflow: auto; max-height: 420px; white-space: pre; } + +.muted { color: var(--muted); } +.spinner { display: inline-block; width: 13px; height: 13px; border: 2px solid var(--border); + border-top-color: var(--accent); border-radius: 50%; animation: spin .7s linear infinite; } +@keyframes spin { to { transform: rotate(360deg); } } +.empty-state { text-align: center; color: var(--muted); padding: 60px 20px; } +.toast { position: fixed; bottom: 20px; left: 50%; transform: translateX(-50%); background: var(--text); + color: var(--bg); padding: 8px 16px; border-radius: 8px; font-size: 13px; z-index: 60; opacity: 0; + transition: opacity .2s; pointer-events: none; } +.toast.show { opacity: 1; } +.legend { display: flex; gap: 14px; color: var(--muted); font-size: 12px; margin: 4px 0 0; flex-wrap: wrap; } +.legend span { display: inline-flex; align-items: center; gap: 5px; } From 2db18abc344ac7736ee3adb5085407e9799bfeb4 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Fri, 10 Jul 2026 23:13:10 +0000 Subject: [PATCH 05/11] tests: bazel cache to speed up builds --- .github/workflows/build_linux.yml | 16 ++ setup.py | 10 + tests/ci/__main__.py | 39 +++- tests/ci/runner.py | 75 ++++++- tests/ci/suites.py | 26 ++- tools/ci-dashboard/README.md | 18 +- tools/ci-dashboard/ci_dashboard.py | 291 +++++++++++++++++++++++++--- tools/ci-dashboard/static/style.css | 16 ++ 8 files changed, 445 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 90333aedf8..f59ec80608 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -254,6 +254,20 @@ jobs: cache-key: ${{ inputs.cache-key }} repository: ${{ inputs.repository }} script: ${{ inputs.pre-script }} + # Persist bazel's action cache across runs so the C++/CUDA compile — the + # serial long pole — reuses unchanged action outputs instead of rebuilding + # from scratch. Lives under RUNNER_TEMP (the "Clean workspace" step wipes + # GITHUB_WORKSPACE). setup.py picks up BAZEL_DISK_CACHE (exported below). + # Key is COARSE (toolchain/deps, not the commit) so most PRs restore a warm + # cache; the restore-keys prefix makes even a dep bump an incremental build. + - name: Cache the bazel C++/CUDA build + if: ${{ inputs.architecture != 'aarch64' }} + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/trt-bazel-disk-cache + key: trt-bazel-${{ inputs.architecture }}-${{ matrix.desired_cuda }}-rtx${{ inputs.use-rtx }}-${{ hashFiles('**/MODULE.bazel', '**/WORKSPACE', '**/.bazelrc', '**/dev_dep_versions.yml') }} + restore-keys: | + trt-bazel-${{ inputs.architecture }}-${{ matrix.desired_cuda }}-rtx${{ inputs.use-rtx }}- - name: Build the wheel (python-build-package) if: ${{ inputs.build-platform == 'python-build-package' }} working-directory: ${{ inputs.repository }} @@ -272,6 +286,8 @@ jobs: run: | set -x source "${BUILD_ENV_FILE}" + # Reuse the cached bazel action outputs (see "Cache the bazel..." step). + export BAZEL_DISK_CACHE="${RUNNER_TEMP}/trt-bazel-disk-cache" export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" ${CONDA_RUN} python setup.py clean echo "Successfully ran `python setup.py clean`" diff --git a/setup.py b/setup.py index 91651c7e15..3b170e16d5 100644 --- a/setup.py +++ b/setup.py @@ -270,6 +270,16 @@ def build_libtorchtrt_cxx11_abi( else: cmd.append("--platforms=//toolchains:ci_rhel_x86_64_linux") + # Persist bazel's action cache across builds when a cache dir is provided. + # The C++/CUDA compile is the serial long pole; with a warm --disk_cache an + # unchanged (or partially-changed) tree reuses cached action outputs, turning + # a from-scratch rebuild into seconds. CI sets BAZEL_DISK_CACHE and restores/ + # saves the dir via actions/cache (coarse key); no-op locally unless opted in. + disk_cache = os.environ.get("BAZEL_DISK_CACHE") + if disk_cache: + cmd.append(f"--disk_cache={disk_cache}") + print(f"Using bazel --disk_cache={disk_cache}") + env = os.environ.copy() if "TORCH_PATH" not in env: stable_torch_path = resolve_torch_path() diff --git a/tests/ci/__main__.py b/tests/ci/__main__.py index fc59e2ecbc..4f2f6c4f8a 100644 --- a/tests/ci/__main__.py +++ b/tests/ci/__main__.py @@ -59,11 +59,30 @@ def _cmd_run(args: argparse.Namespace) -> int: return rc +def _read_changed(args: argparse.Namespace) -> list[str] | None: + """Changed-file list for path-based selection, from ``--changed-from`` (a + file path, or ``-`` for stdin; one repo-relative path per line). Returns + ``None`` when not supplied → no narrowing (run everything the filters select).""" + src = getattr(args, "changed_from", None) + if not src: + return None + if src == "-": + text = sys.stdin.read() + else: + with open(src, encoding="utf-8") as fh: + text = fh.read() + return [ln.strip() for ln in text.splitlines() if ln.strip()] + + def _cmd_run_lane(args: argparse.Namespace) -> int: """Run every suite in a lane/tier, continuing past failures (so one consolidated report sees them all). Returns non-zero if any suite failed.""" jobs = select( - lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform + lane=args.lane, + tier=args.tier, + variant=args.variant, + platform=args.platform, + changed=_read_changed(args), ) if not jobs: print("::warning::no suites match the given filters", file=sys.stderr) @@ -76,7 +95,11 @@ def _cmd_run_lane(args: argparse.Namespace) -> int: def _cmd_matrix(args: argparse.Namespace) -> int: include = matrix( - lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform + lane=args.lane, + tier=args.tier, + variant=args.variant, + platform=args.platform, + changed=_read_changed(args), ) if not include: print("::warning::matrix is empty for the given filters", file=sys.stderr) @@ -161,6 +184,12 @@ def main(argv: list[str] | None = None) -> int: sp.add_argument("--variant", choices=("standard", "rtx")) sp.add_argument("--platform", choices=("linux-x86_64", "windows")) sp.add_argument("--dry-run", action="store_true") + sp.add_argument( + "--changed-from", + metavar="FILE", + help="narrow to suites affected by these changed files (a file, or '-' " + "for stdin; one repo-relative path per line). Broad changes → no narrowing.", + ) sp.set_defaults(fn=_cmd_run_lane) sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON") @@ -169,6 +198,12 @@ def main(argv: list[str] | None = None) -> int: g.add_argument("--tier", choices=("l0", "l1", "l2")) sp.add_argument("--variant", choices=("standard", "rtx")) sp.add_argument("--platform", choices=("linux-x86_64", "windows")) + sp.add_argument( + "--changed-from", + metavar="FILE", + help="narrow to suites affected by these changed files (a file, or '-' " + "for stdin; one repo-relative path per line). Broad changes → no narrowing.", + ) sp.set_defaults(fn=_cmd_matrix) sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor) diff --git a/tests/ci/runner.py b/tests/ci/runner.py index 96fa3ddd69..70974e87a9 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -212,6 +212,69 @@ def run_suite( return 0 +# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching +# ONLY these can skip the whole test matrix. +_NO_TEST_IMPACT = ( + "docs/", + "docsrc/", + "tools/", + "notebooks/", + "examples/", + ".devcontainer/", + "README", + "CHANGELOG", + "LICENSE", + ".gitignore", + ".pre-commit", + ".flake8", +) + + +def _owned_dirs(s: Suite) -> set[str]: + """Repo-relative directories a suite owns — the directory of each path target + (glob tails dropped, file targets mapped to their dir). Coarse on purpose: + change-based selection must OVER-select (never miss a suite a change breaks).""" + dirs: set[str] = set() + for var in s.variants: + fv = s.for_variant(var) + cwd = fv["cwd"].rstrip("/") + for p in fv["paths"] or (".",): + t = p.split("*")[0].strip("/") # drop any glob tail + full = f"{cwd}/{t}" if t and t != "." else cwd + base = full.rsplit("/", 1)[-1] + dirs.add(full.rsplit("/", 1)[0] if "." in base else full) + return dirs + + +def affected_suites(changed: list[str]) -> set[str] | None: + """Suite names a `changed` file set can affect. + + Returns ``None`` when the change set can affect ANYTHING — any file that is + neither a no-test-impact path nor owned by a specific suite (i.e. library + source, build files, a shared conftest/harness). The caller then runs the + full selection (safe default). An empty set means "nothing to test" (e.g. a + docs-only PR). This never narrows away a suite a change could break. + """ + owned = {s.name: _owned_dirs(s) for s in SUITES} + hit: set[str] = set() + for f in changed: + f = f.strip() + if not f: + continue + if any(f.startswith(p) for p in _NO_TEST_IMPACT): + continue # no test impact + matched = { + name + for name, dirs in owned.items() + if any(f == d or f.startswith(d + "/") for d in dirs) + } + if matched: + hit |= matched + else: + return None # source / build / shared-test change → cannot narrow + return hit + + def select( *, lane: str | None = None, @@ -219,10 +282,20 @@ def select( variant: str | None = None, platform: str | None = None, names: list[str] | None = None, + changed: list[str] | None = None, ) -> list[tuple[Suite, Variant]]: - """All (suite, variant) jobs matching the filters. No filter on an axis = all.""" + """All (suite, variant) jobs matching the filters. No filter on an axis = all. + + ``changed`` (a list of repo-relative changed paths) narrows to only the + affected suites — but ONLY when every change is confined to test dirs; any + source/build/shared change keeps the full pool (see ``affected_suites``). + """ jobs: list[tuple[Suite, Variant]] = [] pool = [by_name(n) for n in names] if names else list(SUITES) + if changed is not None: + aff = affected_suites(changed) + if aff is not None: # None → a broad change → do not narrow + pool = [s for s in pool if s.name in aff] for s in pool: if lane is not None and lane not in s.lanes: continue diff --git a/tests/ci/suites.py b/tests/ci/suites.py index 4e87e53898..800803b7e7 100644 --- a/tests/ci/suites.py +++ b/tests/ci/suites.py @@ -43,6 +43,16 @@ ALL_VARIANTS: tuple[Variant, ...] = ("standard", "rtx") ALL_PLATFORMS: tuple[Platform, ...] = ("linux-x86_64", "windows") +# xdist worker counts (`-n`). GPU tests are MEMORY-bound, not CPU-bound: each +# worker builds its own TensorRT engine, so N workers ≈ N× peak GPU memory. +# `-n auto` (= CPU count, up to ~48 on big runners) is a footgun here — it OOMs +# the model/runtime suites on real 24 GB runners, and an OOM'd engine build fails +# `assert cuda_engine` and then *reruns*, doubling wall-clock. So workers are +# tiered by worst-case engine size, not by core count: +_LIGHT = "8" # small single-op / graph tests (conversion, partitioning, hlo) +_HEAVY = "4" # full compile+execute suites (lowering, runtime) — OOM'd at -n 8 +_MODEL = "2" # real models / LLMs (ResNet, BERT, Llama) — biggest engines + @dataclass(frozen=True) class Suite: @@ -115,7 +125,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: tier="l0", lanes=("fast", "full"), paths=("runtime/test_000_*",), - jobs="8", + jobs=_HEAVY, ), Suite( "dynamo-partitioning-smoke", @@ -131,7 +141,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: tier="l0", lanes=("fast", "full"), paths=("lowering/",), - jobs="8", + jobs=_HEAVY, ), Suite( "py-core", @@ -159,7 +169,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: tier="l1", lanes=("full",), paths=("runtime/test_001_*",), - jobs="8", + jobs=_HEAVY, ), Suite( "dynamo-partitioning", @@ -218,7 +228,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: paths=("models/test_models.py", "models/test_dyn_models.py"), markers="not critical", ir="torch_compile", - jobs="auto", + jobs=_MODEL, ), Suite( "dynamo-models", @@ -226,14 +236,14 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: lanes=("full", "nightly"), paths=("models/",), markers="not critical", - jobs="auto", + jobs=_MODEL, ), Suite( "dynamo-llm", tier="l2", lanes=("nightly",), paths=("llm/",), - jobs="auto", + jobs=_MODEL, ), Suite( "dynamo-runtime-full", @@ -241,7 +251,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: lanes=("full", "nightly"), paths=("runtime/",), keyword="not test_000_ and not test_001_", - jobs="auto", + jobs=_HEAVY, ), Suite( "executorch", @@ -330,7 +340,7 @@ def for_variant(self, variant: Variant) -> dict[str, Any]: tier="l1", lanes=("python-only",), paths=("runtime/",), - jobs="8", + jobs=_HEAVY, # Runs for BOTH backends: the PYTHON_ONLY=1 wheel is validated against # standard TensorRT and TensorRT-RTX (variants default to both). ), diff --git a/tools/ci-dashboard/README.md b/tools/ci-dashboard/README.md index 6381d56753..09345fe74a 100644 --- a/tools/ci-dashboard/README.md +++ b/tools/ci-dashboard/README.md @@ -37,13 +37,13 @@ reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN U the RTX and python-only variants, jetpack…), sorted worst-first, with a live status badge. A summary strip up top counts failing / running / queued / passing at a glance. -- **Drill in** — open a platform to see its jobs as a **python × cuda × tier** - grid. Green/red cells; L0/L1/L2 tiers are labelled with the pytest paths they - run (pulled from `tests/py/utils/ci_helpers.sh`). +- **Drill in** — open a platform to see its jobs as a **python × cuda × suite** + grid. Green/red cells; each suite is labelled with the pytest paths it runs + (read from the `tests/ci` manifest — the same data CI runs). - **Click a red cell** — a drawer shows every failing test with its error, the **source file:line it maps to** (local path + a GitHub link pinned to the run's - commit), and a **copy-paste command to reproduce it locally** via the same - tier function CI used. + commit), and a **copy-paste command to reproduce it locally** — the exact + `python -m tests.ci run ` command CI used, narrowed to the failing tests. - **Relevant logs, captured** — the drawer then pulls that job's log and extracts just the **pytest FAILURES block per failing test** (traceback + captured stdout/stderr), so you read the actual failure without scrolling a 1 MB log. @@ -76,8 +76,12 @@ platform on open, out-of-band badge polling, the failure drawer). the failing test name + traceback is one cheap API call (`…/check-runs/{id}/annotations`) — no wheel or junit download. We then `git grep` the test symbol in `tests/` to resolve it to `file:line`. -- **Tier → paths** live in `TIER_MAP`, a mirror of the `trt_tier_*` selectors in - `tests/py/utils/ci_helpers.sh`. If a tier is added/renamed there, add it here. +- **Suite → paths / reproduce command** come straight from the `tests/ci` + manifest (`tests/ci/suites.py`) via `info_for()` — the CI names each test job + `-`, which is exactly a manifest suite, so there is nothing to + keep in sync. A small `TIER_MAP` remains only as a fallback so historical, + pre-migration runs (tier-named) still resolve; it can be dropped once those age + out. ## Limitations diff --git a/tools/ci-dashboard/ci_dashboard.py b/tools/ci-dashboard/ci_dashboard.py index e8c4923118..68a04e2154 100755 --- a/tools/ci-dashboard/ci_dashboard.py +++ b/tools/ci-dashboard/ci_dashboard.py @@ -14,9 +14,11 @@ name + traceback is one cheap API call away — no wheel/junit download needed. Job names encode the whole matrix, e.g. - core / L2 dynamo distributed tests / L2-dynamo-distributed-tests--3.12-cu130 -We parse that into {group/tier, python, cuda, kind} and map the tier back to the -pytest paths in tests/py/utils/ci_helpers.sh so a red cell points at code. + test / dynamo-converters-standard / dynamo-converters-standard--3.12-cu130 +We parse that into {group, python, cuda, kind}; the group is a `-` +from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red +cell maps to the suite's pytest paths and its exact `python -m tests.ci run` +reproduce command. Pre-migration runs (tier-named) fall back to the legacy map. """ from __future__ import annotations @@ -41,11 +43,24 @@ STATIC = os.path.join(HERE, "static") REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) -# ── Tier → source mapping ──────────────────────────────────────────────────── -# Mirrors the trt_tier_* selectors in tests/py/utils/ci_helpers.sh (the single -# source of truth for "what does each CI tier run"). Keys are normalized job -# group names (see _norm()); `paths` are repo-relative; `fn` is the shell tier -# function you'd invoke locally to reproduce. +# ── tests/ci manifest (single source of truth) ─────────────────────────────── +# The dashboard resolves a red CI cell back to "what ran + how to reproduce" from +# the SAME manifest CI and the `just` recipes use (tests/ci/suites.py). CI names +# each test job `-` (tests/ci → linux-test.yml), so the group the +# dashboard parses out of a job IS a manifest suite — no duplicated path lists to +# drift. Imported best-effort: a checkout predating the manifest falls back to the +# legacy TIER_MAP below. +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) +try: + from tests.ci import SUITES as _SUITES # noqa: E402 +except Exception: # pragma: no cover — manifest absent on pre-migration branches + _SUITES = () + +# ── Legacy tier → source mapping (fallback) ────────────────────────────────── +# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not +# "-") still resolve. New runs go through the manifest above. +# Keys are normalized job group names (see _norm()); `fn` is the old shell tier. TIER_MAP = { "l0 dynamo converter tests": dict( fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"] @@ -123,8 +138,71 @@ def _norm(s: str) -> str: return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip() -def tier_for(group: str): - return TIER_MAP.get(_norm(group)) +# ── manifest-backed resolution (group → what ran + how to reproduce) ────────── +def _suite_group_index(): + """Map a normalized CI job group to ``(suite, variant)``. + + CI names each test job ``-`` so the parsed group is exactly a + manifest suite. Index both the ``-`` and bare ```` + forms; per-variant ``paths`` overrides (e.g. RTX) are applied at lookup.""" + idx = {} + for s in _SUITES: + for v in s.variants: + idx.setdefault(_norm(f"{s.name}-{v}"), (s, v)) + idx.setdefault(_norm(s.name), (s, s.variants[0])) + return idx + + +_SUITE_INDEX = _suite_group_index() +# Longest keys first so 'dynamo runtime smoke' wins over 'dynamo runtime'. +_SUITE_KEYS = sorted(_SUITE_INDEX, key=len, reverse=True) + + +def _suite_repro(suite, variant, kexpr=""): + """The exact command CI ran for this suite (what `just suite` invokes).""" + var = f" --variant {variant}" if variant and variant != "standard" else "" + ka = f' -- -k "{kexpr}"' if kexpr else "" + return ( + f"TRT_PYTEST_RERUNS=0 uv run --no-sync python -m tests.ci " + f"run {suite.name}{var}{ka}" + ) + + +def info_for(group: str): + """Resolve a CI job group → ``{paths, repro, label}`` from the tests/ci + manifest, falling back to the legacy ``TIER_MAP`` for pre-migration runs. + + ``paths`` are repo-relative (``cwd`` joined with the suite's pytest + positionals) so a red cell still points at code. ``repro(kexpr)`` returns the + local reproduce command, narrowed to the failing tests when ``kexpr`` given. + """ + g = _norm(group) + hit = _SUITE_INDEX.get(g) or next( + (_SUITE_INDEX[k] for k in _SUITE_KEYS if k in g), None + ) + if hit: + s, v = hit + fv = s.for_variant(v) + cwd = fv["cwd"].rstrip("/") + paths = [cwd if p in (".", "./") else f"{cwd}/{p}" for p in fv["paths"]] + paths = paths or [cwd] + return dict( + paths=paths, + repro=lambda kexpr="", _s=s, _v=v: _suite_repro(_s, _v, kexpr), + label=f"{s.name} · {v}", + ) + t = TIER_MAP.get(g) + if t: + return dict( + paths=t["paths"], + repro=lambda kexpr="", _t=t: ( + 'source tests/py/utils/ci_helpers.sh && PYTHON="uv run ' + f'--no-sync python" TRT_PYTEST_RERUNS=0 {_t["fn"]}' + + (f' -k "{kexpr}"' if kexpr else "") + ), + label=None, + ) + return None # ── gh plumbing (cached) ───────────────────────────────────────────────────── @@ -514,6 +592,133 @@ def classify(status, conclusion): DIDNT_RUN = ("skip", "cancel") +# ── failure categorization ─────────────────────────────────────────────────── +# Triage-at-a-glance: the judgment a human makes reading each traceback — is this +# infra (OOM/resource → retry/parallelism, not a code bug), an env/import gap, a +# converter gap, or a real regression (numerical/assertion)? First pattern wins, +# so order is most-specific → most-generic. `kind` drives the tag colour: +# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real. +# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM +# signatures below are listed first on purpose so they win over the generic assert. +_FAIL_CATS = [ + ( + "oom", + "GPU / resource", + "infra", + re.compile( + r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|" + r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|" + r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|" + r"CUDA error|cudaMalloc|device-side assert", + re.I, + ), + ), + ( + "timeout", + "timeout", + "infra", + re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I), + ), + ( + "import", + "import / collection", + "env", + re.compile( + r"ModuleNotFoundError|ImportError|No module named|cannot import name|" + r"error collecting|errors during collection|failed to import", + re.I, + ), + ), + ( + "unsupported", + "unsupported op", + "env", + re.compile( + r"no converter|not support|unsupported|Could not find any implementation|" + r"UnsupportedOperator|no implementation for", + re.I, + ), + ), + ( + "numerical", + "numerical / accuracy", + "bug", + re.compile( + r"not close|Mismatched elements|cosine|tolerance|allclose|" + r"Max absolute difference|relative difference|accuracy|atol|rtol", + re.I, + ), + ), + ( + "assertion", + "assertion", + "bug", + re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M), + ), + ( + "typeerr", + "type / shape", + "bug", + re.compile( + r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|" + r"shape mismatch|size mismatch|dtype", + re.I, + ), + ), + ( + "runtime", + "runtime error", + "bug", + re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I), + ), +] + + +# Roll-up labels for the per-run category tally (kind → human summary word). +_KIND_LABEL = { + "bug": "real bug", + "env": "env / gap", + "infra": "infra / OOM", + "unknown": "uncategorized", +} + + +def categorize_failure(message): + """Classify a failure's error text → {key, label, kind}. `kind` ∈ + {infra, env, bug, unknown} and drives the tag colour.""" + text = message or "" + for key, label, kind, rx in _FAIL_CATS: + if rx.search(text): + return dict(key=key, label=label, kind=kind) + return dict(key="other", label="error", kind="unknown") + + +def _cat_tag(cat): + return ( + f'{e(cat["label"])}' + ) + + +def _config_pattern(occ, all_pys, all_cudas): + """Which configuration a failure correlates with — the strongest diagnostic + signal after the category. Only asserts "X only" when X is a strict subset of + what actually ran (so we never claim a pattern that isn't real).""" + fpys = {o["py"] for o in occ if o.get("py")} + fcudas = {o["cuda"] for o in occ if o.get("cuda")} + fplats = {o["plat"] for o in occ} + bits = [] + if fplats and all("RTX" in p or "rtx" in p for p in fplats): + bits.append("RTX only") + if fplats and all("py-only" in p for p in fplats): + bits.append("python-only") + if len(all_cudas) > 1 and len(fcudas) == 1 and fcudas < all_cudas: + bits.append(f"{next(iter(fcudas))} only") + if len(all_pys) > 1 and len(fpys) == 1 and fpys < all_pys: + bits.append(f"py{next(iter(fpys))} only") + return bits + + def rel_time(iso): if not iso: return "" @@ -688,10 +893,10 @@ def render_platform(run_id, sha, platform, refresh=False): for (_, gname), gjobs in sorted( groups.items(), key=lambda kv: (kv[0][0], kv[0][1]) ): - tier = tier_for(gname) + info = info_for(gname) tierpath = ( - f'{e(", ".join(tier["paths"]))}' - if tier + f'{e(", ".join(info["paths"]))}' + if info else "" ) cells, rows = [], [] @@ -770,7 +975,7 @@ def render_failures(job_id, sha, platform, tier, py, cuda, url): + f'
Could not load annotations: {e(data["error"])}
' ) fails = data["failures"] - tinfo = tier_for(tier) + info = info_for(tier) body = [] if not fails: loglink = ( @@ -792,11 +997,13 @@ def render_failures(job_id, sha, platform, tier, py, cuda, url): f'
↳ ' f'{e(f["file"])}:{f["line"]} ↗
' ) + cat = categorize_failure(f["message"]) body.append( - f'
{e(f["test"] or "failure")}
' + f'
{e(f["test"] or "failure")}' + f" {_cat_tag(cat)}
" f'{loc}
{e(f["message"])}
' ) - if tinfo: + if info: leaves = [] for f in fails: if f["test"]: @@ -804,11 +1011,8 @@ def render_failures(job_id, sha, platform, tier, py, cuda, url): leaf = re.sub(r"\[.*$", "", leaf) if leaf and leaf not in leaves: leaves.append(leaf) - k = f' -k "{" or ".join(leaves[:6])}"' if leaves else "" - cmd = ( - f"source tests/py/utils/ci_helpers.sh && " - f'PYTHON="uv run --no-sync python" TRT_PYTEST_RERUNS=0 {tinfo["fn"]}{k}' - ) + kexpr = " or ".join(leaves[:6]) + cmd = info["repro"](kexpr) body.append( f'
' f'
reproduce locally
{e(cmd)}
' @@ -887,29 +1091,60 @@ def fails_of(rj): ) ) + # Config universe (what actually ran) so "X only" hints are real subsets. + test_jobs = [j for _, j in alljobs if j["kind"] == "test"] + all_pys = {j["python"] for j in test_jobs if j["python"]} + all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]} + + # Categorize once per row (from the sample traceback) so we can tally + tag. + for a in agg.values(): + a["sample"] = next((o["msg"] for o in a["occ"] if o["msg"]), "") + a["cat"] = categorize_failure(a["sample"] or a["test"]) + + # Real regressions (bug) first, then env/gap, infra last; ties → most cells. + _KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3} rows = sorted( agg.values(), - key=lambda a: (-len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]), + key=lambda a: ( + _KIND_RANK.get(a["cat"]["kind"], 2), + -len({o["plat"] for o in a["occ"]}), + -len(a["occ"]), + a["test"], + ), ) out = [] for a in rows: plats = sorted({o["plat"] for o in a["occ"]}) + ncells = len(a["occ"]) loc = "" if a["file"]: gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}' loc = f'
↳ {e(a["file"])}:{a["line"]} ↗
' chips = "".join(f'{e(p)}' for p in plats) - sample = next((o["msg"] for o in a["occ"] if o["msg"]), "") + pat = _config_pattern(a["occ"], all_pys, all_cudas) + patchips = "".join(f'{e(p)}' for p in pat) out.append(f"""
-
{e(a["test"])}
{loc}
- {len(plats)} platform{"s" if len(plats)!=1 else ""} +
{e(a["test"])} {_cat_tag(a["cat"])}{patchips}
{loc}
+ {len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""} -
{chips}
{f"
{e(sample)}
" if sample else ""}
+
{chips}
{f"
{e(a['sample'])}
" if a["sample"] else ""}
""") + # Category tally — an instant read on the run's character (infra flakes vs regressions). + tally = {} + for a in rows: + tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1 + order = ["bug", "env", "unknown", "infra"] + tstr = " · ".join( + f'{tally[k]["n"]} {e(_KIND_LABEL[k])}' + for k in order + if k in tally + ) header = ( - f'
{len(rows)} distinct failing test' - f'{"s" if len(rows)!=1 else ""} across {len({o["plat"] for a in rows for o in a["occ"]})} platforms' + f'
{len(rows)} distinct failing test' + f'{"s" if len(rows)!=1 else ""} across ' + f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms' + f'{tstr}' f'
' ) diff --git a/tools/ci-dashboard/static/style.css b/tools/ci-dashboard/static/style.css index e8cb475723..f3fd67bc8e 100644 --- a/tools/ci-dashboard/static/style.css +++ b/tools/ci-dashboard/static/style.css @@ -112,6 +112,22 @@ h2.sec { font-size: 12px; text-transform: uppercase; letter-spacing: .08em; colo .chip { font-size: 11px; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--border); background: var(--surface); color: var(--muted); } +/* failure category tags — colour encodes triage kind: + bug=real regression (red) · infra=OOM/resource (purple) · env=setup/feature gap (amber) */ +.cat { display: inline-block; font-size: 10.5px; font-weight: 650; letter-spacing: .02em; + padding: 1px 7px; border-radius: 999px; border: 1px solid transparent; + vertical-align: middle; white-space: nowrap; } +.cat.bug { color: var(--fail); background: var(--fail-bg); border-color: var(--fail-line); } +.cat.infra { color: var(--cancel); background: var(--cancel-bg); border-color: var(--cancel-line); } +.cat.env { color: var(--run); background: var(--run-bg); border-color: var(--run-line); } +.cat.unknown { color: var(--skip); background: var(--skip-bg); border-color: var(--skip-line); } +/* config-correlation hint (e.g. "RTX only", "cu132 only") */ +.cfgpat { display: inline-block; font-size: 10.5px; font-weight: 600; padding: 1px 7px; + border-radius: 999px; margin-left: 4px; vertical-align: middle; white-space: nowrap; + color: var(--accent); background: transparent; border: 1px dashed var(--accent); } +.agg-tally { display: inline-flex; gap: 6px; flex-wrap: wrap; align-items: center; } +.agg-tally .cat { font-weight: 700; } + /* platform board */ .board { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 12px; } .card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); From 7bae5171ac401b4a240e4027b70e878f8b2b59e2 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Fri, 17 Jul 2026 20:45:34 +0000 Subject: [PATCH 06/11] fix: better concurrency rules --- .github/workflows/_decide.yml | 4 +++- .github/workflows/ci-linux-x86_64.yml | 12 +++++++++++- .github/workflows/ci-sbsa.yml | 12 +++++++++++- .github/workflows/ci-windows.yml | 12 +++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_decide.yml b/.github/workflows/_decide.yml index bcfc32289b..2bc1e1539e 100644 --- a/.github/workflows/_decide.yml +++ b/.github/workflows/_decide.yml @@ -30,7 +30,9 @@ jobs: env: EVENT: ${{ github.event_name }} REVIEW_STATE: ${{ github.event.review.state }} - HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') }}" + # Either the explicit ci:full label or the repo's "Force All Tests" label + # requests the full lane (the latter was previously a no-op here). + HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]') }}" # Exact array-element match: 'backend: TensorRT' != 'backend: TensorRT-RTX'. HAS_RTX_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT-RTX') }}" HAS_STD_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT') }}" diff --git a/.github/workflows/ci-linux-x86_64.yml b/.github/workflows/ci-linux-x86_64.yml index f3b0569f49..8a89381c34 100644 --- a/.github/workflows/ci-linux-x86_64.yml +++ b/.github/workflows/ci-linux-x86_64.yml @@ -27,7 +27,11 @@ on: default: both concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Label churn (auto-labels + ci:full / Force-All) each fire a `labeled` event → + # a fresh run; cancel-in-progress would then have each cancel the previous. Give + # label/unlabel events their OWN per-label group so they never cancel the commit + # pipeline — only real pushes (synchronize) supersede each other. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('-label-{0}', github.event.label.name) || '' }} cancel-in-progress: true permissions: @@ -36,6 +40,12 @@ permissions: jobs: decide: + # A label event only warrants running the pipeline when the label controls CI; + # auto-labels (component:*, cla signed) must not spawn work — and would each + # start a full run once ci:full is present. Non-label events always proceed. + if: >- + github.event.action != 'labeled' || + contains(fromJSON('["ci: full", "Force All Tests[L0+L1+L2]", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml # Generate the build matrix ONCE. generate_binary_build_matrix's concurrency diff --git a/.github/workflows/ci-sbsa.yml b/.github/workflows/ci-sbsa.yml index 3f7cdaae58..10e0641f6d 100644 --- a/.github/workflows/ci-sbsa.yml +++ b/.github/workflows/ci-sbsa.yml @@ -27,7 +27,11 @@ on: default: both concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Label churn (auto-labels + ci:full / Force-All) each fire a `labeled` event → + # a fresh run; cancel-in-progress would then have each cancel the previous. Give + # label/unlabel events their OWN per-label group so they never cancel the commit + # pipeline — only real pushes (synchronize) supersede each other. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('-label-{0}', github.event.label.name) || '' }} cancel-in-progress: true permissions: @@ -36,6 +40,12 @@ permissions: jobs: decide: + # A label event only warrants running the pipeline when the label controls CI; + # auto-labels (component:*, cla signed) must not spawn work — and would each + # start a full run once ci:full is present. Non-label events always proceed. + if: >- + github.event.action != 'labeled' || + contains(fromJSON('["ci: full", "Force All Tests[L0+L1+L2]", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml # Generate the aarch64 build matrix once (shared by both build channels). diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 5b0d4f13d0..1bd6096c3d 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -27,7 +27,11 @@ on: default: both concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Label churn (auto-labels + ci:full / Force-All) each fire a `labeled` event → + # a fresh run; cancel-in-progress would then have each cancel the previous. Give + # label/unlabel events their OWN per-label group so they never cancel the commit + # pipeline — only real pushes (synchronize) supersede each other. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('-label-{0}', github.event.label.name) || '' }} cancel-in-progress: true permissions: @@ -36,6 +40,12 @@ permissions: jobs: decide: + # A label event only warrants running the pipeline when the label controls CI; + # auto-labels (component:*, cla signed) must not spawn work — and would each + # start a full run once ci:full is present. Non-label events always proceed. + if: >- + github.event.action != 'labeled' || + contains(fromJSON('["ci: full", "Force All Tests[L0+L1+L2]", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml # Generate the windows build matrix once (shared across channels; From cd358d4ba9ac1c60293a8cc6953f4f8cd205e718 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Fri, 17 Jul 2026 21:21:17 +0000 Subject: [PATCH 07/11] infra: fix aarch64 ci --- .github/workflows/_decide.yml | 3 +++ .github/workflows/ci-linux-x86_64.yml | 12 +++++++----- .github/workflows/ci-sbsa.yml | 12 +++++++----- .github/workflows/ci-windows.yml | 12 +++++++----- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/workflows/_decide.yml b/.github/workflows/_decide.yml index 2bc1e1539e..c108807a1d 100644 --- a/.github/workflows/_decide.yml +++ b/.github/workflows/_decide.yml @@ -36,6 +36,8 @@ jobs: # Exact array-element match: 'backend: TensorRT' != 'backend: TensorRT-RTX'. HAS_RTX_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT-RTX') }}" HAS_STD_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT') }}" + # "Force All Tests" means every backend too, not just the full tiers. + HAS_FORCE_ALL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]') }}" DISPATCH_LANE: ${{ github.event.inputs.lane }} DISPATCH_BACKEND: ${{ github.event.inputs.backend }} run: | @@ -57,6 +59,7 @@ jobs: if [ "$HAS_RTX_LABEL" = "true" ] && [ "$HAS_STD_LABEL" = "true" ]; then backend=both elif [ "$HAS_RTX_LABEL" = "true" ]; then backend=rtx elif [ "$HAS_STD_LABEL" = "true" ]; then backend=standard + elif [ "$HAS_FORCE_ALL_LABEL" = "true" ]; then backend=both # Force All → every backend else backend=standard; fi ;; *) backend=both ;; # push / approval / schedule esac diff --git a/.github/workflows/ci-linux-x86_64.yml b/.github/workflows/ci-linux-x86_64.yml index 8a89381c34..68c623320b 100644 --- a/.github/workflows/ci-linux-x86_64.yml +++ b/.github/workflows/ci-linux-x86_64.yml @@ -27,11 +27,13 @@ on: default: both concurrency: - # Label churn (auto-labels + ci:full / Force-All) each fire a `labeled` event → - # a fresh run; cancel-in-progress would then have each cancel the previous. Give - # label/unlabel events their OWN per-label group so they never cancel the commit - # pipeline — only real pushes (synchronize) supersede each other. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('-label-{0}', github.event.label.name) || '' }} + # Only NON-CI label events (component:*, cla signed, …) get their own per-label + # group, so auto-label churn never cancels the running pipeline. CI-control labels + # (ci:full / Force-All / backend:*) and real pushes share the PR group so they + # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would + # otherwise collide inside build_linux.yml's own concurrency and cancel a + # half-built wheel (that is what cancelled the SBSA aarch64 build). + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"Force All Tests[L0+L1+L2]\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" cancel-in-progress: true permissions: diff --git a/.github/workflows/ci-sbsa.yml b/.github/workflows/ci-sbsa.yml index 10e0641f6d..9b2cfdde12 100644 --- a/.github/workflows/ci-sbsa.yml +++ b/.github/workflows/ci-sbsa.yml @@ -27,11 +27,13 @@ on: default: both concurrency: - # Label churn (auto-labels + ci:full / Force-All) each fire a `labeled` event → - # a fresh run; cancel-in-progress would then have each cancel the previous. Give - # label/unlabel events their OWN per-label group so they never cancel the commit - # pipeline — only real pushes (synchronize) supersede each other. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('-label-{0}', github.event.label.name) || '' }} + # Only NON-CI label events (component:*, cla signed, …) get their own per-label + # group, so auto-label churn never cancels the running pipeline. CI-control labels + # (ci:full / Force-All / backend:*) and real pushes share the PR group so they + # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would + # otherwise collide inside build_linux.yml's own concurrency and cancel a + # half-built wheel (that is what cancelled the SBSA aarch64 build). + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"Force All Tests[L0+L1+L2]\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" cancel-in-progress: true permissions: diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 1bd6096c3d..9b380c10d3 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -27,11 +27,13 @@ on: default: both concurrency: - # Label churn (auto-labels + ci:full / Force-All) each fire a `labeled` event → - # a fresh run; cancel-in-progress would then have each cancel the previous. Give - # label/unlabel events their OWN per-label group so they never cancel the commit - # pipeline — only real pushes (synchronize) supersede each other. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && format('-label-{0}', github.event.label.name) || '' }} + # Only NON-CI label events (component:*, cla signed, …) get their own per-label + # group, so auto-label churn never cancels the running pipeline. CI-control labels + # (ci:full / Force-All / backend:*) and real pushes share the PR group so they + # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would + # otherwise collide inside build_linux.yml's own concurrency and cancel a + # half-built wheel (that is what cancelled the SBSA aarch64 build). + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"Force All Tests[L0+L1+L2]\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" cancel-in-progress: true permissions: From ee86bc0e4f3175fe688d393a25144f8731e624bf Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Fri, 17 Jul 2026 22:21:58 +0000 Subject: [PATCH 08/11] some commands to manage CI --- .github/workflows/_decide.yml | 23 +++-- .github/workflows/build_linux.yml | 7 +- .github/workflows/ci-linux-x86_64.yml | 6 +- .github/workflows/ci-sbsa.yml | 6 +- .github/workflows/ci-windows.yml | 6 +- .github/workflows/retrigger-ci.yml | 119 ++++++++++++++++---------- tools/ci-dashboard/README.md | 18 ++++ tools/ci-dashboard/static/index.html | 12 +++ tools/ci-dashboard/static/style.css | 12 +++ 9 files changed, 145 insertions(+), 64 deletions(-) diff --git a/.github/workflows/_decide.yml b/.github/workflows/_decide.yml index c108807a1d..62f6dfa07c 100644 --- a/.github/workflows/_decide.yml +++ b/.github/workflows/_decide.yml @@ -30,14 +30,15 @@ jobs: env: EVENT: ${{ github.event_name }} REVIEW_STATE: ${{ github.event.review.state }} - # Either the explicit ci:full label or the repo's "Force All Tests" label - # requests the full lane (the latter was previously a no-op here). - HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]') }}" - # Exact array-element match: 'backend: TensorRT' != 'backend: TensorRT-RTX'. + # Lane labels (consolidated): `ci: full` runs all tiers (L0+L1+L2); + # `ci: nightly` also adds the nightly-only suites. Neither → fast (L0 + # smoke) on PR pushes. (The old `Force All Tests[L0+L1+L2]` label is gone.) + HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') }}" + HAS_NIGHTLY_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: nightly') }}" + # Backend labels narrow the engine; on a full/nightly run their absence + # means BOTH. Exact match: 'backend: TensorRT' != 'backend: TensorRT-RTX'. HAS_RTX_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT-RTX') }}" HAS_STD_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT') }}" - # "Force All Tests" means every backend too, not just the full tiers. - HAS_FORCE_ALL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]') }}" DISPATCH_LANE: ${{ github.event.inputs.lane }} DISPATCH_BACKEND: ${{ github.event.inputs.backend }} run: | @@ -49,18 +50,22 @@ jobs: pull_request_review) [ "$REVIEW_STATE" = "approved" ] && lane=full || lane=skip ;; pull_request) - [ "$HAS_FULL_LABEL" = "true" ] && lane=full || lane=fast ;; + if [ "$HAS_NIGHTLY_LABEL" = "true" ]; then lane=nightly + elif [ "$HAS_FULL_LABEL" = "true" ]; then lane=full + else lane=fast; fi ;; *) lane=fast ;; esac echo "lane=$lane" >> "$GITHUB_OUTPUT" + # Backend: explicit labels win; otherwise a fast PR push stays + # standard-only (cheap), while full/nightly default to BOTH engines. case "$EVENT" in workflow_dispatch) backend="${DISPATCH_BACKEND:-both}" ;; pull_request) if [ "$HAS_RTX_LABEL" = "true" ] && [ "$HAS_STD_LABEL" = "true" ]; then backend=both elif [ "$HAS_RTX_LABEL" = "true" ]; then backend=rtx elif [ "$HAS_STD_LABEL" = "true" ]; then backend=standard - elif [ "$HAS_FORCE_ALL_LABEL" = "true" ]; then backend=both # Force All → every backend - else backend=standard; fi ;; + elif [ "$lane" = "fast" ]; then backend=standard + else backend=both; fi ;; *) backend=both ;; # push / approval / schedule esac echo "backend=$backend" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index f59ec80608..de62327eaf 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -429,5 +429,10 @@ jobs: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{inputs.is-release-wheel}}-${{inputs.is-release-tarball}}-${{inputs.use-rtx}}-${{inputs.architecture}}-${{inputs.is-jetpack}}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'no-tag' }}-${{ inputs.use-rtx }} + # env-var-script distinguishes the python-only build (env_vars_python_only.txt) + # from the standard build (env_vars.txt). Without it, the standard and + # python-only channels of the SAME run share this group and cancel-in-progress + # kills one of them — which is why "run everything" never completed. (Local + # addition to the test-infra-synced key; keep on re-sync.) + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{inputs.is-release-wheel}}-${{inputs.is-release-tarball}}-${{inputs.use-rtx}}-${{inputs.architecture}}-${{inputs.is-jetpack}}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'no-tag' }}-${{ inputs.use-rtx }}-${{ inputs.env-var-script }} cancel-in-progress: true diff --git a/.github/workflows/ci-linux-x86_64.yml b/.github/workflows/ci-linux-x86_64.yml index 68c623320b..27f497eb11 100644 --- a/.github/workflows/ci-linux-x86_64.yml +++ b/.github/workflows/ci-linux-x86_64.yml @@ -29,11 +29,11 @@ on: concurrency: # Only NON-CI label events (component:*, cla signed, …) get their own per-label # group, so auto-label churn never cancels the running pipeline. CI-control labels - # (ci:full / Force-All / backend:*) and real pushes share the PR group so they + # (ci:full / ci:nightly / backend:*) and real pushes share the PR group so they # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would # otherwise collide inside build_linux.yml's own concurrency and cancel a # half-built wheel (that is what cancelled the SBSA aarch64 build). - group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"Force All Tests[L0+L1+L2]\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" cancel-in-progress: true permissions: @@ -47,7 +47,7 @@ jobs: # start a full run once ci:full is present. Non-label events always proceed. if: >- github.event.action != 'labeled' || - contains(fromJSON('["ci: full", "Force All Tests[L0+L1+L2]", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) + contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml # Generate the build matrix ONCE. generate_binary_build_matrix's concurrency diff --git a/.github/workflows/ci-sbsa.yml b/.github/workflows/ci-sbsa.yml index 9b2cfdde12..a4be4e2ce1 100644 --- a/.github/workflows/ci-sbsa.yml +++ b/.github/workflows/ci-sbsa.yml @@ -29,11 +29,11 @@ on: concurrency: # Only NON-CI label events (component:*, cla signed, …) get their own per-label # group, so auto-label churn never cancels the running pipeline. CI-control labels - # (ci:full / Force-All / backend:*) and real pushes share the PR group so they + # (ci:full / ci:nightly / backend:*) and real pushes share the PR group so they # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would # otherwise collide inside build_linux.yml's own concurrency and cancel a # half-built wheel (that is what cancelled the SBSA aarch64 build). - group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"Force All Tests[L0+L1+L2]\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" cancel-in-progress: true permissions: @@ -47,7 +47,7 @@ jobs: # start a full run once ci:full is present. Non-label events always proceed. if: >- github.event.action != 'labeled' || - contains(fromJSON('["ci: full", "Force All Tests[L0+L1+L2]", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) + contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml # Generate the aarch64 build matrix once (shared by both build channels). diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 9b380c10d3..1220df1279 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -29,11 +29,11 @@ on: concurrency: # Only NON-CI label events (component:*, cla signed, …) get their own per-label # group, so auto-label churn never cancels the running pipeline. CI-control labels - # (ci:full / Force-All / backend:*) and real pushes share the PR group so they + # (ci:full / ci:nightly / backend:*) and real pushes share the PR group so they # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would # otherwise collide inside build_linux.yml's own concurrency and cancel a # half-built wheel (that is what cancelled the SBSA aarch64 build). - group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"Force All Tests[L0+L1+L2]\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" cancel-in-progress: true permissions: @@ -47,7 +47,7 @@ jobs: # start a full run once ci:full is present. Non-label events always proceed. if: >- github.event.action != 'labeled' || - contains(fromJSON('["ci: full", "Force All Tests[L0+L1+L2]", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) + contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml # Generate the windows build matrix once (shared across channels; diff --git a/.github/workflows/retrigger-ci.yml b/.github/workflows/retrigger-ci.yml index da3fd921b5..8a184ad65a 100644 --- a/.github/workflows/retrigger-ci.yml +++ b/.github/workflows/retrigger-ci.yml @@ -1,12 +1,22 @@ name: Retrigger CI -# Re-run CI on a PR without pushing a new commit. +# Re-run or cancel CI on a PR without pushing a new commit. # # Usage — post a comment on any PR: -# /rerun re-run only failed / cancelled jobs -# /rerun all re-run all jobs from scratch +# /rerun re-run only failed / cancelled jobs (current commit) +# /rerun all re-run all jobs from scratch (current commit) +# /cancel cancel stale "zombie" runs — anything still in-flight on an +# OLD commit of this PR (leaves the current commit's run alone) +# /cancel all cancel EVERY in-flight run for this PR (full stop) +# /test [backend] +# dispatch a fresh run on this PR's branch — lane ∈ +# fast|full|nightly, backend ∈ standard|rtx|both (default both). +# e.g. `/test full rtx`, `/test nightly`, `/test full` # # The comment author must have write access to the repo. +# +# NOTE: issue_comment workflows always run from the DEFAULT branch, so changes to +# this file only take effect once merged to main. on: issue_comment: @@ -17,16 +27,17 @@ permissions: pull-requests: read jobs: - retrigger: + command: if: | github.event.issue.pull_request != null && ( - startsWith(github.event.comment.body, '/rerun') + startsWith(github.event.comment.body, '/rerun') || + startsWith(github.event.comment.body, '/cancel') || + startsWith(github.event.comment.body, '/test') ) runs-on: ubuntu-latest steps: - name: Check commenter has write access - id: auth uses: actions/github-script@v7 with: script: | @@ -35,68 +46,86 @@ jobs: repo: context.repo.repo, username: context.actor, }); - const level = perm.permission; - if (!["admin", "write"].includes(level)) { - core.setFailed(`${context.actor} has permission '${level}' — write access required to retrigger CI`); + if (!["admin", "write"].includes(perm.permission)) { + core.setFailed(`${context.actor} has permission '${perm.permission}' — write access required to control CI`); } - return level; - - name: Parse command - id: cmd + - name: Run command uses: actions/github-script@v7 with: script: | const body = context.payload.comment.body.trim(); - const rerunAll = body === '/rerun all'; - core.setOutput('rerun_all', rerunAll ? 'true' : 'false'); - return rerunAll; - - - name: Re-run CI - uses: actions/github-script@v7 - with: - script: | - const rerunAll = '${{ steps.cmd.outputs.rerun_all }}' === 'true'; - + const { owner, repo } = context.repo; const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, + owner, repo, pull_number: context.issue.number, }); - const sha = pr.head.sha; + const headSha = pr.head.sha; + // ── /test [backend] — dispatch a run with an exact lane ── + if (body.startsWith('/test')) { + const parts = body.split(/\s+/).slice(1); + const lane = ['fast', 'full', 'nightly'].includes(parts[0]) ? parts[0] : 'full'; + const backend = ['standard', 'rtx', 'both'].includes(parts[1]) ? parts[1] : 'both'; + let dispatched = 0; + for (const wf of ['ci-linux-x86_64.yml', 'ci-windows.yml', 'ci-sbsa.yml']) { + try { + await github.rest.actions.createWorkflowDispatch({ + owner, repo, workflow_id: wf, ref: pr.head.ref, + inputs: { lane, backend }, + }); + dispatched++; + } catch (e) { + core.info(`Skipped ${wf}: ${e.message}`); + } + } + core.notice(`Dispatched ${dispatched} workflow(s) — lane=${lane} backend=${backend} on ${pr.head.ref}`); + return; + } + + // ── /cancel [all] — kill zombie runs ─────────────────────────── + if (body.startsWith('/cancel')) { + const all = body === '/cancel all'; + const { data: { workflow_runs: runs } } = + await github.rest.actions.listWorkflowRunsForRepo({ + owner, repo, branch: pr.head.ref, per_page: 100, + }); + let cancelled = 0; + for (const run of runs) { + if (run.status === 'completed') continue; // already done + if (!all && run.head_sha === headSha) continue; // keep current commit + try { + await github.rest.actions.cancelWorkflowRun({ owner, repo, run_id: run.id }); + cancelled++; + } catch (e) { + core.info(`Skipped ${run.id} (${run.name}): ${e.message}`); + } + } + core.notice(`Cancelled ${cancelled} ${all ? 'in-flight' : 'stale (old-commit)'} run(s).`); + return; + } + + // ── /rerun [all] ─────────────────────────────────────────────── + const rerunAll = body === '/rerun all'; const { data: { workflow_runs: runs } } = await github.rest.actions.listWorkflowRunsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: sha, - per_page: 50, + owner, repo, head_sha: headSha, per_page: 50, }); - if (runs.length === 0) { - core.warning(`No workflow runs found for SHA ${sha}`); + core.warning(`No workflow runs found for SHA ${headSha}`); return; } - let retriggered = 0; for (const run of runs) { try { if (rerunAll) { - await github.rest.actions.reRunWorkflow({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run.id, - }); + await github.rest.actions.reRunWorkflow({ owner, repo, run_id: run.id }); } else { - await github.rest.actions.reRunWorkflowFailedJobs({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run.id, - }); + await github.rest.actions.reRunWorkflowFailedJobs({ owner, repo, run_id: run.id }); } retriggered++; } catch (e) { - // A run that is still in progress can't be re-run; skip it. + // A run still in progress can't be re-run; skip it. core.info(`Skipped run ${run.id} (${run.name}): ${e.message}`); } } - core.info(`Retriggered ${retriggered}/${runs.length} workflow runs`); + core.notice(`Retriggered ${retriggered}/${runs.length} workflow runs.`); diff --git a/tools/ci-dashboard/README.md b/tools/ci-dashboard/README.md index 09345fe74a..c4a2e764d6 100644 --- a/tools/ci-dashboard/README.md +++ b/tools/ci-dashboard/README.md @@ -61,6 +61,24 @@ reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN U collapsing whatever you've expanded. `↻ Refresh` (or `r`) forces a full reload; `/` focuses the branch box; the *failing only* toggle hides the green cards. +## PR commands + +Comment these on a PR to control its CI without pushing a new commit (write +access required; handled by `.github/workflows/retrigger-ci.yml`). They're also +listed in the dashboard header under **PR commands** with copy buttons. + +| comment | effect | +|---|---| +| `/rerun` | re-run only the failed / cancelled jobs on the current commit | +| `/rerun all` | re-run every job from scratch on the current commit | +| `/cancel` | cancel stale “zombie” runs still in-flight on **old** commits of the PR | +| `/cancel all` | cancel **every** in-flight run for the PR (full stop) | +| `/test [backend]` | dispatch a fresh run at an exact lane — `lane` ∈ `fast\|full\|nightly`, `backend` ∈ `standard\|rtx\|both` (default `both`). e.g. `/test full rtx`, `/test nightly` | + +Typical unstick: `/cancel` to clear zombies, then `/rerun all` for a clean wave. +Run everything on demand: `/test full` (or `/test nightly` for llm/kernels/distributed). +(These take effect from `main`, since `issue_comment` workflows always run there.) + ## How it works The server (`ci_dashboard.py`) is a thin, caching proxy over `gh`; all rendering diff --git a/tools/ci-dashboard/static/index.html b/tools/ci-dashboard/static/index.html index 23c0aa8475..89ac3b5d1d 100644 --- a/tools/ci-dashboard/static/index.html +++ b/tools/ci-dashboard/static/index.html @@ -32,6 +32,18 @@

Torch-TensorRT CI

didn’t run · open a platform for its python×cuda×tier grid · click a red cell for failing tests + code
+
+ PR commands — comment these on a PR to control its CI (write access) +
+
/rerunre-run only the failed / cancelled jobs (current commit)
+
/rerun allre-run every job from scratch (current commit)
+
/cancelcancel stale “zombie” runs still in-flight on old commits
+
/cancel allcancel every in-flight run for the PR (full stop)
+
/test fullrun the full suite (all tiers, both engines)
+
/test full rtxfull suite, RTX engine only (also standard)
+
/test nightlyeverything incl. llm / kernels / distributed
+
+
loading {{BRANCH}}…
diff --git a/tools/ci-dashboard/static/style.css b/tools/ci-dashboard/static/style.css index f3fd67bc8e..25146987eb 100644 --- a/tools/ci-dashboard/static/style.css +++ b/tools/ci-dashboard/static/style.css @@ -236,3 +236,15 @@ h2.sec { font-size: 12px; text-transform: uppercase; letter-spacing: .08em; colo .toast.show { opacity: 1; } .legend { display: flex; gap: 14px; color: var(--muted); font-size: 12px; margin: 4px 0 0; flex-wrap: wrap; } .legend span { display: inline-flex; align-items: center; gap: 5px; } + +/* PR command reference (collapsible) */ +.cmds { margin: 8px 0 0; font-size: 12px; } +.cmds > summary { cursor: pointer; color: var(--muted); user-select: none; padding: 2px 0; } +.cmds > summary:hover { color: var(--text); } +.cmd-list { display: flex; flex-direction: column; gap: 6px; margin: 8px 0 2px; padding: 10px 12px; + background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; } +.cmd { display: flex; align-items: center; gap: 10px; } +.cmd code { font-family: var(--mono); font-size: 12px; color: var(--text); background: var(--surface); + border: 1px solid var(--border); border-radius: 6px; padding: 2px 8px; white-space: nowrap; min-width: 82px; } +.cmd .copy { float: none; } +.cmd .lbl { color: var(--muted); } From e2a00e0ef42c67fb232e260c2e76eeb34f3471c0 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Tue, 21 Jul 2026 20:45:09 +0000 Subject: [PATCH 09/11] Some UI improvements to the local dashboard --- tools/ci-dashboard/README.md | 26 +- tools/ci-dashboard/ci_dashboard.py | 336 ++++++++++++++++++++---- tools/ci-dashboard/static/index.html | 27 +- tools/ci-dashboard/static/style.css | 371 ++++++++++++++++++--------- 4 files changed, 581 insertions(+), 179 deletions(-) diff --git a/tools/ci-dashboard/README.md b/tools/ci-dashboard/README.md index c4a2e764d6..728c18abd5 100644 --- a/tools/ci-dashboard/README.md +++ b/tools/ci-dashboard/README.md @@ -44,11 +44,22 @@ reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN U **source file:line it maps to** (local path + a GitHub link pinned to the run's commit), and a **copy-paste command to reproduce it locally** — the exact `python -m tests.ci run ` command CI used, narrowed to the failing tests. -- **Relevant logs, captured** — the drawer then pulls that job's log and extracts - just the **pytest FAILURES block per failing test** (traceback + captured - stdout/stderr), so you read the actual failure without scrolling a 1 MB log. - A **raw log ↗** link opens the full plain-text log (grep/save it); a - build/env failure with no test annotations falls back to the end of the log. +- **Relevant logs, captured** — the drawer pulls that job's log (ANSI codes and + `##[group]` markers stripped) and shows just the part that matters: + - the **pytest FAILURES block per failing test** (traceback + captured output), + so you read the actual failure without scrolling a ~0.5 MB / ~8k-line log; + - **root-cause collapse** — when N failures share one reason (e.g. 5 cumsum tests + with `build_serialized_network returned None`, or 29 tests that all failed to + import), they fold into **one** `N× …` block + a banner, not N identical ones; + - **install/build failures surfaced** — an import storm (every test fails to + import) points at the actual `pip … subprocess-exited-with-error`, not the + misleading per-test `ModuleNotFoundError`s; + - **smart anchoring** for non-pytest failures — instead of the blind log tail, it + anchors on the real error (`short test summary`, a build error, or a backward + walk from `exit code N` to the traceback), skipping echoed `::error::` script noise; + - **search / jump / highlight** — a search box (with a *matches only* filter), an + error **jump-list** that scrolls to each error line, and severity-coloured lines. + A **raw log ↗** link still opens the full plain-text log to grep/save. - **Failures across platforms** — a rollup that dedupes a failing test across every platform it breaks on ("fails on 3 platforms → likely this code"), so a systemic break stands out from a one-off. @@ -104,8 +115,9 @@ platform on open, out-of-band badge polling, the failure drawer). ## Limitations - Results are only as granular as the CI annotations. A **build/env/setup** - failure (not a test assertion) has no pytest annotation — the drawer says so - and links you to the raw job log. + failure (not a test assertion) has no pytest annotation — the drawer says so, + then anchors the log on the real error (and, for an import storm, points at the + install failure) and links you to the raw job log. - Expanded platform grids don't auto-refresh (only the top-level badges do). Open a stale platform again, or hit `↻ Refresh`, to re-pull its jobs. - The cross-platform rollup fans out `gh` calls for every failing test job; on a diff --git a/tools/ci-dashboard/ci_dashboard.py b/tools/ci-dashboard/ci_dashboard.py index 68a04e2154..9742b146db 100755 --- a/tools/ci-dashboard/ci_dashboard.py +++ b/tools/ci-dashboard/ci_dashboard.py @@ -367,18 +367,31 @@ def get_jobs(run_id, refresh=False): r"might have been added implicitly", re.I, ) -TESTLINE = re.compile(r"^(?P[\w./-]+(?:\.[\w\[\]-]+)+)\s*$") +# A test id is the annotation's first line: a single token that names a test — +# a bare `test_x[param]`, a dotted `Class.test_x`, or a `file.py::Class::test_x` +# nodeid. (The old regex required a dot, so it missed bare parametrized names.) +TESTLINE = re.compile(r"^[\w./\[\]:-]+$") -@lru_cache(maxsize=4096) -def grep_test(symbol): - """Resolve a failing-test symbol (Class.method / module.func) to (file, line) - via `git grep`. Returns None if not found.""" - leaf = re.split(r"[.:]", symbol.strip())[-1] - leaf = re.sub(r"\[.*$", "", leaf) - if not re.match(r"^[A-Za-z_]\w+$", leaf): +def _test_name(head): + """Pull a test id out of an annotation's first line, or None.""" + if not head or " " in head or not TESTLINE.match(head): return None - pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b" + tok = head.split("::")[-1] if "::" in head else head # nodeid → last segment + return tok if re.search(r"test", tok, re.I) else None + + +def _annot_loc(a): + """Exact (repo-relative file, line) straight from the annotation — better than + grep, which only finds the def line. The path is prefixed (pytorch/tensorrt/…); + keep from `tests/` on and confirm it exists locally.""" + m = re.search(r"(tests/.+)$", a.get("path") or "") + if not m or not os.path.exists(os.path.join(REPO_ROOT, m.group(1))): + return None + return (m.group(1), a.get("start_line") or 0) + + +def _git_grep(pat): try: out = subprocess.run( ["git", "grep", "-n", "-E", pat, "--", "tests/"], @@ -395,6 +408,22 @@ def grep_test(symbol): return None +@lru_cache(maxsize=4096) +def grep_test(symbol): + """Resolve a failing-test symbol (Class.method / module.func) to (file, line) + via `git grep`. Tries the method, then the enclosing class — parametrized names + (`@parameterized.expand` → test_x_5) have no literal `def`, so the class is the + best anchor. Returns None if nothing matches.""" + parts = [re.sub(r"\[.*$", "", p) for p in re.split(r"[.:]+", symbol.strip()) if p] + for token in reversed(parts): # method first, then its class + if not re.match(r"^[A-Za-z_]\w+$", token): + continue + pat = f"def {token}\\b" if token.startswith("test") else f"class {token}\\b" + if hit := _git_grep(pat): + return hit + return None + + def get_failures(job_id, refresh=False): key = f"fail:{job_id}" if not refresh and (c := CACHE.get(key)) is not None: @@ -411,13 +440,14 @@ def get_failures(job_id, refresh=False): if not msg or NOISE.search(msg.splitlines()[0]): continue head = msg.splitlines()[0].strip() - m = TESTLINE.match(head) - test = m.group("test") if m else None + test = _test_name(head) dedupe = test or head if dedupe in seen: continue seen.add(dedupe) - loc = grep_test(test) if test else None + loc = _annot_loc(a) or ( + grep_test(test) if test else None + ) # annotation line first failures.append( dict( test=test, @@ -433,6 +463,9 @@ def get_failures(job_id, refresh=False): # ── job logs (fetch + per-test extraction) ────────────────────────────────── _TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix +_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") # SGR / CSI colour codes +_GROUP = re.compile(r"^##\[(?:group|section|command)\]") # fold marker → keep title +_ENDGROUP = re.compile(r"^##\[endgroup\]\s*$") # pure noise _SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep _SECT = re.compile(r"^={4,}") # ==== section ==== @@ -455,7 +488,15 @@ def get_job_log(job_id, refresh=False): def _clean_lines(raw): - return [_TS.sub("", ln) for ln in raw.splitlines()] + """Strip the GH timestamp prefix, ANSI colour codes, and the ##[group] / + ##[endgroup] fold markers that otherwise leak into the 
."""
+    out = []
+    for ln in raw.splitlines():
+        ln = _ANSI.sub("", _TS.sub("", ln)).replace("\r", "")
+        if _ENDGROUP.match(ln):
+            continue
+        out.append(_GROUP.sub("", ln))
+    return out
 
 
 def extract_test_log(lines, test, max_lines=160):
@@ -504,14 +545,195 @@ def extract_test_log(lines, test, max_lines=160):
     return "\n".join(block).strip()
 
 
-def summary_reasons(lines, test):
-    """The concise `FAILED …/ERROR … - ` lines from pytest's summary."""
-    leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
-    return [
-        ln
-        for ln in lines
-        if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
-    ]
+# ── anchoring: find the *real* error in a non-pytest (build/install) failure ──
+# In GH logs the first `##[error]` is almost always echoed shell text
+# (`echo "::error::…"`), so we never anchor on it. The reliable signals, in order:
+# the pytest summary, a pip/build metadata error, or a walk backward from the
+# `Process completed with exit code N` line to the nearest genuine error.
+_SUMMARY_HDR = re.compile(r"^=+\s*short test summary info\s*=+", re.I)
+_PIP_ERR = re.compile(
+    r"error: subprocess-exited-with-error|did not run successfully|^\s*╰─>"
+)
+_EXITCODE = re.compile(r"(?:Process completed with exit code|exit code:?)\s*[1-9]")
+_ECHO_NOISE = re.compile(
+    r'echo\s+["\']?::|Specified script file|^\s*(?:if|else|elif|fi|then|source|for|do|done)\b'
+)
+_REAL_ERR = re.compile(
+    r"Traceback \(most recent call last\)|\b\w*(?:Error|Exception)\b\s*:|"
+    r"^E\s{2,}\S|^\s*(?:FAILED|ERROR)\s|error: subprocess-exited-with-error|"
+    r"CMake Error|fatal error:|ninja: build stopped|Segmentation fault|OutOfMemory|Killed\b"
+)
+
+
+def find_error_window(lines, max_lines=140):
+    """Anchor a non-pytest failure on its real error instead of the blind tail.
+    Returns (title, start, end)."""
+    n = len(lines)
+    if not n:
+        return ("end of job log", 0, 0)
+    for i, ln in enumerate(lines):  # 1. pytest summary
+        if _SUMMARY_HDR.search(ln):
+            end = next(
+                (
+                    j
+                    for j in range(i + 1, n)
+                    if _SECT.match(lines[j]) and not _SUMMARY_HDR.search(lines[j])
+                ),
+                n,
+            )
+            return ("short test summary", i, min(end, i + max_lines))
+    for i, ln in enumerate(lines):  # 2. pip / build error
+        if _PIP_ERR.search(ln):
+            return ("install / build error", max(0, i - 3), min(n, i + max_lines))
+    exit_i = next(
+        (i for i in range(n - 1, -1, -1) if _EXITCODE.search(lines[i])), n - 1
+    )
+    anchor = next(
+        (
+            i
+            for i in range(exit_i, -1, -1)  # 3. walk back to real error
+            if _REAL_ERR.search(lines[i]) and not _ECHO_NOISE.search(lines[i])
+        ),
+        None,
+    )
+    if anchor is not None:
+        start = anchor
+        for j in range(
+            anchor, max(0, anchor - 80), -1
+        ):  # widen up to the traceback head
+            if "Traceback (most recent call last)" in lines[j]:
+                start = j
+                break
+        return ("error", max(0, start - 2), min(n, exit_i + 1))
+    return ("end of job log", max(0, n - 180), n)  # 4. fallback: tail
+
+
+# ── root-cause collapse: many identical failures → one line + a pointer ───────
+_EXC_RE = re.compile(r"\b([A-Z]\w*(?:Error|Exception|Warning))\b\s*:?\s*([^\n]*)")
+
+
+def _reason_key(msg):
+    """Stable, comparable essence of a failure message, for de-duplication.
+    Volatile bits (quoted names, addresses, numbers, paths) are stripped so
+    parametrized variants of one failure collapse together."""
+    m = _EXC_RE.search(msg or "")
+    if m:
+        detail = re.sub(r"['\"][^'\"]*['\"]|0x[0-9a-fA-F]+|\d+|/\S+", "", m.group(2))
+        detail = re.sub(r"\s+", " ", detail).strip()
+        return f"{m.group(1)}: {detail}"[:90].rstrip(": ")
+    head = next((l.strip() for l in (msg or "").splitlines() if l.strip()), "")
+    return re.sub(r"\d+", "", head)[:90]
+
+
+def _reason_first(msg):
+    """The single most informative line of a message, for a block subtitle."""
+    m = _EXC_RE.search(msg or "")
+    if m:
+        return m.group(0)[:300]
+    return next((l.strip() for l in (msg or "").splitlines() if l.strip()), "")[:300]
+
+
+def _group_failures(fails):
+    """Group failures by normalized reason, biggest group first (a systemic
+    break surfaces above one-offs). Returns [(reason, [failure, …]), …]."""
+    groups, order = {}, []
+    for f in fails:
+        k = _reason_key(f["message"])
+        if k not in groups:
+            groups[k] = []
+            order.append(k)
+        groups[k].append(f)
+    return sorted(((k, groups[k]) for k in order), key=lambda kv: -len(kv[1]))
+
+
+# ── severity rendering: per-line spans so the client can search / jump / colour ─
+_SEV_ERR = re.compile(
+    r"^##\[error\]|Traceback \(most recent call last\)|\b\w*(?:Error|Exception)\b\s*:|"
+    r"^E\s{2,}\S|^\s*(?:FAILED|ERROR)\s|error: subprocess-exited-with-error|"
+    r"CMake Error|fatal error:|Segmentation fault|\bAssertionError\b|exit code:?\s*[1-9]"
+)
+_SEV_WARN = re.compile(r"^##\[warning\]|\bwarning\b|\bdeprecat|\bWARN\b", re.I)
+_SEV_PASS = re.compile(r"\bPASSED\b|=+\s*\d+ passed|^\s*ok\s*$", re.I)
+
+
+def _render_logtext(text):
+    """Emit log text as one  per line, tagged by severity.
+    Returns (html, error_hits) where error_hits = [(line_index, short_text)]
+    for the jump-list."""
+    spans, hits = [], []
+    for i, ln in enumerate(text.split("\n")):
+        sev = ""
+        if _SEV_ERR.search(ln) and not _ECHO_NOISE.search(ln):
+            sev, _ = "err", hits.append((i, ln.strip()[:90] or "error"))
+        elif _SEV_WARN.search(ln):
+            sev = "warn"
+        elif _SEV_PASS.search(ln):
+            sev = "pass"
+        attr = f' data-sev="{sev}"' if sev else ""
+        spans.append(f'{e(ln) or " "}')
+    return "".join(spans), hits  # .ll is display:block; no newlines between
+
+
+def _logblock(title, subtitle, excerpt, is_open=False):
+    """A collapsible log excerpt: severity-highlighted 
 + an error jump-list."""
+    if not excerpt:
+        return (
+            f'
{e(title)}{subtitle}'
+            '
No matching block '
+            "in the log — see the raw log.
"
+        )
+    body, hits = _render_logtext(excerpt)
+    jumps = ""
+    if hits:
+        chips = "".join(
+            f''
+            for i, short in hits[:8]
+        )
+        jumps = f'
jump{chips}
'
+    op = " open" if is_open else ""
+    return (
+        f'
{e(title)}{subtitle}'
+        f'{jumps}
{body}
'
+    )
+
+
+_IMPORT_FAIL = re.compile(
+    r"ModuleNotFoundError|ImportError|No module named|"
+    r"error collecting|during collection|failed to import",
+    re.I,
+)
+
+
+def _install_error_block(lines):
+    """Locate a pip/build install error in the log and render it, or return ''."""
+    pin = next((i for i, ln in enumerate(lines) if _PIP_ERR.search(ln)), None)
+    if pin is None:
+        return ""
+    body, _h = _render_logtext("\n".join(lines[max(0, pin - 3) : pin + 40]).strip())
+    return (
+        '
'
+        "likely root cause · install / build error"
+        f'
{body}
'
+    )
+
+
+def _root_cause_banner(lines, fails, groups):
+    """When most failures share one reason, say so once and point at the cause."""
+    reason, members = groups[0]
+    n = len(members)
+    msg = (
+        f"{n} of {len(fails)} failures share one cause: "
+        f"{e(reason)}."
+    )
+    extra = ""
+    if _IMPORT_FAIL.search(reason):
+        msg += (
+            f" That is an install / build failure, not {n} test bugs — "
+            "the package never imported."
+        )
+        extra = _install_error_block(lines)
+    return f'
{msg}
{extra}'
 
 
 def render_joblog(job_id, url):
@@ -529,33 +751,61 @@ def render_joblog(job_id, url):
             f"running, or GitHub expired it. {ghlink}
" ) lines = _clean_lines(log) + tools = ( + '
' + '' + '' + '
' + ) fails = get_failures(job_id).get("failures", []) blocks = [] - for f in fails: - excerpt = extract_test_log(lines, f["test"]) - reasons = summary_reasons(lines, f["test"]) - title = e(f["test"] or "failure") - reason_html = ( - f'
{e(reasons[0][:300])}
' if reasons else "" + if fails: + groups = _group_failures(fails) + if len(fails) >= 3 and any(len(m) > 1 for _, m in groups): + blocks.append(_root_cause_banner(lines, fails, groups)) + for reason, members in groups: + if len(members) == 1: + f = members[0] + sub = f'
{e(_reason_first(f["message"]))}
' + blocks.append( + _logblock( + f["test"] or "failure", + sub, + extract_test_log(lines, f["test"]), + is_open=True, + ) + ) + else: # collapsed: N parametrized / duplicate failures → one block + names = ", ".join(m["test"] or "?" for m in members[:10]) + more = f" +{len(members) - 10} more" if len(members) > 10 else "" + sub = ( + f'
{e(reason)}
' + f'
{len(members)} tests · {e(names)}{e(more)}
' + ) + blocks.append( + _logblock( + f"{len(members)}× {reason}", + sub, + extract_test_log(lines, members[0]["test"]), + is_open=True, + ) + ) + else: # no annotations → anchor on the real error; surface install failures + title, s, en = find_error_window(lines) + window = "\n".join(lines[s:en]).strip() + import_dominated = ( + title == "short test summary" and len(_IMPORT_FAIL.findall(window)) >= 2 ) - if excerpt: - blocks.append( - f'
{title}' - f"{reason_html}
{e(excerpt)}
" - ) - else: + inst = _install_error_block(lines) if import_dominated else "" + if inst: + blocks.append(inst) blocks.append( - f'
{title}' - f'{reason_html}
No matching block in the log — ' - f"see the raw log.
" + '
Every test failed at import — this is ' + "an install / build failure (above), not test bugs.
" ) - if not fails: # build/env/setup failure: no test annotations → show the tail - tail = "\n".join(lines[-180:]).strip() - blocks.append( - '
end of job log' - f"
{e(tail)}
" - ) - return head + "".join(blocks) + blocks.append(_logblock(title, "", window, is_open=not inst)) + return head + tools + "".join(blocks) # ── status helpers ─────────────────────────────────────────────────────────── diff --git a/tools/ci-dashboard/static/index.html b/tools/ci-dashboard/static/index.html index 89ac3b5d1d..193fff7d67 100644 --- a/tools/ci-dashboard/static/index.html +++ b/tools/ci-dashboard/static/index.html @@ -9,7 +9,7 @@
-

Torch-TensorRT CI

+

torch-tensorrt // ci

Torch-TensorRT CI const code = btn.parentElement.querySelector('code'); navigator.clipboard.writeText(code.innerText).then(() => toast('copied')); } + // ── log fishing: search / filter / jump within the drawer's log section ── + function logSearch(inp) { + const q = inp.value.trim().toLowerCase(); + const sec = inp.closest('.logsec'); + let n = 0, first = null; + sec.querySelectorAll('.logtext .ll').forEach(ll => { + const hit = q && ll.textContent.toLowerCase().includes(q); + ll.classList.toggle('match', hit); + if (hit) { n++; if (!first) first = ll; } + }); + const c = sec.querySelector('.logcount'); + if (c) c.textContent = q ? (n + (n === 1 ? ' match' : ' matches')) : ''; + if (first) first.scrollIntoView({ block: 'center', behavior: 'smooth' }); + } + function logFilter(cb) { + cb.closest('.logsec').querySelectorAll('.logtext') + .forEach(p => p.classList.toggle('filtered', cb.checked)); + } + function jumpLine(btn) { + const pre = btn.closest('.logblock').querySelector('.logtext'); + const t = pre.querySelectorAll('.ll')[+btn.dataset.line]; + if (!t) return; + t.scrollIntoView({ block: 'center', behavior: 'smooth' }); + t.classList.remove('flash'); void t.offsetWidth; t.classList.add('flash'); + } function toast(m) { const t = document.getElementById('toast'); t.textContent = m; t.classList.add('show'); diff --git a/tools/ci-dashboard/static/style.css b/tools/ci-dashboard/static/style.css index 25146987eb..029d16982a 100644 --- a/tools/ci-dashboard/static/style.css +++ b/tools/ci-dashboard/static/style.css @@ -1,89 +1,140 @@ +/* ── Torch-TensorRT CI · pytorch × mission-control × terminal ────────────────── + Dark-only, phosphor-on-black. PyTorch flame (#ee4c2c) is the single accent. + Status palette validated with the dataviz validator: contrast ≥3:1 on the + surface, normal-vision ΔE 17; the red↔green CVD pair (ΔE 6.5) is covered by the + always-present text labels (secondary encoding), never colour alone. */ :root { - --bg: #f6f7f9; --surface: #fff; --surface-2: #f0f2f5; --border: #e2e6ea; - --text: #1c2024; --muted: #5b6570; --accent: #0969da; - --pass: #1a7f37; --pass-bg: #e6f4ea; --pass-line: #b7dcc0; - --fail: #cf222e; --fail-bg: #ffebe9; --fail-line: #f3c0c2; - --run: #9a6700; --run-bg: #fff8e6; --run-line: #e6d9a8; - --queue: #6e7781; --queue-bg: #eef0f2; --queue-line: #dbe0e4; - --skip: #7d8590; --skip-bg: #eef1f4; --skip-line: #d3d9df; - --cancel: #8250df; --cancel-bg: #f4f0fb; --cancel-line: #e1d5f5; - --radius: 10px; --shadow: 0 1px 2px rgba(0,0,0,.06), 0 3px 12px rgba(0,0,0,.04); - --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + --bg: #080b0f; /* deep space */ + --surface: #0e1319; /* panel */ + --surface-2: #141b23; /* raised / cell */ + --border: #202a35; /* hairline */ + --border-bri: #33475a; /* active edge */ + --text: #cdd9e5; /* phosphor white */ + --muted: #647688; /* dim readout */ + --accent: #ee4c2c; /* PyTorch flame */ + --accent-2: #ff6a49; /* hot flame (hover) */ + --pass: #35d07f; --pass-bg: #0c1c15; --pass-line: #17402c; + --fail: #d90429; --fail-bg: #260810; --fail-line: #551126; /* deep alarm red, held clear of the flame orange (validated ΔE 9.1) */ + --fail-glow: 235,20,60; /* brightened crimson for the phosphor glow */ + --run: #f5a623; --run-bg: #221a0a; --run-line: #453410; + --queue:#59c2ff; --queue-bg:#0a1a26; --queue-line:#183545; + --skip: #7d8590; --skip-bg: #151b22; --skip-line: #2a333d; + --cancel:#b78cff; --cancel-bg:#181328; --cancel-line:#332748; + --radius: 2px; + --shadow: none; + --mono: ui-monospace, "SF Mono", "JetBrains Mono", "Cascadia Code", Menlo, Consolas, monospace; + --glow: 0 0 7px; /* phosphor glow */ + color-scheme: dark; } -@media (prefers-color-scheme: dark) { + +/* ── light mode : "daylight printout" ──────────────────────────────────────── + Same mono / hard-edges / flame identity, but no phosphor glow or CRT + scanlines (they only read on black). Status inks go dark for paper contrast; + the palette is its own validated set, not a flipped dark theme. */ +@media (prefers-color-scheme: light) { :root { - --bg: #0d1117; --surface: #161b22; --surface-2: #1c2128; --border: #30363d; - --text: #e6edf3; --muted: #8b949e; --accent: #58a6ff; - --pass: #3fb950; --pass-bg: #12261a; --pass-line: #23552f; - --fail: #f85149; --fail-bg: #2c1518; --fail-line: #5c2a2a; - --run: #d29922; --run-bg: #2a230f; --run-line: #574a1e; - --queue: #8b949e; --queue-bg: #1c2128; --queue-line: #363c44; - --skip: #8b949e; --skip-bg: #1a1f26; --skip-line: #363c44; - --cancel: #a371f7; --cancel-bg: #201a2c; --cancel-line: #3d3357; - --shadow: 0 1px 2px rgba(0,0,0,.4), 0 3px 14px rgba(0,0,0,.3); + --bg: #f4f2ee; /* warm paper */ + --surface: #ffffff; + --surface-2: #efece6; + --border: #ddd7cd; + --border-bri: #c7bfb1; + --text: #1f2328; /* ink */ + --muted: #6b7280; + --accent: #d8431f; /* flame, darkened for contrast on paper */ + --accent-2: #ee4c2c; + --pass: #1a7f37; --pass-bg: #e8f5ec; --pass-line: #b7dcc0; + --fail: #970c28; --fail-bg: #fbe9ec; --fail-line: #edc2c9; /* deep crimson, ΔE 16.9 from the flame */ + --fail-glow: 151,12,40; + --run: #9a6700; --run-bg: #fbf3e0; --run-line: #e6d9a8; + --queue:#0969da; --queue-bg:#eaf2fd; --queue-line:#cfe0f5; + --skip: #6b7280; --skip-bg: #eef0f2; --skip-line: #d7dbe0; + --cancel:#7c3aed; --cancel-bg:#f2ecfc; --cancel-line:#dccff5; + --shadow: none; + --glow: 0 0 0; /* no phosphor in daylight */ + color-scheme: light; } + /* drop the CRT scanlines; keep only a faint flame wash up top */ + body { background-image: radial-gradient(140% 90% at 50% -20%, rgba(216,67,31,.05), transparent 55%); } } * { box-sizing: border-box; } body { margin: 0; background: var(--bg); color: var(--text); - font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + font: 13px/1.55 var(--mono); letter-spacing: .1px; + /* faint flame glow up top + hairline scanlines = CRT */ + background-image: + radial-gradient(140% 90% at 50% -20%, rgba(238,76,44,.06), transparent 55%), + repeating-linear-gradient(0deg, rgba(205,217,229,.014) 0 1px, transparent 1px 3px); + background-attachment: fixed; } a { color: var(--accent); text-decoration: none; } -a:hover { text-decoration: underline; } +a:hover { color: var(--accent-2); text-decoration: underline; } button, input { font: inherit; color: inherit; } +code, .mono { font-family: var(--mono); } -/* header */ +/* ── header : system titlebar ─────────────────────────────────────────────── */ header { position: sticky; top: 0; z-index: 20; background: var(--surface); - border-bottom: 1px solid var(--border); padding: 10px 18px; - display: flex; align-items: center; gap: 14px; flex-wrap: wrap; + border-bottom: 1px solid var(--border); box-shadow: inset 0 2px 0 var(--accent); + padding: 9px 16px; display: flex; align-items: center; gap: 14px; flex-wrap: wrap; } -header h1 { font-size: 15px; margin: 0; font-weight: 650; letter-spacing: .2px; } -header h1 .dim { color: var(--muted); font-weight: 400; } +header h1 { font-size: 12.5px; margin: 0; font-weight: 600; letter-spacing: 1px; + text-transform: uppercase; display: inline-flex; align-items: center; gap: 7px; } +header h1::before { content: "▸"; color: var(--accent); text-shadow: var(--glow) var(--accent); } +header h1 .dim { color: var(--accent); font-weight: 700; } +.cursor { width: 8px; height: 15px; background: var(--accent); display: inline-block; + box-shadow: var(--glow) var(--accent); animation: blink 1.1s steps(1) infinite; } +@keyframes blink { 50% { opacity: 0; } } + .controls { display: flex; align-items: center; gap: 8px; margin-left: auto; flex-wrap: wrap; } .controls input[type=text] { - background: var(--bg); border: 1px solid var(--border); border-radius: 8px; - padding: 6px 10px; width: 190px; + background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); + padding: 6px 10px; width: 190px; font-family: var(--mono); color: var(--text); } -.controls input[type=text]:focus { outline: 2px solid var(--accent); border-color: transparent; } +.controls input[type=text]::placeholder { color: var(--muted); } +.controls input[type=text]:focus { outline: none; border-color: var(--accent); + box-shadow: 0 0 0 1px var(--accent), var(--glow) rgba(238,76,44,.35); } .btn { - background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; - padding: 6px 12px; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; + background: var(--surface-2); border: 1px solid var(--border); border-radius: var(--radius); + padding: 5px 11px; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; + font-size: 12px; text-transform: uppercase; letter-spacing: .05em; color: var(--muted); } -.btn:hover { border-color: var(--accent); } -.btn.primary { background: var(--accent); color: #fff; border-color: transparent; } -.btn.primary:hover { filter: brightness(1.08); } -.toggle { display: inline-flex; align-items: center; gap: 6px; color: var(--muted); font-size: 13px; cursor: pointer; } - -main { max-width: 1280px; margin: 0 auto; padding: 18px; } - -/* summary strip */ -.summary { display: flex; gap: 10px; flex-wrap: wrap; margin-bottom: 8px; align-items: center; } -.stat { display: inline-flex; align-items: center; gap: 7px; padding: 5px 12px; border-radius: 999px; - border: 1px solid var(--border); background: var(--surface); font-weight: 600; } -.stat .n { font-variant-numeric: tabular-nums; } +.btn:hover { border-color: var(--accent); color: var(--accent); } +.btn.primary { background: var(--accent); color: #0b0b0b; border-color: transparent; font-weight: 700; } +.btn.primary:hover { background: var(--accent-2); box-shadow: var(--glow) rgba(238,76,44,.4); } +.toggle { display: inline-flex; align-items: center; gap: 6px; color: var(--muted); + font-size: 12px; text-transform: uppercase; letter-spacing: .04em; cursor: pointer; } +.toggle input { accent-color: var(--accent); } + +main { max-width: 1320px; margin: 0 auto; padding: 18px; } + +/* ── summary strip : status bar ───────────────────────────────────────────── */ +.summary { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 8px; align-items: center; } +.stat { display: inline-flex; align-items: center; gap: 7px; padding: 4px 11px; border-radius: var(--radius); + border: 1px solid var(--border); background: var(--surface); font-weight: 600; + text-transform: uppercase; letter-spacing: .04em; font-size: 12px; } +.stat .n { font-variant-numeric: tabular-nums; font-weight: 700; } .stat.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } .stat.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } .stat.run { color: var(--run); border-color: var(--run-line); background: var(--run-bg); } .stat.skip { color: var(--skip); border-color: var(--skip-line); background: var(--skip-bg); border-style: dashed; } -.commit { color: var(--muted); font-size: 13px; margin-left: auto; } -.commit code { font-family: var(--mono); color: var(--text); } - -/* dot / pill */ -.dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; flex: none; } -.dot.pass { background: var(--pass); } -.dot.fail { background: var(--fail); } -.dot.run { background: var(--run); animation: pulse 1.4s ease-in-out infinite; } -.dot.queue{ background: var(--queue); } +.commit { color: var(--muted); font-size: 12px; margin-left: auto; text-transform: uppercase; letter-spacing: .04em; } +.commit code { font-family: var(--mono); color: var(--accent); text-transform: none; } + +/* ── status lights ────────────────────────────────────────────────────────── */ +.dot { width: 8px; height: 8px; border-radius: 50%; display: inline-block; flex: none; } +.dot.pass { background: var(--pass); box-shadow: var(--glow) var(--pass); } +.dot.fail { background: var(--fail); box-shadow: var(--glow) rgba(var(--fail-glow),.95); } +.dot.run { background: var(--run); box-shadow: var(--glow) var(--run); animation: pulse 1.2s ease-in-out infinite; } +.dot.queue{ background: var(--queue); box-shadow: var(--glow) var(--queue); } .dot.skip { background: transparent; box-shadow: inset 0 0 0 1.5px var(--skip); } -.dot.cancel { background: var(--cancel); } -@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .35; } } +.dot.cancel { background: var(--cancel); box-shadow: var(--glow) var(--cancel); } +@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .3; } } -/* section headings */ -h2.sec { font-size: 12px; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); - margin: 22px 0 10px; font-weight: 700; } +/* section headings : readout markers */ +h2.sec { font-size: 11px; text-transform: uppercase; letter-spacing: .12em; color: var(--muted); + margin: 22px 0 10px; font-weight: 700; padding-left: 9px; border-left: 2px solid var(--accent); } -/* aggregate failures panel */ +/* ── aggregate failures panel ─────────────────────────────────────────────── */ .agg { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); overflow: hidden; margin-bottom: 8px; } .agg-row-d { border-top: 1px solid var(--border); } @@ -94,157 +145,221 @@ h2.sec { font-size: 12px; text-transform: uppercase; letter-spacing: .08em; colo padding: 11px 14px; cursor: pointer; } .agg-row:hover { background: var(--surface-2); } .agg-head-row { display: flex; align-items: center; gap: 10px; color: var(--muted); - font-size: 13px; margin: 2px 0 8px; } + font-size: 12px; margin: 2px 0 8px; text-transform: uppercase; letter-spacing: .04em; } .agg-loading { color: var(--muted); padding: 12px 2px; } -.agg-ok { color: var(--pass); font-weight: 600; padding: 12px 2px; } -.btn.small { padding: 3px 9px; font-size: 12px; margin-left: auto; } +.agg-ok { color: var(--pass); font-weight: 700; padding: 12px 2px; text-transform: uppercase; + letter-spacing: .06em; text-shadow: var(--glow) rgba(53,208,127,.4); } +.btn.small { padding: 3px 9px; font-size: 11px; margin-left: auto; } #content.filter-fail .card:not(.fail) { display: none; } -.agg-test { font-family: var(--mono); font-size: 13px; word-break: break-all; } +.agg-test { font-family: var(--mono); font-size: 13px; word-break: break-all; color: var(--text); } .agg-file { color: var(--muted); font-size: 12px; margin-top: 2px; } -.agg-file code { font-family: var(--mono); } +.agg-file code { font-family: var(--mono); color: var(--accent); } .agg-count { display: inline-flex; align-items: center; gap: 6px; white-space: nowrap; - color: var(--fail); font-weight: 650; } + color: var(--fail); font-weight: 700; text-transform: uppercase; letter-spacing: .03em; } .agg-detail { padding: 10px 14px 12px 14px; border-top: 1px dashed var(--border); background: var(--surface-2); font-size: 13px; } -.agg-detail pre { background: var(--bg); border: 1px solid var(--border); border-radius: 8px; - padding: 10px; overflow: auto; font-family: var(--mono); font-size: 12px; margin: 10px 0; max-height: 240px; } +.agg-detail pre { background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); + padding: 10px; overflow: auto; font-family: var(--mono); font-size: 12px; margin: 10px 0; max-height: 240px; + color: var(--fail); } .chips { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 8px; } -.chip { font-size: 11px; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--border); +.chip { font-size: 11px; padding: 2px 8px; border-radius: var(--radius); border: 1px solid var(--border); background: var(--surface); color: var(--muted); } -/* failure category tags — colour encodes triage kind: - bug=real regression (red) · infra=OOM/resource (purple) · env=setup/feature gap (amber) */ -.cat { display: inline-block; font-size: 10.5px; font-weight: 650; letter-spacing: .02em; - padding: 1px 7px; border-radius: 999px; border: 1px solid transparent; - vertical-align: middle; white-space: nowrap; } +/* ── failure category tags — colour encodes triage kind ─────────────────────── + bug=regression(red) · infra=OOM/resource(purple) · env=setup/gap(amber) · + always paired with the label text, so identity is never colour-alone. */ +.cat { display: inline-block; font-size: 10px; font-weight: 700; letter-spacing: .06em; + text-transform: uppercase; padding: 1px 6px; border-radius: var(--radius); + border: 1px solid transparent; vertical-align: middle; white-space: nowrap; } .cat.bug { color: var(--fail); background: var(--fail-bg); border-color: var(--fail-line); } .cat.infra { color: var(--cancel); background: var(--cancel-bg); border-color: var(--cancel-line); } .cat.env { color: var(--run); background: var(--run-bg); border-color: var(--run-line); } .cat.unknown { color: var(--skip); background: var(--skip-bg); border-color: var(--skip-line); } /* config-correlation hint (e.g. "RTX only", "cu132 only") */ -.cfgpat { display: inline-block; font-size: 10.5px; font-weight: 600; padding: 1px 7px; - border-radius: 999px; margin-left: 4px; vertical-align: middle; white-space: nowrap; - color: var(--accent); background: transparent; border: 1px dashed var(--accent); } +.cfgpat { display: inline-block; font-size: 10px; font-weight: 600; letter-spacing: .04em; + text-transform: uppercase; padding: 1px 6px; border-radius: var(--radius); margin-left: 4px; + vertical-align: middle; white-space: nowrap; color: var(--accent); background: transparent; + border: 1px dashed var(--accent); } .agg-tally { display: inline-flex; gap: 6px; flex-wrap: wrap; align-items: center; } .agg-tally .cat { font-weight: 700; } -/* platform board */ -.board { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 12px; } +/* ── platform board ───────────────────────────────────────────────────────── */ +.board { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 10px; } .card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); overflow: hidden; } -.card.fail { border-color: var(--fail-line); } -.card-head { display: flex; align-items: center; gap: 10px; padding: 12px 14px; cursor: pointer; } +.card.fail { border-color: var(--fail-line); box-shadow: inset 3px 0 0 var(--fail); } +.card-head { display: flex; align-items: center; gap: 10px; padding: 11px 14px; cursor: pointer; } .card-head:hover { background: var(--surface-2); } -.card-title { font-weight: 620; flex: 1; min-width: 0; } -.card-title .sub { color: var(--muted); font-weight: 400; font-size: 12px; margin-top: 1px; } -.card-badge { font-size: 12px; font-weight: 650; padding: 3px 9px; border-radius: 999px; white-space: nowrap; } +.card-title { font-weight: 650; flex: 1; min-width: 0; letter-spacing: .02em; } +.card-title .sub { color: var(--muted); font-weight: 400; font-size: 11px; margin-top: 1px; + text-transform: uppercase; letter-spacing: .03em; } +.card-badge { font-size: 11px; font-weight: 700; padding: 3px 9px; border-radius: var(--radius); + white-space: nowrap; text-transform: uppercase; letter-spacing: .04em; } .card-badge.pass { color: var(--pass); background: var(--pass-bg); border: 1px solid var(--pass-line); } -.card-badge.fail { color: var(--fail); background: var(--fail-bg); border: 1px solid var(--fail-line); } +.card-badge.fail { color: var(--fail); background: var(--fail-bg); border: 1px solid var(--fail-line); + text-shadow: var(--glow) rgba(var(--fail-glow),.5); } .card-badge.run { color: var(--run); background: var(--run-bg); border: 1px solid var(--run-line); } .card-badge.queue{ color: var(--queue);background: var(--queue-bg);border: 1px solid var(--queue-line); } .card-badge.skip { color: var(--skip); background: var(--skip-bg); border: 1px dashed var(--skip-line); } .card-badge.cancel { color: var(--cancel); background: var(--cancel-bg); border: 1px solid var(--cancel-line); } -.progress { height: 4px; background: var(--surface-2); display: flex; } +.progress { height: 3px; background: var(--surface-2); display: flex; } .progress > span { height: 100%; } -.progress .p-pass { background: var(--pass); } -.progress .p-fail { background: var(--fail); } +.progress .p-pass { background: var(--pass); box-shadow: var(--glow) var(--pass); } +.progress .p-fail { background: var(--fail); box-shadow: var(--glow) rgba(var(--fail-glow),.85); } .progress .p-run { background: var(--run); } .card-body { padding: 6px 14px 14px; border-top: 1px solid var(--border); } .card > summary { list-style: none; } .card > summary::-webkit-details-marker { display: none; } -.caret { color: var(--muted); transition: transform .15s; } +.caret { color: var(--accent); transition: transform .15s; } .card[open] .caret { transform: rotate(90deg); } -/* job groups + matrix */ +/* ── job groups + matrix ──────────────────────────────────────────────────── */ .jobgroup { margin-top: 12px; } -.jobgroup h3 { font-size: 12px; font-weight: 650; margin: 0 0 6px; display: flex; align-items: center; gap: 8px; } -.jobgroup h3 .tierpath { color: var(--muted); font-weight: 400; font-family: var(--mono); font-size: 11px; } -.matrix { display: flex; gap: 6px; flex-wrap: wrap; } -.cell { min-width: 84px; border: 1px solid var(--border); border-radius: 8px; padding: 6px 9px; - display: flex; flex-direction: column; gap: 3px; cursor: default; background: var(--surface); } +.jobgroup h3 { font-size: 11px; font-weight: 700; margin: 0 0 6px; display: flex; align-items: center; + gap: 8px; text-transform: uppercase; letter-spacing: .06em; } +.jobgroup h3 .tierpath { color: var(--muted); font-weight: 400; font-family: var(--mono); + font-size: 11px; text-transform: none; letter-spacing: 0; } +.matrix { display: flex; gap: 5px; flex-wrap: wrap; } +.cell { min-width: 84px; border: 1px solid var(--border); border-radius: var(--radius); padding: 6px 9px; + display: flex; flex-direction: column; gap: 3px; cursor: default; background: var(--surface-2); } .cell.pass { background: var(--pass-bg); border-color: var(--pass-line); } .cell.fail { background: var(--fail-bg); border-color: var(--fail-line); cursor: pointer; } -.cell.fail:hover { outline: 2px solid var(--fail); } +.cell.fail:hover { outline: 1px solid var(--fail); box-shadow: var(--glow) rgba(var(--fail-glow),.45); } .cell.run { background: var(--run-bg); border-color: var(--run-line); } -.cell.queue{ background: var(--queue-bg); } +.cell.queue{ background: var(--queue-bg); border-color: var(--queue-line); } /* "didn't run" — dashed + muted so it reads as inactive, never as pass or fail */ .cell.skip { background: var(--surface); border-style: dashed; border-color: var(--skip-line); color: var(--muted); } .cell.skip .k, .cell.cancel .k { color: var(--muted); font-weight: 500; } .cell.cancel { background: var(--cancel-bg); border-style: dashed; border-color: var(--cancel-line); } -.cell .k { font-size: 11px; font-weight: 650; font-variant-numeric: tabular-nums; display: flex; gap: 5px; align-items: center; } -.cell .v { font-size: 10px; color: var(--muted); font-family: var(--mono); } +.cell .k { font-size: 11px; font-weight: 700; font-variant-numeric: tabular-nums; display: flex; gap: 5px; align-items: center; } +.cell .v { font-size: 10px; color: var(--muted); font-family: var(--mono); text-transform: uppercase; letter-spacing: .03em; } .rows { display: flex; flex-direction: column; gap: 4px; } -.jobrow { display: flex; align-items: center; gap: 8px; padding: 4px 6px; border-radius: 6px; font-size: 13px; } +.jobrow { display: flex; align-items: center; gap: 8px; padding: 4px 6px; border-radius: var(--radius); font-size: 13px; } .jobrow:hover { background: var(--surface-2); } .jobrow .name { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .jobrow.fail .name { color: var(--fail); } -.jobstate { font-size: 11px; padding: 1px 8px; border-radius: 999px; white-space: nowrap; color: var(--muted); border: 1px solid var(--border); } +.jobstate { font-size: 10px; padding: 1px 8px; border-radius: var(--radius); white-space: nowrap; + color: var(--muted); border: 1px solid var(--border); text-transform: uppercase; letter-spacing: .04em; } .jobstate.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } .jobstate.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } .jobstate.skip { border-style: dashed; } .jobstate.cancel { color: var(--cancel); border-color: var(--cancel-line); border-style: dashed; } -/* drawer */ -.drawer-bg { position: fixed; inset: 0; background: rgba(0,0,0,.35); z-index: 40; display: none; } +/* ── drawer ───────────────────────────────────────────────────────────────── */ +.drawer-bg { position: fixed; inset: 0; background: rgba(2,4,6,.6); z-index: 40; display: none; } .drawer-bg.open { display: block; } -.drawer { position: fixed; top: 0; right: 0; height: 100%; width: min(560px, 94vw); z-index: 41; - background: var(--surface); border-left: 1px solid var(--border); box-shadow: -8px 0 30px rgba(0,0,0,.2); +.drawer { position: fixed; top: 0; right: 0; height: 100%; width: min(580px, 94vw); z-index: 41; + background: var(--surface); border-left: 1px solid var(--accent); transform: translateX(100%); transition: transform .2s ease; display: flex; flex-direction: column; } .drawer.open { transform: translateX(0); } -.drawer-head { padding: 14px 16px; border-bottom: 1px solid var(--border); display: flex; align-items: flex-start; gap: 10px; } +.drawer-head { padding: 13px 16px; border-bottom: 1px solid var(--border); display: flex; align-items: flex-start; gap: 10px; } .drawer-head .t { flex: 1; } -.drawer-head .t .title { font-weight: 650; } +.drawer-head .t .title { font-weight: 700; text-transform: uppercase; letter-spacing: .05em; color: var(--accent); } .drawer-head .t .sub { color: var(--muted); font-size: 12px; margin-top: 2px; } .drawer-head .x { cursor: pointer; border: none; background: none; color: var(--muted); font-size: 20px; line-height: 1; } +.drawer-head .x:hover { color: var(--accent); } .drawer-body { padding: 14px 16px; overflow: auto; } -.fail-item { border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; margin-bottom: 10px; background: var(--surface); } +.fail-item { border: 1px solid var(--border); border-radius: var(--radius); padding: 10px 12px; + margin-bottom: 10px; background: var(--surface); border-left: 2px solid var(--fail); } .fail-item .ftest { font-family: var(--mono); font-size: 13px; font-weight: 600; word-break: break-all; } .fail-item .floc { font-size: 12px; margin-top: 4px; } .fail-item .floc code { font-family: var(--mono); } -.fail-item pre { background: var(--bg); border: 1px solid var(--border); border-radius: 6px; - padding: 9px; font-family: var(--mono); font-size: 12px; overflow: auto; max-height: 260px; margin: 8px 0 0; white-space: pre-wrap; } -.repro { background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; margin-top: 10px; } -.repro .lbl { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); margin-bottom: 6px; } -.repro code { font-family: var(--mono); font-size: 12px; word-break: break-all; display: block; } -.copy { float: right; font-size: 11px; cursor: pointer; color: var(--accent); background: none; border: none; } +.fail-item pre { background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); + padding: 9px; font-family: var(--mono); font-size: 12px; overflow: auto; max-height: 260px; + margin: 8px 0 0; white-space: pre-wrap; color: #ec97a3; } +.repro { background: var(--surface-2); border: 1px solid var(--border); border-radius: var(--radius); + padding: 10px 12px; margin-top: 10px; box-shadow: inset 2px 0 0 var(--accent); } +.repro .lbl { font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: var(--accent); margin-bottom: 6px; } +.repro code { font-family: var(--mono); font-size: 12px; word-break: break-all; display: block; color: var(--text); } +.repro code::before { content: "$ "; color: var(--muted); } +.copy { float: right; font-size: 10px; cursor: pointer; color: var(--accent); background: none; border: none; + text-transform: uppercase; letter-spacing: .05em; } +.copy:hover { color: var(--accent-2); } /* relevant logs (in drawer) */ .logsec { margin-top: 14px; border-top: 1px solid var(--border); padding-top: 12px; } .logsec-head { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--muted); margin-bottom: 8px; flex-wrap: wrap; } -.logsec-head .lbl { text-transform: uppercase; letter-spacing: .06em; font-weight: 600; +.logsec-head .lbl { text-transform: uppercase; letter-spacing: .08em; font-weight: 700; color: var(--text); margin-right: auto; } -.logblock { border: 1px solid var(--border); border-radius: 8px; margin-bottom: 8px; overflow: hidden; } +.logblock { border: 1px solid var(--border); border-radius: var(--radius); margin-bottom: 8px; overflow: hidden; } .logblock > summary { cursor: pointer; padding: 8px 12px; font-family: var(--mono); font-size: 12px; font-weight: 600; background: var(--surface-2); word-break: break-all; list-style: none; } .logblock > summary::-webkit-details-marker { display: none; } -.logblock > summary::before { content: "▸ "; color: var(--muted); } +.logblock > summary::before { content: "▸ "; color: var(--accent); } .logblock[open] > summary::before { content: "▾ "; } .logblock .logreason { padding: 8px 12px 0; font-family: var(--mono); font-size: 12px; color: var(--fail); word-break: break-word; } +.logblock .logmeta { padding: 4px 12px 0; font-family: var(--mono); font-size: 11px; color: var(--muted); word-break: break-word; } .logblock pre { margin: 8px 12px 12px; background: var(--bg); border: 1px solid var(--border); - border-radius: 6px; padding: 10px; font-family: var(--mono); font-size: 12px; line-height: 1.45; + border-radius: var(--radius); padding: 10px; font-family: var(--mono); font-size: 12px; line-height: 1.45; overflow: auto; max-height: 420px; white-space: pre; } +/* root-cause collapse: one banner instead of N identical blocks */ +.rootbanner { border: 1px solid var(--fail-line); background: var(--fail-bg); border-radius: var(--radius); + padding: 9px 12px; margin-bottom: 8px; font-size: 12.5px; color: var(--text); line-height: 1.5; } +.rootbanner code { font-family: var(--mono); color: rgb(var(--fail-glow)); word-break: break-word; } +.rootbanner strong { color: rgb(var(--fail-glow)); } +.logblock.rootcause { border-color: var(--fail-line); } +.logblock.rootcause > summary { background: var(--fail-bg); color: rgb(var(--fail-glow)); } +.logblock.rootcause > summary::before { color: rgb(var(--fail-glow)); } + +/* log toolbar: search + filter, sticky above the blocks */ +.logtools { position: sticky; top: 0; z-index: 3; display: flex; align-items: center; gap: 10px; + padding: 8px 0; margin-bottom: 6px; background: var(--surface); flex-wrap: wrap; } +.logfind { flex: 1; min-width: 140px; font-family: var(--mono); font-size: 12px; padding: 6px 9px; + background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); } +.logfind:focus { outline: none; border-color: var(--accent); } +.logfind::placeholder { color: var(--muted); } +.logfilt { display: inline-flex; align-items: center; gap: 5px; font-size: 11px; color: var(--muted); + text-transform: uppercase; letter-spacing: .04em; white-space: nowrap; cursor: pointer; } +.logcount { font-size: 11px; white-space: nowrap; } + +/* per-block error jump-list */ +.logjumps { display: flex; flex-wrap: wrap; align-items: center; gap: 5px; padding: 8px 12px 0; } +.logjumps .lbl { font-size: 10px; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); } +.jump { font-family: var(--mono); font-size: 11px; max-width: 100%; padding: 2px 7px; cursor: pointer; + color: rgb(var(--fail-glow)); background: var(--fail-bg); border: 1px solid var(--fail-line); + border-radius: var(--radius); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.jump:hover { border-color: var(--fail); box-shadow: var(--glow) rgba(var(--fail-glow), .35); } + +/* per-line spans: severity colour + search highlight */ +.logtext .ll { display: block; } +.logtext .ll[data-sev="err"] { color: rgb(var(--fail-glow)); box-shadow: inset 2px 0 0 var(--fail); } +.logtext .ll[data-sev="warn"] { color: var(--run); box-shadow: inset 2px 0 0 var(--run); } +.logtext .ll[data-sev="pass"] { color: var(--pass); } +.logtext .ll.match { background: rgba(245, 166, 35, .22); } +.logtext.filtered .ll:not(.match) { display: none; } +.logtext .ll.flash { animation: llflash .9s ease-out; } +@keyframes llflash { 0% { background: rgba(245, 166, 35, .55); } 100% { background: transparent; } } + .muted { color: var(--muted); } .spinner { display: inline-block; width: 13px; height: 13px; border: 2px solid var(--border); border-top-color: var(--accent); border-radius: 50%; animation: spin .7s linear infinite; } @keyframes spin { to { transform: rotate(360deg); } } -.empty-state { text-align: center; color: var(--muted); padding: 60px 20px; } -.toast { position: fixed; bottom: 20px; left: 50%; transform: translateX(-50%); background: var(--text); - color: var(--bg); padding: 8px 16px; border-radius: 8px; font-size: 13px; z-index: 60; opacity: 0; - transition: opacity .2s; pointer-events: none; } +.empty-state { text-align: center; color: var(--muted); padding: 60px 20px; text-transform: uppercase; letter-spacing: .06em; } +.toast { position: fixed; bottom: 20px; left: 50%; transform: translateX(-50%); background: var(--accent); + color: #0b0b0b; padding: 7px 16px; border-radius: var(--radius); font-size: 12px; font-weight: 700; z-index: 60; + opacity: 0; transition: opacity .2s; pointer-events: none; text-transform: uppercase; letter-spacing: .06em; + box-shadow: var(--glow) rgba(238,76,44,.5); } .toast.show { opacity: 1; } -.legend { display: flex; gap: 14px; color: var(--muted); font-size: 12px; margin: 4px 0 0; flex-wrap: wrap; } +.legend { display: flex; gap: 14px; color: var(--muted); font-size: 11px; margin: 4px 0 0; flex-wrap: wrap; + text-transform: uppercase; letter-spacing: .04em; } .legend span { display: inline-flex; align-items: center; gap: 5px; } +.legend .muted { text-transform: none; letter-spacing: 0; } -/* PR command reference (collapsible) */ +/* ── PR command reference (collapsible) ───────────────────────────────────── */ .cmds { margin: 8px 0 0; font-size: 12px; } -.cmds > summary { cursor: pointer; color: var(--muted); user-select: none; padding: 2px 0; } -.cmds > summary:hover { color: var(--text); } +.cmds > summary { cursor: pointer; color: var(--muted); user-select: none; padding: 2px 0; + text-transform: uppercase; letter-spacing: .05em; } +.cmds > summary::marker, .cmds > summary::-webkit-details-marker { color: var(--accent); } +.cmds > summary:hover { color: var(--accent); } .cmd-list { display: flex; flex-direction: column; gap: 6px; margin: 8px 0 2px; padding: 10px 12px; - background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; } + background: var(--surface-2); border: 1px solid var(--border); border-radius: var(--radius); + box-shadow: inset 2px 0 0 var(--accent); } .cmd { display: flex; align-items: center; gap: 10px; } -.cmd code { font-family: var(--mono); font-size: 12px; color: var(--text); background: var(--surface); - border: 1px solid var(--border); border-radius: 6px; padding: 2px 8px; white-space: nowrap; min-width: 82px; } +.cmd > code { font-family: var(--mono); font-size: 12px; color: var(--accent); background: var(--bg); + border: 1px solid var(--border); border-radius: var(--radius); padding: 2px 8px; white-space: nowrap; min-width: 96px; } +.cmd > code::before { content: "$ "; color: var(--muted); } .cmd .copy { float: none; } .cmd .lbl { color: var(--muted); } +.cmd .lbl code { color: var(--text); background: var(--bg); padding: 0 4px; border-radius: var(--radius); } From a89ca8d69220a7b49517bab0becfc99f4113adb6 Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Tue, 21 Jul 2026 22:12:25 +0000 Subject: [PATCH 10/11] feat: Add test plan visualization --- .github/workflows/ci-linux-x86_64.yml | 9 +- tools/ci-dashboard/README.md | 16 +- tools/ci-dashboard/ci_dashboard.py | 491 +++++++++++++++++++++++++- tools/ci-dashboard/static/index.html | 232 +++++++++++- tools/ci-dashboard/static/style.css | 119 +++++++ 5 files changed, 843 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci-linux-x86_64.yml b/.github/workflows/ci-linux-x86_64.yml index 27f497eb11..11c1d03723 100644 --- a/.github/workflows/ci-linux-x86_64.yml +++ b/.github/workflows/ci-linux-x86_64.yml @@ -64,7 +64,7 @@ jobs: with-rocm: false with-cpu: false - # Standard — the only channel on the fast lane (every PR push). + # Standard — runs on every non-skip lane, incl. the fast lane (every PR push). standard: needs: [decide, generate-matrix] if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'rtx' @@ -84,9 +84,11 @@ jobs: name-prefix: "RTX - " raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + # python-only runs on EVERY non-skip lane incl. fast: PYTHON_ONLY=1 skips Bazel, + # so the wheel builds cheaply and each push smoke-tests the no-C++-runtime path. python-only: needs: [decide, generate-matrix] - if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'rtx' uses: ./.github/workflows/_test-linux.yml with: lane: python-only @@ -95,9 +97,10 @@ jobs: raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} # python-only against TensorRT-RTX (so backend=both runs BOTH python-only variants). + # Runs on any non-skip lane, but only when an RTX backend is selected. python-only-rtx: needs: [decide, generate-matrix] - if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'standard' uses: ./.github/workflows/_test-linux.yml with: lane: python-only diff --git a/tools/ci-dashboard/README.md b/tools/ci-dashboard/README.md index 728c18abd5..96c8e6115f 100644 --- a/tools/ci-dashboard/README.md +++ b/tools/ci-dashboard/README.md @@ -33,6 +33,13 @@ reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN U ## What you get +- **Test plan — "what will run"** — a panel that resolves the branch's PR labels + into the exact plan CI will execute, *before* it runs: `labels → lane + backend` + (mirroring `.github/workflows/_decide.yml`) → the per-platform channels (Linux + x86_64 / Windows / SBSA-build-only, standard + RTX + python-only) → the suites + each channel runs (from `tests.ci matrix`). It shows the job count and a hint for + which label widens it (`ci: full`, `ci: nightly`, `backend: TensorRT-RTX`). `main` + is shown as its push canary (full / both). - **Platform board** — one card per workflow (Linux x86_64, aarch64, Windows, the RTX and python-only variants, jetpack…), sorted worst-first, with a live status badge. A summary strip up top counts failing / running / queued / @@ -75,8 +82,13 @@ reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN U ## PR commands Comment these on a PR to control its CI without pushing a new commit (write -access required; handled by `.github/workflows/retrigger-ci.yml`). They're also -listed in the dashboard header under **PR commands** with copy buttons. +access required; handled by `.github/workflows/retrigger-ci.yml`). They're +listed in the dashboard header under **PR commands** with copy buttons, and the +**test-plan panel has live buttons that post them for you** — one click *arms* +the button, a second *confirms* and posts the comment via the dashboard's +authenticated `gh` (a `POST /api/pr-comment`, guarded by a strict command +whitelist). The two-step confirm keeps a stray click from launching or cancelling +a run. Since posting triggers real CI, this only appears when there's an open PR. | comment | effect | |---|---| diff --git a/tools/ci-dashboard/ci_dashboard.py b/tools/ci-dashboard/ci_dashboard.py index 9742b146db..828d851d77 100755 --- a/tools/ci-dashboard/ci_dashboard.py +++ b/tools/ci-dashboard/ci_dashboard.py @@ -56,6 +56,10 @@ from tests.ci import SUITES as _SUITES # noqa: E402 except Exception: # pragma: no cover — manifest absent on pre-migration branches _SUITES = () +try: + from tests.ci.runner import matrix as _ci_matrix # noqa: E402 +except Exception: # pragma: no cover + _ci_matrix = None # ── Legacy tier → source mapping (fallback) ────────────────────────────────── # Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not @@ -272,6 +276,229 @@ def default_branch(): return "main" +# ── test-plan preview: what a branch's labels will actually run ─────────────── +# This mirrors, in Python, exactly what CI derives from the triggering event: +# 1. labels → (lane, backend) — .github/workflows/_decide.yml +# 2. (lane, backend) → per-platform channels — the `if:` gates in the +# ci-linux-x86_64 / ci-windows / ci-sbsa entry workflows +# 3. (lane, variant, platform) → suites — `tests.ci matrix` (_test-linux.yml) +# Keep in sync with those files if the gating changes. + +# The CI-control labels (the ONLY ones that change what runs) + the axis each +# drives and a one-line effect. Order = display order. Kept in sync with _decide.yml. +CI_LABELS = [ + ("ci: full", "lane", "all tiers (L0–L2) on Linux + Windows"), + ("ci: nightly", "lane", "everything, incl. llm / kernels / distributed"), + ("backend: TensorRT", "std", "test the standard TensorRT engine"), + ("backend: TensorRT-RTX", "rtx", "test the TensorRT-RTX engine"), +] +_CI_LABEL_NAMES = {n for n, _, _ in CI_LABELS} +_INERT_LABEL = "Force All Tests[L0+L1+L2]" # replaced by ci: full / ci: nightly + + +def _ci_label_chips(labels, pr): + """The four CI-control labels as on/off toggles — clicking (then confirming) + adds/removes the label on the PR live. Disabled when there's no PR to edit. + (Adding a CI label triggers a run, hence the confirm.)""" + present = set(labels) + num = pr["number"] if pr else None + out = [] + for name, group, effect in CI_LABELS: + on = name in present + title = f'{"remove" if on else "add"} — {effect}' + out.append( + f'' + ) + return "".join(out) + + +def resolve_lane_backend(labels, event="pull_request"): + """labels → (lane, backend), per _decide.yml. `contains()` in Actions is an + exact array-element match, so the two backend labels never alias.""" + L = set(labels or []) + has_full, has_nightly = "ci: full" in L, "ci: nightly" in L + has_rtx, has_std = "backend: TensorRT-RTX" in L, "backend: TensorRT" in L + if event == "schedule": + lane = "nightly" + elif event == "push": # main canary + lane = "full" + else: # pull_request + lane = "nightly" if has_nightly else "full" if has_full else "fast" + if event != "pull_request": + backend = "both" + elif has_rtx and has_std: + backend = "both" + elif has_rtx: + backend = "rtx" + elif has_std: + backend = "standard" + elif lane == "fast": # cheap default on a plain PR push + backend = "standard" + else: + backend = "both" + return lane, backend + + +def compute_plan(lane, backend): + """The channels that will run for (lane, backend), each with its suite list. + Mirrors the entry workflows' channel `if:` gates. Returns [{platform, engine, + kind, suites}], suites empty for build-only channels.""" + if _ci_matrix is None: + return [] + run, fullish = lane != "skip", lane in ("full", "nightly") + std, rtx = backend != "rtx", backend != "standard" # which engine channels run + plan = [] + + def add(platform, engine, kind, suite_lane, suite_platform): + suites = ( + [ + m["suite"] + for m in _ci_matrix( + lane=suite_lane, variant=engine, platform=suite_platform + ) + ] + if suite_platform + else [] + ) + plan.append(dict(platform=platform, engine=engine, kind=kind, suites=suites)) + + # Linux x86_64 (ci-linux-x86_64.yml): tests AND python-only on any non-skip lane + # (incl. fast — PYTHON_ONLY=1 skips Bazel, so it's a cheap per-push smoke). + if run and std: + add("Linux x86_64", "standard", "tests", lane, "linux-x86_64") + if run and rtx: + add("Linux x86_64", "rtx", "tests", lane, "linux-x86_64") + if run and std: + add("Linux x86_64", "standard", "python-only", "python-only", "linux-x86_64") + if run and rtx: + add("Linux x86_64", "rtx", "python-only", "python-only", "linux-x86_64") + # Windows (ci-windows.yml): full/nightly only — never on the fast lane + if fullish and std: + add("Windows", "standard", "tests", lane, "windows") + if fullish and rtx: + add("Windows", "rtx", "tests", lane, "windows") + if fullish and std: + add("Windows", "standard", "python-only", "python-only", "windows") + if fullish and rtx: + add("Windows", "rtx", "python-only", "python-only", "windows") + # SBSA aarch64 (ci-sbsa.yml): BUILD-ONLY (no GPU runners), full/nightly, standard + if fullish and std: + add("Linux aarch64 · SBSA", "standard", "build-only", lane, None) + add( + "Linux aarch64 · SBSA", + "standard", + "build-only · py-only", + "python-only", + None, + ) + return plan + + +def pr_for(branch, refresh=False): + """The PR whose head is `branch` — an OPEN one if there is one, else the most + recent of any state (so a merged/closed PR's number still shows). Fields: + number/title/url/labels/baseRefName/state, or None. Cached briefly so the board + banner + plan panel share one lookup (a `{}` sentinel caches the 'no PR' answer).""" + key = f"pr:{branch}" + if not refresh and (c := CACHE.get(key)) is not None: + return c or None + try: + prs = gh( + [ + "pr", + "list", + "--head", + branch, + "--state", + "all", + "--json", + "number,title,url,labels,baseRefName,state", + "--limit", + "5", + ] + ) + pr = next((p for p in prs if p.get("state") == "OPEN"), prs[0] if prs else None) + except Exception: + pr = None + CACHE.put(key, pr or {}, ttl=120) + return pr + + +def resolve_pr_branch(num): + """A PR number → its head branch name, so `#4414` typed in the search box (or a + `?branch=%234414` URL) loads the right branch. Cached; None if no such PR.""" + num = str(num or "").lstrip("#").strip() + if not re.fullmatch(r"\d+", num): + return None + key = f"prbranch:{num}" + if (c := CACHE.get(key)) is not None: + return c or None + try: + b = gh( + [ + "pr", + "view", + num, + "--repo", + repo_slug(), + "--json", + "headRefName", + "-q", + ".headRefName", + ], + parse=False, + ).strip() + except Exception: + b = None + CACHE.put(key, b or "", ttl=300) + return b or None + + +# CI-control comment commands the dashboard can POST to a PR (handled by +# retrigger-ci.yml). (command, effect, is_destructive) — the whitelist is also the +# server-side guard, so only these exact strings can ever be posted. +CI_ACTIONS = [ + ("/rerun", "re-run only the failed / cancelled jobs", False), + ("/rerun all", "re-run every job from scratch", False), + ("/cancel", "cancel stale zombie runs on old commits", False), + ("/cancel all", "cancel EVERY in-flight run for the PR", True), + ("/test full", "run the full suite (all tiers, both engines)", False), + ("/test full rtx", "full suite, RTX engine only", False), + ("/test nightly", "everything incl. llm / kernels / distributed", False), +] +_ALLOWED_CMDS = {c for c, _, _ in CI_ACTIONS} + + +def post_pr_comment(pr, cmd): + """Post one whitelisted CI command as a PR comment (which triggers + retrigger-ci.yml). Raises on a bad PR number or non-whitelisted command — + the whitelist is the guard against arbitrary comments.""" + if cmd not in _ALLOWED_CMDS: + raise ValueError(f"command not allowed: {cmd!r}") + if not re.fullmatch(r"\d+", str(pr or "")): + raise ValueError(f"bad PR number: {pr!r}") + return gh( + ["pr", "comment", str(pr), "--repo", repo_slug(), "--body", cmd], parse=False + ).strip() + + +def set_pr_label(pr, label, add): + """Add or remove one CI-control label on a PR (adding triggers a run via the + `labeled` event). Whitelisted to the CI labels — the guard against editing + arbitrary labels. The client reloads with refresh=1 so the plan re-resolves.""" + if label not in _CI_LABEL_NAMES: + raise ValueError(f"label not allowed: {label!r}") + if not re.fullmatch(r"\d+", str(pr or "")): + raise ValueError(f"bad PR number: {pr!r}") + flag = "--add-label" if add else "--remove-label" + return gh( + ["pr", "edit", str(pr), "--repo", repo_slug(), flag, label], parse=False + ).strip() + + # ── data layer ─────────────────────────────────────────────────────────────── def get_runs(branch, limit=40, refresh=False): key = f"runs:{branch}:{limit}" @@ -309,9 +536,14 @@ def _parse_job(j): group_parts = rest if rest else (parts[:-1] if is_matrix else parts) group = " / ".join(group_parts) if group_parts else raw gl = _norm(group) + last_word = gl.rsplit(" ", 1)[-1] if gl else "" if "build-wheel" in last or gl.startswith("build "): kind = "build" - elif re.match(r"l[012]\b", gl): + elif ( + re.search(r"\bl[012]\b", gl) # tier token (old `L2 …` names, new-format tier) + or " / test (" in group # new ` / test (…)` format + or last_word in ("test", "tests") # descriptive `… dynamo runtime tests` groups + ): kind = "test" elif "matrix" in gl or "generate" in gl or group == "filter-matrix": kind = "setup" @@ -1023,9 +1255,18 @@ def _summary_html(runs, oob=False): counts[cls] = counts.get(cls, 0) + 1 didnt_run = counts["skip"] + counts["cancel"] head = runs[0] if runs else {} - sha = (head.get("headSha") or "")[:9] + full_sha = head.get("headSha") or "" + sha = full_sha[:9] + msg = head.get("displayTitle", "") + sha_html = ( + f'{e(sha)}' + if full_sha + else f"{e(sha)}" + ) commit = ( - f'
{e(sha)} · {e(head.get("displayTitle", ""))[:80]}' + f'
' + f"{sha_html} · {e(msg)[:80]}" f' · {e(rel_time(head.get("createdAt")))}
' ) @@ -1051,6 +1292,141 @@ def stat(cls, n, word): ) +def _plan_hint(lane, backend): + if lane == "fast": + h = ( + "fast = L0 smoke + python-only on Linux x86_64. Add ci: full " + "for all tiers on Linux + Windows, or ci: nightly to also run " + "llm / kernels / distributed." + ) + elif lane == "full": + h = ( + "full runs L0–L2. Add ci: nightly to also run " + "llm / kernels / distributed." + ) + else: + h = "nightly runs everything." + if backend == "standard": + h += " Add backend: TensorRT-RTX to also test RTX." + elif backend == "rtx": + h += " Add backend: TensorRT to also test standard." + return f'{h}' + + +def render_plan(branch, refresh=False): + """'What will run' panel: branch labels → lane/backend → the exact per-platform + channels + suites CI will execute. Cached briefly (labels change rarely).""" + key = f"plan:{branch}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + try: + html = _render_plan(branch) + except Exception as ex: # best-effort — never break the board + html = ( + f'
plan unavailable: {e(ex)}
' + ) + CACHE.put(key, html, ttl=120) + return html + + +def _render_plan(branch): + pr = None + if branch == "main": # the push canary — not any head=main PR + labels, event, src = [], "push", "push → main" + elif pr := pr_for(branch): + labels = [l.get("name", "") for l in pr.get("labels", [])] + event, src = "pull_request", f'PR #{pr["number"]}' + else: + labels, event, src = [], "pull_request", "no open PR" + lane, backend = resolve_lane_backend(labels, event) + plan = compute_plan(lane, backend) + njobs = sum(len(c["suites"]) for c in plan) + + srchtml = ( + f'{e(src)} ↗' + if pr + else e(src) + ) + # CI-control labels as on/off state; the rest shown muted as "other". + if event == "push": + labelblock = ( + '
push to main — ' + "runs the full lane on both engines regardless of labels.
" + ) + else: + others = [x for x in labels if x not in _CI_LABEL_NAMES and x != _INERT_LABEL] + inert = ( + f'' + if _INERT_LABEL in labels + else "" + ) + other_html = ( + ( + '
other: ' + + " ".join(f'{e(x)}' for x in others) + + "
" + ) + if others + else "" + ) + labelblock = ( + f'
CI labels' + f"{_ci_label_chips(labels, pr)}{inert}
" + f'
{_plan_hint(lane, backend)}
{other_html}' + ) + + # Live CI-control buttons: post a command as a PR comment (with a click-to-confirm + # guard so a stray click can't trigger a run). Only when there's a PR to post to. + actions_html = "" + if pr: + acts = "".join( + f'" + for cmd, desc, danger in CI_ACTIONS + ) + actions_html = ( + f'
CI actions{acts}' + f'click, then confirm — posts to ' + f'PR #{pr["number"]}
' + ) + labelblock += actions_html + rows = [] + for c in plan: + if c["suites"]: + detail, cnt = ( + f'{e(", ".join(c["suites"]))}', + str(len(c["suites"])), + ) + else: + detail, cnt = 'wheel build · no tests', "—" + rows.append( + f'{e(c["platform"])}' + f'{e(c["engine"])}' + f'{e(c["kind"])}' + f'{cnt}{detail}' + ) + table = ( + ( + '' + "" + f'{"".join(rows)}
platformenginekind#suites
' + ) + if rows + else '
Manifest not available on this checkout.
' + ) + summary = ( + f'test plan' + f'lane {e(lane)} · backend {e(backend)}' + f'{njobs} test jobs · {srchtml}' + ) + return ( + f'
{summary}' + f'
{labelblock}{table}
' + ) + + def render_board(branch, refresh=False): try: runs = get_runs(branch, refresh=refresh) @@ -1103,7 +1479,28 @@ def render_board(branch, refresh=False): f'
scanning failing tests…
' ) board = f'

Platforms

{"".join(cards)}
' - return summary + agg + board + poller + # main is the push canary, not a PR (ignore any stray head=main fork PR) — matches + # render_plan. pr_for still warms the cache render_plan reuses. + pr = pr_for(branch, refresh=refresh) if branch != "main" else None + banner = "" + if pr: + base = pr.get("baseRefName", "") + state = (pr.get("state") or "").upper() + state_tag = ( + f'{e(state.lower())}' + if state and state != "OPEN" + else "" + ) + base_html = f'→ {e(base)}' if base else "" + banner = ( + f'
' + f'PR #{pr["number"]} ↗{state_tag}{base_html}' + f'{e(pr.get("title", ""))}
' + ) + return ( + banner + summary + render_plan(branch, refresh=refresh) + agg + board + poller + ) def render_status(branch): @@ -1126,6 +1523,19 @@ def render_status(branch): KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4} +def fail_kind(job): + """A FAILED test/build job → 'test' (it produced pytest failure annotations, i.e. + real test failures) or 'infra' (build/setup/env/OOM/runner died with no test + verdict). Build failures are always infra; test jobs are probed via their (cached) + annotations. Lets the grid show a broken runner differently from a broken test.""" + if job.get("kind") != "test": + return "infra" + try: + return "test" if get_failures(str(job["id"])).get("failures") else "infra" + except Exception: + return "test" # if we truly can't tell, don't cry wolf about infra + + def render_platform(run_id, sha, platform, refresh=False): try: jobs = get_jobs(run_id, refresh=refresh) @@ -1134,6 +1544,20 @@ def render_platform(run_id, sha, platform, refresh=False): if not jobs: return '
No jobs.
' + # Split test failures from infra/runner failures for the failing cells — one + # cached annotation call each, fanned out so a platform with many reds stays snappy. + failing = [ + j + for j in jobs + if classify(j["status"], j["conclusion"])[0] == "fail" + and j["kind"] in ("test", "build") + ] + fk = {} + if failing: + with ThreadPoolExecutor(max_workers=8) as ex: + for j, k in zip(failing, ex.map(fail_kind, failing)): + fk[j["id"]] = k + # group by (kind, group) groups = {} for j in jobs: @@ -1153,6 +1577,12 @@ def render_platform(run_id, sha, platform, refresh=False): for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")): cls, label = classify(j["status"], j["conclusion"]) failable = cls == "fail" and j["kind"] in ("test", "build") + infra = failable and fk.get(j["id"]) == "infra" + if infra: + label = "infra" + fcls = cls + ( + " infra" if infra else "" + ) # add .infra shading when a runner (not a test) died if j["python"] and j["cuda"]: inner = ( f'{e(j["python"])}·{e(j["cuda"])}' @@ -1169,11 +1599,11 @@ def render_platform(run_id, sha, platform, refresh=False): url=j["url"], ) cells.append( - f'
{inner}
' ) else: - cells.append(f'
{inner}
') + cells.append(f'
{inner}
') else: name = j["raw"].split(" / ")[-1] or j["group"] link = ( @@ -1195,9 +1625,9 @@ def render_platform(run_id, sha, platform, refresh=False): f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details' ) rows.append( - f'
' + f'
' f'{e(name)}' - f'{e(label)}{link}{extra}
' + f'{e(label)}{link}{extra}
' ) body = "" if cells: @@ -1413,6 +1843,10 @@ def _send(self, code, body, ctype="text/html; charset=utf-8"): self.send_response(code) self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(len(body))) + # Never let the browser serve a stale page/fragment/asset — this is a + # live, rapidly-changing tool and cached HTML/CSS/JS causes "it broke + # again" ghosts. gh data is already cached server-side (Cache class). + self.send_header("Cache-Control", "no-store, must-revalidate") self.end_headers() self.wfile.write(body) @@ -1423,7 +1857,7 @@ def do_GET(self): try: p = u.path if p in ("/", "/index.html"): - return self._index() + return self._index(q.get("branch") or default_branch()) if p.startswith("/static/"): return self._static(p[len("/static/") :]) if p == "/ui/board": @@ -1454,6 +1888,9 @@ def do_GET(self): q.get("url", ""), ), ) + if p == "/api/resolve-pr": + b = resolve_pr_branch(q.get("pr", "")) + return self._send(200, {"branch": b} if b else {"error": "not found"}) if p == "/ui/aggregate": return self._send( 200, render_aggregate(q.get("branch") or default_branch(), refresh) @@ -1473,10 +1910,42 @@ def do_GET(self): except Exception as ex: self._send(500, f'
error: {e(ex)}
') - def _index(self): + def do_POST(self): + u = urlparse(self.path) + try: + length = int(self.headers.get("Content-Length") or 0) + form = { + k: v[0] for k, v in parse_qs(self.rfile.read(length).decode()).items() + } + if u.path == "/api/pr-comment": + try: + url = post_pr_comment(form.get("pr", ""), form.get("cmd", "")) + return self._send(200, {"ok": True, "url": url}) + except Exception as ex: # bad cmd / gh failure → readable msg + return self._send(200, {"ok": False, "error": str(ex)}) + if u.path == "/api/pr-label": + try: + out = set_pr_label( + form.get("pr", ""), + form.get("label", ""), + form.get("add") == "1", + ) + return self._send(200, {"ok": True, "out": out}) + except Exception as ex: # bad label / gh failure → readable msg + return self._send(200, {"ok": False, "error": str(ex)}) + return self._send(404, {"ok": False, "error": "not found"}) + except Exception as ex: + return self._send(500, {"ok": False, "error": str(ex)}) + + def _index(self, branch): + """Full page bootstrapped to `branch` — so `/?branch=` is a real, + reloadable, shareable URL (and history-restore lands on a valid page). + A bare `#` in the URL is a PR number → resolve to its head branch.""" + if branch and branch.startswith("#") and branch[1:].isdigit(): + branch = resolve_pr_branch(branch) or branch with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f: html_s = f.read() - html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace( + html_s = html_s.replace("{{BRANCH}}", e(branch)).replace( "{{REPO}}", e(repo_slug()) ) self._send(200, html_s) diff --git a/tools/ci-dashboard/static/index.html b/tools/ci-dashboard/static/index.html index 193fff7d67..74fd388381 100644 --- a/tools/ci-dashboard/static/index.html +++ b/tools/ci-dashboard/static/index.html @@ -10,10 +10,12 @@

torch-tensorrt // ci

- - + +
+ + +
re-run only the failed / cancelled jobs (current commit)
/rerun allre-run every job from scratch (current commit)
@@ -71,10 +74,215 @@

torch-tensorrt // ci { + const ta = document.createElement('textarea'); + ta.value = text; ta.style.position = 'fixed'; ta.style.opacity = '0'; + document.body.appendChild(ta); ta.focus(); ta.select(); + let ok = false; + try { ok = document.execCommand('copy'); } catch (e) { ok = false; } + document.body.removeChild(ta); + ok ? resolve() : reject(new Error('copy blocked')); + }); + } function copyPre(btn) { const code = btn.parentElement.querySelector('code'); - navigator.clipboard.writeText(code.innerText).then(() => toast('copied')); + copyText(code.innerText).then(() => toast('copied')).catch(() => toast('copy blocked — select + ⌘/Ctrl-C')); + } + // Toggle a CI label live (add/remove on the PR) — two-step: click arms, click confirms. + function disarmLabel(btn) { + if (btn._t) clearTimeout(btn._t); + btn.dataset.armed = ''; + btn.classList.remove('arming'); + if (btn._orig != null) { btn.innerHTML = btn._orig; btn._orig = null; } + } + function toggleLabel(btn) { + if (btn.dataset.armed !== '1') { // arm + document.querySelectorAll('.cilabel.arming').forEach(disarmLabel); + btn.dataset.armed = '1'; + btn._orig = btn.innerHTML; + btn.classList.add('arming'); + btn.textContent = (btn.dataset.add === '1' ? 'confirm add ' : 'confirm remove ') + btn.dataset.label; + btn._t = setTimeout(() => disarmLabel(btn), 5000); + return; + } + disarmLabel(btn); // confirmed → apply + const pr = btn.dataset.pr, label = btn.dataset.label, add = btn.dataset.add; + btn.disabled = true; + fetch('/api/pr-label', { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ pr: pr, label: label, add: add }), + }).then(r => r.json()) + .then(d => { + btn.disabled = false; + if (!d.ok) { toast('failed: ' + (d.error || '?')); return; } + toast((add === '1' ? 'added ' : 'removed ') + label); + const b = document.getElementById('branch').value; // refresh plan with new labels + htmx.ajax('GET', '/ui/board?branch=' + encodeURIComponent(b) + '&refresh=1', + { target: '#content', swap: 'innerHTML' }); + }) + .catch(err => { btn.disabled = false; toast('failed: ' + err); }); + } + // Post a CI command as a PR comment — two-step: first click arms, second confirms. + function disarmCiact(btn) { + if (btn._t) clearTimeout(btn._t); + btn.dataset.armed = ''; + btn.classList.remove('armed'); + if (btn._orig != null) { btn.textContent = btn._orig; btn._orig = null; } + } + function postCmd(btn) { + if (btn.dataset.armed !== '1') { // arm + document.querySelectorAll('.ciact.armed').forEach(disarmCiact); + btn.dataset.armed = '1'; + btn._orig = btn.textContent; + btn.classList.add('armed'); + btn.textContent = 'confirm: ' + btn.dataset.cmd; + btn._t = setTimeout(() => disarmCiact(btn), 5000); + return; + } + disarmCiact(btn); // confirmed → post + const cmd = btn.dataset.cmd, pr = btn.dataset.pr, label = btn.textContent; + btn.disabled = true; btn.textContent = 'posting…'; + fetch('/api/pr-comment', { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ pr: pr, cmd: cmd }), + }).then(r => r.json()) + .then(d => toast(d.ok ? ('posted ' + cmd + ' to #' + pr) : ('failed: ' + (d.error || '?')))) + .catch(err => toast('failed: ' + err)) + .finally(() => { btn.disabled = false; btn.textContent = label; }); + } + // ── branch navigation with real browser history (back/forward) ── + function loadBranch(branch, push) { + branch = (branch || '').trim(); + if (!branch) return; + const c = document.getElementById('content'); + const m = branch.match(/^#(\d+)$/); // a PR number → resolve its head branch + if (m) { + c.innerHTML = '
resolving PR #' + m[1] + '…
'; + fetch('/api/resolve-pr?pr=' + m[1]).then(r => r.json()).then(d => { + if (d && d.branch) loadBranch(d.branch, push); + else c.innerHTML = '
No branch found for PR #' + m[1] + '.
'; + }).catch(() => { c.innerHTML = '
PR lookup failed.
'; }); + return; + } + document.getElementById('branch').value = branch; + if (push) history.pushState({ branch: branch }, '', '/?branch=' + encodeURIComponent(branch)); + c.innerHTML = '
loading…
'; + htmx.ajax('GET', '/ui/board?branch=' + encodeURIComponent(branch), { target: '#content', swap: 'innerHTML' }); + } + window.addEventListener('popstate', function (e) { + const b = (e.state && e.state.branch) || + new URLSearchParams(location.search).get('branch') || + document.getElementById('branch').value; + loadBranch(b, false); // back/forward: restore, don't push a new entry + }); + // Seed a state for the branch the page booted with, so the first Back has a target. + history.replaceState({ branch: document.getElementById('branch').value }, '', + location.pathname + location.search); + + // ── client-side branch history + fuzzy autocomplete (with #PR search) ── + const AC = { items: [], index: -1 }; + function escHtml(s) { + return (s + '').replace(/[&<>"']/g, c => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); + } + function loadHistory() { + try { return JSON.parse(localStorage.getItem('branchHistory') || '[]'); } catch (e) { return []; } + } + function recordBranch(rec) { + const branch = (rec.branch || '').trim(); + if (!branch) return; + const h = loadHistory().filter(x => x.branch !== branch); + h.unshift({ branch: branch, pr: rec.pr || null, base: rec.base || null, + head: rec.head || '', msg: rec.msg || '', ts: Date.now() }); + localStorage.setItem('branchHistory', JSON.stringify(h.slice(0, 50))); + } + function fuzzy(q, s) { // subsequence match, contiguity-weighted + s = (s || '').toLowerCase(); + let qi = 0, score = 0, streak = 0; + for (let i = 0; i < s.length && qi < q.length; i++) { + if (s[i] === q[qi]) { qi++; streak++; score += streak * 2; } else streak = 0; + } + return qi === q.length ? score : -1; + } + function acSearch(query) { + const h = loadHistory(); + query = query.trim().toLowerCase(); + if (!query) return h.slice(0, 8); // recent + if (query[0] === '#') { // by PR number + const n = query.slice(1); + return h.filter(x => x.pr != null && String(x.pr).includes(n)).slice(0, 8); + } + return h.map(x => ({ x: x, s: Math.max(fuzzy(query, x.branch), fuzzy(query, x.msg || x.title) - 1) })) + .filter(r => r.s >= 0).sort((a, b) => b.s - a.s).slice(0, 8).map(r => r.x); } + function acRender() { + const box = document.getElementById('branch-ac'); + if (!AC.items.length) { box.hidden = true; return; } + box.innerHTML = AC.items.map((it, i) => { + const msg = it.msg || it.title || ''; // format: branch #PR →base HEAD msg + return '
' + + '' + escHtml(it.branch) + '' + + (it.pr ? '#' + escHtml(String(it.pr)) + '' : '') + + (it.base ? '→ ' + escHtml(it.base) + '' : '') + + (it.head ? '' + escHtml(String(it.head).slice(0, 7)) + '' : '') + + (msg ? '' + escHtml(msg) + '' : '') + + '
'; + }).join(''); + box.hidden = false; + } + function acOpen() { AC.items = acSearch(document.getElementById('branch').value); AC.index = -1; acRender(); } + function acClose() { document.getElementById('branch-ac').hidden = true; AC.index = -1; } + function acPick(e, i) { + if (e) e.preventDefault(); + const it = AC.items[i]; + if (!it) return; + acClose(); + loadBranch(it.branch, true); + } + (function () { + const bi = document.getElementById('branch'); + bi.addEventListener('input', acOpen); + bi.addEventListener('focus', acOpen); + bi.addEventListener('blur', () => setTimeout(acClose, 120)); // let a row mousedown land first + bi.addEventListener('keydown', e => { + const box = document.getElementById('branch-ac'); + const open = box && !box.hidden; + if (e.key === 'ArrowDown') { + if (!open) acOpen(); + AC.index = Math.min(AC.index + 1, AC.items.length - 1); acRender(); e.preventDefault(); + } else if (e.key === 'ArrowUp') { + AC.index = Math.max(AC.index - 1, 0); acRender(); e.preventDefault(); + } else if (e.key === 'Enter' && open && AC.index >= 0) { + e.preventDefault(); e.stopPropagation(); acPick(e, AC.index); + } else if (e.key === 'Escape' && open) { + acClose(); e.stopPropagation(); + } + }); + // Record every branch that actually loaded a board (skip empty/error states), + // capturing its PR number + title from the rendered banner. + document.body.addEventListener('htmx:afterSwap', e => { + if (!e.target || e.target.id !== 'content' || !e.target.querySelector('#board')) return; + const banner = e.target.querySelector('.pr-banner'); + const commit = e.target.querySelector('.commit'); + recordBranch({ + branch: document.getElementById('branch').value, + pr: banner ? banner.dataset.pr : null, + base: banner ? banner.dataset.base : null, + head: commit ? (commit.dataset.sha || '') : '', + msg: commit ? (commit.dataset.msg || '') : '', + }); + }); + })(); + // ── log fishing: search / filter / jump within the drawer's log section ── function logSearch(inp) { const q = inp.value.trim().toLowerCase(); @@ -105,6 +313,14 @@

torch-tensorrt // ci t.classList.remove('show'), 1400); } + // PR commands: shown by default, but remember if the user closes it. + (function () { + const c = document.getElementById('cmds'); + if (!c) return; + const saved = localStorage.getItem('cmdsOpen'); + if (saved !== null) c.open = saved === '1'; + c.addEventListener('toggle', () => localStorage.setItem('cmdsOpen', c.open ? '1' : '0')); + })(); document.addEventListener('keydown', e => { const typing = /^(INPUT|TEXTAREA)$/.test(document.activeElement.tagName); if (e.key === 'Escape') closeDrawer(); diff --git a/tools/ci-dashboard/static/style.css b/tools/ci-dashboard/static/style.css index 029d16982a..37360ee81a 100644 --- a/tools/ci-dashboard/static/style.css +++ b/tools/ci-dashboard/static/style.css @@ -118,6 +118,8 @@ main { max-width: 1320px; margin: 0 auto; padding: 18px; } .stat.run { color: var(--run); border-color: var(--run-line); background: var(--run-bg); } .stat.skip { color: var(--skip); border-color: var(--skip-line); background: var(--skip-bg); border-style: dashed; } .commit { color: var(--muted); font-size: 12px; margin-left: auto; text-transform: uppercase; letter-spacing: .04em; } +.commit a.sha { color: var(--accent); text-decoration: none; } +.commit a.sha:hover { text-decoration: underline; } .commit code { font-family: var(--mono); color: var(--accent); text-transform: none; } /* ── status lights ────────────────────────────────────────────────────────── */ @@ -243,6 +245,18 @@ h2.sec { font-size: 11px; text-transform: uppercase; letter-spacing: .12em; colo .jobstate.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } .jobstate.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } .jobstate.skip { border-style: dashed; } +/* infra / runner failure — a diagonal amber hatch over the fail red + amber "infra" + label, so a broken runner/build reads apart from a clean red test failure (and the + reddish base keeps it distinct from amber "running"). */ +.cell.fail.infra, .jobrow.fail.infra { + border-color: var(--run-line); + background-image: repeating-linear-gradient(45deg, + transparent 0, transparent 6px, var(--run-line) 6px, var(--run-line) 8px); +} +.cell.fail.infra .v { color: var(--run); } +.cell.fail.infra:hover { outline: 1px solid var(--run); box-shadow: var(--glow) var(--run); } +.jobrow.fail.infra .name { color: var(--run); } +.jobstate.fail.infra { color: var(--run); border-color: var(--run-line); background: var(--run-bg); } .jobstate.cancel { color: var(--cancel); border-color: var(--cancel-line); border-style: dashed; } /* ── drawer ───────────────────────────────────────────────────────────────── */ @@ -346,6 +360,39 @@ h2.sec { font-size: 11px; text-transform: uppercase; letter-spacing: .12em; colo text-transform: uppercase; letter-spacing: .04em; } .legend span { display: inline-flex; align-items: center; gap: 5px; } .legend .muted { text-transform: none; letter-spacing: 0; } +.leg-infra { width: 10px; height: 10px; border-radius: 2px; display: inline-block; flex: none; + border: 1px solid var(--run-line); background: var(--fail-bg); + background-image: repeating-linear-gradient(45deg, transparent 0, transparent 2px, var(--run-line) 2px, var(--run-line) 3px); } + +/* branch search: fuzzy-autocomplete dropdown over the client-side visit history */ +.branch-wrap { position: relative; display: inline-block; } +.branch-ac { position: absolute; top: calc(100% + 4px); left: 0; z-index: 40; width: max-content; + min-width: 100%; max-width: 480px; background: var(--surface); border: 1px solid var(--accent); + border-radius: var(--radius); overflow: hidden; } +.ac-row { display: flex; align-items: baseline; gap: 8px; padding: 6px 10px; cursor: pointer; + font-size: 12px; border-bottom: 1px solid var(--border); } +.ac-row:last-child { border-bottom: none; } +.ac-row.sel, .ac-row:hover { background: var(--surface-2); } +.ac-branch { font-family: var(--mono); color: var(--text); font-weight: 600; white-space: nowrap; } +.ac-pr { font-family: var(--mono); font-size: 11px; color: var(--accent); white-space: nowrap; } +.ac-base { font-family: var(--mono); font-size: 11px; color: var(--muted); white-space: nowrap; } +.ac-head { font-family: var(--mono); font-size: 11px; color: var(--run); white-space: nowrap; } +.ac-title { color: var(--muted); font-size: 11px; overflow: hidden; text-overflow: ellipsis; + white-space: nowrap; min-width: 0; } + +/* PR banner — the open PR for the current branch, number + title + link */ +.pr-banner { display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; margin: 10px 0 0; + padding: 8px 12px; background: var(--surface-2); border: 1px solid var(--border); + border-radius: var(--radius); box-shadow: inset 2px 0 0 var(--accent); } +.pr-num { font-family: var(--mono); font-weight: 700; color: var(--accent); white-space: nowrap; + text-decoration: none; } +.pr-num:hover { text-decoration: underline; } +.pr-base { font-family: var(--mono); font-size: 12px; color: var(--muted); white-space: nowrap; } +.pr-state { font-family: var(--mono); font-size: 10px; text-transform: uppercase; letter-spacing: .05em; + padding: 1px 6px; border-radius: var(--radius); white-space: nowrap; } +.pr-state.merged { color: var(--cancel); background: var(--cancel-bg); border: 1px solid var(--cancel-line); } +.pr-state.closed { color: var(--fail); background: var(--fail-bg); border: 1px solid var(--fail-line); } +.pr-title { color: var(--text); font-size: 13px; word-break: break-word; } /* ── PR command reference (collapsible) ───────────────────────────────────── */ .cmds { margin: 8px 0 0; font-size: 12px; } @@ -363,3 +410,75 @@ h2.sec { font-size: 11px; text-transform: uppercase; letter-spacing: .12em; colo .cmd .copy { float: none; } .cmd .lbl { color: var(--muted); } .cmd .lbl code { color: var(--text); background: var(--bg); padding: 0 4px; border-radius: var(--radius); } + +/* test-plan panel: what a branch's labels will actually run */ +.plan { margin: 10px 0 0; border: 1px solid var(--border); border-radius: var(--radius); + background: var(--surface); box-shadow: inset 2px 0 0 var(--accent); } +.plan > summary { cursor: pointer; user-select: none; list-style: none; display: flex; + align-items: baseline; gap: 12px; flex-wrap: wrap; padding: 9px 12px; font-size: 12px; } +.plan > summary::-webkit-details-marker { display: none; } +.plan > summary::before { content: "▸ "; color: var(--accent); } +.plan[open] > summary::before { content: "▾ "; } +.plan > summary .lbl { text-transform: uppercase; letter-spacing: .1em; font-weight: 700; + color: var(--accent); font-size: 11px; } +.pverdict { font-family: var(--mono); color: var(--text); } +.pverdict b { color: var(--accent); text-transform: uppercase; } +.plan > summary .muted { margin-left: auto; } +.plan-body { padding: 0 12px 12px; } +.plan-labels { font-size: 12px; color: var(--muted); line-height: 1.6; margin-bottom: 10px; } +/* passive metadata tags (non-CI labels) — flat faint fill, no border, so they + read as info, not buttons like the CI-label / action chips */ +.plabel { display: inline-block; font-family: var(--mono); font-size: 11px; color: var(--muted); + background: var(--surface-2); border-radius: var(--radius); padding: 0 5px; margin: 0 2px; opacity: .8; } +.plan-hint { color: var(--muted); } +.plan-hint code { font-family: var(--mono); color: var(--accent); background: var(--bg); + border: 1px solid var(--border); border-radius: var(--radius); padding: 0 4px; } +.plan-table { width: 100%; border-collapse: collapse; font-size: 12px; } +.plan-table th { text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: .07em; + color: var(--muted); font-weight: 600; padding: 4px 10px 4px 0; border-bottom: 1px solid var(--border); } +.plan-table td { padding: 5px 10px 5px 0; border-bottom: 1px solid var(--border); vertical-align: top; } +.plan-table tr:last-child td { border-bottom: none; } +.plan-table .pplat { font-weight: 600; white-space: nowrap; } +.plan-table .pkind { color: var(--muted); white-space: nowrap; } +.plan-table .pcnt { font-family: var(--mono); text-align: right; color: var(--text); } +.peng { font-family: var(--mono); font-size: 10px; font-weight: 700; text-transform: uppercase; + letter-spacing: .04em; padding: 1px 6px; border-radius: var(--radius); border: 1px solid transparent; white-space: nowrap; } +.peng.standard { color: var(--queue); background: var(--queue-bg); border-color: var(--queue-line); } +.peng.rtx { color: var(--accent); background: var(--fail-bg); border-color: var(--fail-line); } +.psuites { font-family: var(--mono); font-size: 11px; color: var(--muted); word-break: break-word; } + +/* CI-control labels — on/off state, colour-coded by axis (lane / std / rtx) */ +.ci-labels { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; margin-bottom: 8px; } +.cil-head { font-size: 10px; text-transform: uppercase; letter-spacing: .1em; font-weight: 700; + color: var(--muted); margin-right: 4px; } +.cilabel { font-family: var(--mono); font-size: 11px; display: inline-flex; align-items: center; gap: 5px; + padding: 2px 8px; border-radius: var(--radius); border: 1px solid var(--border); background: transparent; + color: var(--muted); cursor: pointer; } +.cilabel .cilmark { font-weight: 700; opacity: .9; } +.cilabel.off { border-style: dashed; color: var(--muted); } +.cilabel.off .cilmark { color: var(--muted); } +.cilabel.on.lane { color: var(--cancel); background: var(--cancel-bg); border-color: var(--cancel-line); } +.cilabel.on.std { color: var(--queue); background: var(--queue-bg); border-color: var(--queue-line); } +.cilabel.on.rtx { color: var(--accent); background: var(--fail-bg); border-color: var(--fail-line); } +.cilabel.inert { color: var(--run); background: var(--run-bg); border-color: var(--run-line); border-style: dashed; } +.cilabel:not([disabled]):hover { border-color: currentColor; box-shadow: var(--glow) currentColor; } +.cilabel[disabled] { cursor: default; opacity: .55; } +.cilabel.arming { color: #fff; background: var(--fail); border-color: var(--fail); + box-shadow: var(--glow) rgba(var(--fail-glow), .6); } +.cilabel.arming .cilmark { display: none; } +.plan-hint-row { font-size: 12px; margin-bottom: 10px; } +.plan-other { font-size: 11px; margin: 6px 0 2px; } + +/* live CI actions — post a command as a PR comment (click, then confirm) */ +.ci-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; margin-bottom: 10px; } +.ciact { font-family: var(--mono); font-size: 11px; padding: 3px 9px; cursor: pointer; + color: var(--accent); background: var(--surface-2); border: 1px solid var(--border); + border-radius: var(--radius); white-space: nowrap; } +.ciact::before { content: "▸ "; color: var(--muted); } +.ciact:not([disabled]):hover { border-color: var(--accent); } +.ciact.danger { color: var(--fail); } +.ciact.armed { color: #fff; background: var(--fail); border-color: var(--fail); + box-shadow: var(--glow) rgba(var(--fail-glow), .6); } +.ciact.armed::before { content: "⚠ "; color: #fff; } +.ciact[disabled] { opacity: .6; cursor: default; } +.ciact-note { font-size: 10px; margin-left: 4px; } From 11b77bf23e796c4ee9bfe7c2273d650043ff400c Mon Sep 17 00:00:00 2001 From: Naren Dasan Date: Wed, 22 Jul 2026 20:30:39 +0000 Subject: [PATCH 11/11] update for rtx sbsa builds --- .../build-test-linux-aarch64_rtx.yml | 91 ------------------- .github/workflows/ci-sbsa.yml | 38 +++++++- tools/ci-dashboard/ci_dashboard.py | 12 ++- 3 files changed, 45 insertions(+), 96 deletions(-) delete mode 100644 .github/workflows/build-test-linux-aarch64_rtx.yml diff --git a/.github/workflows/build-test-linux-aarch64_rtx.yml b/.github/workflows/build-test-linux-aarch64_rtx.yml deleted file mode 100644 index eb98862081..0000000000 --- a/.github/workflows/build-test-linux-aarch64_rtx.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: RTX - Build Linux aarch64 wheels - -# Basic RTX support for aarch64 (SBSA): mirrors build-test-linux-aarch64.yml but -# selects TensorRT-RTX (use-rtx: true). BUILD-ONLY — there are no aarch64 GPU test -# runners, so this just validates the RTX aarch64 wheel builds. The full -# manifest-based wiring lives in the CI redesign (test-dx) on top of this. - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: RTX - Build SBSA torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - use-rtx: true - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/ci-sbsa.yml b/.github/workflows/ci-sbsa.yml index a4be4e2ce1..30d242f50c 100644 --- a/.github/workflows/ci-sbsa.yml +++ b/.github/workflows/ci-sbsa.yml @@ -1,8 +1,10 @@ name: CI SBSA # Per-platform entry — SBSA (linux-aarch64) is BUILD-ONLY (no aarch64 GPU test -# runners), so this just validates the wheel builds (standard + python-only) on -# the full/nightly lanes. No test/report jobs since nothing runs. +# runners), so this just validates the wheel builds — standard AND TensorRT-RTX, +# each with a python-only variant — on the full/nightly lanes. No test/report jobs +# since nothing runs. (TensorRT-RTX ships an aarch64/SBSA build as of 1.5, so the +# RTX channels build @tensorrt_rtx_sbsa via USE_TRT_RTX=1.) on: pull_request: @@ -50,7 +52,7 @@ jobs: contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) uses: ./.github/workflows/_decide.yml - # Generate the aarch64 build matrix once (shared by both build channels). + # Generate the aarch64 build matrix once (shared by all build channels). generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: @@ -84,8 +86,36 @@ jobs: name-prefix: "Python-only SBSA " raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + # RTX channels — run when an RTX backend is selected (backend != 'standard'). + # TensorRT-RTX 1.5 ships an aarch64 (SBSA) build, so USE_TRT_RTX=1 resolves + # @tensorrt_rtx_sbsa. Build-only, same as the standard SBSA channels. + build-rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + architecture: aarch64 + use-rtx: true + run-tests: false + name-prefix: "RTX SBSA " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only-rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + architecture: aarch64 + use-rtx: true + run-tests: false + name-prefix: "RTX Python-only SBSA " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + gate: - needs: [decide, build, python-only] + needs: [decide, build, python-only, build-rtx, python-only-rtx] if: always() runs-on: ubuntu-latest steps: diff --git a/tools/ci-dashboard/ci_dashboard.py b/tools/ci-dashboard/ci_dashboard.py index 828d851d77..f59464eadf 100755 --- a/tools/ci-dashboard/ci_dashboard.py +++ b/tools/ci-dashboard/ci_dashboard.py @@ -384,7 +384,8 @@ def add(platform, engine, kind, suite_lane, suite_platform): add("Windows", "standard", "python-only", "python-only", "windows") if fullish and rtx: add("Windows", "rtx", "python-only", "python-only", "windows") - # SBSA aarch64 (ci-sbsa.yml): BUILD-ONLY (no GPU runners), full/nightly, standard + # SBSA aarch64 (ci-sbsa.yml): BUILD-ONLY (no GPU runners), full/nightly. + # Standard AND RTX (TensorRT-RTX 1.5 ships an aarch64 build), each w/ python-only. if fullish and std: add("Linux aarch64 · SBSA", "standard", "build-only", lane, None) add( @@ -394,6 +395,15 @@ def add(platform, engine, kind, suite_lane, suite_platform): "python-only", None, ) + if fullish and rtx: + add("Linux aarch64 · SBSA", "rtx", "build-only", lane, None) + add( + "Linux aarch64 · SBSA", + "rtx", + "build-only · py-only", + "python-only", + None, + ) return plan