tool: A UI to look at CI results that doesnt suck#4397
Open
narendasan wants to merge 9 commits into
Open
Conversation
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-10 22:02:48.795897+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-10 22:03:16.163949+00:00
@@ -16,10 +16,11 @@
Job names encode the whole matrix, e.g.
core / L2 dynamo distributed tests / L2-dynamo-distributed-tests--3.12-cu130
We parse that into {group/tier, python, cuda, kind} and map the tier back to the
pytest paths in tests/py/utils/ci_helpers.sh so a red cell points at code.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -45,42 +46,76 @@
# source of truth for "what does each CI tier run"). Keys are normalized job
# group names (see _norm()); `paths` are repo-relative; `fn` is the shell tier
# function you'd invoke locally to reproduce.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -120,22 +155,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -143,25 +181,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -189,40 +235,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -232,12 +300,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -265,31 +338,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -301,12 +383,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -327,53 +410,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -382,11 +487,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -422,27 +529,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -453,24 +567,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -479,38 +604,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -521,12 +659,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -543,38 +683,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
tier = tier_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(tier["paths"]))}</span>' if tier else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(tier["paths"]))}</span>'
+ if tier
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -582,48 +753,75 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
tinfo = tier_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}</div>'
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if tinfo:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
k = f' -k "{" or ".join(leaves[:6])}"' if leaves else ""
- cmd = (f'source tests/py/utils/ci_helpers.sh && '
- f'PYTHON="uv run --no-sync python" TRT_PYTEST_RERUNS=0 {tinfo["fn"]}{k}')
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ cmd = (
+ f"source tests/py/utils/ci_helpers.sh && "
+ f'PYTHON="uv run --no-sync python" TRT_PYTEST_RERUNS=0 {tinfo["fn"]}{k}'
+ )
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -639,61 +837,84 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
-
- rows = sorted(agg.values(),
- key=lambda a: (-len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
+
+ rows = sorted(
+ agg.values(),
+ key=lambda a: (-len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
loc = ""
if a["file"]:
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
sample = next((o["msg"] for o in a["occ"] if o["msg"]), "")
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(sample)}</pre>" if sample else ""}</div>
- </details>''')
- header = (f'<div class="agg-head-row">{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across {len({o["plat"] for a in rows for o in a["occ"]})} platforms'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ </details>""")
+ header = (
+ f'<div class="agg-head-row">{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across {len({o["plat"] for a in rows for o in a["occ"]})} platforms'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -717,49 +938,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -771,12 +1017,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -804,13 +1055,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -823,15 +1077,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
narendasan
force-pushed
the
narendasan/push-msnkmsyslqxq
branch
from
July 16, 2026 22:08
9f53412 to
0809065
Compare
github-actions
Bot
added
component: tests
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-16 22:10:30.838957+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-16 22:10:56.916213+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-16 22:10:30.838957+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-16 22:10:57.010234+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-16 22:10:30.867969+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-16 22:11:03.352495+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
narendasan
added
ci: full
Force All Tests[L0+L1+L2]
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-17 20:46:09.336555+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-17 20:46:35.902660+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-17 20:46:09.336555+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-17 20:46:35.994478+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-17 20:46:09.366248+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-17 20:46:42.776365+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-17 21:21:53.921996+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-17 21:22:17.516504+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-17 21:21:53.921996+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-17 21:22:17.600928+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-17 21:21:53.947873+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-17 21:22:23.527141+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-17 22:22:51.104520+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-17 22:23:14.509432+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-17 22:22:51.104520+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-17 22:23:14.602399+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-17 22:22:51.129202+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-17 22:23:20.355265+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
…esign
Add tests/ci/{suites,runner,__main__}.py — a typed, declarative suite manifest
plus one runner (`python -m tests.ci`) that replaces the trt_tier_* bash
functions as the source of truth for what each test job runs. Faithfully
reproduces today's tier selection (verified via --dry-run) while fixing the
junit-path collisions, the double-run of hlo/, and giving every suite the
uniform flake-rerun+repro wrapper.
- pyproject: register smoke/flaky markers; norecursedirs += ci
- justfile: doctor / suites / suite / lane / report recipes (thin callers)
- TESTING_AND_CI_DESIGN.md: full local + CI design (manifest+runner, lanes
without merge queue, GHA-cache caching, aggregated agent-friendly reports)
ci(test-dx): grant id-token: write at ci.yml top level The reusable build jobs (build_linux/build_windows) request 'id-token: write' for OIDC; a reusable workflow can't exceed its caller's permission ceiling, so ci.yml's top-level 'contents: read' made GitHub reject it (startup_failure: "nested job 'build' is requesting 'id-token: write', but is only allowed 'id-token: none'"). Grant id-token: write + contents: read, matching the original per-platform entry workflows.
narendasan
force-pushed
the
narendasan/push-msnkmsyslqxq
branch
from
July 18, 2026 21:14
2416215 to
846937d
Compare
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-18 21:15:07.853299+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-18 21:15:32.195485+00:00
@@ -2,11 +2,10 @@
import torch
from torch.fx import GraphModule, Node
from .pass_utils import clean_up_graph_after_modifications
-
_INT64_MAX = 2**63 - 1
_SYM_MIN = getattr(torch, "sym_min", None)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-18 21:15:07.869118+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-18 21:15:33.824130+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-18 21:15:07.869118+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-18 21:15:33.927876+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-18 21:15:07.894925+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-18 21:15:40.361608+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
narendasan
force-pushed
the
narendasan/push-msnkmsyslqxq
branch
from
July 20, 2026 19:15
846937d to
37750b2
Compare
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-20 19:15:34.348297+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-20 19:16:00.519880+00:00
@@ -2,11 +2,10 @@
import torch
from torch.fx import GraphModule, Node
from .pass_utils import clean_up_graph_after_modifications
-
_INT64_MAX = 2**63 - 1
_SYM_MIN = getattr(torch, "sym_min", None)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-20 19:15:34.364355+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-20 19:16:02.091556+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-20 19:15:34.364355+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-20 19:16:02.178838+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-20 19:15:34.390222+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-20 19:16:08.483870+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
Collaborator
There was a problem hiding this comment.
this job is missing python-only flag?
Collaborator
Author
There was a problem hiding this comment.
I see the python-only jobs in CI Windows? They should be sharing some core
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-21 20:45:45.177284+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-21 20:46:07.915778+00:00
@@ -2,11 +2,10 @@
import torch
from torch.fx import GraphModule, Node
from .pass_utils import clean_up_graph_after_modifications
-
_INT64_MAX = 2**63 - 1
_SYM_MIN = getattr(torch, "sym_min", None)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-21 20:45:45.193321+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-21 20:46:09.425363+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-21 20:45:45.193321+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-21 20:46:09.503224+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-21 20:45:45.218845+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-21 20:46:15.596862+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,40 +313,62 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
TESTLINE = re.compile(r"^(?P<test>[\w./-]+(?:\.[\w\[\]-]+)+)\s*$")
@lru_cache(maxsize=4096)
def grep_test(symbol):
@@ -310,12 +378,17 @@
leaf = re.sub(r"\[.*$", "", leaf)
if not re.match(r"^[A-Za-z_]\w+$", leaf):
return None
pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b"
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -343,31 +416,40 @@
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
loc = grep_test(test) if test else None
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -379,12 +461,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -405,53 +488,75 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
def summary_reasons(lines, test):
"""The concise `FAILED …/ERROR … - <reason>` lines from pytest's summary."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- return [ln for ln in lines
- if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)]
+ return [
+ ln
+ for ln in lines
+ if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)
+ ]
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
fails = get_failures(job_id).get("failures", [])
blocks = []
for f in fails:
excerpt = extract_test_log(lines, f["test"])
reasons = summary_reasons(lines, f["test"])
title = e(f["test"] or "failure")
- reason_html = (f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else "")
+ reason_html = (
+ f'<div class="logreason">{e(reasons[0][:300])}</div>' if reasons else ""
+ )
if excerpt:
- blocks.append(f'<details class="logblock" open><summary>{title}</summary>'
- f'{reason_html}<pre>{e(excerpt)}</pre></details>')
+ blocks.append(
+ f'<details class="logblock" open><summary>{title}</summary>'
+ f"{reason_html}<pre>{e(excerpt)}</pre></details>"
+ )
else:
- blocks.append(f'<details class="logblock"><summary>{title}</summary>'
- f'{reason_html}<div class="muted">No matching block in the log — '
- f'see the raw log.</div></details>')
+ blocks.append(
+ f'<details class="logblock"><summary>{title}</summary>'
+ f'{reason_html}<div class="muted">No matching block in the log — '
+ f"see the raw log.</div></details>"
+ )
if not fails: # build/env/setup failure: no test annotations → show the tail
tail = "\n".join(lines[-180:]).strip()
- blocks.append('<details class="logblock" open><summary>end of job log</summary>'
- f'<pre>{e(tail)}</pre></details>')
+ blocks.append(
+ '<details class="logblock" open><summary>end of job log</summary>'
+ f"<pre>{e(tail)}</pre></details>"
+ )
return head + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
def classify(status, conclusion):
@@ -460,11 +565,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -492,33 +599,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -538,12 +692,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -578,27 +734,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -609,24 +772,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -635,38 +809,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -677,12 +864,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -699,38 +888,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -738,49 +958,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -796,38 +1041,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -839,12 +1103,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -852,31 +1121,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -900,49 +1173,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -954,12 +1252,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -987,13 +1290,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1006,15 +1312,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
narendasan
force-pushed
the
narendasan/push-msnkmsyslqxq
branch
from
July 21, 2026 21:48
9e5a5be to
c4c4e86
Compare
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-21 21:48:20.452222+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-21 21:48:46.327215+00:00
@@ -2,11 +2,10 @@
import torch
from torch.fx import GraphModule, Node
from .pass_utils import clean_up_graph_after_modifications
-
_INT64_MAX = 2**63 - 1
_SYM_MIN = getattr(torch, "sym_min", None)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-21 21:48:20.470972+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-21 21:48:48.022342+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-21 21:48:20.470972+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-21 21:48:48.105328+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-21 21:48:20.497579+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-21 21:48:54.906906+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -60,42 +61,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -198,22 +233,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -221,25 +259,33 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -267,51 +313,73 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
# A test id is the annotation's first line: a single token that names a test —
# a bare `test_x[param]`, a dotted `Class.test_x`, or a `file.py::Class::test_x`
# nodeid. (The old regex required a dot, so it missed bare parametrized names.)
TESTLINE = re.compile(r"^[\w./\[\]:-]+$")
def _test_name(head):
"""Pull a test id out of an annotation's first line, or None."""
if not head or " " in head or not TESTLINE.match(head):
return None
- tok = head.split("::")[-1] if "::" in head else head # nodeid → last segment
+ tok = head.split("::")[-1] if "::" in head else head # nodeid → last segment
return tok if re.search(r"test", tok, re.I) else None
def _annot_loc(a):
"""Exact (repo-relative file, line) straight from the annotation — better than
@@ -323,12 +391,17 @@
return (m.group(1), a.get("start_line") or 0)
def _git_grep(pat):
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -340,15 +413,15 @@
"""Resolve a failing-test symbol (Class.method / module.func) to (file, line)
via `git grep`. Tries the method, then the enclosing class — parametrized names
(`@parameterized.expand` → test_x_5) have no literal `def`, so the class is the
best anchor. Returns None if nothing matches."""
parts = [re.sub(r"\[.*$", "", p) for p in re.split(r"[.:]+", symbol.strip()) if p]
- for token in reversed(parts): # method first, then its class
+ for token in reversed(parts): # method first, then its class
if not re.match(r"^[A-Za-z_]\w+$", token):
continue
pat = f"def {token}\\b" if token.startswith("test") else f"class {token}\\b"
- if (hit := _git_grep(pat)):
+ if hit := _git_grep(pat):
return hit
return None
def get_failures(job_id, refresh=False):
@@ -370,35 +443,46 @@
test = _test_name(head)
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
- loc = _annot_loc(a) or (grep_test(test) if test else None) # annotation line first
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ loc = _annot_loc(a) or (
+ grep_test(test) if test else None
+ ) # annotation line first
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") # SGR / CSI colour codes
-_GROUP = re.compile(r"^##\[(?:group|section|command)\]") # fold marker → keep title
-_ENDGROUP = re.compile(r"^##\[endgroup\]\s*$") # pure noise
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") # SGR / CSI colour codes
+_GROUP = re.compile(r"^##\[(?:group|section|command)\]") # fold marker → keep title
+_ENDGROUP = re.compile(r"^##\[endgroup\]\s*$") # pure noise
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -418,12 +502,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -444,59 +529,85 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
# ── anchoring: find the *real* error in a non-pytest (build/install) failure ──
# In GH logs the first `##[error]` is almost always echoed shell text
# (`echo "::error::…"`), so we never anchor on it. The reliable signals, in order:
# the pytest summary, a pip/build metadata error, or a walk backward from the
# `Process completed with exit code N` line to the nearest genuine error.
_SUMMARY_HDR = re.compile(r"^=+\s*short test summary info\s*=+", re.I)
-_PIP_ERR = re.compile(r"error: subprocess-exited-with-error|did not run successfully|^\s*╰─>")
+_PIP_ERR = re.compile(
+ r"error: subprocess-exited-with-error|did not run successfully|^\s*╰─>"
+)
_EXITCODE = re.compile(r"(?:Process completed with exit code|exit code:?)\s*[1-9]")
-_ECHO_NOISE = re.compile(r'echo\s+["\']?::|Specified script file|^\s*(?:if|else|elif|fi|then|source|for|do|done)\b')
+_ECHO_NOISE = re.compile(
+ r'echo\s+["\']?::|Specified script file|^\s*(?:if|else|elif|fi|then|source|for|do|done)\b'
+)
_REAL_ERR = re.compile(
r"Traceback \(most recent call last\)|\b\w*(?:Error|Exception)\b\s*:|"
r"^E\s{2,}\S|^\s*(?:FAILED|ERROR)\s|error: subprocess-exited-with-error|"
- r"CMake Error|fatal error:|ninja: build stopped|Segmentation fault|OutOfMemory|Killed\b")
+ r"CMake Error|fatal error:|ninja: build stopped|Segmentation fault|OutOfMemory|Killed\b"
+)
def find_error_window(lines, max_lines=140):
"""Anchor a non-pytest failure on its real error instead of the blind tail.
Returns (title, start, end)."""
n = len(lines)
if not n:
return ("end of job log", 0, 0)
- for i, ln in enumerate(lines): # 1. pytest summary
+ for i, ln in enumerate(lines): # 1. pytest summary
if _SUMMARY_HDR.search(ln):
- end = next((j for j in range(i + 1, n)
- if _SECT.match(lines[j]) and not _SUMMARY_HDR.search(lines[j])), n)
+ end = next(
+ (
+ j
+ for j in range(i + 1, n)
+ if _SECT.match(lines[j]) and not _SUMMARY_HDR.search(lines[j])
+ ),
+ n,
+ )
return ("short test summary", i, min(end, i + max_lines))
- for i, ln in enumerate(lines): # 2. pip / build error
+ for i, ln in enumerate(lines): # 2. pip / build error
if _PIP_ERR.search(ln):
return ("install / build error", max(0, i - 3), min(n, i + max_lines))
- exit_i = next((i for i in range(n - 1, -1, -1) if _EXITCODE.search(lines[i])), n - 1)
- anchor = next((i for i in range(exit_i, -1, -1) # 3. walk back to real error
- if _REAL_ERR.search(lines[i]) and not _ECHO_NOISE.search(lines[i])), None)
+ exit_i = next(
+ (i for i in range(n - 1, -1, -1) if _EXITCODE.search(lines[i])), n - 1
+ )
+ anchor = next(
+ (
+ i
+ for i in range(exit_i, -1, -1) # 3. walk back to real error
+ if _REAL_ERR.search(lines[i]) and not _ECHO_NOISE.search(lines[i])
+ ),
+ None,
+ )
if anchor is not None:
start = anchor
- for j in range(anchor, max(0, anchor - 80), -1): # widen up to the traceback head
+ for j in range(
+ anchor, max(0, anchor - 80), -1
+ ): # widen up to the traceback head
if "Traceback (most recent call last)" in lines[j]:
start = j
break
return ("error", max(0, start - 2), min(n, exit_i + 1))
- return ("end of job log", max(0, n - 180), n) # 4. fallback: tail
+ return ("end of job log", max(0, n - 180), n) # 4. fallback: tail
# ── root-cause collapse: many identical failures → one line + a pointer ───────
_EXC_RE = re.compile(r"\b([A-Z]\w*(?:Error|Exception|Warning))\b\s*:?\s*([^\n]*)")
@@ -537,11 +648,12 @@
# ── severity rendering: per-line spans so the client can search / jump / colour ─
_SEV_ERR = re.compile(
r"^##\[error\]|Traceback \(most recent call last\)|\b\w*(?:Error|Exception)\b\s*:|"
r"^E\s{2,}\S|^\s*(?:FAILED|ERROR)\s|error: subprocess-exited-with-error|"
- r"CMake Error|fatal error:|Segmentation fault|\bAssertionError\b|exit code:?\s*[1-9]")
+ r"CMake Error|fatal error:|Segmentation fault|\bAssertionError\b|exit code:?\s*[1-9]"
+)
_SEV_WARN = re.compile(r"^##\[warning\]|\bwarning\b|\bdeprecat|\bWARN\b", re.I)
_SEV_PASS = re.compile(r"\bPASSED\b|=+\s*\d+ passed|^\s*ok\s*$", re.I)
def _render_logtext(text):
@@ -557,101 +669,143 @@
sev = "warn"
elif _SEV_PASS.search(ln):
sev = "pass"
attr = f' data-sev="{sev}"' if sev else ""
spans.append(f'<span class="ll"{attr}>{e(ln) or " "}</span>')
- return "".join(spans), hits # .ll is display:block; no newlines between
+ return "".join(spans), hits # .ll is display:block; no newlines between
def _logblock(title, subtitle, excerpt, is_open=False):
"""A collapsible log excerpt: severity-highlighted <pre> + an error jump-list."""
if not excerpt:
- return (f'<details class="logblock"><summary>{e(title)}</summary>{subtitle}'
- '<div class="muted" style="padding:8px 12px 12px">No matching block '
- 'in the log — see the raw log.</div></details>')
+ return (
+ f'<details class="logblock"><summary>{e(title)}</summary>{subtitle}'
+ '<div class="muted" style="padding:8px 12px 12px">No matching block '
+ "in the log — see the raw log.</div></details>"
+ )
body, hits = _render_logtext(excerpt)
jumps = ""
if hits:
- chips = "".join(f'<button class="jump" data-line="{i}" onclick="jumpLine(this)" '
- f'title="{e(short)}">{e(short[:44])}</button>' for i, short in hits[:8])
+ chips = "".join(
+ f'<button class="jump" data-line="{i}" onclick="jumpLine(this)" '
+ f'title="{e(short)}">{e(short[:44])}</button>'
+ for i, short in hits[:8]
+ )
jumps = f'<div class="logjumps"><span class="lbl">jump</span>{chips}</div>'
op = " open" if is_open else ""
- return (f'<details class="logblock"{op}><summary>{e(title)}</summary>{subtitle}'
- f'{jumps}<pre class="logtext">{body}</pre></details>')
-
-
-_IMPORT_FAIL = re.compile(r"ModuleNotFoundError|ImportError|No module named|"
- r"error collecting|during collection|failed to import", re.I)
+ return (
+ f'<details class="logblock"{op}><summary>{e(title)}</summary>{subtitle}'
+ f'{jumps}<pre class="logtext">{body}</pre></details>'
+ )
+
+
+_IMPORT_FAIL = re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|"
+ r"error collecting|during collection|failed to import",
+ re.I,
+)
def _install_error_block(lines):
"""Locate a pip/build install error in the log and render it, or return ''."""
pin = next((i for i, ln in enumerate(lines) if _PIP_ERR.search(ln)), None)
if pin is None:
return ""
- body, _h = _render_logtext("\n".join(lines[max(0, pin - 3):pin + 40]).strip())
- return ('<details class="logblock rootcause" open><summary>'
- 'likely root cause · install / build error</summary>'
- f'<pre class="logtext">{body}</pre></details>')
+ body, _h = _render_logtext("\n".join(lines[max(0, pin - 3) : pin + 40]).strip())
+ return (
+ '<details class="logblock rootcause" open><summary>'
+ "likely root cause · install / build error</summary>"
+ f'<pre class="logtext">{body}</pre></details>'
+ )
def _root_cause_banner(lines, fails, groups):
"""When most failures share one reason, say so once and point at the cause."""
reason, members = groups[0]
n = len(members)
- msg = (f'<strong>{n} of {len(fails)}</strong> failures share one cause: '
- f'<code>{e(reason)}</code>.')
+ msg = (
+ f"<strong>{n} of {len(fails)}</strong> failures share one cause: "
+ f"<code>{e(reason)}</code>."
+ )
extra = ""
if _IMPORT_FAIL.search(reason):
- msg += (f' That is an <em>install / build</em> failure, not {n} test bugs — '
- 'the package never imported.')
+ msg += (
+ f" That is an <em>install / build</em> failure, not {n} test bugs — "
+ "the package never imported."
+ )
extra = _install_error_block(lines)
return f'<div class="rootbanner">{msg}</div>{extra}'
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
- tools = ('<div class="logtools">'
- '<input class="logfind" type="search" placeholder="search these logs…" '
- 'oninput="logSearch(this)" aria-label="search logs">'
- '<label class="logfilt"><input type="checkbox" onchange="logFilter(this)"> matches only</label>'
- '<span class="logcount muted"></span></div>')
+ tools = (
+ '<div class="logtools">'
+ '<input class="logfind" type="search" placeholder="search these logs…" '
+ 'oninput="logSearch(this)" aria-label="search logs">'
+ '<label class="logfilt"><input type="checkbox" onchange="logFilter(this)"> matches only</label>'
+ '<span class="logcount muted"></span></div>'
+ )
fails = get_failures(job_id).get("failures", [])
blocks = []
if fails:
groups = _group_failures(fails)
if len(fails) >= 3 and any(len(m) > 1 for _, m in groups):
blocks.append(_root_cause_banner(lines, fails, groups))
for reason, members in groups:
if len(members) == 1:
f = members[0]
sub = f'<div class="logreason">{e(_reason_first(f["message"]))}</div>'
- blocks.append(_logblock(f["test"] or "failure", sub,
- extract_test_log(lines, f["test"]), is_open=True))
+ blocks.append(
+ _logblock(
+ f["test"] or "failure",
+ sub,
+ extract_test_log(lines, f["test"]),
+ is_open=True,
+ )
+ )
else: # collapsed: N parametrized / duplicate failures → one block
names = ", ".join(m["test"] or "?" for m in members[:10])
more = f" +{len(members) - 10} more" if len(members) > 10 else ""
- sub = (f'<div class="logreason">{e(reason)}</div>'
- f'<div class="logmeta">{len(members)} tests · {e(names)}{e(more)}</div>')
- blocks.append(_logblock(f"{len(members)}× {reason}", sub,
- extract_test_log(lines, members[0]["test"]), is_open=True))
+ sub = (
+ f'<div class="logreason">{e(reason)}</div>'
+ f'<div class="logmeta">{len(members)} tests · {e(names)}{e(more)}</div>'
+ )
+ blocks.append(
+ _logblock(
+ f"{len(members)}× {reason}",
+ sub,
+ extract_test_log(lines, members[0]["test"]),
+ is_open=True,
+ )
+ )
else: # no annotations → anchor on the real error; surface install failures
title, s, en = find_error_window(lines)
window = "\n".join(lines[s:en]).strip()
- import_dominated = title == "short test summary" and len(_IMPORT_FAIL.findall(window)) >= 2
+ import_dominated = (
+ title == "short test summary" and len(_IMPORT_FAIL.findall(window)) >= 2
+ )
inst = _install_error_block(lines) if import_dominated else ""
if inst:
blocks.append(inst)
- blocks.append('<div class="rootbanner">Every test failed at <em>import</em> — this is '
- 'an <em>install / build</em> failure (above), not test bugs.</div>')
+ blocks.append(
+ '<div class="rootbanner">Every test failed at <em>import</em> — this is '
+ "an <em>install / build</em> failure (above), not test bugs.</div>"
+ )
blocks.append(_logblock(title, "", window, is_open=not inst))
return head + tools + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
@@ -661,11 +815,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -693,33 +849,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -739,12 +942,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -779,27 +984,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -810,24 +1022,35 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -836,38 +1059,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + agg + board + poller
def render_status(branch):
@@ -878,12 +1114,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -900,38 +1138,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -939,49 +1208,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -997,38 +1291,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -1040,12 +1353,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -1053,31 +1371,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -1101,49 +1423,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -1155,12 +1502,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -1188,13 +1540,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1207,15 +1562,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
There was a problem hiding this comment.
There are some changes that do not conform to Python style guidelines:
--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-21 22:12:48.848442+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/lowering/passes/eliminate_sym_min_int64_max.py 2026-07-21 22:13:14.812240+00:00
@@ -2,11 +2,10 @@
import torch
from torch.fx import GraphModule, Node
from .pass_utils import clean_up_graph_after_modifications
-
_INT64_MAX = 2**63 - 1
_SYM_MIN = getattr(torch, "sym_min", None)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-21 22:12:48.864351+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/__main__.py 2026-07-21 22:13:16.474069+00:00
@@ -76,11 +76,14 @@
def _cmd_run_lane(args: argparse.Namespace) -> int:
"""Run every suite in a lane/tier, continuing past failures (so one consolidated
report sees them all). Returns non-zero if any suite failed."""
jobs = select(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not jobs:
print("::warning::no suites match the given filters", file=sys.stderr)
return 0
@@ -90,11 +93,14 @@
return rc
def _cmd_matrix(args: argparse.Namespace) -> int:
include = matrix(
- lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform,
+ lane=args.lane,
+ tier=args.tier,
+ variant=args.variant,
+ platform=args.platform,
changed=_read_changed(args),
)
if not include:
print("::warning::matrix is empty for the given filters", file=sys.stderr)
print(json.dumps({"include": include}))
@@ -177,26 +183,28 @@
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument("--dry-run", action="store_true")
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_run_lane)
sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON")
g = sp.add_mutually_exclusive_group()
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
g.add_argument("--tier", choices=("l0", "l1", "l2"))
sp.add_argument("--variant", choices=("standard", "rtx"))
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
sp.add_argument(
- "--changed-from", metavar="FILE",
+ "--changed-from",
+ metavar="FILE",
help="narrow to suites affected by these changed files (a file, or '-' "
- "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
+ "for stdin; one repo-relative path per line). Broad changes → no narrowing.",
)
sp.set_defaults(fn=_cmd_matrix)
sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor)
--- /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-21 22:12:48.864351+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/ci/runner.py 2026-07-21 22:13:16.564546+00:00
@@ -213,12 +213,22 @@
# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching
# ONLY these can skip the whole test matrix.
_NO_TEST_IMPACT = (
- "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/",
- "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8",
+ "docs/",
+ "docsrc/",
+ "tools/",
+ "notebooks/",
+ "examples/",
+ ".devcontainer/",
+ "README",
+ "CHANGELOG",
+ "LICENSE",
+ ".gitignore",
+ ".pre-commit",
+ ".flake8",
)
def _owned_dirs(s: Suite) -> set[str]:
"""Repo-relative directories a suite owns — the directory of each path target
--- /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-21 22:12:48.891643+00:00
+++ /home/runner/work/TensorRT/TensorRT/tools/ci-dashboard/ci_dashboard.py 2026-07-21 22:13:23.720936+00:00
@@ -18,10 +18,11 @@
We parse that into {group, python, cuda, kind}; the group is a `<suite>-<variant>`
from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red
cell maps to the suite's pytest paths and its exact `python -m tests.ci run`
reproduce command. Pre-migration runs (tier-named) fall back to the legacy map.
"""
+
from __future__ import annotations
import argparse
import html
import json
@@ -64,42 +65,76 @@
# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not
# "<suite>-<variant>") still resolve. New runs go through the manifest above.
# Keys are normalized job group names (see _norm()); `fn` is the old shell tier.
TIER_MAP = {
"l0 dynamo converter tests": dict(
- fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]),
+ fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]
+ ),
"l0 dynamo core tests": dict(
fn="trt_tier_l0_core",
- paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*",
- "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_000_*",
+ "tests/py/dynamo/partitioning/test_000_*",
+ "tests/py/dynamo/lowering/",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]),
"l0 torchscript tests": dict(
- fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]),
+ fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]
+ ),
"l1 dynamo core tests": dict(
fn="trt_tier_l1_dynamo_core",
- paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*",
- "tests/py/dynamo/hlo/"]),
+ paths=[
+ "tests/py/dynamo/runtime/test_001_*",
+ "tests/py/dynamo/partitioning/test_001_*",
+ "tests/py/dynamo/hlo/",
+ ],
+ ),
"l1 dynamo compile tests": dict(
- fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]),
+ fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]
+ ),
"l1 torch compile tests": dict(
fn="trt_tier_l1_torch_compile",
- paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py",
- "tests/py/dynamo/models/test_dyn_models.py"]),
- "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]),
+ paths=[
+ "tests/py/dynamo/backend/",
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
+ "l1 torch script tests": dict(
+ fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]
+ ),
"l2 torch compile tests": dict(
fn="trt_tier_l2_torch_compile",
- paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]),
+ paths=[
+ "tests/py/dynamo/models/test_models.py",
+ "tests/py/dynamo/models/test_dyn_models.py",
+ ],
+ ),
"l2 dynamo compile tests": dict(
- fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]),
+ fn="trt_tier_l2_dynamo_compile",
+ paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"],
+ ),
"l2 dynamo core tests": dict(
- fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]),
+ fn="trt_tier_l2_dynamo_core",
+ paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"],
+ ),
"l2 dynamo plugin tests": dict(
fn="trt_tier_l2_plugin",
- paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]),
- "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]),
+ paths=[
+ "tests/py/dynamo/conversion/",
+ "tests/py/dynamo/automatic_plugin/",
+ "tests/py/kernels/",
+ ],
+ ),
+ "l2 torch script tests": dict(
+ fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]
+ ),
"l2 dynamo distributed tests": dict(
- fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]),
+ fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]
+ ),
}
def _norm(s: str) -> str:
"""Lowercase and collapse separators so 'L2-dynamo-core-tests' and
@@ -202,22 +237,25 @@
CACHE = Cache()
def gh(args, parse=True, timeout=90):
- proc = subprocess.run(["gh", *args], capture_output=True, text=True,
- timeout=timeout, cwd=REPO_ROOT)
+ proc = subprocess.run(
+ ["gh", *args], capture_output=True, text=True, timeout=timeout, cwd=REPO_ROOT
+ )
if proc.returncode != 0:
raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip())
return json.loads(proc.stdout) if parse else proc.stdout
@lru_cache(maxsize=1)
def repo_slug():
try:
- return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
- parse=False).strip()
+ return gh(
+ ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"],
+ parse=False,
+ ).strip()
except Exception:
return "pytorch/TensorRT"
_DEFAULT_BRANCH = None
@@ -225,12 +263,16 @@
def default_branch():
if _DEFAULT_BRANCH:
return _DEFAULT_BRANCH
try:
- b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
- capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip()
+ b = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ ).stdout.strip()
return b if b and b != "HEAD" else "main"
except Exception:
return "main"
@@ -249,27 +291,32 @@
("ci: nightly", "lane", "everything, incl. llm / kernels / distributed"),
("backend: TensorRT", "std", "test the standard TensorRT engine"),
("backend: TensorRT-RTX", "rtx", "test the TensorRT-RTX engine"),
]
_CI_LABEL_NAMES = {n for n, _, _ in CI_LABELS}
-_INERT_LABEL = "Force All Tests[L0+L1+L2]" # replaced by ci: full / ci: nightly
+_INERT_LABEL = "Force All Tests[L0+L1+L2]" # replaced by ci: full / ci: nightly
def _ci_label_chips(labels, pr):
"""The four CI-control labels as on/off state chips; each copies the `gh pr edit`
command to toggle it (disabled when there's no PR to edit)."""
present = set(labels)
num = pr["number"] if pr else None
out = []
for name, group, effect in CI_LABELS:
on = name in present
- cmd = f'gh pr edit {num} --{"remove" if on else "add"}-label "{name}"' if num else ""
- title = effect + (f' · click to copy: {cmd}' if cmd else "")
+ cmd = (
+ f'gh pr edit {num} --{"remove" if on else "add"}-label "{name}"'
+ if num
+ else ""
+ )
+ title = effect + (f" · click to copy: {cmd}" if cmd else "")
out.append(
f'<button class="cilabel {group} {"on" if on else "off"}" data-cmd="{e(cmd)}" '
f'onclick="copyCmd(this)" title="{e(title)}"{"" if cmd else " disabled"}>'
- f'<span class="cilmark">{"✓" if on else "+"}</span>{e(name)}</button>')
+ f'<span class="cilmark">{"✓" if on else "+"}</span>{e(name)}</button>'
+ )
return "".join(out)
def resolve_lane_backend(labels, event="pull_request"):
"""labels → (lane, backend), per _decide.yml. `contains()` in Actions is an
@@ -277,23 +324,23 @@
L = set(labels or [])
has_full, has_nightly = "ci: full" in L, "ci: nightly" in L
has_rtx, has_std = "backend: TensorRT-RTX" in L, "backend: TensorRT" in L
if event == "schedule":
lane = "nightly"
- elif event == "push": # main canary
+ elif event == "push": # main canary
lane = "full"
- else: # pull_request
+ else: # pull_request
lane = "nightly" if has_nightly else "full" if has_full else "fast"
if event != "pull_request":
backend = "both"
elif has_rtx and has_std:
backend = "both"
elif has_rtx:
backend = "rtx"
elif has_std:
backend = "standard"
- elif lane == "fast": # cheap default on a plain PR push
+ elif lane == "fast": # cheap default on a plain PR push
backend = "standard"
else:
backend = "both"
return lane, backend
@@ -303,17 +350,24 @@
Mirrors the entry workflows' channel `if:` gates. Returns [{platform, engine,
kind, suites}], suites empty for build-only channels."""
if _ci_matrix is None:
return []
run, fullish = lane != "skip", lane in ("full", "nightly")
- std, rtx = backend != "rtx", backend != "standard" # which engine channels run
+ std, rtx = backend != "rtx", backend != "standard" # which engine channels run
plan = []
def add(platform, engine, kind, suite_lane, suite_platform):
- suites = ([m["suite"] for m in _ci_matrix(lane=suite_lane, variant=engine,
- platform=suite_platform)]
- if suite_platform else [])
+ suites = (
+ [
+ m["suite"]
+ for m in _ci_matrix(
+ lane=suite_lane, variant=engine, platform=suite_platform
+ )
+ ]
+ if suite_platform
+ else []
+ )
plan.append(dict(platform=platform, engine=engine, kind=kind, suites=suites))
# Linux x86_64 (ci-linux-x86_64.yml): tests on any non-skip lane; py-only full/nightly
if run and std:
add("Linux x86_64", "standard", "tests", lane, "linux-x86_64")
@@ -333,32 +387,54 @@
if fullish and rtx:
add("Windows", "rtx", "python-only", "python-only", "windows")
# SBSA aarch64 (ci-sbsa.yml): BUILD-ONLY (no GPU runners), full/nightly, standard
if fullish and std:
add("Linux aarch64 · SBSA", "standard", "build-only", lane, None)
- add("Linux aarch64 · SBSA", "standard", "build-only · py-only", "python-only", None)
+ add(
+ "Linux aarch64 · SBSA",
+ "standard",
+ "build-only · py-only",
+ "python-only",
+ None,
+ )
return plan
def pr_for(branch):
"""The open PR whose head is `branch` (number/title/url/labels), or None."""
try:
- prs = gh(["pr", "list", "--head", branch, "--state", "open",
- "--json", "number,title,url,labels", "--limit", "1"])
+ prs = gh(
+ [
+ "pr",
+ "list",
+ "--head",
+ branch,
+ "--state",
+ "open",
+ "--json",
+ "number,title,url,labels",
+ "--limit",
+ "1",
+ ]
+ )
except Exception:
return None
return prs[0] if prs else None
# ── data layer ───────────────────────────────────────────────────────────────
def get_runs(branch, limit=40, refresh=False):
key = f"runs:{branch}:{limit}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status,"
- "conclusion,event,createdAt,updatedAt,url,number")
- runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields])
+ fields = (
+ "databaseId,workflowName,displayTitle,headBranch,headSha,status,"
+ "conclusion,event,createdAt,updatedAt,url,number"
+ )
+ runs = gh(
+ ["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]
+ )
newest = {}
for r in runs:
r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId)
wf = r.get("workflowName") or "?"
if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]:
@@ -386,51 +462,73 @@
kind = "build"
elif re.match(r"l[012]\b", gl):
kind = "test"
elif "matrix" in gl or "generate" in gl or group == "filter-matrix":
kind = "setup"
- elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl:
+ elif (
+ raw.startswith("CI /")
+ or "aggregate" in gl
+ or "collect results" in gl
+ or "rollup" in gl
+ ):
kind = "rollup"
else:
kind = "other"
return dict(
- id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind,
- python=py.group(0) if py else None, cuda=cu.group(0) if cu else None,
- status=j.get("status"), conclusion=j.get("conclusion"),
+ id=j.get("databaseId") or j.get("id"),
+ raw=raw,
+ group=group,
+ kind=kind,
+ python=py.group(0) if py else None,
+ cuda=cu.group(0) if cu else None,
+ status=j.get("status"),
+ conclusion=j.get("conclusion"),
url=j.get("html_url") or j.get("url"),
- failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"],
+ failedSteps=[
+ s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"
+ ],
)
def get_jobs(run_id, refresh=False):
key = f"jobs:{run_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
- data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
- "--paginate", "-q", ".jobs[]"], parse=False)
+ data = gh(
+ [
+ "api",
+ f"repos/{repo_slug()}/actions/runs/{run_id}/jobs",
+ "--paginate",
+ "-q",
+ ".jobs[]",
+ ],
+ parse=False,
+ )
jobs = [json.loads(line) for line in data.splitlines() if line.strip()]
parsed = [_parse_job(j) for j in jobs]
live = any(j["status"] != "completed" for j in parsed)
CACHE.put(key, parsed, ttl=15 if live else 300)
return parsed
NOISE = re.compile(
r"Process completed with exit code|Node\.js \d+ is deprecated|"
r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|"
- r"might have been added implicitly", re.I)
+ r"might have been added implicitly",
+ re.I,
+)
# A test id is the annotation's first line: a single token that names a test —
# a bare `test_x[param]`, a dotted `Class.test_x`, or a `file.py::Class::test_x`
# nodeid. (The old regex required a dot, so it missed bare parametrized names.)
TESTLINE = re.compile(r"^[\w./\[\]:-]+$")
def _test_name(head):
"""Pull a test id out of an annotation's first line, or None."""
if not head or " " in head or not TESTLINE.match(head):
return None
- tok = head.split("::")[-1] if "::" in head else head # nodeid → last segment
+ tok = head.split("::")[-1] if "::" in head else head # nodeid → last segment
return tok if re.search(r"test", tok, re.I) else None
def _annot_loc(a):
"""Exact (repo-relative file, line) straight from the annotation — better than
@@ -442,12 +540,17 @@
return (m.group(1), a.get("start_line") or 0)
def _git_grep(pat):
try:
- out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"],
- capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout
+ out = subprocess.run(
+ ["git", "grep", "-n", "-E", pat, "--", "tests/"],
+ capture_output=True,
+ text=True,
+ cwd=REPO_ROOT,
+ timeout=15,
+ ).stdout
except Exception:
return None
for line in out.splitlines():
path, lineno, _ = line.split(":", 2)
return (path, int(lineno))
@@ -459,15 +562,15 @@
"""Resolve a failing-test symbol (Class.method / module.func) to (file, line)
via `git grep`. Tries the method, then the enclosing class — parametrized names
(`@parameterized.expand` → test_x_5) have no literal `def`, so the class is the
best anchor. Returns None if nothing matches."""
parts = [re.sub(r"\[.*$", "", p) for p in re.split(r"[.:]+", symbol.strip()) if p]
- for token in reversed(parts): # method first, then its class
+ for token in reversed(parts): # method first, then its class
if not re.match(r"^[A-Za-z_]\w+$", token):
continue
pat = f"def {token}\\b" if token.startswith("test") else f"class {token}\\b"
- if (hit := _git_grep(pat)):
+ if hit := _git_grep(pat):
return hit
return None
def get_failures(job_id, refresh=False):
@@ -489,35 +592,46 @@
test = _test_name(head)
dedupe = test or head
if dedupe in seen:
continue
seen.add(dedupe)
- loc = _annot_loc(a) or (grep_test(test) if test else None) # annotation line first
- failures.append(dict(test=test, message=msg[:1400],
- file=loc[0] if loc else None, line=loc[1] if loc else None))
+ loc = _annot_loc(a) or (
+ grep_test(test) if test else None
+ ) # annotation line first
+ failures.append(
+ dict(
+ test=test,
+ message=msg[:1400],
+ file=loc[0] if loc else None,
+ line=loc[1] if loc else None,
+ )
+ )
result = dict(failures=failures)
CACHE.put(key, result, ttl=600)
return result
# ── job logs (fetch + per-test extraction) ──────────────────────────────────
-_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
-_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") # SGR / CSI colour codes
-_GROUP = re.compile(r"^##\[(?:group|section|command)\]") # fold marker → keep title
-_ENDGROUP = re.compile(r"^##\[endgroup\]\s*$") # pure noise
-_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
-_SECT = re.compile(r"^={4,}") # ==== section ====
+_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix
+_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") # SGR / CSI colour codes
+_GROUP = re.compile(r"^##\[(?:group|section|command)\]") # fold marker → keep title
+_ENDGROUP = re.compile(r"^##\[endgroup\]\s*$") # pure noise
+_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep
+_SECT = re.compile(r"^={4,}") # ==== section ====
def get_job_log(job_id, refresh=False):
"""Raw plain-text log for one job (immutable once complete → cached 1h)."""
key = f"log:{job_id}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
- raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
- parse=False, timeout=60)
+ raw = gh(
+ ["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"],
+ parse=False,
+ timeout=60,
+ )
except Exception:
raw = None
CACHE.put(key, raw, ttl=3600 if raw else 20)
return raw
@@ -537,12 +651,13 @@
def extract_test_log(lines, test, max_lines=160):
"""Pull the pytest FAILURES block for `test`. Falls back to a context window
around the test's last mention (covers custom harnesses w/o a FAILURES sep).
Returns the excerpt string, or None."""
leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None
- seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines)
- if (m := _SEP.match(ln))]
+ seps = [
+ (i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _SEP.match(ln))
+ ]
start = None
if test:
# Prefer an exact title match — the leaf method name (e.g. test_trt_compile)
# is shared across many classes, so fuzzy matching would grab the wrong block.
for i, title in seps:
@@ -563,59 +678,85 @@
end = j
break
block = lines[start:end]
else:
needle = leaf or test
- hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None)
+ hit = next(
+ (i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]),
+ None,
+ )
if hit is None:
return None
- block = lines[max(0, hit - 4): hit + 44]
+ block = lines[max(0, hit - 4) : hit + 44]
if len(block) > max_lines:
- block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"]
+ block = block[:max_lines] + [
+ f"… (+{len(block) - max_lines} more lines — open the raw log)"
+ ]
return "\n".join(block).strip()
# ── anchoring: find the *real* error in a non-pytest (build/install) failure ──
# In GH logs the first `##[error]` is almost always echoed shell text
# (`echo "::error::…"`), so we never anchor on it. The reliable signals, in order:
# the pytest summary, a pip/build metadata error, or a walk backward from the
# `Process completed with exit code N` line to the nearest genuine error.
_SUMMARY_HDR = re.compile(r"^=+\s*short test summary info\s*=+", re.I)
-_PIP_ERR = re.compile(r"error: subprocess-exited-with-error|did not run successfully|^\s*╰─>")
+_PIP_ERR = re.compile(
+ r"error: subprocess-exited-with-error|did not run successfully|^\s*╰─>"
+)
_EXITCODE = re.compile(r"(?:Process completed with exit code|exit code:?)\s*[1-9]")
-_ECHO_NOISE = re.compile(r'echo\s+["\']?::|Specified script file|^\s*(?:if|else|elif|fi|then|source|for|do|done)\b')
+_ECHO_NOISE = re.compile(
+ r'echo\s+["\']?::|Specified script file|^\s*(?:if|else|elif|fi|then|source|for|do|done)\b'
+)
_REAL_ERR = re.compile(
r"Traceback \(most recent call last\)|\b\w*(?:Error|Exception)\b\s*:|"
r"^E\s{2,}\S|^\s*(?:FAILED|ERROR)\s|error: subprocess-exited-with-error|"
- r"CMake Error|fatal error:|ninja: build stopped|Segmentation fault|OutOfMemory|Killed\b")
+ r"CMake Error|fatal error:|ninja: build stopped|Segmentation fault|OutOfMemory|Killed\b"
+)
def find_error_window(lines, max_lines=140):
"""Anchor a non-pytest failure on its real error instead of the blind tail.
Returns (title, start, end)."""
n = len(lines)
if not n:
return ("end of job log", 0, 0)
- for i, ln in enumerate(lines): # 1. pytest summary
+ for i, ln in enumerate(lines): # 1. pytest summary
if _SUMMARY_HDR.search(ln):
- end = next((j for j in range(i + 1, n)
- if _SECT.match(lines[j]) and not _SUMMARY_HDR.search(lines[j])), n)
+ end = next(
+ (
+ j
+ for j in range(i + 1, n)
+ if _SECT.match(lines[j]) and not _SUMMARY_HDR.search(lines[j])
+ ),
+ n,
+ )
return ("short test summary", i, min(end, i + max_lines))
- for i, ln in enumerate(lines): # 2. pip / build error
+ for i, ln in enumerate(lines): # 2. pip / build error
if _PIP_ERR.search(ln):
return ("install / build error", max(0, i - 3), min(n, i + max_lines))
- exit_i = next((i for i in range(n - 1, -1, -1) if _EXITCODE.search(lines[i])), n - 1)
- anchor = next((i for i in range(exit_i, -1, -1) # 3. walk back to real error
- if _REAL_ERR.search(lines[i]) and not _ECHO_NOISE.search(lines[i])), None)
+ exit_i = next(
+ (i for i in range(n - 1, -1, -1) if _EXITCODE.search(lines[i])), n - 1
+ )
+ anchor = next(
+ (
+ i
+ for i in range(exit_i, -1, -1) # 3. walk back to real error
+ if _REAL_ERR.search(lines[i]) and not _ECHO_NOISE.search(lines[i])
+ ),
+ None,
+ )
if anchor is not None:
start = anchor
- for j in range(anchor, max(0, anchor - 80), -1): # widen up to the traceback head
+ for j in range(
+ anchor, max(0, anchor - 80), -1
+ ): # widen up to the traceback head
if "Traceback (most recent call last)" in lines[j]:
start = j
break
return ("error", max(0, start - 2), min(n, exit_i + 1))
- return ("end of job log", max(0, n - 180), n) # 4. fallback: tail
+ return ("end of job log", max(0, n - 180), n) # 4. fallback: tail
# ── root-cause collapse: many identical failures → one line + a pointer ───────
_EXC_RE = re.compile(r"\b([A-Z]\w*(?:Error|Exception|Warning))\b\s*:?\s*([^\n]*)")
@@ -656,11 +797,12 @@
# ── severity rendering: per-line spans so the client can search / jump / colour ─
_SEV_ERR = re.compile(
r"^##\[error\]|Traceback \(most recent call last\)|\b\w*(?:Error|Exception)\b\s*:|"
r"^E\s{2,}\S|^\s*(?:FAILED|ERROR)\s|error: subprocess-exited-with-error|"
- r"CMake Error|fatal error:|Segmentation fault|\bAssertionError\b|exit code:?\s*[1-9]")
+ r"CMake Error|fatal error:|Segmentation fault|\bAssertionError\b|exit code:?\s*[1-9]"
+)
_SEV_WARN = re.compile(r"^##\[warning\]|\bwarning\b|\bdeprecat|\bWARN\b", re.I)
_SEV_PASS = re.compile(r"\bPASSED\b|=+\s*\d+ passed|^\s*ok\s*$", re.I)
def _render_logtext(text):
@@ -676,101 +818,143 @@
sev = "warn"
elif _SEV_PASS.search(ln):
sev = "pass"
attr = f' data-sev="{sev}"' if sev else ""
spans.append(f'<span class="ll"{attr}>{e(ln) or " "}</span>')
- return "".join(spans), hits # .ll is display:block; no newlines between
+ return "".join(spans), hits # .ll is display:block; no newlines between
def _logblock(title, subtitle, excerpt, is_open=False):
"""A collapsible log excerpt: severity-highlighted <pre> + an error jump-list."""
if not excerpt:
- return (f'<details class="logblock"><summary>{e(title)}</summary>{subtitle}'
- '<div class="muted" style="padding:8px 12px 12px">No matching block '
- 'in the log — see the raw log.</div></details>')
+ return (
+ f'<details class="logblock"><summary>{e(title)}</summary>{subtitle}'
+ '<div class="muted" style="padding:8px 12px 12px">No matching block '
+ "in the log — see the raw log.</div></details>"
+ )
body, hits = _render_logtext(excerpt)
jumps = ""
if hits:
- chips = "".join(f'<button class="jump" data-line="{i}" onclick="jumpLine(this)" '
- f'title="{e(short)}">{e(short[:44])}</button>' for i, short in hits[:8])
+ chips = "".join(
+ f'<button class="jump" data-line="{i}" onclick="jumpLine(this)" '
+ f'title="{e(short)}">{e(short[:44])}</button>'
+ for i, short in hits[:8]
+ )
jumps = f'<div class="logjumps"><span class="lbl">jump</span>{chips}</div>'
op = " open" if is_open else ""
- return (f'<details class="logblock"{op}><summary>{e(title)}</summary>{subtitle}'
- f'{jumps}<pre class="logtext">{body}</pre></details>')
-
-
-_IMPORT_FAIL = re.compile(r"ModuleNotFoundError|ImportError|No module named|"
- r"error collecting|during collection|failed to import", re.I)
+ return (
+ f'<details class="logblock"{op}><summary>{e(title)}</summary>{subtitle}'
+ f'{jumps}<pre class="logtext">{body}</pre></details>'
+ )
+
+
+_IMPORT_FAIL = re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|"
+ r"error collecting|during collection|failed to import",
+ re.I,
+)
def _install_error_block(lines):
"""Locate a pip/build install error in the log and render it, or return ''."""
pin = next((i for i, ln in enumerate(lines) if _PIP_ERR.search(ln)), None)
if pin is None:
return ""
- body, _h = _render_logtext("\n".join(lines[max(0, pin - 3):pin + 40]).strip())
- return ('<details class="logblock rootcause" open><summary>'
- 'likely root cause · install / build error</summary>'
- f'<pre class="logtext">{body}</pre></details>')
+ body, _h = _render_logtext("\n".join(lines[max(0, pin - 3) : pin + 40]).strip())
+ return (
+ '<details class="logblock rootcause" open><summary>'
+ "likely root cause · install / build error</summary>"
+ f'<pre class="logtext">{body}</pre></details>'
+ )
def _root_cause_banner(lines, fails, groups):
"""When most failures share one reason, say so once and point at the cause."""
reason, members = groups[0]
n = len(members)
- msg = (f'<strong>{n} of {len(fails)}</strong> failures share one cause: '
- f'<code>{e(reason)}</code>.')
+ msg = (
+ f"<strong>{n} of {len(fails)}</strong> failures share one cause: "
+ f"<code>{e(reason)}</code>."
+ )
extra = ""
if _IMPORT_FAIL.search(reason):
- msg += (f' That is an <em>install / build</em> failure, not {n} test bugs — '
- 'the package never imported.')
+ msg += (
+ f" That is an <em>install / build</em> failure, not {n} test bugs — "
+ "the package never imported."
+ )
extra = _install_error_block(lines)
return f'<div class="rootbanner">{msg}</div>{extra}'
def render_joblog(job_id, url):
log = get_job_log(job_id)
- ghlink = f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>' if url else ""
+ ghlink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">job on GitHub ↗</a>'
+ if url
+ else ""
+ )
rawlink = f'<a href="/ui/joblog?job={e(job_id)}&raw=1" target="_blank" rel="noopener">raw log ↗</a>'
head = f'<div class="logsec-head"><span class="lbl">relevant logs</span>{rawlink} · {ghlink}</div>'
if not log:
- return head + ('<div class="muted">Log not available yet — the job may still be '
- f'running, or GitHub expired it. {ghlink}</div>')
+ return head + (
+ '<div class="muted">Log not available yet — the job may still be '
+ f"running, or GitHub expired it. {ghlink}</div>"
+ )
lines = _clean_lines(log)
- tools = ('<div class="logtools">'
- '<input class="logfind" type="search" placeholder="search these logs…" '
- 'oninput="logSearch(this)" aria-label="search logs">'
- '<label class="logfilt"><input type="checkbox" onchange="logFilter(this)"> matches only</label>'
- '<span class="logcount muted"></span></div>')
+ tools = (
+ '<div class="logtools">'
+ '<input class="logfind" type="search" placeholder="search these logs…" '
+ 'oninput="logSearch(this)" aria-label="search logs">'
+ '<label class="logfilt"><input type="checkbox" onchange="logFilter(this)"> matches only</label>'
+ '<span class="logcount muted"></span></div>'
+ )
fails = get_failures(job_id).get("failures", [])
blocks = []
if fails:
groups = _group_failures(fails)
if len(fails) >= 3 and any(len(m) > 1 for _, m in groups):
blocks.append(_root_cause_banner(lines, fails, groups))
for reason, members in groups:
if len(members) == 1:
f = members[0]
sub = f'<div class="logreason">{e(_reason_first(f["message"]))}</div>'
- blocks.append(_logblock(f["test"] or "failure", sub,
- extract_test_log(lines, f["test"]), is_open=True))
+ blocks.append(
+ _logblock(
+ f["test"] or "failure",
+ sub,
+ extract_test_log(lines, f["test"]),
+ is_open=True,
+ )
+ )
else: # collapsed: N parametrized / duplicate failures → one block
names = ", ".join(m["test"] or "?" for m in members[:10])
more = f" +{len(members) - 10} more" if len(members) > 10 else ""
- sub = (f'<div class="logreason">{e(reason)}</div>'
- f'<div class="logmeta">{len(members)} tests · {e(names)}{e(more)}</div>')
- blocks.append(_logblock(f"{len(members)}× {reason}", sub,
- extract_test_log(lines, members[0]["test"]), is_open=True))
+ sub = (
+ f'<div class="logreason">{e(reason)}</div>'
+ f'<div class="logmeta">{len(members)} tests · {e(names)}{e(more)}</div>'
+ )
+ blocks.append(
+ _logblock(
+ f"{len(members)}× {reason}",
+ sub,
+ extract_test_log(lines, members[0]["test"]),
+ is_open=True,
+ )
+ )
else: # no annotations → anchor on the real error; surface install failures
title, s, en = find_error_window(lines)
window = "\n".join(lines[s:en]).strip()
- import_dominated = title == "short test summary" and len(_IMPORT_FAIL.findall(window)) >= 2
+ import_dominated = (
+ title == "short test summary" and len(_IMPORT_FAIL.findall(window)) >= 2
+ )
inst = _install_error_block(lines) if import_dominated else ""
if inst:
blocks.append(inst)
- blocks.append('<div class="rootbanner">Every test failed at <em>import</em> — this is '
- 'an <em>install / build</em> failure (above), not test bugs.</div>')
+ blocks.append(
+ '<div class="rootbanner">Every test failed at <em>import</em> — this is '
+ "an <em>install / build</em> failure (above), not test bugs.</div>"
+ )
blocks.append(_logblock(title, "", window, is_open=not inst))
return head + tools + "".join(blocks)
# ── status helpers ───────────────────────────────────────────────────────────
@@ -780,11 +964,13 @@
and a cancelled job didn't produce a verdict. Only `failure`/`timed_out`
(actually executed and failed) are `fail`."""
if status and status != "completed":
if status in ("in_progress", "running"):
return "run", "running"
- return "queue", status.replace("_", " ") # queued / waiting / pending / requested
+ return "queue", status.replace(
+ "_", " "
+ ) # queued / waiting / pending / requested
c = conclusion or ""
return {
"success": ("pass", "passed"),
"failure": ("fail", "failed"),
"timed_out": ("fail", "timed out"),
@@ -812,33 +998,80 @@
# so order is most-specific → most-generic. `kind` drives the tag colour:
# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real.
# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM
# signatures below are listed first on purpose so they win over the generic assert.
_FAIL_CATS = [
- ("oom", "GPU / resource", "infra", re.compile(
- r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
- r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
- r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
- r"CUDA error|cudaMalloc|device-side assert", re.I)),
- ("timeout", "timeout", "infra", re.compile(
- r"timed out|timeout|deadline exceeded|took too long", re.I)),
- ("import", "import / collection", "env", re.compile(
- r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
- r"error collecting|errors during collection|failed to import", re.I)),
- ("unsupported", "unsupported op", "env", re.compile(
- r"no converter|not support|unsupported|Could not find any implementation|"
- r"UnsupportedOperator|no implementation for", re.I)),
- ("numerical", "numerical / accuracy", "bug", re.compile(
- r"not close|Mismatched elements|cosine|tolerance|allclose|"
- r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)),
- ("assertion", "assertion", "bug", re.compile(
- r"\bAssertionError\b|^\s*assert\s", re.I | re.M)),
- ("typeerr", "type / shape", "bug", re.compile(
- r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
- r"shape mismatch|size mismatch|dtype", re.I)),
- ("runtime", "runtime error", "bug", re.compile(
- r"\b(RuntimeError|NotImplementedError)\b", re.I)),
+ (
+ "oom",
+ "GPU / resource",
+ "infra",
+ re.compile(
+ r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|"
+ r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|"
+ r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|"
+ r"CUDA error|cudaMalloc|device-side assert",
+ re.I,
+ ),
+ ),
+ (
+ "timeout",
+ "timeout",
+ "infra",
+ re.compile(r"timed out|timeout|deadline exceeded|took too long", re.I),
+ ),
+ (
+ "import",
+ "import / collection",
+ "env",
+ re.compile(
+ r"ModuleNotFoundError|ImportError|No module named|cannot import name|"
+ r"error collecting|errors during collection|failed to import",
+ re.I,
+ ),
+ ),
+ (
+ "unsupported",
+ "unsupported op",
+ "env",
+ re.compile(
+ r"no converter|not support|unsupported|Could not find any implementation|"
+ r"UnsupportedOperator|no implementation for",
+ re.I,
+ ),
+ ),
+ (
+ "numerical",
+ "numerical / accuracy",
+ "bug",
+ re.compile(
+ r"not close|Mismatched elements|cosine|tolerance|allclose|"
+ r"Max absolute difference|relative difference|accuracy|atol|rtol",
+ re.I,
+ ),
+ ),
+ (
+ "assertion",
+ "assertion",
+ "bug",
+ re.compile(r"\bAssertionError\b|^\s*assert\s", re.I | re.M),
+ ),
+ (
+ "typeerr",
+ "type / shape",
+ "bug",
+ re.compile(
+ r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|"
+ r"shape mismatch|size mismatch|dtype",
+ re.I,
+ ),
+ ),
+ (
+ "runtime",
+ "runtime error",
+ "bug",
+ re.compile(r"\b(RuntimeError|NotImplementedError)\b", re.I),
+ ),
]
# Roll-up labels for the per-run category tally (kind → human summary word).
_KIND_LABEL = {
@@ -858,12 +1091,14 @@
return dict(key=key, label=label, kind=kind)
return dict(key="other", label="error", kind="unknown")
def _cat_tag(cat):
- return (f'<span class="cat {cat["kind"]}" '
- f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>')
+ return (
+ f'<span class="cat {cat["kind"]}" '
+ f'title="{e(cat["label"])} — {cat["kind"]}">{e(cat["label"])}</span>'
+ )
def _config_pattern(occ, all_pys, all_cudas):
"""Which configuration a failure correlates with — the strongest diagnostic
signal after the category. Only asserts "X only" when X is a strict subset of
@@ -898,27 +1133,34 @@
return f"{int(s // n)}{unit} ago"
return "just now"
def pretty_platform(name):
- n = (name.replace("Python-only build and test ", "py-only ")
- .replace("RTX - Build and test ", "RTX ")
- .replace("RTX - Python-only build and test ", "RTX py-only ")
- .replace("Build and test ", "").replace(" wheels", "")
- .replace(" for Jetpack", " · jetpack"))
+ n = (
+ name.replace("Python-only build and test ", "py-only ")
+ .replace("RTX - Build and test ", "RTX ")
+ .replace("RTX - Python-only build and test ", "RTX py-only ")
+ .replace("Build and test ", "")
+ .replace(" wheels", "")
+ .replace(" for Jetpack", " · jetpack")
+ )
return n.strip()
def e(s):
return html.escape(str(s if s is not None else ""))
def qp(**kw):
"""Build a URL-encoded, HTML-attribute-safe query string from kwargs, so
values with spaces/&/ (platform + tier names, job URLs) survive intact."""
- return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}"
- for k, v in kw.items()))
+ return e(
+ "&".join(
+ f"{k}={quote(str(v if v is not None else ''), safe='')}"
+ for k, v in kw.items()
+ )
+ )
# ── HTML fragment renderers ──────────────────────────────────────────────────
def _summary_html(runs, oob=False):
"""The top counts strip. Shared by the board and the live poller so the two
@@ -929,39 +1171,54 @@
cls, _ = classify(r["status"], r["conclusion"])
counts[cls] = counts.get(cls, 0) + 1
didnt_run = counts["skip"] + counts["cancel"]
head = runs[0] if runs else {}
sha = (head.get("headSha") or "")[:9]
- commit = (f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
- f' · {e(rel_time(head.get("createdAt")))}</div>')
+ commit = (
+ f'<div class="commit"><code>{e(sha)}</code> · {e(head.get("displayTitle", ""))[:80]}'
+ f' · {e(rel_time(head.get("createdAt")))}</div>'
+ )
def stat(cls, n, word):
- return (f'<span class="stat {cls}"><span class="dot {cls}"></span>'
- f'<span class="n">{n}</span> {word}</span>') if n else ""
- oob_attr = ' hx-swap-oob="true"' if oob else ''
- return (f'<div id="summary" class="summary"{oob_attr}>'
- f'{stat("fail", counts["fail"], "failing")}'
- f'{stat("run", counts["run"], "running")}'
- f'{stat("queue", counts["queue"], "queued")}'
- f'{stat("pass", counts["pass"], "passing")}'
- f'{stat("skip", didnt_run, "didn’t run")}'
- f'{commit}</div>')
+ return (
+ (
+ f'<span class="stat {cls}"><span class="dot {cls}"></span>'
+ f'<span class="n">{n}</span> {word}</span>'
+ )
+ if n
+ else ""
+ )
+
+ oob_attr = ' hx-swap-oob="true"' if oob else ""
+ return (
+ f'<div id="summary" class="summary"{oob_attr}>'
+ f'{stat("fail", counts["fail"], "failing")}'
+ f'{stat("run", counts["run"], "running")}'
+ f'{stat("queue", counts["queue"], "queued")}'
+ f'{stat("pass", counts["pass"], "passing")}'
+ f'{stat("skip", didnt_run, "didn’t run")}'
+ f"{commit}</div>"
+ )
def _plan_hint(lane, backend):
if lane == "fast":
- h = ('<code>fast</code> L0 smoke only. Add <code>ci: full</code> for all tiers on '
- 'Linux + Windows, or <code>ci: nightly</code> to also run llm / kernels / distributed.')
+ h = (
+ "<code>fast</code> L0 smoke only. Add <code>ci: full</code> for all tiers on "
+ "Linux + Windows, or <code>ci: nightly</code> to also run llm / kernels / distributed."
+ )
elif lane == "full":
- h = ('<code>full</code> runs L0–L2. Add <code>ci: nightly</code> to also run '
- 'llm / kernels / distributed.')
+ h = (
+ "<code>full</code> runs L0–L2. Add <code>ci: nightly</code> to also run "
+ "llm / kernels / distributed."
+ )
else:
- h = '<code>nightly</code> runs everything.'
+ h = "<code>nightly</code> runs everything."
if backend == "standard":
- h += ' Add <code>backend: TensorRT-RTX</code> to also test RTX.'
+ h += " Add <code>backend: TensorRT-RTX</code> to also test RTX."
elif backend == "rtx":
- h += ' Add <code>backend: TensorRT</code> to also test standard.'
+ h += " Add <code>backend: TensorRT</code> to also test standard."
return f'<span class="plan-hint">{h}</span>'
def render_plan(branch, refresh=False):
"""'What will run' panel: branch labels → lane/backend → the exact per-platform
@@ -969,65 +1226,97 @@
key = f"plan:{branch}"
if not refresh and (c := CACHE.get(key)) is not None:
return c
try:
html = _render_plan(branch)
- except Exception as ex: # best-effort — never break the board
- html = f'<div class="muted" style="padding:4px 0">plan unavailable: {e(ex)}</div>'
+ except Exception as ex: # best-effort — never break the board
+ html = (
+ f'<div class="muted" style="padding:4px 0">plan unavailable: {e(ex)}</div>'
+ )
CACHE.put(key, html, ttl=120)
return html
def _render_plan(branch):
pr = None
- if branch == "main": # the push canary — not any head=main PR
+ if branch == "main": # the push canary — not any head=main PR
labels, event, src = [], "push", "push → main"
- elif (pr := pr_for(branch)):
+ elif pr := pr_for(branch):
labels = [l.get("name", "") for l in pr.get("labels", [])]
event, src = "pull_request", f'PR #{pr["number"]}'
else:
labels, event, src = [], "pull_request", "no open PR"
lane, backend = resolve_lane_backend(labels, event)
plan = compute_plan(lane, backend)
njobs = sum(len(c["suites"]) for c in plan)
- srchtml = (f'<a href="{e(pr["url"])}" target="_blank" rel="noopener">{e(src)} ↗</a>'
- if pr else e(src))
+ srchtml = (
+ f'<a href="{e(pr["url"])}" target="_blank" rel="noopener">{e(src)} ↗</a>'
+ if pr
+ else e(src)
+ )
# CI-control labels as on/off state; the rest shown muted as "other".
if event == "push":
- labelblock = ('<div class="plan-labels"><span class="muted">push to <code>main</code> — '
- 'runs the full lane on both engines regardless of labels.</span></div>')
+ labelblock = (
+ '<div class="plan-labels"><span class="muted">push to <code>main</code> — '
+ "runs the full lane on both engines regardless of labels.</span></div>"
+ )
else:
others = [x for x in labels if x not in _CI_LABEL_NAMES and x != _INERT_LABEL]
- inert = (f'<button class="cilabel inert" disabled title="inert — superseded by '
- f'ci: full / ci: nightly">⚠ {e(_INERT_LABEL)}</button>'
- if _INERT_LABEL in labels else "")
- other_html = ('<div class="plan-other"><span class="muted">other:</span> '
- + " ".join(f'<span class="plabel">{e(x)}</span>' for x in others)
- + '</div>') if others else ""
- labelblock = (f'<div class="ci-labels"><span class="cil-head">CI labels</span>'
- f'{_ci_label_chips(labels, pr)}{inert}</div>'
- f'<div class="plan-hint-row">{_plan_hint(lane, backend)}</div>{other_html}')
+ inert = (
+ f'<button class="cilabel inert" disabled title="inert — superseded by '
+ f'ci: full / ci: nightly">⚠ {e(_INERT_LABEL)}</button>'
+ if _INERT_LABEL in labels
+ else ""
+ )
+ other_html = (
+ (
+ '<div class="plan-other"><span class="muted">other:</span> '
+ + " ".join(f'<span class="plabel">{e(x)}</span>' for x in others)
+ + "</div>"
+ )
+ if others
+ else ""
+ )
+ labelblock = (
+ f'<div class="ci-labels"><span class="cil-head">CI labels</span>'
+ f"{_ci_label_chips(labels, pr)}{inert}</div>"
+ f'<div class="plan-hint-row">{_plan_hint(lane, backend)}</div>{other_html}'
+ )
rows = []
for c in plan:
if c["suites"]:
- detail, cnt = f'<span class="psuites">{e(", ".join(c["suites"]))}</span>', str(len(c["suites"]))
+ detail, cnt = (
+ f'<span class="psuites">{e(", ".join(c["suites"]))}</span>',
+ str(len(c["suites"])),
+ )
else:
detail, cnt = '<span class="muted">wheel build · no tests</span>', "—"
- rows.append(f'<tr><td class="pplat">{e(c["platform"])}</td>'
- f'<td><span class="peng {e(c["engine"])}">{e(c["engine"])}</span></td>'
- f'<td class="pkind">{e(c["kind"])}</td>'
- f'<td class="pcnt">{cnt}</td><td class="pdet">{detail}</td></tr>')
- table = ('<table class="plan-table"><thead><tr><th>platform</th><th>engine</th>'
- '<th>kind</th><th>#</th><th>suites</th></tr></thead>'
- f'<tbody>{"".join(rows)}</tbody></table>') if rows else \
- '<div class="muted">Manifest not available on this checkout.</div>'
- summary = (f'<summary><span class="lbl">test plan</span>'
- f'<span class="pverdict">lane <b>{e(lane)}</b> · backend <b>{e(backend)}</b></span>'
- f'<span class="muted">{njobs} test jobs · {srchtml}</span></summary>')
- return (f'<details class="plan" open>{summary}'
- f'<div class="plan-body">{labelblock}{table}</div></details>')
+ rows.append(
+ f'<tr><td class="pplat">{e(c["platform"])}</td>'
+ f'<td><span class="peng {e(c["engine"])}">{e(c["engine"])}</span></td>'
+ f'<td class="pkind">{e(c["kind"])}</td>'
+ f'<td class="pcnt">{cnt}</td><td class="pdet">{detail}</td></tr>'
+ )
+ table = (
+ (
+ '<table class="plan-table"><thead><tr><th>platform</th><th>engine</th>'
+ "<th>kind</th><th>#</th><th>suites</th></tr></thead>"
+ f'<tbody>{"".join(rows)}</tbody></table>'
+ )
+ if rows
+ else '<div class="muted">Manifest not available on this checkout.</div>'
+ )
+ summary = (
+ f'<summary><span class="lbl">test plan</span>'
+ f'<span class="pverdict">lane <b>{e(lane)}</b> · backend <b>{e(backend)}</b></span>'
+ f'<span class="muted">{njobs} test jobs · {srchtml}</span></summary>'
+ )
+ return (
+ f'<details class="plan" open>{summary}'
+ f'<div class="plan-body">{labelblock}{table}</div></details>'
+ )
def render_board(branch, refresh=False):
try:
runs = get_runs(branch, refresh=refresh)
@@ -1036,38 +1325,51 @@
if not runs:
return f'<div class="empty-state">No CI runs found for <code>{e(branch)}</code>.</div>'
for r in runs:
r["_cls"], r["_label"] = classify(r["status"], r["conclusion"])
- runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower()))
+ runs.sort(
+ key=lambda r: (
+ RANK.get(r["_cls"], 9),
+ pretty_platform(r["workflowName"]).lower(),
+ )
+ )
summary = _summary_html(runs)
cards = []
for r in runs:
rid, cls, label = r["id"], r["_cls"], r["_label"]
plat = pretty_platform(r["workflowName"])
sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}'
- q = qp(run=rid, sha=r.get("headSha", ""), platform=plat,
- refresh="1" if refresh else "0")
- cards.append(f'''
+ q = qp(
+ run=rid,
+ sha=r.get("headSha", ""),
+ platform=plat,
+ refresh="1" if refresh else "0",
+ )
+ cards.append(f"""
<details class="card {'fail' if cls=='fail' else ''}"
hx-get="/ui/platform?{q}" hx-trigger="toggle once"
hx-target="find .card-body" hx-swap="innerHTML">
<summary class="card-head">
<span class="dot {cls}"></span>
<span class="card-title">{e(plat)}<div class="sub">{sub}</div></span>
<span id="badge-{rid}" class="card-badge {cls}">{e(label)}</span>
<span class="caret">▸</span>
</summary>
<div class="card-body"><div class="muted" style="padding:10px 0"><span class="spinner"></span> loading jobs…</div></div>
- </details>''')
-
- poller = (f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
- f'hx-swap="none"></div>')
- agg = (f'<h2 class="sec">Failures across platforms</h2>'
- f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>')
+ </details>""")
+
+ poller = (
+ f'<div hx-get="/ui/status?branch={e(branch)}" hx-trigger="load delay:20s, every 25s" '
+ f'hx-swap="none"></div>'
+ )
+ agg = (
+ f'<h2 class="sec">Failures across platforms</h2>'
+ f'<div id="agg" hx-get="/ui/aggregate?branch={e(branch)}" hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="agg-loading"><span class="spinner"></span> scanning failing tests…</div></div>'
+ )
board = f'<h2 class="sec">Platforms</h2><div id="board" class="board">{"".join(cards)}</div>'
return summary + render_plan(branch, refresh=refresh) + agg + board + poller
def render_status(branch):
@@ -1078,12 +1380,14 @@
except Exception:
return ""
spans = []
for r in runs:
cls, label = classify(r["status"], r["conclusion"])
- spans.append(f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
- f'class="card-badge {cls}">{e(label)}</span>')
+ spans.append(
+ f'<span id="badge-{r["id"]}" hx-swap-oob="true" '
+ f'class="card-badge {cls}">{e(label)}</span>'
+ )
return _summary_html(runs, oob=True) + "".join(spans)
KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4}
@@ -1100,38 +1404,69 @@
groups = {}
for j in jobs:
groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j)
out = []
- for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])):
+ for (_, gname), gjobs in sorted(
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1])
+ ):
info = info_for(gname)
- tierpath = f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>' if info else ""
+ tierpath = (
+ f'<span class="tierpath">{e(", ".join(info["paths"]))}</span>'
+ if info
+ else ""
+ )
cells, rows = [], []
for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")):
cls, label = classify(j["status"], j["conclusion"])
failable = cls == "fail" and j["kind"] in ("test", "build")
if j["python"] and j["cuda"]:
- inner = (f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
- f'<span class="v">{e(label)}</span>')
+ inner = (
+ f'<span class="k"><span class="dot {cls}"></span>{e(j["python"])}·{e(j["cuda"])}</span>'
+ f'<span class="v">{e(label)}</span>'
+ )
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname,
- py=j["python"], cuda=j["cuda"], url=j["url"])
- cells.append(f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ )
+ cells.append(
+ f'<div class="cell fail" tabindex="0" onclick="openDrawer()" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">{inner}</div>'
+ )
else:
cells.append(f'<div class="cell {cls}">{inner}</div>')
else:
name = j["raw"].split(" / ")[-1] or j["group"]
- link = f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>' if j["url"] else ""
+ link = (
+ f' <a href="{e(j["url"])}" target="_blank" rel="noopener">log ↗</a>'
+ if j["url"]
+ else ""
+ )
extra = ""
if failable:
- q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"])
- extra = (f' · <a href="#" onclick="openDrawer();return false" '
- f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>')
- rows.append(f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
- f'<span class="name">{e(name)}</span>'
- f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>')
+ q = qp(
+ job=j["id"],
+ sha=sha,
+ platform=platform,
+ tier=gname,
+ url=j["url"],
+ )
+ extra = (
+ f' · <a href="#" onclick="openDrawer();return false" '
+ f'hx-get="/ui/failures?{q}" hx-target="#drawer-body" hx-swap="innerHTML">details</a>'
+ )
+ rows.append(
+ f'<div class="jobrow {cls}"><span class="dot {cls}"></span>'
+ f'<span class="name">{e(name)}</span>'
+ f'<span class="jobstate {cls}">{e(label)}</span>{link}{extra}</div>'
+ )
body = ""
if cells:
body += f'<div class="matrix">{"".join(cells)}</div>'
if rows:
body += f'<div class="rows">{"".join(rows)}</div>'
@@ -1139,49 +1474,74 @@
return "".join(out)
def render_failures(job_id, sha, platform, tier, py, cuda, url):
data = get_failures(job_id)
- joblink = (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>') if url else ""
- oob = (f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
- f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>')
+ joblink = (
+ (f' · <a href="{e(url)}" target="_blank" rel="noopener">job log ↗</a>')
+ if url
+ else ""
+ )
+ oob = (
+ f'<span id="drawer-title" hx-swap-oob="true">{e(platform)} — {e(tier)}</span>'
+ f'<span id="drawer-sub" hx-swap-oob="true">{e(py)} {e(cuda)}{joblink}</span>'
+ )
if data.get("error"):
- return oob + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ return (
+ oob
+ + f'<div class="muted">Could not load annotations: {e(data["error"])}</div>'
+ )
fails = data["failures"]
info = info_for(tier)
body = []
if not fails:
- loglink = f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>' if url else ""
- body.append('<div class="muted">No pytest failure annotations — this is likely a '
- f'build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>')
+ loglink = (
+ f'<a href="{e(url)}" target="_blank" rel="noopener">Open the job log ↗</a>'
+ if url
+ else ""
+ )
+ body.append(
+ '<div class="muted">No pytest failure annotations — this is likely a '
+ f"build / environment / setup failure rather than a test assertion.<br><br>{loglink}</div>"
+ )
for f in fails:
loc = ""
if f["file"]:
- gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
- loc = (f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
- f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>')
+ gh_url = (
+ f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}'
+ )
+ loc = (
+ f'<div class="floc">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener">'
+ f'<code>{e(f["file"])}:{f["line"]}</code> ↗</a></div>'
+ )
cat = categorize_failure(f["message"])
- body.append(f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
- f' {_cat_tag(cat)}</div>'
- f'{loc}<pre>{e(f["message"])}</pre></div>')
+ body.append(
+ f'<div class="fail-item"><div class="ftest">{e(f["test"] or "failure")}'
+ f" {_cat_tag(cat)}</div>"
+ f'{loc}<pre>{e(f["message"])}</pre></div>'
+ )
if info:
leaves = []
for f in fails:
if f["test"]:
leaf = re.split(r"[.:]", f["test"])[-1]
leaf = re.sub(r"\[.*$", "", leaf)
if leaf and leaf not in leaves:
leaves.append(leaf)
kexpr = " or ".join(leaves[:6])
cmd = info["repro"](kexpr)
- body.append(f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
- f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>')
+ body.append(
+ f'<div class="repro"><button class="copy" onclick="copyPre(this)">copy</button>'
+ f'<div class="lbl">reproduce locally</div><code>{e(cmd)}</code></div>'
+ )
# Lazy: fetch the job log + extract the relevant block(s) once the drawer is in.
- body.append(f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
- f'hx-trigger="load" hx-swap="innerHTML">'
- f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
- f'fetching relevant logs…</div></div>')
+ body.append(
+ f'<div class="logsec" hx-get="/ui/joblog?{qp(job=job_id, url=url)}" '
+ f'hx-trigger="load" hx-swap="innerHTML">'
+ f'<div class="muted" style="padding:8px 0"><span class="spinner"></span> '
+ f"fetching relevant logs…</div></div>"
+ )
return oob + "".join(body)
def render_aggregate(branch, refresh=False):
"""Cross-platform failure rollup: dedupe a failing test across every platform
@@ -1197,38 +1557,57 @@
except Exception:
return []
with ThreadPoolExecutor(max_workers=8) as ex:
alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub]
- failed = [(r, j) for r, j in alljobs
- if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"]
+ failed = [
+ (r, j)
+ for r, j in alljobs
+ if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"
+ ]
if not failed:
healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs)
- msg = ("No failing test jobs 🎉" if healthy
- else "No test-level failures found (failures are build/setup — see the platform grids).")
+ msg = (
+ "No failing test jobs 🎉"
+ if healthy
+ else "No test-level failures found (failures are build/setup — see the platform grids)."
+ )
return f'<div class="agg-ok">{msg}</div>'
def fails_of(rj):
r, j = rj
return (r, j, get_failures(j["id"], refresh=refresh).get("failures", []))
+
with ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(fails_of, failed))
agg = {}
for r, j, fails in results:
plat = pretty_platform(r["workflowName"])
if not fails: # failed job but no parseable test → bucket under the tier
key = f"[{j['group']}]"
a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[]))
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=""))
+ a["occ"].append(
+ dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")
+ )
continue
for f in fails:
key = f["test"] or f["message"].splitlines()[0]
- a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[]))
+ a = agg.setdefault(
+ key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])
+ )
a["file"] = a["file"] or f["file"]
a["line"] = a["line"] or f["line"]
- a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"]))
+ a["occ"].append(
+ dict(
+ plat=plat,
+ py=j["python"],
+ cuda=j["cuda"],
+ url=j["url"],
+ msg=f["message"],
+ )
+ )
# Config universe (what actually ran) so "X only" hints are real subsets.
test_jobs = [j for _, j in alljobs if j["kind"] == "test"]
all_pys = {j["python"] for j in test_jobs if j["python"]}
all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]}
@@ -1240,12 +1619,17 @@
# Real regressions (bug) first, then env/gap, infra last; ties → most cells.
_KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3}
rows = sorted(
agg.values(),
- key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2),
- -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"]))
+ key=lambda a: (
+ _KIND_RANK.get(a["cat"]["kind"], 2),
+ -len({o["plat"] for o in a["occ"]}),
+ -len(a["occ"]),
+ a["test"],
+ ),
+ )
out = []
for a in rows:
plats = sorted({o["plat"] for o in a["occ"]})
ncells = len(a["occ"])
loc = ""
@@ -1253,31 +1637,35 @@
gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}'
loc = f'<div class="agg-file">↳ <a href="{e(gh_url)}" target="_blank" rel="noopener"><code>{e(a["file"])}:{a["line"]}</code> ↗</a></div>'
chips = "".join(f'<span class="chip">{e(p)}</span>' for p in plats)
pat = _config_pattern(a["occ"], all_pys, all_cudas)
patchips = "".join(f'<span class="cfgpat">{e(p)}</span>' for p in pat)
- out.append(f'''<details class="agg-row-d">
+ out.append(f"""<details class="agg-row-d">
<summary class="agg-row">
<div><div class="agg-test">{e(a["test"])} {_cat_tag(a["cat"])}{patchips}</div>{loc}</div>
<span class="agg-count"><span class="dot fail"></span>{len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""}</span>
</summary>
<div class="agg-detail"><div class="chips">{chips}</div>{f"<pre>{e(a['sample'])}</pre>" if a["sample"] else ""}</div>
- </details>''')
+ </details>""")
# Category tally — an instant read on the run's character (infra flakes vs regressions).
tally = {}
for a in rows:
tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1
order = ["bug", "env", "unknown", "infra"]
tstr = " · ".join(
f'<span class="cat {k}">{tally[k]["n"]} {e(_KIND_LABEL[k])}</span>'
- for k in order if k in tally)
- header = (f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
- f'{"s" if len(rows)!=1 else ""} across '
- f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
- f'<span class="agg-tally">{tstr}</span>'
- f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
- f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>')
+ for k in order
+ if k in tally
+ )
+ header = (
+ f'<div class="agg-head-row"><span>{len(rows)} distinct failing test'
+ f'{"s" if len(rows)!=1 else ""} across '
+ f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms</span>'
+ f'<span class="agg-tally">{tstr}</span>'
+ f'<button class="btn small" hx-get="/ui/aggregate?branch={e(branch)}&refresh=1" '
+ f'hx-target="#agg" hx-swap="innerHTML">↻ rescan</button></div>'
+ )
return header + f'<div class="agg">{"".join(out)}</div>'
# ── HTTP server ──────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
@@ -1301,49 +1689,74 @@
try:
p = u.path
if p in ("/", "/index.html"):
return self._index()
if p.startswith("/static/"):
- return self._static(p[len("/static/"):])
+ return self._static(p[len("/static/") :])
if p == "/ui/board":
- return self._send(200, render_board(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_board(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/status":
- return self._send(200, render_status(q.get("branch") or default_branch()))
+ return self._send(
+ 200, render_status(q.get("branch") or default_branch())
+ )
if p == "/ui/platform":
- return self._send(200, render_platform(q["run"], q.get("sha", ""),
- q.get("platform", ""), refresh))
+ return self._send(
+ 200,
+ render_platform(
+ q["run"], q.get("sha", ""), q.get("platform", ""), refresh
+ ),
+ )
if p == "/ui/failures":
- return self._send(200, render_failures(
- q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""),
- q.get("py", ""), q.get("cuda", ""), q.get("url", "")))
+ return self._send(
+ 200,
+ render_failures(
+ q["job"],
+ q.get("sha", ""),
+ q.get("platform", ""),
+ q.get("tier", ""),
+ q.get("py", ""),
+ q.get("cuda", ""),
+ q.get("url", ""),
+ ),
+ )
if p == "/ui/aggregate":
- return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh))
+ return self._send(
+ 200, render_aggregate(q.get("branch") or default_branch(), refresh)
+ )
if p == "/ui/joblog":
if q.get("raw") == "1":
log = get_job_log(q["job"], refresh)
- return self._send(200 if log else 404,
- log or "log not available", "text/plain; charset=utf-8")
+ return self._send(
+ 200 if log else 404,
+ log or "log not available",
+ "text/plain; charset=utf-8",
+ )
return self._send(200, render_joblog(q["job"], q.get("url", "")))
return self._send(404, "not found", "text/plain")
except KeyError as ex:
self._send(400, f"missing param {ex}", "text/plain")
except Exception as ex:
self._send(500, f'<div class="muted">error: {e(ex)}</div>')
def _index(self):
with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f:
html_s = f.read()
- html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug()))
+ html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace(
+ "{{REPO}}", e(repo_slug())
+ )
self._send(200, html_s)
def _static(self, rel):
rel = rel.split("?")[0].lstrip("/")
path = os.path.normpath(os.path.join(STATIC, rel))
if not path.startswith(STATIC) or not os.path.isfile(path):
return self._send(404, "not found", "text/plain")
- ctype = {"html": "text/html", "js": "text/javascript",
- "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream")
+ ctype = {"html": "text/html", "js": "text/javascript", "css": "text/css"}.get(
+ rel.rsplit(".", 1)[-1], "application/octet-stream"
+ )
with open(path, "rb") as f:
self._send(200, f.read(), ctype + "; charset=utf-8")
def preflight():
@@ -1355,12 +1768,17 @@
def tailscale_ip():
"""Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present,
otherwise sniffs local interfaces. Returns None if not on a tailnet."""
try:
- out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True,
- text=True, timeout=3).stdout.strip().splitlines()
+ out = (
+ subprocess.run(
+ ["tailscale", "ip", "-4"], capture_output=True, text=True, timeout=3
+ )
+ .stdout.strip()
+ .splitlines()
+ )
if out and out[0]:
return out[0].strip()
except Exception:
pass
try:
@@ -1388,13 +1806,16 @@
def main():
global _DEFAULT_BRANCH
ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard")
ap.add_argument("-b", "--branch", default=None, help="default branch to show")
ap.add_argument("-p", "--port", type=int, default=8712)
- ap.add_argument("--host", default="0.0.0.0",
- help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
- "use 127.0.0.1 to keep it local-only)")
+ ap.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; "
+ "use 127.0.0.1 to keep it local-only)",
+ )
ap.add_argument("--no-open", action="store_true")
args = ap.parse_args()
preflight()
if args.branch:
_DEFAULT_BRANCH = args.branch
@@ -1407,15 +1828,18 @@
if ts:
print(f" tailscale http://{ts}:{args.port}/")
lan = lan_ip()
if lan and lan != ts:
print(f" lan http://{lan}:{args.port}/")
- print(" (bound to all interfaces — anyone who can reach this host can view CI status)")
+ print(
+ " (bound to all interfaces — anyone who can reach this host can view CI status)"
+ )
print("Ctrl-C to stop.", flush=True)
if not args.no_open:
try:
import webbrowser
+
webbrowser.open(local) # always open the loopback URL locally
except Exception:
pass
try:
srv.serve_forever()
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Description
Got sick of digging through the Github CI, so added a tool that leverages the gh cli to show the current ci status of a branch
Fixes # (issue)
Type of change
Please delete options that are not relevant and/or add your own.
Checklist: