feat(run): name the criteria a task failed in the summary, and fail CI with --fail-under

src/agent_env/bundle/run.py

@@ -29,7 +29,7 @@
 from agent_env.store.routing import namespace_routing, run_scope
 from agent_env.task import Task, record_task_cancelled
 from agent_env.task.interrupts import Interrupts
-from agent_env.task.task import recorded_scores
+from agent_env.task.task import FailedCriterion, failed_criteria, recorded_scores
 from agent_env.task.teardown import TeardownReport, teardown_run
 from agent_env.task_step.context import TaskStepContext
 from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM
@@ -83,6 +83,11 @@ def scores(self) -> dict[str, float]:
         when that isn't a step's verifier id."""
         return recorded_scores(self.task.steps, self.context)
 
+    @property
+    def failed_criteria(self) -> list[FailedCriterion]:
+        """Each criterion a verification scoring below 1 recorded as failed, with the reason it gave."""
+        return failed_criteria(self.task.steps, self.context)
+
     @property
     def outcome(self) -> Outcome:
         if self.cancelled:
@@ -112,6 +117,11 @@ class BundleRun:
     def failed(self) -> bool:
         return any(run.outcome is Outcome.FAILED for run in self.runs)
 
+    def below(self, threshold: float) -> tuple[TaskRun, ...]:
+        """The runs that finished but scored below ``threshold``, or recorded no score, which can't show it passed."""
+        return tuple(run for run in self.runs if run.outcome in (Outcome.PASSED, Outcome.BELOW_ONE, Outcome.UNSCORED)
+                     and (not run.scores or min(run.scores.values()) < threshold))
+
     def path(self, entry: BundleEntry) -> str:
         """``entry``'s path in the bundle, as the summary names it (``tasks/hello.json``)."""
         return relative(self.materialization.plan.bundle.bundle.root, entry.path)

src/agent_env/cli/run.py

@@ -4,11 +4,13 @@
 
 import os
 import shlex
+import sys
 import time
 import traceback
 from pathlib import Path
 
 import click
+from click.core import ParameterSource
 
 from agent_env.bundle import (BundleError, BundleKind, BundleRun, DryRun, Outcome, RunInterrupted, TaskRun,
                               dry_run_bundle, run_bundle)
@@ -25,6 +27,8 @@
 _ENV_ENDPOINTS = (("mcp", "mcp_url"), ("gateway", "gateway_url"), ("pgweb", "db_web_url"), ("db-mcp", "db_mcp_url"),
                   ("vnc", "vnc_url"), ("expires", "expires_at_utc"))
 _DRY_RUN = "Dry run: nothing is written or run."
+_SHOWN_CRITERIA = 5
+_REASON_CHARS = 160
 
 
 @click.command()
@@ -39,41 +43,48 @@
               help="Keep the sandboxes up after the runs, print their endpoints, and tear them down on Ctrl-C.")
 @click.option("--dry-run", is_flag=True,
               help="Check the bundle as the run does and show what it would write and run; write and run nothing.")
+@click.option("--fail-under", type=click.FloatRange(0, 1), default=None, metavar="SCORE",
+              help="Exit 1 when a task scored below SCORE, or recorded no score; --fail-under 1 fails on any miss.")
 @click.pass_context
 def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: tuple[str, ...], model: str | None,
-        sandbox: str | None, keep: bool, dry_run: bool):
+        sandbox: str | None, keep: bool, dry_run: bool, fail_under: float | None):
     """Run a bundle's tasks and evals, or list the installed bundles.
 
     BUNDLE is a folder, or the name of a bundle an installed package provides (PACKAGE/NAME when several
     packages provide that name). Without BUNDLE, it lists the installed bundles.
 
     With no --task or --eval, it runs every eval or, in a bundle without evals, every task. Each task runs
-    once, at most four at a time, and the run exits 1 if any of them failed. With --verbose, a run that
-    raised prints its traceback.
+    once, at most four at a time, and the run exits 1 if any of them failed. A task that scored below 1 lists
+    the criteria that failed and why, and exits 1 only under --fail-under, which also fails a task that
+    recorded no score. With --verbose, a run that raised prints its traceback.
 
     --dry-run makes the checks the run makes before its first task, reading the stores as the run does, and
     prints what it would write and run, writing and running nothing. It exits 1 on a problem the run would
-    stop at, and 0 otherwise; --model and --keep change nothing it shows. It takes no lock, so another run
-    can write first and change what it shows.
+    stop at, and 0 otherwise; --model, --keep and --fail-under change nothing it shows. It takes no lock, so
+    another run can write first and change what it shows.
 
     Each run's sandboxes are torn down as it ends; its instance and outputs stay. --keep holds them up
     until Ctrl-C instead. Ctrl-C or SIGTERM mid-run cancels the runs, tears them down, prints what ran and
     exits 130 (143 for SIGTERM); a second one stops the teardown and prints what is still up.
     """
     if bundle is None:
-        if tasks or evals or model or sandbox or keep or dry_run:
-            raise click.UsageError("--task, --eval, --model, --sandbox, --keep and --dry-run need a BUNDLE to run")
+        given = [param.opts[0] for param in ctx.command.params if isinstance(param, click.Option)
+                 and ctx.get_parameter_source(param.name) not in (None, ParameterSource.DEFAULT)]
+        if given:
+            *rest, last = given
+            raise click.UsageError(f"{', '.join(rest)} and {last} need a BUNDLE to run" if rest
+                                   else f"{last} needs a BUNDLE to run")
         _list_installed()
         return
     root, id_root = _locate(bundle)
     verbose = ctx.find_root().params.get("verbose")
     try:
         if dry_run:
-            click.echo(_DRY_RUN)
-            _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, on_progress=click.echo,
-                                           id_root=id_root))
+            _echo(_DRY_RUN)
+            dry = dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, on_progress=_echo, id_root=id_root)
+            _report_dry_run(dry)
             return
-        result = run_bundle(root, tasks=tasks, evals=evals, model=model, sandbox=sandbox, on_progress=click.echo,
+        result = run_bundle(root, tasks=tasks, evals=evals, model=model, sandbox=sandbox, on_progress=_echo,
                             id_root=id_root, keep=keep)
     except BundleError as e:
         _note_an_installed_namesake(e, bundle, root, id_root)
@@ -88,10 +99,30 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t
         if keep:
             result = _hold(result, interrupts)
         _report_teardown(result)
-    if result.failed:
+    below = result.below(fail_under) if fail_under is not None else ()
+    if below:
+        _report_below(result, below, fail_under)
+    if result.failed or below:
         raise SystemExit(1)
 
 
+def _echo(message: str | None = None, err: bool = False) -> None:
+    """``click.echo``, which drops the line once its stream's reader has gone, as when piped into head, so the run
+    still finishes and tears down."""
+    try:
+        click.echo(message, err=err)
+    except BrokenPipeError:
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, (sys.stderr if err else sys.stdout).fileno())
+        os.close(devnull)
+
+
+def _report_below(result: BundleRun, below: tuple[TaskRun, ...], threshold: float) -> None:
+    shown = ", ".join(f"{result.path(run.entry)} ({min(run.scores.values()):g})" if run.scores
+                      else f"{result.path(run.entry)} (no score)" for run in below)
+    _echo(f"\nBelow --fail-under {threshold:g}: {shown}")
+
+
 def _path_like(text: str) -> bool:
     """Text that names a folder even when there is none there."""
     return text.startswith((".", "/", "~"))
@@ -134,21 +165,21 @@ def _installed() -> tuple[InstalledBundle, ...]:
 def _list_installed() -> None:
     bundles = _installed()
     if not bundles:
-        click.echo("No bundles are installed. agent-env run PATH runs a folder.")
+        _echo("No bundles are installed. agent-env run PATH runs a folder.")
         return
     header = ("NAME", "PACKAGE", "CONTENTS", "DESCRIPTION")
     rows = [(run_name(bundle, bundles), f"{bundle.package} {bundle.version}", *_contents(bundle)) for bundle in bundles]
     widths = [max(len(row[i]) for row in (header, *rows)) for i in range(3)]
 
     def echo(name: str, package: str, contents: str, description: str) -> None:
-        click.echo(f"{name:<{widths[0]}}  {package:<{widths[1]}}  {contents:<{widths[2]}}  {description}".rstrip())
+        _echo(f"{name:<{widths[0]}}  {package:<{widths[1]}}  {contents:<{widths[2]}}  {description}".rstrip())
 
     echo(*header)
     for bundle, row in zip(bundles, rows):
         echo(*row)
         if bundle.root is not None:
-            click.echo(f"{'':<{widths[0]}}  {bundle.root}")
-    click.echo("\nRun one with agent-env run NAME. To change one, copy its folder and run the copy.")
+            _echo(f"{'':<{widths[0]}}  {bundle.root}")
+    _echo("\nRun one with agent-env run NAME. To change one, copy its folder and run the copy.")
 
 
 def _contents(bundle: InstalledBundle) -> tuple[str, str]:
@@ -169,67 +200,84 @@ def _count(entries, kind: BundleKind, noun: str) -> str:
 
 
 def _summarize(result: BundleRun, verbose: bool) -> None:
-    click.echo()
-    click.echo("Tasks:")
+    _echo()
+    _echo("Tasks:")
     for run in result.runs:
-        click.echo(click.style(f"  {_described(result, run)}", fg=_COLORS[run.outcome]))
+        _echo(click.style(f"  {_described(result, run)}", fg=_COLORS[run.outcome]))
+        for line in _criteria_lines(run):
+            _echo(f"    {line}")
     if result.evals:
-        click.echo("Evals:")
+        _echo("Evals:")
     for eval_run in result.evals:
         passed, failed, cancelled = (sum(run.outcome is outcome for run in eval_run.runs)
                                      for outcome in (Outcome.PASSED, Outcome.FAILED, Outcome.CANCELLED))
         line = f"  {result.path(eval_run.entry)} v{eval_run.version}: {passed} of {len(eval_run.runs)} passed"
         rest = [f"{failed} failed" if failed else "", f"{cancelled} cancelled" if cancelled else ""]
-        click.echo(", ".join([line, *filter(None, rest)]))
+        _echo(", ".join([line, *filter(None, rest)]))
     for entry in result.skipped:
-        click.echo(f"{result.path(entry)} isn't named by any eval, so it didn't run; run it with "
-                   f"--task {shlex.quote(entry.name)}")
+        _echo(f"{result.path(entry)} isn't named by any eval, so it didn't run; run it with "
+              f"--task {shlex.quote(entry.name)}")
     if verbose:
         _print_tracebacks(result)
 
 
+def _criteria_lines(run: TaskRun) -> list[str]:
+    """A line for each criterion the run failed, its step, the criterion and the reason it gave, up to
+    ``_SHOWN_CRITERIA``, then a count of the rest."""
+    failed = run.failed_criteria
+    lines = []
+    for criterion in failed[:_SHOWN_CRITERIA]:
+        reason = " ".join(criterion.reason.split())
+        if len(reason) > _REASON_CHARS:
+            reason = reason[:_REASON_CHARS - 1].rstrip() + "…"
+        lines.append(f"{criterion.step}: {criterion.criterion}" + (f": {reason}" if reason else ""))
+    if len(failed) > _SHOWN_CRITERIA:
+        lines.append(f"and {len(failed) - _SHOWN_CRITERIA} more")
+    return lines
+
+
 def _report_dry_run(dry: DryRun) -> None:
     materialization = dry.materialization
     plan = materialization.plan
-    click.echo()
+    _echo()
     if plan.store_refs:
-        click.echo("Store refs:")
+        _echo("Store refs:")
     for ref in plan.store_refs:
         if ref.version is None:
-            click.echo(f"  {ref.kind} {ref.id} v{plan.store_latest[ref.kind, ref.id]}, the latest")
+            _echo(f"  {ref.kind} {ref.id} v{plan.store_latest[ref.kind, ref.id]}, the latest")
         else:
-            click.echo(f"  {ref.kind} {ref.id} v{ref.version}")
+            _echo(f"  {ref.kind} {ref.id} v{ref.version}")
     if images := dry.images():
-        click.echo("Images:")
+        _echo("Images:")
     for line in images:
-        click.echo(f"  {line}")
+        _echo(f"  {line}")
     if dry.infra:
-        click.echo("Would build first:")
+        _echo("Would build first:")
     for build in dry.infra:
-        click.echo(f"  {build}")
-    click.echo("Would run:")
+        _echo(f"  {build}")
+    _echo("Would run:")
     for entry in dry.runs:
-        click.echo(f"  {dry.path(entry)} v{materialization.version_of('task', entry.id)}")
+        _echo(f"  {dry.path(entry)} v{materialization.version_of('task', entry.id)}")
     if plan.evals:
-        click.echo("Evals:")
+        _echo("Evals:")
     for entry in plan.evals:
         named = ", ".join(dry.path(ref.local) for ref in entry.references)
-        click.echo(f"  {dry.path(entry.entry)} v{materialization.version_of('eval', entry.entry.id)}: {named}")
+        _echo(f"  {dry.path(entry.entry)} v{materialization.version_of('eval', entry.entry.id)}: {named}")
     for entry in dry.skipped:
-        click.echo(f"{dry.path(entry)} isn't named by any eval, so it wouldn't run; run it with "
-                   f"--task {shlex.quote(entry.name)}")
+        _echo(f"{dry.path(entry)} isn't named by any eval, so it wouldn't run; run it with "
+              f"--task {shlex.quote(entry.name)}")
     if materialization.not_preflighted:
-        click.echo("Not preflighted, since each reads what the run would write first:")
+        _echo("Not preflighted, since each reads what the run would write first:")
     for write, step in materialization.not_preflighted:
-        click.echo(f"  {dry.path(write.source.entry)}: step {step.id!r} ({step.type})")
-    click.echo(_DRY_RUN)
+        _echo(f"  {dry.path(write.source.entry)}: step {step.id!r} ({step.type})")
+    _echo(_DRY_RUN)
 
 
 def _print_tracebacks(result: BundleRun) -> None:
     for run in result.runs:
         if run.error is not None:
-            click.echo(f"\n{result.path(run.entry)} raised:", err=True)
-            click.echo("".join(traceback.format_exception(run.error)).rstrip(), err=True)
+            _echo(f"\n{result.path(run.entry)} raised:", err=True)
+            _echo("".join(traceback.format_exception(run.error)).rstrip(), err=True)
 
 
 def _described(result: BundleRun, run: TaskRun) -> str:
@@ -254,22 +302,22 @@ def _hold(result: BundleRun, interrupts: Interrupts) -> BundleRun:
     kept = [run for run in result.runs if run.torn_down is None]
     count = sum(len(TeardownReport.skipped(run.context).left) for run in kept)
     if not count:
-        click.echo("\nNothing to keep up: no run left a sandbox.")
+        _echo("\nNothing to keep up: no run left a sandbox.")
         return result
-    click.echo("\nKept up:")
+    _echo("\nKept up:")
     for run in kept:
         lines = _kept_up(run.context)
         if lines:
-            click.echo(f"  {result.path(run.entry)}")
+            _echo(f"  {result.path(run.entry)}")
             for line in lines:
-                click.echo(f"    {line}")
+                _echo(f"    {line}")
     them = "it" if count == 1 else "them"
-    click.echo(f"\nHolding {_sandboxes(count)} up; Ctrl-C tears {them} down.")
+    _echo(f"\nHolding {_sandboxes(count)} up; Ctrl-C tears {them} down.")
     while not interrupts.count:
         time.sleep(0.2)
     if interrupts.count > 1:
         return result
-    click.echo(f"\nTearing down {_sandboxes(count)} (Ctrl-C again to stop now)")
+    _echo(f"\nTearing down {_sandboxes(count)} (Ctrl-C again to stop now)")
     try:
         return result.teardown()
     except RunInterrupted as stopped:
@@ -318,13 +366,13 @@ def _report_teardown(result: BundleRun) -> None:
     still_up = [sandbox for report in reports for sandbox in report.still_up
                 if sandbox.sandbox_type != "local" or LocalSandbox.find_work_dir(sandbox.sandbox_id)]
     if terminated or still_up:
-        click.echo()
+        _echo()
     if terminated:
-        click.echo(f"Tore down {_sandboxes(terminated)}.")
-    if still_up:
-        click.echo(click.style(f"Still up, {_sandboxes(len(still_up))}:", fg="yellow"))
+        _echo(f"Tore down {_sandboxes(terminated)}.")
+    if still_up:  # on stderr, which still reaches the user when stdout is piped away
+        _echo(click.style(f"Still up, {_sandboxes(len(still_up))}:", fg="yellow"), err=True)
         for sandbox in still_up:
-            click.echo(f"  {_still_up(sandbox)}")
+            _echo(f"  {_still_up(sandbox)}", err=True)
 
 
 def _still_up(sandbox: RecordedSandbox) -> str: