From 922c0a957e0764d6f251955f2a0cb24ab125aa34 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 5 Nov 2026 03:42:48 +0000 Subject: [PATCH 1/2] fix(python): accurate LLM hint and exit 124 on Monty time limit The hint said "No classes" after Monty 0.0.19 added simple classes and never mentioned that `re` is disabled. Rewrite it and pin it with a test that imports every module it names and checks the class/match claims. A Monty time-limit hit returned exit 1 and a TimeoutError traceback, so callers had to parse stderr to tell a limit from a script error. Report it like the CPython runtime: exit 124, 'python3: execution timed out'. --- crates/bashkit/docs/python.md | 7 +- crates/bashkit/src/builtins/python.rs | 79 ++++++++++++++++--- .../integration/python_integration_tests.rs | 28 +++++++ knowledge/log.md | 1 + knowledge/runtimes/python-builtin.md | 12 ++- 5 files changed, 111 insertions(+), 16 deletions(-) diff --git a/crates/bashkit/docs/python.md b/crates/bashkit/docs/python.md index bc288bf04..79f2ed3de 100644 --- a/crates/bashkit/docs/python.md +++ b/crates/bashkit/docs/python.md @@ -199,7 +199,7 @@ let help = tool.help(); // Includes a Markdown Notes section with Python hints ``` The builtin's `llm_hint()` is automatically included in the tool's documentation, -so LLMs know file I/O is VFS-scoped and HTTP requests/classes are unavailable. +so LLMs know file I/O is VFS-scoped, HTTP is unavailable and classes are limited. `BashTool::builder().python_with_external_handler(limits, names, handler)` exposes the same named callbacks through the Rust tool builder. The Python runtime gate still applies. Foreign bindings can use `bashkit::runtime_value` for bounded JSON conversion; native Rust callbacks keep their Monty value types. @@ -220,10 +220,11 @@ content = Path('/tmp/data.txt').read_text() **No HTTP/network.** No `socket`, `urllib`, `requests`, or `http.client` modules. Monty has no network primitives and no OsCall variants for network operations. -**No classes.** Class definitions are not yet supported by Monty (planned upstream). +**Simple classes only.** Classes with methods and attributes work; inheritance, +metaclasses and `match` statements are not supported by Monty yet. **No third-party imports.** Only builtin modules (`sys`, `typing`, `os`, `pathlib`, -`math`, `json`, `datetime`) are available. No `pip install`, no `import numpy`. +`math`, `json`, `datetime`, `unicodedata`) are available. No `pip install`, no `import numpy`. **No `re` module.** Disabled due to catastrophic backtracking DoS risk in untrusted code execution. diff --git a/crates/bashkit/src/builtins/python.rs b/crates/bashkit/src/builtins/python.rs index 0ea5c9f68..7b7a9ffed 100644 --- a/crates/bashkit/src/builtins/python.rs +++ b/crates/bashkit/src/builtins/python.rs @@ -429,10 +429,11 @@ fn is_disabled_import(code: &str) -> Option<&'static str> { impl Builtin for Python { fn llm_hint(&self) -> Option<&'static str> { Some( - "python/python3: Embedded Python (Monty). \ - Stdlib: math, pathlib, os.getenv, sys, typing. \ - File I/O via pathlib.Path and open() against the VFS. \ - No HTTP/network. No classes. No third-party imports.", + "python/python3: Embedded Python (Monty), not CPython. \ + Stdlib: json, math, datetime, pathlib, os.getenv, sys, typing, unicodedata; \ + `re` is disabled. File I/O via pathlib.Path and open() against the VFS. \ + Simple classes only (no inheritance), no match statements. \ + No HTTP/network, subprocess or third-party imports. Output via print().", ) } @@ -671,6 +672,7 @@ async fn run_python( // `max_memory` keeps one number to reason about: tightening the memory // limit tightens the output cap with it. let print_cap = Some(py_limits.common.max_memory); + let time_limit = py_limits.common.max_duration; // Important security decision: cap awaited host callbacks with the same wall-clock // budget as Monty so external functions cannot pin execution between VM steps. let python_deadline = Instant::now().checked_add(py_limits.common.max_duration); @@ -686,7 +688,7 @@ async fn run_python( ) { Ok(p) => (p, buf), Err(e) => { - return Ok(format_exception_with_output(e, &buf)); + return Ok(format_exception_with_output(e, &buf, time_limit)); } } }; @@ -704,7 +706,7 @@ async fn run_python( progress = next; } Err(e) => { - return Ok(format_exception_with_output(e, &buf)); + return Ok(format_exception_with_output(e, &buf, time_limit)); } } } @@ -736,7 +738,7 @@ async fn run_python( progress = next; } Err(e) => { - return Ok(format_exception_with_output(e, &buf)); + return Ok(format_exception_with_output(e, &buf, time_limit)); } } } @@ -763,7 +765,7 @@ async fn run_python( progress = next; } Err(e) => { - return Ok(format_exception_with_output(e, &buf)); + return Ok(format_exception_with_output(e, &buf, time_limit)); } } } @@ -773,7 +775,7 @@ async fn run_python( ExcType::RuntimeError, Some("async operations not supported in virtual mode".into()), ); - return Ok(format_exception_with_output(err, &buf)); + return Ok(format_exception_with_output(err, &buf, time_limit)); } RunProgress::Complete(_) => { return Ok(ExecResult::ok(buf)); @@ -1257,7 +1259,29 @@ fn format_exception(e: MontyException) -> ExecResult { } /// Format exception, preserving any output produced before the error. -fn format_exception_with_output(e: MontyException, printed: &str) -> ExecResult { +/// +/// Decision: Monty's time limit reports like the CPython runtime and coreutils +/// `timeout` (exit 124, `python3: execution timed out after Ns`), so callers can +/// tell a limit from a script error without parsing a traceback. Monty raises +/// the limit as an uncatchable `TimeoutError` whose message starts with +/// `time limit exceeded:` (`ResourceError::Time`'s Display). +fn format_exception_with_output( + e: MontyException, + printed: &str, + time_limit: Duration, +) -> ExecResult { + if e.exc_type() == ExcType::TimeoutError + && e.message() + .is_some_and(|message| message.starts_with("time limit exceeded:")) + { + let mut result = ExecResult::with_code(printed.to_string(), 124); + result.stderr = format!( + "python3: execution timed out after {:.1}s\n", + time_limit.as_secs_f64() + ) + .into(); + return result; + } let stderr = format!("{e}\n"); let mut result = ExecResult::err(stderr, 1); if !printed.is_empty() { @@ -2012,6 +2036,41 @@ mod tests { assert_eq!(r.stdout, "import re \"re\"\n"); } + /// The LLM hint must describe what Monty really does: every module it + /// names imports, simple classes work, and what it rules out fails. + /// Regression: the hint said "No classes" long after Monty added them. + #[tokio::test] + async fn test_llm_hint_matches_runtime() { + let hint = Python::new().llm_hint().unwrap(); + let stdlib = hint + .split("Stdlib: ") + .nth(1) + .and_then(|rest| rest.split(';').next()) + .unwrap(); + for module in stdlib.split(", ") { + let module = module.split('.').next().unwrap(); + let r = run(&["-c", &format!("import {module}")], None).await; + assert_eq!(r.exit_code, 0, "hint names {module}: {}", r.stderr); + } + assert!(hint.contains("Simple classes only (no inheritance)")); + let r = run( + &[ + "-c", + "class A:\n def __init__(self):\n self.v = 1\nprint(A().v)", + ], + None, + ) + .await; + assert_eq!(r.stdout, "1\n", "{}", r.stderr); + let r = run(&["-c", "class A: pass\nclass B(A): pass"], None).await; + assert_ne!(r.exit_code, 0, "inheritance now works; update the hint"); + let r = run(&["-c", "match 1:\n case 1: pass"], None).await; + assert_ne!(r.exit_code, 0, "match now works; update the hint"); + assert!(hint.contains("`re` is disabled")); + let r = run(&["-c", "import re"], None).await; + assert_ne!(r.exit_code, 0); + } + #[tokio::test] async fn test_filter_builtin() { let r = run( diff --git a/crates/bashkit/tests/integration/python_integration_tests.rs b/crates/bashkit/tests/integration/python_integration_tests.rs index 4a3edd279..9d6e98e08 100644 --- a/crates/bashkit/tests/integration/python_integration_tests.rs +++ b/crates/bashkit/tests/integration/python_integration_tests.rs @@ -1227,6 +1227,34 @@ mod resource_limits { let r = bash.exec("python3 -c \"while True: pass\"").await.unwrap(); assert_ne!(r.exit_code, 0, "Infinite loop should be killed by timeout"); } + + /// A Monty time-limit hit reports like the CPython runtime and `timeout`: + /// exit 124 and `python3: execution timed out`, keeping earlier output. + /// It used to be exit 1 with a `TimeoutError` traceback, so callers could + /// only tell a limit from a script error by parsing stderr. + #[tokio::test] + async fn timeout_limit_exits_124() { + let limits = PythonLimits::default().max_duration(Duration::from_millis(200)); + let mut bash = bash_python_limits(limits); + let r = bash + .exec("python3 -c \"print('before')\nwhile True: pass\"; echo \"rc=$?\"") + .await + .unwrap(); + assert_eq!(r.stdout, "before\nrc=124\n"); + assert_eq!(r.stderr, "python3: execution timed out after 0.2s\n"); + } + + /// A script's own TimeoutError is an ordinary exception (exit 1). + #[tokio::test] + async fn user_timeout_error_is_not_a_limit() { + let mut bash = bash_python(); + let r = bash + .exec("python3 -c \"raise TimeoutError('slow')\"") + .await + .unwrap(); + assert_eq!(r.exit_code, 1); + assert!(r.stderr.contains("TimeoutError: slow"), "{}", r.stderr); + } } // ============================================================================= diff --git a/knowledge/log.md b/knowledge/log.md index 995eb5245..34517d899 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -4,6 +4,7 @@ * **Python `re` gate parses code**: the embedded Monty `python3` blocked `import re` by matching lines that start with `import`/`from`, so `pass; import re` or `if True: import re` loaded the module anyway (found by a downstream integration review). The gate now walks the parsed statement tree. See [Python Builtin](runtimes/python-builtin.md). * **Python call cap and final-value hang**: each embedded `python3` call reserved `max_memory / 64` work units up front, so a default script failed on its 96th call; heap growth is now charged as it happens. The last expression is no longer echoed (CPython never does for `-c`/files/stdin): exporting it ran outside Monty's limits, and `x = [x, x]` thirty times hung past the timeout. Found reviewing a downstream integration. See [Python Builtin](runtimes/python-builtin.md). +* **Python hint and timeouts**: the Monty `python3` LLM hint said "No classes" after Monty 0.0.19 added simple classes, and never mentioned the `re` block; it now matches the runtime and `test_llm_hint_matches_runtime` imports every module it names. A Monty time-limit hit now exits 124 with `python3: execution timed out after Ns`, like the CPython runtime, instead of exit 1 and a `TimeoutError` traceback. See [Python Builtin](runtimes/python-builtin.md). ## 2026-11-03 diff --git a/knowledge/runtimes/python-builtin.md b/knowledge/runtimes/python-builtin.md index 4c2d22cd4..b1fa41ab3 100644 --- a/knowledge/runtimes/python-builtin.md +++ b/knowledge/runtimes/python-builtin.md @@ -66,6 +66,11 @@ the value REPL-style. ### Resource Limits +A Monty time-limit hit (uncatchable `TimeoutError: time limit exceeded: ...`) +exits 124 with `python3: execution timed out after Ns` and keeps earlier +`print` output, matching the CPython runtime and coreutils `timeout`. A +script's own `TimeoutError` stays an ordinary exit-1 exception. + Monty enforces its own limits independent of Bashkit's shell limits, configurable via `PythonLimits`: @@ -162,8 +167,8 @@ isinstance, open, input, ...), and stdlib modules: sys, typing, math, pathlib, os (getenv/environ), json, datetime (incl. `date.today()`, `datetime.now(tz)`). -Not supported (Monty limitations): classes (planned upstream), match -statements, third-party imports, most stdlib modules, and HTTP/network I/O, +Simple classes (methods, attributes) work since Monty 0.0.19. Not supported +(Monty limitations): class inheritance and metaclasses, match statements, third-party imports, most stdlib modules, and HTTP/network I/O, no `socket`/`urllib`/`requests`/`http.client`; Monty has no OsCall variants for network operations, so there is no way to bridge these. @@ -234,7 +239,8 @@ file not found, missing `-c` argument, unknown option. Registration via `BashToolBuilder::python()` contributes a hint to `help()` / `system_prompt()` documenting the limitations (stdlib subset, VFS-only file -I/O, no network/classes/third-party imports) through the general +I/O, no network/third-party imports, simple classes only, `re` disabled); +`test_llm_hint_matches_runtime` fails if the hint drifts from what Monty runs through the general `Builtin::llm_hint()` mechanism (hints deduplicated automatically). The regex module `re` is intentionally disabled in Bashkit due to From 8a6974c8e5eefa1da7b63d331bbaf2488857c876 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 5 Nov 2026 03:59:14 +0000 Subject: [PATCH 2/2] fix(python): detect Monty time limit by tracker, not message text A script could raise TimeoutError('time limit exceeded: ...') and be reported as a limit hit (exit 124). Record in the tracker when its own time check fires and key the exit-124 path on that. --- crates/bashkit/src/builtins/python.rs | 59 +++++++++++++++---- .../integration/python_integration_tests.rs | 7 +++ knowledge/runtimes/python-builtin.md | 3 +- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/crates/bashkit/src/builtins/python.rs b/crates/bashkit/src/builtins/python.rs index 7b7a9ffed..dca56fdc2 100644 --- a/crates/bashkit/src/builtins/python.rs +++ b/crates/bashkit/src/builtins/python.rs @@ -33,6 +33,7 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use super::{Builtin, Context, ExecutionDeadline, RuntimeLimits, resolve_path}; @@ -62,6 +63,9 @@ struct BudgetTracker { vm_checkpoints: Cell, /// Highest Monty heap size already charged to `execution`, in bytes. charged_memory: Cell, + /// Set when Monty's own time limit fired. Script code can raise a + /// look-alike `TimeoutError`, so only this flag marks a real limit hit. + time_limit_hit: Arc, } impl BudgetTracker { @@ -125,7 +129,11 @@ impl ResourceTracker for BudgetTracker { fn check_time(&self) -> std::result::Result<(), ResourceError> { self.check_vm()?; - self.runtime.check_time() + let result = self.runtime.check_time(); + if matches!(result, Err(ResourceError::Time { .. })) { + self.time_limit_hit.store(true, Ordering::Relaxed); + } + result } fn check_recursion_depth( @@ -658,11 +666,13 @@ async fn run_python( .max_memory(py_limits.common.max_memory) .max_recursion_depth(Some(py_limits.common.max_call_depth)); + let time_limit_hit = Arc::new(AtomicBool::new(false)); let tracker = BudgetTracker { runtime: LimitedTracker::new(limits), execution: execution_budget.clone(), vm_checkpoints: Cell::new(0), charged_memory: Cell::new(0), + time_limit_hit: time_limit_hit.clone(), }; // Important security decision: cap collected print output at the same // memory budget as the VM heap. Monty 0.0.19 added a byte cap on @@ -688,7 +698,11 @@ async fn run_python( ) { Ok(p) => (p, buf), Err(e) => { - return Ok(format_exception_with_output(e, &buf, time_limit)); + return Ok(format_exception_with_output( + e, + &buf, + timed_out(&time_limit_hit, time_limit), + )); } } }; @@ -706,7 +720,11 @@ async fn run_python( progress = next; } Err(e) => { - return Ok(format_exception_with_output(e, &buf, time_limit)); + return Ok(format_exception_with_output( + e, + &buf, + timed_out(&time_limit_hit, time_limit), + )); } } } @@ -738,7 +756,11 @@ async fn run_python( progress = next; } Err(e) => { - return Ok(format_exception_with_output(e, &buf, time_limit)); + return Ok(format_exception_with_output( + e, + &buf, + timed_out(&time_limit_hit, time_limit), + )); } } } @@ -765,7 +787,11 @@ async fn run_python( progress = next; } Err(e) => { - return Ok(format_exception_with_output(e, &buf, time_limit)); + return Ok(format_exception_with_output( + e, + &buf, + timed_out(&time_limit_hit, time_limit), + )); } } } @@ -775,7 +801,11 @@ async fn run_python( ExcType::RuntimeError, Some("async operations not supported in virtual mode".into()), ); - return Ok(format_exception_with_output(err, &buf, time_limit)); + return Ok(format_exception_with_output( + err, + &buf, + timed_out(&time_limit_hit, time_limit), + )); } RunProgress::Complete(_) => { return Ok(ExecResult::ok(buf)); @@ -1258,21 +1288,26 @@ fn format_exception(e: MontyException) -> ExecResult { ExecResult::err(format!("{e}\n"), 1) } +/// The limit to report when Monty's own time limit fired, else `None`. +fn timed_out(hit: &AtomicBool, limit: Duration) -> Option { + hit.load(Ordering::Relaxed).then_some(limit) +} + /// Format exception, preserving any output produced before the error. /// /// Decision: Monty's time limit reports like the CPython runtime and coreutils /// `timeout` (exit 124, `python3: execution timed out after Ns`), so callers can /// tell a limit from a script error without parsing a traceback. Monty raises -/// the limit as an uncatchable `TimeoutError` whose message starts with -/// `time limit exceeded:` (`ResourceError::Time`'s Display). +/// the limit as an uncatchable `TimeoutError`; the tracker records that its +/// check fired, because script code can raise a `TimeoutError` with the same +/// message. fn format_exception_with_output( e: MontyException, printed: &str, - time_limit: Duration, + time_limit: Option, ) -> ExecResult { - if e.exc_type() == ExcType::TimeoutError - && e.message() - .is_some_and(|message| message.starts_with("time limit exceeded:")) + if let Some(time_limit) = time_limit + && e.exc_type() == ExcType::TimeoutError { let mut result = ExecResult::with_code(printed.to_string(), 124); result.stderr = format!( diff --git a/crates/bashkit/tests/integration/python_integration_tests.rs b/crates/bashkit/tests/integration/python_integration_tests.rs index 9d6e98e08..0abb6884e 100644 --- a/crates/bashkit/tests/integration/python_integration_tests.rs +++ b/crates/bashkit/tests/integration/python_integration_tests.rs @@ -1254,6 +1254,13 @@ mod resource_limits { .unwrap(); assert_eq!(r.exit_code, 1); assert!(r.stderr.contains("TimeoutError: slow"), "{}", r.stderr); + // Even one that copies Monty's limit message is a script error. + let r = bash + .exec("python3 -c \"raise TimeoutError('time limit exceeded: 1s > 1s')\"") + .await + .unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(r.stderr.contains("Traceback"), "{}", r.stderr); } } diff --git a/knowledge/runtimes/python-builtin.md b/knowledge/runtimes/python-builtin.md index b1fa41ab3..b1a6e9a2d 100644 --- a/knowledge/runtimes/python-builtin.md +++ b/knowledge/runtimes/python-builtin.md @@ -69,7 +69,8 @@ the value REPL-style. A Monty time-limit hit (uncatchable `TimeoutError: time limit exceeded: ...`) exits 124 with `python3: execution timed out after Ns` and keeps earlier `print` output, matching the CPython runtime and coreutils `timeout`. A -script's own `TimeoutError` stays an ordinary exit-1 exception. +script's own `TimeoutError` stays an ordinary exit-1 exception, even one +that copies the limit message: the tracker records when its time check fired. Monty enforces its own limits independent of Bashkit's shell limits, configurable via `PythonLimits`: