From 33c05b2fe1aa879c5c128abfe7655a1c6dd76196 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 29 Jun 2026 00:42:50 -0400 Subject: [PATCH 1/3] revert: "revert: add life support to handles cast to string_view (#6092)" This re-applies #6092 (reverting #6097) so the follow-up fixes in this PR can build on it. Assisted-by: ClaudeCode:claude-opus-4.8 --- include/pybind11/cast.h | 9 +++++++++ tests/test_stl.cpp | 10 ++++++++++ tests/test_stl.py | 12 ++++++++++++ 3 files changed, 31 insertions(+) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 62ca45a09a..fb67ff8d56 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -525,6 +525,9 @@ struct string_caster { return false; } value = StringType(buffer, static_cast(size)); + if (IsView) { + loader_life_support::add_patient(src); + } return true; } @@ -602,6 +605,9 @@ struct string_caster { pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); } value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + if (IsView) { + loader_life_support::add_patient(src); + } return true; } if (PyByteArray_Check(src.ptr())) { @@ -612,6 +618,9 @@ struct string_caster { pybind11_fail("Unexpected PyByteArray_AsString() failure."); } value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + if (IsView) { + loader_life_support::add_patient(src); + } return true; } diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 8bddbb1f38..526c7643ac 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -582,6 +582,16 @@ TEST_SUBMODULE(stl, m) { [](const std::list &) { return 2; }); m.def("func_with_string_or_vector_string_arg_overload", [](const std::string &) { return 3; }); +#ifdef PYBIND11_HAS_STRING_VIEW + m.def("func_with_string_views", [](const std::vector &svs) { + py::list l; + for (std::string_view sv : svs) { + l.append(sv); + } + return l; + }); +#endif + class Placeholder { public: Placeholder() { print_created(this); } diff --git a/tests/test_stl.py b/tests/test_stl.py index b04f55c9f8..8b97c76195 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -28,6 +28,18 @@ def test_vector(doc): # Test regression caused by 936: pointers to stl containers weren't castable assert m.cast_ptr_vector() == ["lvalue", "lvalue"] + if hasattr(m, "func_with_string_views"): + + def gen(): + return ("a" + str(x) for x in range(10000, 10010)) + + expected = list(gen()) + assert m.func_with_string_views(gen()) == expected + assert m.func_with_string_views(x.encode() for x in gen()) == expected + assert ( + m.func_with_string_views(bytearray(x.encode()) for x in gen()) == expected + ) + def test_deque(): """std::deque <-> list""" From 23fd3af2c42a89d6049dfcad7b29967ae49df84d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 26 Jun 2026 10:58:42 -0400 Subject: [PATCH 2/3] fix: don't throw from string_view life support outside a bound function PR #6092 added loader_life_support::add_patient(src) to keep the source object alive when loading a string view, fixing a real use-after-free when a container of views is built from a non-sequence iterable (e.g. a generator): list_caster materializes a temporary tuple that owns the strings and destroys it when load() returns, before the bound function body runs. add_patient throws when there is no life support frame, so casting to a view outside a bound function (e.g. a manual py::cast) now raises instead of relying on the caller-owned source, a regression from #6092. For these view-into-src cases registration is best effort: inside a bound function it keeps src alive (fixing the UAF), and outside one the caller owns src's lifetime as before. Add try_add_patient(), which returns false instead of throwing when there is no frame, and use it at the three view load sites. add_patient() keeps its strict contract for value-creating conversions. Assisted-by: ClaudeCode:claude-opus-4.8 --- include/pybind11/cast.h | 8 +++++--- include/pybind11/detail/type_caster_base.h | 23 ++++++++++++++++------ tests/test_with_catch/test_interpreter.cpp | 16 +++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index fb67ff8d56..9ab6e332d1 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -526,7 +526,9 @@ struct string_caster { } value = StringType(buffer, static_cast(size)); if (IsView) { - loader_life_support::add_patient(src); + // `src` owns the buffer; keep it alive if inside a bound function, + // otherwise the caller is responsible for its lifetime. + loader_life_support::try_add_patient(src); } return true; } @@ -606,7 +608,7 @@ struct string_caster { } value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); if (IsView) { - loader_life_support::add_patient(src); + loader_life_support::try_add_patient(src); } return true; } @@ -619,7 +621,7 @@ struct string_caster { } value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); if (IsView) { - loader_life_support::add_patient(src); + loader_life_support::try_add_patient(src); } return true; } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 8fbf700e12..b6d03ca903 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -81,11 +81,26 @@ class loader_life_support { } } + /// Keep `h` alive until the current patient frame is destroyed, if there is one. + /// Returns false when called outside a bound function (no frame). Use this, rather + /// than `add_patient`, when failing to register is acceptable because the caller + /// owns the source's lifetime outside the call framework (e.g. a view that points + /// into an existing Python object, as opposed to a freshly created temporary). + PYBIND11_NOINLINE static bool try_add_patient(handle h) { + loader_life_support *frame = tls_current_frame(); + if (!frame) { + return false; + } + if (frame->keep_alive.insert(h.ptr()).second) { + Py_INCREF(h.ptr()); + } + return true; + } + /// This can only be used inside a pybind11-bound function, either by `argument_loader` /// at argument preparation time or by `py::cast()` at execution time. PYBIND11_NOINLINE static void add_patient(handle h) { - loader_life_support *frame = tls_current_frame(); - if (!frame) { + if (!try_add_patient(h)) { // NOTE: It would be nice to include the stack frames here, as this indicates // use of pybind11::cast<> outside the normal call framework, finding such // a location is challenging. Developers could consider printing out @@ -94,10 +109,6 @@ class loader_life_support { "do Python -> C++ conversions which require the creation " "of temporary values"); } - - if (frame->keep_alive.insert(h.ptr()).second) { - Py_INCREF(h.ptr()); - } } }; diff --git a/tests/test_with_catch/test_interpreter.cpp b/tests/test_with_catch/test_interpreter.cpp index 4103c0f5ff..dd712f7ba9 100644 --- a/tests/test_with_catch/test_interpreter.cpp +++ b/tests/test_with_catch/test_interpreter.cpp @@ -509,3 +509,19 @@ TEST_CASE("make_iterator can be called before then after finalizing an interpret py::initialize_interpreter(); } + +#ifdef PYBIND11_HAS_STRING_VIEW +TEST_CASE("Casting to a string_view outside a bound function") { + // Regression for PR #6092: view casters add the source to loader_life_support, but + // outside a bound function there is no frame. The caller owns the source's lifetime + // here, so the cast must succeed rather than throw. + py::str unicode("hello"); + py::bytes bytes_obj("world", 5); + auto bytearray_obj + = py::reinterpret_steal(PyByteArray_FromStringAndSize("bytes", 5)); + + REQUIRE(py::cast(unicode) == "hello"); + REQUIRE(py::cast(bytes_obj) == "world"); + REQUIRE(py::cast(bytearray_obj) == "bytes"); +} +#endif From e18b8346a28c125964fcb9d192e6643d5ef3b420 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 26 Jun 2026 12:04:38 -0400 Subject: [PATCH 3/3] fix: only add string_view life support for transient sources Refine the previous commit. Best-effort registration (try_add_patient) silently produces a dangling view when a container of views is built from a generator outside a bound function: there the materialized temporary is released before the view is used, and with no frame nothing keeps it alive. Such a cast cannot be made safe, so it should fail loudly, while a view into a durable, caller-owned object needs no life support at all. The view caster cannot tell a durable source from a pybind11-managed transient one; that provenance lives in the container caster. Introduce an ambient transient_source_guard that the list, set, map, and array casters set around their generator/materialized paths, and have the string caster keep the source alive only when loading from a transient source (via the throwing add_patient, so try_add_patient is no longer needed). This means: - views into durable sources (direct arguments, sequences, manual casts) add no life support and no longer throw outside a bound function, and - a generator used outside a frame throws, rather than silently dangling. The guard restores (rather than clears) the previous value, so a durable container nested in a transient one is correctly treated as transient. Verified with AddressSanitizer: the in-frame generator case is clean, the out-of-frame durable cases succeed, and the out-of-frame generator case throws. Assisted-by: ClaudeCode:claude-opus-4.8 --- include/pybind11/cast.h | 16 +++---- include/pybind11/detail/type_caster_base.h | 51 ++++++++++++++-------- include/pybind11/stl.h | 12 +++++ tests/test_with_catch/test_interpreter.cpp | 28 ++++++++++-- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9ab6e332d1..9572f95356 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -525,10 +525,10 @@ struct string_caster { return false; } value = StringType(buffer, static_cast(size)); - if (IsView) { - // `src` owns the buffer; keep it alive if inside a bound function, - // otherwise the caller is responsible for its lifetime. - loader_life_support::try_add_patient(src); + if (IsView && loading_from_transient_source()) { + // The view points into `src`, which is owned by a transient source that + // will be released before the view is used, so keep `src` alive. + loader_life_support::add_patient(src); } return true; } @@ -607,8 +607,8 @@ struct string_caster { pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); } value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); - if (IsView) { - loader_life_support::try_add_patient(src); + if (IsView && loading_from_transient_source()) { + loader_life_support::add_patient(src); } return true; } @@ -620,8 +620,8 @@ struct string_caster { pybind11_fail("Unexpected PyByteArray_AsString() failure."); } value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); - if (IsView) { - loader_life_support::try_add_patient(src); + if (IsView && loading_from_transient_source()) { + loader_life_support::add_patient(src); } return true; } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index b6d03ca903..e0f2ed11b4 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -81,26 +81,11 @@ class loader_life_support { } } - /// Keep `h` alive until the current patient frame is destroyed, if there is one. - /// Returns false when called outside a bound function (no frame). Use this, rather - /// than `add_patient`, when failing to register is acceptable because the caller - /// owns the source's lifetime outside the call framework (e.g. a view that points - /// into an existing Python object, as opposed to a freshly created temporary). - PYBIND11_NOINLINE static bool try_add_patient(handle h) { - loader_life_support *frame = tls_current_frame(); - if (!frame) { - return false; - } - if (frame->keep_alive.insert(h.ptr()).second) { - Py_INCREF(h.ptr()); - } - return true; - } - /// This can only be used inside a pybind11-bound function, either by `argument_loader` /// at argument preparation time or by `py::cast()` at execution time. PYBIND11_NOINLINE static void add_patient(handle h) { - if (!try_add_patient(h)) { + loader_life_support *frame = tls_current_frame(); + if (!frame) { // NOTE: It would be nice to include the stack frames here, as this indicates // use of pybind11::cast<> outside the normal call framework, finding such // a location is challenging. Developers could consider printing out @@ -109,7 +94,39 @@ class loader_life_support { "do Python -> C++ conversions which require the creation " "of temporary values"); } + + if (frame->keep_alive.insert(h.ptr()).second) { + Py_INCREF(h.ptr()); + } + } +}; + +// While set, a type caster is converting elements borrowed from a *transient* source -- +// a generator/iterator, or a temporary container materialized from one -- whose items are +// released before the converted C++ value is used. A view caster (e.g. for std::string_view) +// consults this to decide whether the viewed Python object needs life support: a view into a +// durable, caller-owned object does not, whereas a view into a transient source does. Outside +// a bound function (no life support frame) the latter cannot be made safe, so `add_patient` +// rightly throws rather than producing a dangling view. +inline bool &loading_from_transient_source() { + static thread_local bool flag = false; + return flag; +} + +// RAII guard marking the dynamic scope in which elements are loaded from a transient source. +// Restores (rather than clears) the previous value so that nesting is transitive: a durable +// container nested inside a transient one is itself transient. +class transient_source_guard { +public: + transient_source_guard() : prev(loading_from_transient_source()) { + loading_from_transient_source() = true; } + ~transient_source_guard() { loading_from_transient_source() = prev; } + transient_source_guard(const transient_source_guard &) = delete; + transient_source_guard &operator=(const transient_source_guard &) = delete; + +private: + bool prev; }; // Gets the cache entry for the given type, creating it if necessary. The return value is the pair diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 01be0b47c6..41e1d59074 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -198,6 +198,9 @@ struct set_caster { } assert(isinstance(src)); value.clear(); + // Elements are borrowed from the iterator and released as it advances, so views + // into them need life support. + transient_source_guard guard; return convert_iterable(reinterpret_borrow(src), convert); } @@ -264,6 +267,9 @@ struct map_caster { throw error_already_set(); } assert(isinstance(items)); + // The materialized dict is transient (released when load() returns), so views into + // its keys/values need life support. + transient_source_guard guard; return convert_elements(dict(reinterpret_borrow(items)), convert); } @@ -314,6 +320,9 @@ struct list_caster { // the generator is not left in an unpredictable (to the caller) partially-consumed // state. assert(isinstance(src)); + // The materialized tuple is transient (released when load() returns), so views into + // its elements need life support. + transient_source_guard guard; return convert_elements(tuple(reinterpret_borrow(src)), convert); } @@ -449,6 +458,9 @@ struct array_caster { // the generator is not left in an unpredictable (to the caller) partially-consumed // state. assert(isinstance(src)); + // The materialized tuple is transient (released when load() returns), so views into + // its elements need life support. + transient_source_guard guard; return convert_elements(tuple(reinterpret_borrow(src)), convert); } diff --git a/tests/test_with_catch/test_interpreter.cpp b/tests/test_with_catch/test_interpreter.cpp index dd712f7ba9..ff4e608c07 100644 --- a/tests/test_with_catch/test_interpreter.cpp +++ b/tests/test_with_catch/test_interpreter.cpp @@ -1,5 +1,6 @@ #include #include +#include // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). @@ -11,8 +12,10 @@ PYBIND11_WARNING_DISABLE_MSVC(4996) #include #include #include +#include #include #include +#include namespace py = pybind11; using namespace py::literals; @@ -511,10 +514,10 @@ TEST_CASE("make_iterator can be called before then after finalizing an interpret } #ifdef PYBIND11_HAS_STRING_VIEW -TEST_CASE("Casting to a string_view outside a bound function") { - // Regression for PR #6092: view casters add the source to loader_life_support, but - // outside a bound function there is no frame. The caller owns the source's lifetime - // here, so the cast must succeed rather than throw. +TEST_CASE("Casting to a string_view from a durable source outside a bound function") { + // Outside a bound function there is no loader_life_support frame. When the source is a + // durable, caller-owned object the view does not need life support, so the cast must + // succeed rather than throw (regression from PR #6092). py::str unicode("hello"); py::bytes bytes_obj("world", 5); auto bytearray_obj @@ -523,5 +526,22 @@ TEST_CASE("Casting to a string_view outside a bound function") { REQUIRE(py::cast(unicode) == "hello"); REQUIRE(py::cast(bytes_obj) == "world"); REQUIRE(py::cast(bytearray_obj) == "bytes"); + + // A list is durable too: the views point into elements the list keeps alive. + py::list durable; + durable.append("a"); + durable.append("b"); + auto from_list = py::cast>(durable); + REQUIRE(from_list.size() == 2); + REQUIRE(from_list[0] == "a"); + REQUIRE(from_list[1] == "b"); +} + +TEST_CASE("Casting to a string_view from a transient source outside a bound function") { + // A generator is a transient source: the materialized temporary backing the views is + // released when the cast returns, and there is no frame to keep it alive. This cannot + // be made safe, so it must throw rather than produce dangling views. + auto gen = py::eval("('transient_' + str(x) for x in range(5))"); + REQUIRE_THROWS_AS(py::cast>(gen), py::cast_error); } #endif