diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f1bbe27a..a7fc78d08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,6 +65,9 @@ jobs: - name: Run inline control harness run: BID_DELAY=3 ./scripts/template-cache-local-test.sh inline + - name: Run cache purge harness + run: BID_DELAY=3 ./scripts/template-cache-local-test.sh purge + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 7051f7dab..7a45462b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,6 +202,7 @@ dependencies = [ "compression-core", "futures-io", "pin-project-lite", + "tokio", ] [[package]] @@ -5282,12 +5283,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags 2.13.0", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower 0.5.3", "tower-layer", "tower-service", @@ -5441,6 +5447,7 @@ dependencies = [ "edgezero-cli", "edgezero-core", "error-stack", + "flate2", "futures", "glob", "http", @@ -5451,6 +5458,7 @@ dependencies = [ "rand 0.8.6", "rcgen", "regex", + "reqwest 0.12.28", "rustls", "rustls-pemfile", "scraper", diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 117ab33bd..caba714d6 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -295,6 +295,7 @@ enum NamedRouteHandler { TrustedServerDiscovery, VerifySignature, AdminNotSupported, + CachePurgeNotSupported, AdminEcNotSupported, AdminEidsLookup, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never @@ -324,7 +325,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 16] { +fn named_routes() -> [NamedRoute; 17] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -349,6 +350,14 @@ fn named_routes() -> [NamedRoute; 16] { primary_methods: &[Method::POST], handler: NamedRouteHandler::AdminNotSupported, }, + // Every method, for the same reason as the Fastly adapter: a method this route + // does not claim falls through to the publisher with the caller's `Authorization` + // header still attached. + NamedRoute { + path: "/_ts/admin/cache/purge", + primary_methods: LEGACY_ADMIN_DENY_METHODS, + handler: NamedRouteHandler::CachePurgeNotSupported, + }, // Admin EC lookup routes. Registered explicitly (like the key routes // above) so they never fall through to the publisher fallback, and // they match `Settings::ADMIN_ENDPOINTS` for auth coverage. @@ -449,6 +458,22 @@ fn named_route_handler( NamedRouteHandler::VerifySignature => { handle_verify_signature(&state.settings, &services, req) } + NamedRouteHandler::CachePurgeNotSupported => { + // The Axum dev server has no template cache to purge. 501 rather + // than a fallthrough 404, so a CMS webhook can tell "not supported + // here" from "endpoint does not exist". + let body = edgezero_core::body::Body::from( + "Template cache purge is not supported on the Axum dev server.\n\ + Use the Fastly adapter (via Viceroy or deployed) to purge.\n", + ); + let mut resp = Response::new(body); + *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; + resp.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + Ok(resp) + } NamedRouteHandler::AdminNotSupported => { // Config/secret-store writes are backed by read-only env vars on the // Axum dev server. Returning 501 is clearer than failing on the first diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 1a9911b0c..87b9567e7 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -352,6 +352,20 @@ fn admin_key_management_not_supported() -> Response { response } +fn cache_purge_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Template cache purge is not supported on Cloudflare Workers.\n\ + Use the Fastly adapter (via Viceroy or deployed) to purge.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + fn admin_ec_lookup_not_supported() -> Response { core_admin_ec_lookup_not_supported() } @@ -725,6 +739,18 @@ fn build_router(state: &Arc) -> RouterService { router = router.route(path, Method::OPTIONS, page_bids_preflight.clone()); } + let cache_purge_unsupported = + make_handler(Arc::clone(&state), |_s, _services, _req| async move { + Ok(cache_purge_not_supported()) + }); + for method in publisher_fallback_methods() { + router = router.route( + "/_ts/admin/cache/purge", + method, + cache_purge_unsupported.clone(), + ); + } + let legacy_admin_deny = make_handler(Arc::clone(&state), |_s, _services, _req| async move { Ok(legacy_admin_alias_denied()) diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index cded42a0d..94616b203 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -240,7 +240,7 @@ fn is_hop_by_hop_response_header(name: &str, connection_tokens: &[String]) -> bo } /// Cache policy for the outbound Workers `fetch` derived from -/// [`PlatformHttpRequest::bypass_cache`]. +/// [`PlatformHttpRequest::cache_intent`]. /// /// Workers subrequests are eligible for Cloudflare's cache by default, so an /// ad-stack navigation could otherwise be satisfied from cache (or revalidated @@ -262,8 +262,12 @@ enum OutboundCacheMode { } #[cfg(any(target_arch = "wasm32", test))] -fn outbound_cache_mode(bypass_cache: bool) -> OutboundCacheMode { - if bypass_cache { +fn outbound_cache_mode( + intent: &trusted_server_core::platform::PlatformCacheIntent, +) -> OutboundCacheMode { + // Workers has no surrogate-key concept, so `Shared` falls in with `Default`: let the + // runtime apply its own behavior rather than pretending to honor a key it cannot use. + if intent.is_bypass() { OutboundCacheMode::NoStore } else { OutboundCacheMode::RuntimeDefault @@ -305,7 +309,7 @@ impl CloudflareHttpClient { )); } - let cache_mode = outbound_cache_mode(request.bypass_cache); + let cache_mode = outbound_cache_mode(&request.cache_intent); let uri = request.request.uri().to_string(); // http::Method always stores uppercase; worker 0.7 implements From only. @@ -929,18 +933,18 @@ mod tests { #[test] fn outbound_cache_mode_maps_bypass_to_no_store() { assert_eq!( - outbound_cache_mode(true), + outbound_cache_mode(&trusted_server_core::platform::PlatformCacheIntent::Bypass), OutboundCacheMode::NoStore, - "bypass_cache should force the Workers `no-store` cache mode" + "a bypass intent should force the Workers `no-store` cache mode" ); } #[test] fn outbound_cache_mode_leaves_default_when_not_bypassing() { assert_eq!( - outbound_cache_mode(false), + outbound_cache_mode(&trusted_server_core::platform::PlatformCacheIntent::Default), OutboundCacheMode::RuntimeDefault, - "requests without bypass_cache should keep the runtime default cache behavior" + "a default intent should keep the runtime default cache behavior" ); } } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index fcb4c2856..b19653c9e 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -267,9 +267,9 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) - // Spike-only (#1009). Constructed unconditionally, but only read when the - // assembly mode is a shared-template one — which defaults to Inline, so this - // is inert until an operator opts in. + // Constructed unconditionally, but only read when the assembly mode is a + // shared-template one — which defaults to Inline, so this is inert until an + // operator opts in. .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) @@ -547,6 +547,20 @@ async fn execute_named( return Ok(run_batch_sync(&state, &services, req)); } + // An operator cache purge is not a reader request: running the EC lifecycle would + // attach finalization state and could ingest the operator's cookies into KV. + if matches!(handler, NamedRouteHandler::AdminCachePurge) { + let principal = trusted_server_core::auth::authenticated_username(&req); + let response = trusted_server_core::cache_purge::handle_cache_purge( + &services, + req, + principal.as_deref(), + ) + .await + .unwrap_or_else(|error| http_error(&error)); + return Ok(response); + } + // These diagnostics are read-only. Running the normal EC lifecycle would // attach finalization state and could ingest request cookies into KV after // the handler returns, violating that contract. @@ -622,6 +636,9 @@ async fn run_named_route( NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup => { unreachable!("admin diagnostics should be handled before EC setup") } + NamedRouteHandler::AdminCachePurge => { + unreachable!("cache purge should be handled before EC setup") + } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. @@ -1051,6 +1068,7 @@ enum NamedRouteHandler { DeactivateKey, AdminEcLookup, AdminEidsLookup, + AdminCachePurge, /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, @@ -1072,6 +1090,11 @@ struct NamedRoute { handler: NamedRouteHandler, } +/// Every method an admin route must claim to keep non-primary methods from falling +/// through to the publisher with the `Authorization` header still attached. +/// +/// Named for the legacy `/admin/*` aliases it was introduced for, and reused by every +/// route with the same requirement here and in the Axum and Spin adapters. const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::GET, Method::POST, @@ -1103,6 +1126,15 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, }, + // Every method is claimed, not just POST. A method this route did not claim would + // fall through to the publisher, and `enforce_basic_auth` leaves the `Authorization` + // header in place, so a GET would ship the shared admin credential to the origin. + // The handler answers the non-POST methods with 405 itself. + NamedRoute { + path: "/_ts/admin/cache/purge", + primary_methods: LEGACY_ADMIN_DENY_METHODS, + handler: NamedRouteHandler::AdminCachePurge, + }, // Admin EC lookup: the bare route reads the EC ID from the caller's // `ts-ec` cookie; the parameterized route takes an explicit EC ID. NamedRoute { @@ -1850,6 +1882,40 @@ mod tests { } } + #[test] + fn cache_purge_claims_every_method_that_could_reach_the_publisher() { + // The guard this route exists behind. `enforce_basic_auth` authenticates on the raw + // path and leaves the `Authorization` header attached, so any method this route does + // not claim falls through to the publisher fallback carrying the shared admin + // credential to the origin. Asserted against the fallback list itself rather than a + // copy of it, so a method added there cannot quietly open a hole here. + let route = NAMED_ROUTES + .iter() + .find(|route| route.path == "/_ts/admin/cache/purge") + .expect("cache purge must be a named route"); + + for method in super::publisher_fallback_methods() { + assert!( + route.primary_methods.contains(&method), + "{method} /_ts/admin/cache/purge must be claimed, or it reaches the publisher \ + with the admin credential attached" + ); + } + assert!(matches!(route.handler, NamedRouteHandler::AdminCachePurge)); + } + + #[test] + fn cache_purge_has_no_legacy_unauthenticated_alias() { + // The production basic-auth regex is `^/_ts/admin`. An `/admin/...` spelling would + // not match it, so it must not exist at all. + assert!( + !NAMED_ROUTES + .iter() + .any(|route| route.path == "/admin/cache/purge"), + "an /admin-prefixed alias would sit outside the basic-auth regex" + ); + } + #[test] fn admin_ec_lookup_routes_are_registered() { // Both lookup shapes must be explicitly routed to the admin EC @@ -2824,6 +2890,12 @@ mod tests { Ok(()) } + /// A no-op beyond succeeding: this double stores by cache key, so it cannot + /// resolve a surrogate key to entries the way the platform does. + async fn purge_url_surrogate_key(&self, _key: &str) -> Result<(), TemplateCacheError> { + Ok(()) + } + async fn purge_all(&self) -> Result<(), TemplateCacheError> { self.entries.lock().expect("should lock entries").clear(); Ok(()) diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index c241dd33c..d0f4ee9d0 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -12,10 +12,11 @@ use crate::backend::BackendConfig; pub(crate) use trusted_server_core::platform::UnavailableKvStore; use trusted_server_core::platform::{ BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, - PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, - PlatformImageOptimizerParams, PlatformImageOptimizerRegion, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, + PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, + PlatformHttpRequest, PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, + PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformImageOptimizerRegion, + PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, + StoreName, }; use trusted_server_core::settings::TrustedClientIpConfig; @@ -437,9 +438,37 @@ fn fastly_response_to_platform( // FastlyPlatformHttpClient // --------------------------------------------------------------------------- -fn apply_fastly_cache_bypass(request: &mut fastly::Request, bypass_cache: bool) { - if bypass_cache { - request.set_pass(true); +/// Apply the caller's cache intent to a Fastly request. +/// +/// The two branches are mutually exclusive by construction, which is the point of +/// [`PlatformCacheIntent`]: `set_surrogate_key` "overrides any previous +/// `Request::set_pass` call" (`fastly-0.12.1/src/http/request.rs:2462`), so calling both +/// would silently cancel the bypass. +/// +/// Readthrough is enabled by *omitting* `set_pass`, never by adding a TTL. `set_ttl` +/// carries the same override note and additionally overrides the origin's own +/// `Cache-Control`, including `private` and `no-store` — it would turn the hazard this +/// gate exists to avoid into an API. +fn apply_fastly_cache_intent(request: &mut fastly::Request, intent: &PlatformCacheIntent) { + match intent { + PlatformCacheIntent::Bypass => request.set_pass(true), + PlatformCacheIntent::Shared { surrogate_key } => { + match fastly::http::HeaderValue::from_str(surrogate_key) { + Ok(value) => request.set_surrogate_key(value), + Err(error) => { + // Fail closed. Caching without the key would store an object no purge + // can reach, which is worse than not caching it: the whole rollback + // story for readthrough is "purge the key". + log::error!( + "Surrogate key {surrogate_key:?} is not a valid header value \ + ({error}); bypassing the cache rather than storing an \ + unpurgeable object" + ); + request.set_pass(true); + } + } + } + PlatformCacheIntent::Default => {} } } @@ -482,13 +511,13 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let backend_name = request.backend_name.clone(); let image_optimizer = request.image_optimizer; let stream_response = request.stream_response; - let bypass_cache = request.bypass_cache; + let cache_intent = request.cache_intent.clone(); let request_is_head = request.request.method() == edgezero_core::http::Method::HEAD; let mut fastly_req = edge_request_to_fastly(request.request)?; if let Some(options) = image_optimizer { apply_fastly_image_optimizer(&mut fastly_req, options)?; } - apply_fastly_cache_bypass(&mut fastly_req, bypass_cache); + apply_fastly_cache_intent(&mut fastly_req, &cache_intent); let fastly_resp = fastly_req .send(&backend_name) .change_context(PlatformError::HttpClient)?; @@ -506,9 +535,9 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { } let stream_response = request.stream_response; let request_method = request.request.method().clone(); - let bypass_cache = request.bypass_cache; + let cache_intent = request.cache_intent.clone(); let mut fastly_req = edge_request_to_fastly(request.request)?; - apply_fastly_cache_bypass(&mut fastly_req, bypass_cache); + apply_fastly_cache_intent(&mut fastly_req, &cache_intent); let pending = fastly_req .send_async(&backend_name) .change_context(PlatformError::HttpClient)?; @@ -1189,22 +1218,62 @@ mod tests { } #[test] - fn apply_fastly_cache_bypass_sets_pass_when_enabled() { + fn apply_fastly_cache_intent_sets_pass_for_bypass() { let mut request = fastly::Request::get("https://example.com/"); - apply_fastly_cache_bypass(&mut request, true); + apply_fastly_cache_intent(&mut request, &PlatformCacheIntent::Bypass); assert!( format!("{request:?}").contains("cache_override: Pass"), - "enabled bypass should select Fastly pass mode" + "bypass should select Fastly pass mode" ); } #[test] - fn apply_fastly_cache_bypass_preserves_default_when_disabled() { + fn apply_fastly_cache_intent_leaves_default_alone() { let mut request = fastly::Request::get("https://example.com/"); - apply_fastly_cache_bypass(&mut request, false); + apply_fastly_cache_intent(&mut request, &PlatformCacheIntent::Default); assert!( format!("{request:?}").contains("cache_override: None"), - "disabled bypass should preserve Fastly read-through caching" + "the default intent should preserve Fastly read-through caching" + ); + } + + #[test] + fn apply_fastly_cache_intent_never_passes_on_the_shared_branch() { + // Readthrough is enabled by *omitting* set_pass. If this branch ever set pass as + // well, the surrogate key would reverse it — and the resulting behavior would + // depend on call order rather than on what the code says. + let mut request = fastly::Request::get("https://example.com/"); + apply_fastly_cache_intent( + &mut request, + &PlatformCacheIntent::Shared { + surrogate_key: "ts-origin".to_owned(), + }, + ); + let rendered = format!("{request:?}"); + assert!( + !rendered.contains("cache_override: Pass"), + "the shared branch must not bypass, got: {rendered}" + ); + assert!( + rendered.contains("ts-origin"), + "the shared branch must attach the surrogate key, got: {rendered}" + ); + } + + #[test] + fn a_surrogate_key_that_cannot_be_a_header_value_falls_back_to_bypass() { + // Fail closed: caching without the key would store an object no purge can reach, + // and "purge the key" is the entire rollback story for readthrough. + let mut request = fastly::Request::get("https://example.com/"); + apply_fastly_cache_intent( + &mut request, + &PlatformCacheIntent::Shared { + surrogate_key: "bad\nkey".to_owned(), + }, + ); + assert!( + format!("{request:?}").contains("cache_override: Pass"), + "an unusable key must bypass rather than store something unpurgeable" ); } diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index 9f0d3d9b9..e6f3ba2c0 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -12,9 +12,7 @@ //! under `fastly compute serve`, `cargo test-fastly` and the parity suite. It is also //! silently dead whenever the origin request is in pass mode, and its closure bounds //! (`Fn + Send + Sync`) are incompatible with a platform layer that is `!Send` by -//! construction. Recorded in the spike plan's Task 3 Step 4 so nobody re-proposes it. -//! -//! Spike-only. Remove with the spike. +//! construction. Recorded here so nobody re-proposes it. use fastly::cache::core::{CacheKey, Found, Transaction}; use std::io::Write as _; @@ -282,6 +280,11 @@ impl PlatformTemplateCache for FastlyTemplateCache { .map_err(|e| backend_error(format!("purging invalid template failed: {e:?}"))) } + async fn purge_url_surrogate_key(&self, key: &str) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(key) + .map_err(|e| backend_error(format!("purging surrogate key {key} failed: {e:?}"))) + } + async fn purge_all(&self) -> Result<(), TemplateCacheError> { fastly::http::purge::purge_surrogate_key(TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY) .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) @@ -316,6 +319,7 @@ mod tests { url: url.to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/page".to_string(), origin_identity: "https://origin.example.com\0origin.example.com".to_string(), assembly_mode: AssemblyMode::Esi, vary_values: vec![trusted_server_core::platform::VaryHeaderValues { diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f315a7b56..504dbd9fb 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -408,6 +408,7 @@ mod tests { is_win: None, ad_domain: None, ad_id: None, + origin_cache_shareable: None, } } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 821b52646..b8e8a2492 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -221,7 +221,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 16] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 17] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -230,6 +230,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 16] { ("/_ts/admin/ec", &[Method::GET]), ("/_ts/admin/ec/{id}", &[Method::GET]), ("/_ts/admin/eids", &[Method::GET]), + ("/_ts/admin/cache/purge", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), @@ -429,6 +430,20 @@ fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Reque }) } +fn cache_purge_not_supported() -> Response { + let body = edgezero_core::body::Body::from( + "Template cache purge is not supported on Spin.\n\ + Use the Fastly adapter (via Viceroy or deployed) to purge.\n", + ); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::NOT_IMPLEMENTED; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + fn admin_key_management_not_supported() -> Response { let body = edgezero_core::body::Body::from( "Admin key management is not supported on Fermyon Spin.\n\ @@ -623,6 +638,9 @@ fn build_router(state: &Arc) -> RouterService { Ok::(admin_key_management_not_supported()) }; + let cache_purge_unsupported_handler = + |_ctx: RequestContext| async { Ok::(cache_purge_not_supported()) }; + let admin_ec_not_supported_handler = |_ctx: RequestContext| async { Ok::(admin_ec_lookup_not_supported()) }; @@ -922,6 +940,14 @@ fn build_router(state: &Arc) -> RouterService { .get("/first-party/proxy-rebuild", fp_rebuild_handler) .post("/first-party/proxy-rebuild", fp_rebuild_post_handler); + for method in LEGACY_ADMIN_DENY_METHODS { + builder = builder.route( + "/_ts/admin/cache/purge", + method.clone(), + cache_purge_unsupported_handler, + ); + } + for method in LEGACY_ADMIN_DENY_METHODS { builder = builder.route("/admin/keys/rotate", method.clone(), legacy_admin_deny); builder = builder.route("/admin/keys/deactivate", method.clone(), legacy_admin_deny); diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 0f93651c5..19fb13366 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -26,6 +26,24 @@ http = { workspace = true } log = { workspace = true } rand = { workspace = true } regex = { workspace = true } +# Also in the macOS block for the dev proxy; needed on every host so the probe can install +# the process-level crypto provider its `-no-provider` reqwest build expects. +rustls = { workspace = true } +# Declared directly rather than inherited from the workspace, to control the rustls +# crypto provider. `reqwest`'s plain `rustls-tls` forces `ring`, and this crate already +# links `aws-lc-rs` through `reqwest` 0.13 (via `chromiumoxide` and `edgezero-cli`). +# Compiling both providers makes rustls's process-level default ambiguous, and it then +# panics on first use — which broke the dev-proxy tests. The `-no-provider` variant uses +# whichever provider is already present instead of adding a second. +# +# `gzip` so the probe's Accept-Encoding axis compares decoded bytes: reqwest transparently +# decodes a gzip response, which is what "differ only by content coding" must be tested +# against. +reqwest = { version = "0.12", default-features = false, features = [ + "gzip", + "json", + "rustls-tls-webpki-roots-no-provider", +] } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -67,6 +85,7 @@ webpki-roots = { workspace = true } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +flate2 = { workspace = true } temp-env = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/trusted-server-cli/src/commands/cache/mod.rs b/crates/trusted-server-cli/src/commands/cache/mod.rs new file mode 100644 index 000000000..889852e10 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/cache/mod.rs @@ -0,0 +1,63 @@ +//! `ts cache` — operator control over shared template and origin caches. + +pub mod purge; + +use clap::Subcommand; + +use crate::error::CliResult; + +/// Subcommands under `ts cache`. +#[derive(Debug, Subcommand)] +pub enum CacheCommand { + /// Purge cached templates and tagged origin responses through a deployed service's admin endpoint. + Purge(PurgeArgs), +} + +/// Arguments for `ts cache purge`. +/// +/// # Why this calls the service rather than the Fastly purge API +/// +/// The recorded Fastly API token is config- and secret-store write only, with no purge +/// permission (`adapter-fastly/src/management_api.rs`). The service's own admin endpoint +/// purges from inside the running service, where the platform SDK needs no API token at +/// all, so this path works with the credentials an operator already has. +/// +/// It also leaves one implementation of the purge. The surrogate key is derived +/// server-side from the URL, so the CLI cannot drift out of agreement with the cache it is +/// purging — a class of bug that a second client-side derivation would reintroduce. +#[derive(Debug, clap::Args)] +pub struct PurgeArgs { + /// HTTPS base URL of the service; HTTP is permitted only for loopback development. + #[arg(long)] + pub service: String, + + /// Purge every cached template and tagged origin response. + #[arg(long, conflicts_with = "page")] + pub all: bool, + + /// Purge one reader-facing page URL, including its exact scheme, host, and port. + #[arg(long, conflicts_with = "all")] + pub page: Option, + + /// Admin username for the service's `^/_ts/admin` basic auth. + #[arg(long, default_value = "admin")] + pub username: String, +} + +/// Environment variable carrying the admin password. +/// +/// Read from the environment and never accepted as a flag: an argument is visible to every +/// other process on the host through `ps`, and lands in shell history. +pub const ADMIN_PASSWORD_ENVIRONMENT_VARIABLE: &str = "TRUSTED_SERVER_ADMIN_PASSWORD"; + +/// Run a `ts cache` subcommand. +/// +/// # Errors +/// +/// Returns an error when neither scope is given, when the admin password is absent from +/// the environment, when the service cannot be reached, or when it refuses the purge. +pub fn run(command: CacheCommand, out: &mut impl std::io::Write) -> CliResult<()> { + match command { + CacheCommand::Purge(args) => purge::run_purge(&args, out), + } +} diff --git a/crates/trusted-server-cli/src/commands/cache/purge.rs b/crates/trusted-server-cli/src/commands/cache/purge.rs new file mode 100644 index 000000000..dcfa38880 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/cache/purge.rs @@ -0,0 +1,268 @@ +//! Calling a deployed service's cache-purge endpoint. + +use std::time::Duration; + +use serde::Deserialize; + +use crate::commands::cache::{ADMIN_PASSWORD_ENVIRONMENT_VARIABLE, PurgeArgs}; +use crate::error::{CliResult, cli_error}; + +const PURGE_PATH: &str = "/_ts/admin/cache/purge"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// The purge request body, as the endpoint's parser expects it. +/// +/// `scope: "all"` must carry no `url` field: the endpoint refuses that combination, on the +/// grounds that an operator who meant one page and mistyped the scope should not be handed +/// a silent full flush. +fn request_body(args: &PurgeArgs) -> CliResult { + match (args.all, args.page.as_deref()) { + (true, _) => Ok(r#"{"scope":"all"}"#.to_owned()), + (false, Some(page)) => { + if !reqwest::Url::parse(page).is_ok_and(|parsed| { + matches!(parsed.scheme(), "http" | "https") && parsed.host_str().is_some() + }) { + return cli_error("--page must be an absolute http(s) URL with a host"); + } + Ok(serde_json::json!({ "scope": "url", "url": page }).to_string()) + } + (false, None) => cli_error("specify --all or --page "), + } +} + +/// Join the service base URL and the purge path without doubling or dropping a slash. +fn purge_endpoint(service: &str) -> CliResult { + let mut url = + reqwest::Url::parse(service).map_err(|_| "--service must be an absolute HTTPS URL")?; + crate::url_guard::require_credential_safe_transport(&url, "--service")?; + if url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || !url.path().trim_matches('/').is_empty() + { + return cli_error( + "--service must be a base HTTPS URL without credentials, a path, query, or fragment", + ); + } + url.set_path(PURGE_PATH); + Ok(url.to_string()) +} + +/// The acknowledgment returned by the purge endpoint. +#[derive(Deserialize)] +struct PurgeAcknowledgment { + purged: bool, + scope: String, + surrogate_key: Option, +} + +/// Execute `ts cache purge`. +/// +/// # Errors +/// +/// Returns an error when no scope is given, the admin password is missing from the +/// environment, the service cannot be reached, or the service answers with a non-success +/// status, redirects, or does not acknowledge the requested purge scope. +pub fn run_purge(args: &PurgeArgs, out: &mut impl std::io::Write) -> CliResult<()> { + let body = request_body(args)?; + let endpoint = purge_endpoint(&args.service)?; + + let password = std::env::var(ADMIN_PASSWORD_ENVIRONMENT_VARIABLE).map_err(|_| { + format!( + "set {ADMIN_PASSWORD_ENVIRONMENT_VARIABLE} to the service's admin password \ + (it is read from the environment, never a flag, so it does not reach `ps` \ + output or shell history)" + ) + })?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to start the HTTP runtime: {error}"))?; + + // Without this the first HTTPS request panics with "No provider set". + crate::tls::install_crypto_provider(); + + let (status, response_body) = runtime.block_on(async { + let client = reqwest::Client::builder() + .timeout(REQUEST_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("failed to build the HTTP client: {error}"))?; + let response = client + .post(&endpoint) + .basic_auth(&args.username, Some(&password)) + .header("content-type", "application/json") + .body(body) + .send() + .await + .map_err(|error| format!("could not reach {endpoint}: {error}"))?; + let status = response.status(); + let text = response + .text() + .await + .map_err(|error| format!("could not read the purge acknowledgment: {error}"))?; + Ok::<_, String>((status, text)) + })?; + + if status.is_success() { + let acknowledgment: PurgeAcknowledgment = serde_json::from_str(&response_body) + .map_err(|error| format!("invalid purge acknowledgment: {error}"))?; + let expected_scope = if args.all { "all" } else { "url" }; + if !acknowledgment.purged || acknowledgment.scope != expected_scope { + return cli_error(format!( + "service did not acknowledge a successful {expected_scope} purge" + )); + } + if !args.all + && acknowledgment + .surrogate_key + .as_deref() + .is_none_or(|key| key.trim().is_empty()) + { + return cli_error("URL purge acknowledgment is missing its surrogate key"); + } + writeln!(out, "{response_body}") + .map_err(|error| format!("failed to write the purge result: {error}"))?; + return Ok(()); + } + + // Named individually, because each one sends the operator somewhere different and a + // purge is usually run mid-incident. + let hint = match status.as_u16() { + 401 => " — check the admin username and $TRUSTED_SERVER_ADMIN_PASSWORD", + 404 => " — this service may predate the purge endpoint", + 501 => " — the template cache is Fastly-backed; this adapter cannot purge", + _ => "", + }; + cli_error(format!( + "purge failed: {status}{hint}\n{}", + response_body.trim() + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(all: bool, page: Option<&str>) -> PurgeArgs { + PurgeArgs { + service: "https://edge.example.com".to_owned(), + all, + page: page.map(str::to_owned), + username: "admin".to_owned(), + } + } + + #[test] + fn invalid_page_urls_are_rejected_locally() { + for page in [ + "/article", + "example.com/article", + "ftp://example.com/article", + "", + ] { + assert!( + request_body(&args(false, Some(page))).is_err(), + "should reject {page:?}" + ); + } + assert!( + request_body(&args(false, Some("http://example.com/article"))).is_ok(), + "should allow HTTP reader URLs" + ); + } + + #[test] + fn purge_all_sends_no_url_field() { + // The endpoint refuses scope "all" carrying a url, so emitting one would make + // every --all run fail. + assert_eq!( + request_body(&args(true, None)).expect("should build"), + r#"{"scope":"all"}"# + ); + } + + #[test] + fn purge_page_sends_the_url_verbatim() { + // Canonicalization is the endpoint's job. Normalizing here too would give the two + // sides separate rules to drift apart. + let body = request_body(&args(false, Some("https://example.com/a?b=2&a=1"))) + .expect("should build"); + assert_eq!( + body, + r#"{"scope":"url","url":"https://example.com/a?b=2&a=1"}"# + ); + } + + #[test] + fn a_url_with_json_punctuation_is_escaped_rather_than_breaking_the_body() { + let body = request_body(&args(false, Some(r#"https://example.com/"; drop"#))) + .expect("should build"); + let parsed: serde_json::Value = serde_json::from_str(&body).expect("should stay valid"); + assert_eq!(parsed["url"], r#"https://example.com/"; drop"#); + } + + #[test] + fn neither_scope_is_an_error_rather_than_a_default() { + // Defaulting to --all would make a bare `ts cache purge` flush production. + assert!(request_body(&args(false, None)).is_err()); + } + + #[test] + fn the_endpoint_url_survives_a_trailing_slash() { + for service in [ + "https://edge.example.com", + "https://edge.example.com/", + "https://edge.example.com///", + ] { + assert_eq!( + purge_endpoint(service).expect("should accept a secure base URL"), + "https://edge.example.com/_ts/admin/cache/purge", + "{service} must resolve to one well-formed endpoint" + ); + } + } + + #[test] + fn only_loopback_services_may_use_plaintext_http() { + for service in [ + "http://127.0.0.1:8080", + "http://[::1]:8080", + "http://localhost:8080", + ] { + assert!( + purge_endpoint(service).is_ok(), + "should permit loopback development: {service}" + ); + } + for service in [ + "http://192.0.2.1", + "http://[2001:db8::1]", + "http://localhost.example.com", + ] { + assert!( + purge_endpoint(service).is_err(), + "should refuse remote plaintext transport: {service}" + ); + } + } + + #[test] + fn service_urls_cannot_redirect_the_purge_path_or_embed_credentials() { + for service in [ + "not a URL", + "https://example.com/path", + "https://example.com?query=1", + "https://example.com#fragment", + "https://user:example-password@example.com", + ] { + assert!( + purge_endpoint(service).is_err(), + "should require an unambiguous service base URL" + ); + } + } +} diff --git a/crates/trusted-server-cli/src/commands/mod.rs b/crates/trusted-server-cli/src/commands/mod.rs index a01559fd4..9e456ce87 100644 --- a/crates/trusted-server-cli/src/commands/mod.rs +++ b/crates/trusted-server-cli/src/commands/mod.rs @@ -1,5 +1,10 @@ pub(crate) mod audit; pub(crate) mod config; // `dev` is `pub` so the macOS-gated `tests/proxy_e2e.rs` suite can reach -// `commands::dev::proxy`; the other command modules are crate-internal. +// `commands::dev::proxy`, `origin` is `pub` so `tests/origin_probe.rs` can drive the +// shareability probe against a local fixture, and `cache` is `pub` so +// `tests/cache_purge.rs` can drive a purge against one; the other command modules are +// crate-internal. +pub mod cache; pub mod dev; +pub mod origin; diff --git a/crates/trusted-server-cli/src/commands/origin/mod.rs b/crates/trusted-server-cli/src/commands/origin/mod.rs new file mode 100644 index 000000000..c385881c2 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/mod.rs @@ -0,0 +1,256 @@ +//! `ts origin` — questions about a publisher origin's behaviour. + +pub mod probe; +pub mod report; + +use clap::Subcommand; + +use crate::error::CliResult; + +/// Subcommands under `ts origin`. +#[derive(Debug, Subcommand)] +pub enum OriginCommand { + /// Check whether an origin's responses may be shared between readers. + /// + /// Cookies are read from the environment, never from the command line, because they + /// are credentials: set `TRUSTED_SERVER_PROBE_COOKIES` to the publisher cookies a real + /// reader carries (`name=value; name=value`), and + /// `TRUSTED_SERVER_PROBE_ADMISSION_COOKIE` to a cookie that gets past a bot wall. Every --url must be HTTPS; plain HTTP is + /// accepted only for a loopback development origin. + ProbeShareability(ProbeShareabilityArgs), +} + +/// Arguments for `ts origin probe-shareability`. +#[derive(Debug, clap::Args)] +pub struct ProbeShareabilityArgs { + /// URL to probe. Repeat for several pages; one clean URL is not a statement about + /// the origin. + #[arg(long, required = true)] + pub url: Vec, + + /// How many extra times to repeat the self-identity comparison. + #[arg(long, default_value_t = 3)] + pub repeat: u32, + + /// Extra cookies sent in the cookie arm, read from + /// [`PROBE_COOKIES_ENVIRONMENT_VARIABLE`]. + /// + /// Never a flag. The report asks for the cookies of a genuine authenticated session, + /// and an argument is visible to every other process on the host through `ps` and + /// lands in shell history. The probe always sends a representative Trusted Server + /// cookie set; the environment adds publisher cookies a real reader would also carry. + #[arg(skip)] + pub cookie: Vec, + + /// Request header and representative value to probe as NAME=VALUE. Repeatable. + /// + /// Mirror `creative_opportunities.template_cache_vary` here and supply each value + /// the origin uses, for example x-exp-variant=A and x-exp-variant=B. Values are + /// compared against the absent baseline and one another, with and without RSC. + /// A bare NAME samples `1`; bare built-in axes are not repeated. + #[arg(long = "vary-header", value_name = "NAME[=VALUE]")] + pub vary_header: Vec, + + /// Cookie every request carries to get past a bot wall, read from + /// [`PROBE_ADMISSION_COOKIE_ENVIRONMENT_VARIABLE`]. + /// + /// Also environment-only: a bot-wall admission token is a credential. Distinct from + /// the cookie arm's extras: this one is sent on *every* arm including the baseline, + /// because without it a protected origin answers each arm with a challenge page and + /// the probe would report on those instead of on the origin. It is not part of what + /// the cookie axis varies. These runs are diagnostic only and always fail the safety + /// gate because cookieless responses are untested. Rerun without it against the + /// origin before enabling caching. + #[arg(skip)] + pub admission_cookie: Option, + + /// Emit JSON instead of a human-readable report. + #[arg(long)] + pub json: bool, +} + +/// Run an `ts origin` subcommand. +/// +/// # Errors +/// +/// Returns an error when an origin cannot be reached, or when the probe's verdict is that +/// the origin is not shareable — the caller turns that into a non-zero exit so the command +/// can gate a deploy. +pub fn run(command: OriginCommand, out: &mut impl std::io::Write) -> CliResult<()> { + match command { + OriginCommand::ProbeShareability(mut args) => { + load_cookies_from_environment(&mut args); + run_probe(&args, out) + } + } +} + +/// Environment variable carrying the cookie arm's extra cookies, as one cookie header +/// value: `name=value; name=value`. +/// +/// Read from the environment and never accepted as a flag, for the same reason as the +/// admin password in `ts cache purge`: an argument is visible to every other process on +/// the host through `ps`, and lands in shell history. The report asks operators to probe +/// with a genuine session's cookies, so these values are credentials. +pub const PROBE_COOKIES_ENVIRONMENT_VARIABLE: &str = "TRUSTED_SERVER_PROBE_COOKIES"; + +/// Environment variable carrying the bot-wall admission cookie, as `name=value`. +pub const PROBE_ADMISSION_COOKIE_ENVIRONMENT_VARIABLE: &str = + "TRUSTED_SERVER_PROBE_ADMISSION_COOKIE"; + +/// Split a cookie header value into its `name=value` pairs. +fn split_cookie_header(value: &str) -> Vec { + value + .split(';') + .map(str::trim) + .filter(|pair| !pair.is_empty()) + .map(str::to_owned) + .collect() +} + +/// Fold the environment's cookies into the parsed arguments. +/// +/// Appends rather than replaces so a caller that constructed the arguments directly — the +/// integration suite — keeps what it set. +fn load_cookies_from_environment(args: &mut ProbeShareabilityArgs) { + load_cookies(args, |name| std::env::var(name).ok()); +} + +fn load_cookies(args: &mut ProbeShareabilityArgs, environment: impl Fn(&str) -> Option) { + if let Some(value) = environment(PROBE_COOKIES_ENVIRONMENT_VARIABLE) { + args.cookie.extend(split_cookie_header(&value)); + } + if args.admission_cookie.is_none() + && let Some(value) = environment(PROBE_ADMISSION_COOKIE_ENVIRONMENT_VARIABLE) + { + let value = value.trim(); + if !value.is_empty() { + args.admission_cookie = Some(value.to_owned()); + } + } +} + +fn run_probe(args: &ProbeShareabilityArgs, out: &mut impl std::io::Write) -> CliResult<()> { + // Transport first: every later step sends the cookies below to these URLs, so a URL + // that cannot carry a credential safely must be refused before one is read. + for url in &args.url { + let parsed = reqwest::Url::parse(url) + .map_err(|error| format!("--url must be an absolute URL, got {url:?}: {error}"))?; + crate::url_guard::require_credential_safe_transport(&parsed, "--url")?; + } + + for cookie in &args.cookie { + if !cookie.contains('=') { + return crate::error::cli_error(format!( + "{PROBE_COOKIES_ENVIRONMENT_VARIABLE} expects name=value pairs separated by \ + `;`, got {cookie:?}" + )); + } + } + + if let Some(cookie) = args.admission_cookie.as_deref() + && !cookie.contains('=') + { + return crate::error::cli_error(format!( + "{PROBE_ADMISSION_COOKIE_ENVIRONMENT_VARIABLE} expects name=value, got {cookie:?}" + )); + } + + let report = probe::probe_urls( + &args.url, + args.repeat, + &args.cookie, + &args.vary_header, + args.admission_cookie.as_deref(), + )?; + + let rendered = if args.json { + serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to render the probe report as JSON: {error}"))? + } else { + report.render_text() + }; + writeln!(out, "{rendered}").map_err(|error| format!("failed to write the report: {error}"))?; + + if report.passed() { + Ok(()) + } else { + // A failing verdict is the answer, not a malfunction — but it must not exit zero. + // The gate this probe guards is decided before the origin responds, so this + // result is the only thing standing between it and cross-serving. + crate::error::cli_error( + "origin is not safe to share: do not enable origin_readthrough_enabled or origin_is_cookie_independent", + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cookie_header_splits_trims_and_ignores_empty_segments() { + assert_eq!( + split_cookie_header(" ; session=example ; ; preference=a=b; "), + ["session=example", "preference=a=b"], + "should preserve cookie pairs while removing empty segments" + ); + assert!( + split_cookie_header(" ; ; ").is_empty(), + "should ignore empty cookies" + ); + } + + #[test] + fn environment_appends_cookies_and_preserves_explicit_admission() { + let mut args = ProbeShareabilityArgs { + url: vec![], + repeat: 1, + cookie: vec!["caller=example".to_owned()], + vary_header: vec![], + admission_cookie: Some("admission=caller".to_owned()), + json: false, + }; + load_cookies(&mut args, |name| { + Some(if name == PROBE_COOKIES_ENVIRONMENT_VARIABLE { + " session=environment; ; preference=example ".to_owned() + } else { + " admission=environment ".to_owned() + }) + }); + assert_eq!( + args.cookie, + [ + "caller=example", + "session=environment", + "preference=example" + ], + "should append parsed environment cookies" + ); + assert_eq!( + args.admission_cookie.as_deref(), + Some("admission=caller"), + "should preserve explicit admission cookie" + ); + args.admission_cookie = None; + load_cookies(&mut args, |_| None); + assert!( + args.admission_cookie.is_none(), + "should tolerate an absent environment" + ); + load_cookies(&mut args, |_| Some(" ".to_owned())); + assert!( + args.admission_cookie.is_none(), + "should ignore blank admission cookies" + ); + load_cookies(&mut args, |name| { + (name == PROBE_ADMISSION_COOKIE_ENVIRONMENT_VARIABLE) + .then(|| " admission=environment ".to_owned()) + }); + assert_eq!( + args.admission_cookie.as_deref(), + Some("admission=environment"), + "should trim the environment admission cookie" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/probe.rs b/crates/trusted-server-cli/src/commands/origin/probe.rs new file mode 100644 index 000000000..43d3909b6 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/probe.rs @@ -0,0 +1,1160 @@ +//! Fetching an origin under varied request signals, and judging the results. + +use std::time::Duration; + +use crate::commands::origin::report::{ + AxisResult, ProbeReport, UrlReport, VerdictResult, first_difference, +}; +use crate::error::{CliResult, cli_error}; + +/// Cookies a real repeat visitor carries, which is what the cookie axis must send. +/// +/// Trusted Server sets `ts-ec` itself, which is why the template cache's cookie gate is +/// very nearly a disable switch — every returning reader trips it. +const TS_COOKIES: &[&str] = &[ + "ts-ec=probe-edge-cookie", + "euconsent-v2=probe-consent", + "ts-tester=probe", +]; + +const DESKTOP_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; +const MOBILE_USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"; + +/// A crawler user agent, matching a fragment the runtime itself classifies as a bot. +/// +/// The ad stack is suppressed for bots and prefetches, but shareability is not: neither +/// classification reaches `origin_response_is_shareable`, so a crawler, challenge, or +/// prefetch document an origin serves without `Vary` can be stored and then handed to a +/// human navigation. These two axes are what makes that visible. +const BOT_USER_AGENT: &str = + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.example.com/bot.html)"; + +/// Response headers a platform cache stores with the body and replays to every later +/// reader. +/// +/// Bodies alone are not the cached representation. Two responses with identical HTML, a +/// per-audience `Content-Security-Policy`, and no matching `Vary` are cross-served +/// policies: a weaker one removes a browser protection, a stricter one breaks the page. +/// +/// An allowlist rather than a denylist of volatile fields, because the alternative fails +/// an origin for every `Date`, request id, or trace header it happens to emit, and a probe +/// that cries wolf is one an operator learns to rerun until it passes. Everything here is +/// policy or representation, and none of it is per-request by design. +const POLICY_HEADERS: &[&str] = &[ + "content-language", + "content-security-policy", + "content-security-policy-report-only", + "content-type", + "cross-origin-embedder-policy", + "cross-origin-opener-policy", + "cross-origin-resource-policy", + "permissions-policy", + "referrer-policy", + "strict-transport-security", + "x-content-type-options", + "x-frame-options", +]; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); + +/// One fetch's result, reduced to what the probe judges. +struct Fetched { + status: u16, + body: Vec, + headers: reqwest::header::HeaderMap, + rsc: bool, +} + +impl Fetched { + /// Every instance of a header, in arrival order. + /// + /// The only accessor on purpose. A first-instance-only variant reads as if it returns + /// "the" value, which is wrong for any field a proxy can append to: judging a response + /// on the origin's `Cache-Control` while a later `private` goes unread is a false pass. + fn all(&self, name: &str) -> Vec<&str> { + // Raw values remain in `headers`; `header_encoding_verdict` rejects any + // uninterpretable safety field before these textual checks can certify it. + self.headers + .get_all(name) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect() + } + + /// What a cache would store for this response: its policy headers, then its body. + /// + /// Every axis compares these rather than bodies, so a difference in a cached header is + /// judged by the same `Vary` rules as a difference in the HTML. Header order is this + /// function's, not the wire's, so two responses carrying the same fields in a + /// different order are not reported as differing. + fn canonical(&self) -> Vec { + let mut canonical = Vec::with_capacity(self.body.len() + 256); + for name in POLICY_HEADERS { + for value in self.headers.get_all(*name) { + canonical.extend_from_slice(name.as_bytes()); + canonical.extend_from_slice(b": "); + canonical.extend_from_slice(value.as_bytes().trim_ascii()); + canonical.push(b'\n'); + } + } + canonical.push(b'\n'); + canonical.extend_from_slice(&self.body); + canonical + } +} + +/// Browser request context for navigation and RSC comparisons. +#[derive(Clone, Copy)] +enum RequestProfile { + Navigation, + Fetch, +} + +/// What varies between the two arms of one axis. +struct Arm<'a> { + profile: RequestProfile, + headers: &'a [(&'a str, &'a str)], +} + +/// Representative values for one request signal. +struct HeaderSamples { + name: String, + values: Vec, +} + +fn parse_header_samples(headers: &[String]) -> CliResult> { + let mut samples: Vec = Vec::new(); + for header in headers { + let (name, value) = header.split_once('=').unwrap_or((header, "1")); + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid --vary-header name: {error}"))? + .as_str() + .to_owned(); + reqwest::header::HeaderValue::from_str(value) + .map_err(|error| format!("invalid --vary-header value for {name}: {error}"))?; + if matches!( + name.as_str(), + "self-identity" | "fetch-profile" | "bot" | "prefetch" + ) { + if header.contains('=') { + return cli_error(format!( + "--vary-header {name} conflicts with a reserved probe axis" + )); + } + continue; + } + if !header.contains('=') + && matches!( + name.as_str(), + "cookie" | "accept-encoding" | "user-agent" | "rsc" + ) + { + continue; + } + if let Some(sample) = samples.iter_mut().find(|sample| sample.name == name) { + if !sample.values.iter().any(|existing| existing == value) { + sample.values.push(value.to_owned()); + } + } else { + samples.push(HeaderSamples { + name, + values: vec![value.to_owned()], + }); + } + } + Ok(samples) +} + +/// Probe every URL and return the combined report. +/// +/// # Errors +/// +/// Returns an error when the runtime cannot be built or an origin cannot be reached. A +/// *reachable* origin that fails a check is not an error — it is a failing report. +pub(crate) fn probe_urls( + urls: &[String], + repeat: u32, + extra_cookies: &[String], + vary_headers: &[String], + admission_cookie: Option<&str>, +) -> CliResult { + let vary_headers = parse_header_samples(vary_headers)?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build the Tokio runtime for the probe: {error}"))?; + + crate::tls::install_crypto_provider(); + + runtime.block_on(async { + let client = reqwest::Client::builder() + .timeout(REQUEST_TIMEOUT) + // The probe must see exactly what it asked for. A redirect would silently + // compare two different documents and report them as a difference. + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("failed to build the probe HTTP client: {error}"))?; + + let mut reports = Vec::with_capacity(urls.len()); + for url in urls { + reports.push( + probe_one( + &client, + url, + repeat, + extra_cookies, + &vary_headers, + admission_cookie, + ) + .await?, + ); + } + Ok(ProbeReport { urls: reports }) + }) +} + +async fn probe_one( + client: &reqwest::Client, + url: &str, + repeat: u32, + extra_cookies: &[String], + vary_headers: &[HeaderSamples], + admission_cookie: Option<&str>, +) -> CliResult { + let cookie_jar = cookie_header(extra_cookies, admission_cookie); + + // Baseline: an HTML navigation carrying only the optional admission cookie. It carries that cookie because without it a bot-protected origin answers + // every arm with a challenge page, and the probe would then compare two challenge + // pages and report on those instead of on the origin. + let baseline = fetch( + client, + url, + RequestProfile::Navigation, + &[], + admission_cookie, + ) + .await?; + + // A challenge page is not the origin. Judging one produces a confident verdict about + // content the origin never served — in practice a false FAIL that reads exactly like a + // real one, which is worse than no answer. + if baseline.status != 200 { + return cli_error(format!( + "{url} answered {} rather than 200, so there is nothing to judge.\n\ + A bot wall or redirect returns a page the origin did not compose, and every \ + verdict below it would describe that page.\n\ + Set TRUSTED_SERVER_PROBE_ADMISSION_COOKIE to a session cookie that reaches \ + real content, as name=value.", + baseline.status + )); + } + + // Bodies are compared as the cache would store them: policy headers first, then the + // document. + let baseline_canonical = baseline.canonical(); + + let (self_identity, repeated) = + self_identity_axis(client, url, &baseline_canonical, repeat, admission_cookie).await?; + let mut axes = vec![self_identity]; + let mut samples: Vec<(String, Fetched)> = repeated + .into_iter() + .enumerate() + .map(|(index, response)| (format!("self-identity repeat {}", index + 1), response)) + .collect(); + + // Hold the browser fetch profile constant when toggling RSC. Otherwise an + // Accept-negotiated difference could be incorrectly excused by `Vary: RSC`. + let mut fetch_control = + fetch(client, url, RequestProfile::Fetch, &[], admission_cookie).await?; + let fetch_control_canonical = fetch_control.canonical(); + axes.push(AxisResult { + name: "fetch-profile".to_owned(), + description: "HTML navigation vs. a same-origin browser fetch".to_owned(), + difference: first_difference(&baseline_canonical, &fetch_control_canonical), + covered_by_vary: false, + }); + let mut rsc_canonical = Vec::new(); + // An axis is named for what it varies, which is not always the header it sends: the + // bot arm varies the user agent, and the prefetch arm varies both purpose headers. + for (name, header, description, value) in [ + ( + "cookie", + "cookie", + "bare vs. a representative cookie jar", + cookie_jar.as_str(), + ), + ( + "accept-encoding", + "accept-encoding", + "identity vs. gzip, compared after decoding", + "gzip", + ), + ( + "user-agent", + "user-agent", + "desktop vs. mobile user agent", + MOBILE_USER_AGENT, + ), + ( + "bot", + "user-agent", + "browser vs. crawler user agent", + BOT_USER_AGENT, + ), + ( + "prefetch", + "sec-purpose", + "navigation vs. a prefetch navigation", + "prefetch", + ), + ("rsc", "rsc", "bare vs. an RSC request", "1"), + ] { + let single_header = [(header, value)]; + let (axis, mut response) = compare_axis( + client, + url, + if name == "rsc" { + &fetch_control_canonical + } else { + &baseline_canonical + }, + name, + description, + Arm { + profile: if name == "rsc" { + RequestProfile::Fetch + } else { + RequestProfile::Navigation + }, + headers: if name == "prefetch" { + &[("sec-purpose", "prefetch"), ("purpose", "prefetch")] + } else { + &single_header + }, + }, + admission_cookie, + ) + .await?; + if name == "rsc" { + rsc_canonical = response.canonical(); + } + response.body = Vec::new(); + axes.push(axis); + samples.push((name.to_owned(), response)); + } + + fetch_control.body.clear(); + samples.push(("fetch-profile".to_owned(), fetch_control)); + + // Each configured signal needs its own comparison and Vary declaration. Combining + // these with RSC lets Vary: rsc hide a difference caused by an unrelated header. + for header in vary_headers { + let mut axis = AxisResult { + name: header.name.clone(), + description: format!( + "absent vs. {} values {:?}, with and without RSC", + header.name, header.values + ), + difference: None, + covered_by_vary: false, + }; + for (profile, baseline, rsc) in [ + (RequestProfile::Navigation, &baseline_canonical, false), + (RequestProfile::Fetch, &rsc_canonical, true), + ] { + let mut previous = vec![baseline.clone()]; + for value in &header.values { + let mut headers = Vec::with_capacity(2); + if rsc { + headers.push(("rsc", "1")); + } + headers.push((header.name.as_str(), value.as_str())); + let mut response = fetch(client, url, profile, &headers, admission_cookie).await?; + let canonical = response.canonical(); + for earlier in &previous { + if axis.difference.is_none() { + axis.difference = first_difference(earlier, &canonical); + } + } + previous.push(canonical); + response.body.clear(); + let label = format!( + "{}: {value:?}{}", + header.name, + if rsc { " with RSC" } else { "" } + ); + samples.push((label, response)); + } + } + if let Some(existing) = axes.iter_mut().find(|existing| existing.name == axis.name) { + if existing.difference.is_none() && axis.differs() { + existing.difference = axis.difference; + } + existing + .description + .push_str(&format!("; {}", axis.description)); + } else { + axes.push(axis); + } + } + + let mut baseline = baseline; + baseline.body = Vec::new(); + samples.insert(0, ("baseline".to_owned(), baseline)); + mark_axes_covered_by_vary(&samples, &mut axes); + let mut verdicts = judge_headers(&samples[0].1, &axes); + for (label, sample) in &samples { + for checked in judge_headers(sample, &axes) { + if !checked.passed { + let verdict = verdicts + .iter_mut() + .find(|verdict| verdict.name == checked.name) + .expect("should find every response verdict"); + if verdict.passed { + *verdict = VerdictResult { + detail: format!("{label}: {}", checked.detail), + ..checked + }; + } + } + } + } + + if admission_cookie.is_some() { + verdicts.push(VerdictResult { + name: "cookieless-coverage".to_owned(), + passed: false, + detail: "TRUSTED_SERVER_PROBE_ADMISSION_COOKIE was sent on every request; \ + cookieless responses were not tested. This diagnostic run cannot \ + establish cache safety. Unset it and probe the origin again before \ + enabling caching." + .to_owned(), + }); + } + + Ok(UrlReport { + url: url.to_owned(), + axes, + verdicts, + }) +} + +/// Only excuse a varying signal when every sampled response declares it. +/// +/// Cookie independence and decoded encoding identity are template-cache prerequisites, +/// regardless of Vary. Self-identity varies no request signal at all. +fn mark_axes_covered_by_vary(samples: &[(String, Fetched)], axes: &mut [AxisResult]) { + for axis in axes { + if matches!( + axis.name.as_str(), + "self-identity" | "cookie" | "accept-encoding" + ) { + continue; + } + axis.covered_by_vary = samples.iter().all(|(_, response)| { + let declared: Vec<&str> = response + .all("vary") + .into_iter() + .flat_map(|value| value.split(',')) + .map(str::trim) + .collect(); + !declared.contains(&"*") && vary_covers_axis(&declared, &axis.name) + }); + } +} + +/// Compare every repeat, keeping the first difference and inspecting all later headers. +async fn self_identity_axis( + client: &reqwest::Client, + url: &str, + baseline_canonical: &[u8], + repeat: u32, + admission_cookie: Option<&str>, +) -> CliResult<(AxisResult, Vec)> { + let mut difference = None; + let mut samples = Vec::new(); + for _ in 0..repeat.max(1) { + let mut again = fetch( + client, + url, + RequestProfile::Navigation, + &[], + admission_cookie, + ) + .await?; + if difference.is_none() { + difference = first_difference(baseline_canonical, &again.canonical()); + } + // Only response metadata is needed after comparison; do not retain a page body + // per repeat or per variant. + again.body = Vec::new(); + samples.push(again); + } + Ok(( + AxisResult { + name: "self-identity".to_owned(), + description: format!("the same request {} times", repeat.max(1) + 1), + difference, + covered_by_vary: false, + }, + samples, + )) +} + +async fn compare_axis( + client: &reqwest::Client, + url: &str, + baseline_canonical: &[u8], + name: &str, + description: &str, + arm: Arm<'_>, + admission_cookie: Option<&str>, +) -> CliResult<(AxisResult, Fetched)> { + let varied = fetch(client, url, arm.profile, arm.headers, admission_cookie).await?; + let axis = AxisResult { + name: name.to_owned(), + description: description.to_owned(), + difference: first_difference(baseline_canonical, &varied.canonical()), + covered_by_vary: false, + }; + Ok((axis, varied)) +} + +/// Response checks applied to every sample, all blocking. +fn judge_headers(response: &Fetched, axes: &[AxisResult]) -> Vec { + vec![ + VerdictResult { + name: "status".to_owned(), + passed: response.status == 200, + detail: format!("response status: {}", response.status), + }, + header_encoding_verdict(response), + content_type_verdict(response), + fronting_cache_verdict(response), + freshness_verdict(response), + set_cookie_verdict(response), + csp_nonce_verdict(response), + vary_coverage_verdict(response, axes), + ] +} + +/// Keep undecodable safety evidence from becoming a successful textual check. +fn header_encoding_verdict(response: &Fetched) -> VerdictResult { + let unreadable: Vec<&str> = response + .headers + .iter() + .filter(|(name, value)| { + matches!( + name.as_str(), + "set-cookie" + | "age" + | "cache-control" + | "surrogate-control" + | "pragma" + | "vary" + | "content-security-policy" + | "content-type" + | "x-cache" + | "cf-cache-status" + | "x-cache-status" + ) && value.to_str().is_err() + }) + .map(|(name, _)| name.as_str()) + .collect(); + VerdictResult { + name: "header-encoding".to_owned(), + passed: unreadable.is_empty(), + detail: if unreadable.is_empty() { + "safety headers are readable".to_owned() + } else { + format!( + "cannot interpret safety header(s): {}; raw values retained", + unreadable.join(", ") + ) + }, + } +} + +/// Certify HTML navigations, while allowing flight payloads only when the request sends RSC. +fn content_type_verdict(response: &Fetched) -> VerdictResult { + let types = response.all("content-type"); + let passed = types.len() == 1 + && types.iter().all(|value| { + let media_type = value.split(';').next().unwrap_or("").trim(); + media_type.eq_ignore_ascii_case("text/html") + || (response.rsc && media_type.eq_ignore_ascii_case("text/x-component")) + }); + VerdictResult { + name: "content-type".to_owned(), + passed, + detail: format!( + "expected HTML for navigation, or HTML/flight for requests with RSC: 1; content-type: {types:?}" + ), + } +} + +/// Every signal changed by the profile comparison must be covered. This is +/// deliberately conservative: a combined profile cannot attribute a difference +/// to just one of its headers. +fn vary_covers_axis(declared: &[&str], axis: &str) -> bool { + let required = match axis { + "fetch-profile" => &[ + "accept", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + ][..], + // Named for the classification, keyed on the header the arm actually sent. + "bot" => &["user-agent"][..], + "prefetch" => &["sec-purpose", "purpose"][..], + _ => std::slice::from_ref(&axis), + }; + required.iter().all(|name| { + declared + .iter() + .any(|field| field.eq_ignore_ascii_case(name)) + }) +} + +/// Every axis compares two responses. A cache between this tool and the origin can answer +/// both from one stored object, so the axes read identical and the report goes green +/// on an origin that personalizes freely on a miss. That is the one failure that invalidates +/// the whole run at once, so it is judged before anything else. +/// +/// Detected rather than defeated. Busting the cache would need a query parameter or a +/// `no-cache` request header, and both change what the origin is asked for — the first +/// changes the cache key and the page identity, the second can change the origin's own +/// caching behaviour and with it the freshness verdict. Perturbing the measurement to +/// rescue it would make a green result mean less, not more. Probe the origin directly. +fn fronting_cache_verdict(baseline: &Fetched) -> VerdictResult { + // Even Age: 0 can be a fresh cache hit. Any Age field makes direct-origin + // evidence uncertain; malformed values must not turn that uncertainty into a pass. + let ages = baseline.all("age"); + let served_from_cache = baseline.headers.contains_key("age"); + + const HIT_INDICATORS: &[&str] = &["x-cache", "cf-cache-status", "x-cache-status"]; + let vendor_hit = HIT_INDICATORS.iter().find(|name| { + baseline + .all(name) + .iter() + .any(|value| value.to_ascii_lowercase().contains("hit")) + }); + + let detail = match (served_from_cache, vendor_hit) { + (true, _) => format!( + "a cache may have answered this request (age: {}), so every axis may be comparing one \ + stored object with itself", + ages.join(", ") + ), + (false, Some(name)) => format!( + "a cache answered this request ({name} reports a hit), so every axis may be \ + comparing one stored object with itself" + ), + (false, None) => "no cache reported serving this response".to_owned(), + }; + + VerdictResult { + name: "fronting-cache".to_owned(), + passed: !served_from_cache && vendor_hit.is_none(), + detail, + } +} + +/// Readthrough has no equivalent of the template cache's `NoPositiveFreshness` refusal, so +/// an origin that declares no freshness would be stored on a platform default instead of +/// being declined. +fn freshness_verdict(baseline: &Fetched) -> VerdictResult { + // Every instance, not just the first. A field may arrive as several lines — a proxy + // that appends `Cache-Control: private` after the origin's `public, max-age=300` is + // the case that matters, and reading only the first line would pass it. + let cache_control = baseline.all("cache-control").join(", "); + let surrogate = baseline.all("surrogate-control").join(", "); + let positive = freshness_seconds(&surrogate, "max-age") + .or_else(|| freshness_seconds(&cache_control, "s-maxage")) + .or_else(|| freshness_seconds(&cache_control, "max-age")) + .is_some_and(|seconds| seconds.is_some_and(|seconds| seconds > 0)); + let pragma = baseline.all("pragma").join(", "); + let forbids = [&cache_control, &surrogate, &pragma].iter().any(|value| { + value.split(',').any(|directive| { + let name = directive.split('=').next().unwrap_or("").trim(); + ["no-store", "private", "no-cache"] + .iter() + .any(|forbidden| name.eq_ignore_ascii_case(forbidden)) + }) + }); + + VerdictResult { + name: "freshness".to_owned(), + passed: positive && !forbids, + detail: if cache_control.is_empty() && surrogate.is_empty() { + "origin declared no Cache-Control or Surrogate-Control".to_owned() + } else { + format!( + "cache-control: {cache_control:?}, surrogate-control: {surrogate:?}, pragma: {pragma:?}" + ) + }, + } +} + +/// A cached `Set-Cookie` is replayed to every later cookieless reader, which is +/// cross-reader session fixation rather than a staleness bug. +fn set_cookie_verdict(baseline: &Fetched) -> VerdictResult { + let cookies = baseline.headers.get_all("set-cookie").iter().count(); + VerdictResult { + name: "set-cookie".to_owned(), + passed: cookies == 0, + detail: if cookies == 0 { + "origin set no cookies".to_owned() + } else { + format!("origin set {} cookie(s) on this response", cookies) + }, + } +} + +/// A shared nonce silently defeats the origin's own nonce-based CSP for the cached window. +fn csp_nonce_verdict(baseline: &Fetched) -> VerdictResult { + let has_nonce = baseline + .all("content-security-policy") + .iter() + .any(|value| value.contains("'nonce-")); + VerdictResult { + name: "csp-nonce".to_owned(), + passed: !has_nonce, + detail: if has_nonce { + "Content-Security-Policy carries a per-response nonce".to_owned() + } else { + "no per-response CSP nonce".to_owned() + }, + } +} + +/// The platform cache keys on URL plus whatever the origin declares in `Vary`, so an axis +/// that varies and is not declared is cross-served. +fn vary_coverage_verdict(baseline: &Fetched, axes: &[AxisResult]) -> VerdictResult { + let declared: Vec = baseline + .all("vary") + .iter() + .flat_map(|value| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .filter(|name| !name.is_empty()) + .collect(); + + let uncovered: Vec<&str> = axes + .iter() + // The raw observation, not `passed()`: `passed()` forgives a declared signal, and + // this verdict is what decides whether it is declared. + .filter(|axis| axis.differs()) + .map(|axis| axis.name.as_str()) + // Self-identity is not a request signal, so `Vary` cannot cover it. + .filter(|name| *name != "self-identity") + .filter(|name| { + !vary_covers_axis( + &declared.iter().map(String::as_str).collect::>(), + name, + ) + }) + .collect(); + + VerdictResult { + name: "vary-coverage".to_owned(), + passed: uncovered.is_empty() && !declared.iter().any(|name| name == "*"), + detail: if uncovered.is_empty() { + format!("declared Vary: {declared:?}") + } else { + format!( + "varies on {} but Vary declares {declared:?}", + uncovered.join(", ") + ) + }, + } +} + +/// Distinguish an absent directive from an invalid or conflicting authoritative one. +fn freshness_seconds(value: &str, name: &str) -> Option> { + let mut seconds = None; + for directive in value.split(',') { + let (field, raw) = directive.split_once('=').unwrap_or((directive, "")); + if field.trim().eq_ignore_ascii_case(name) { + let parsed = raw.trim().trim_matches('"').parse::().ok(); + seconds = Some(match seconds { + None => parsed, + Some(existing) if existing == parsed => parsed, + Some(_) => return Some(None), + }); + } + } + seconds +} + +/// The cookie arm's jar: the admission cookie plus the cookies a repeat visitor carries. +/// +/// The admission cookie is included so this arm differs from the baseline by the *added* +/// cookies only. Without it the axis would also be varying whether the request is admitted +/// at all, which is not a question about personalization. +fn cookie_header(extra: &[String], admission_cookie: Option<&str>) -> String { + let mut parts: Vec = admission_cookie.into_iter().map(str::to_owned).collect(); + parts.extend(TS_COOKIES.iter().map(|pair| (*pair).to_owned())); + parts.extend(extra.iter().cloned()); + parts.join("; ") +} + +async fn fetch( + client: &reqwest::Client, + url: &str, + profile: RequestProfile, + headers: &[(&str, &str)], + admission_cookie: Option<&str>, +) -> CliResult { + // Resolved into one map before the request is built, because `RequestBuilder::header` + // *appends*. Layering an arm's override on top of a default would send the header + // twice, and an origin that reads the first instance would never see the override — + // silently turning the user-agent and accept-encoding axes into no-ops that pass. + let mut resolved: Vec<(&str, &str)> = vec![ + ("user-agent", DESKTOP_USER_AGENT), + // Identity unless an arm overrides it, so the encoding axis is the only thing that + // changes what the origin may compress. + ("accept-encoding", "identity"), + ]; + match profile { + RequestProfile::Navigation => resolved.extend([ + ( + "accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + ), + ("sec-fetch-dest", "document"), + ("sec-fetch-mode", "navigate"), + ("sec-fetch-site", "none"), + ("sec-fetch-user", "?1"), + ]), + RequestProfile::Fetch => resolved.extend([ + ("accept", "*/*"), + ("sec-fetch-dest", "empty"), + ("sec-fetch-mode", "cors"), + ("sec-fetch-site", "same-origin"), + ]), + } + // Seeded before the arm's own headers so an arm that sets `cookie` replaces it rather + // than duplicating it — every arm must be admitted, but only the cookie arm varies + // what else it carries. + if let Some(cookie) = admission_cookie { + resolved.push(("cookie", cookie)); + } + for (name, value) in headers { + match resolved + .iter_mut() + .find(|(existing, _)| existing.eq_ignore_ascii_case(name)) + { + Some(slot) => slot.1 = value, + None => resolved.push((*name, *value)), + } + } + + let mut request = client.get(url); + for (name, value) in &resolved { + request = request.header(*name, *value); + } + + let response = match request.send().await { + Ok(response) => response, + Err(error) => return cli_error(format!("could not reach {url}: {error}")), + }; + let status = response.status().as_u16(); + + let collected = response.headers().clone(); + + let body = match response.bytes().await { + Ok(bytes) => bytes.to_vec(), + Err(error) => return cli_error(format!("could not read the body of {url}: {error}")), + }; + + Ok(Fetched { + status, + body, + headers: collected, + rsc: resolved + .iter() + .any(|(name, value)| name.eq_ignore_ascii_case("rsc") && *value == "1"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fetched(headers: &[(&str, &str)]) -> Fetched { + let mut collected = reqwest::header::HeaderMap::new(); + for (name, value) in headers { + collected.append( + reqwest::header::HeaderName::from_bytes(name.as_bytes()) + .expect("should parse test header name"), + reqwest::header::HeaderValue::from_str(value) + .expect("should parse test header value"), + ); + } + Fetched { + status: 200, + body: b"".to_vec(), + headers: collected, + rsc: false, + } + } + + fn failing_axis(name: &str) -> AxisResult { + AxisResult { + name: name.to_owned(), + description: String::new(), + difference: Some(crate::commands::origin::report::Difference { + offset: 0, + left: "a".to_owned(), + right: "b".to_owned(), + }), + covered_by_vary: false, + } + } + + #[test] + fn header_samples_preserve_values_and_deduplicate_case_insensitive_names() { + let inputs = [ + "X-Layout=A=B", + "x-layout=A=B", + "x-layout=", + "x-layout", + "user-agent", + ] + .map(str::to_owned); + + let samples = parse_header_samples(&inputs).expect("should parse representative headers"); + + assert_eq!( + samples.len(), + 1, + "should group names and skip bare built-in axes" + ); + assert_eq!( + samples[0].name, "x-layout", + "should normalize only header names" + ); + assert_eq!( + samples[0].values, + ["A=B", "", "1"], + "should preserve empty and equals-containing values and sample bare names as 1" + ); + } + + #[test] + fn shared_freshness_overrides_browser_freshness() { + for value in [ + "public, s-maxage=0, max-age=300", + "public, max-age=300, s-maxage=0", + "public, s-maxage=0", + "max-age=300, s-maxage=invalid", + ] { + assert!( + !freshness_verdict(&fetched(&[("cache-control", value)])).passed, + "should reject shared freshness in {value}" + ); + } + for value in ["s-maxage=60, max-age=300", "max-age=300", "s-maxage=\"60\""] { + assert!( + freshness_verdict(&fetched(&[("cache-control", value)])).passed, + "should accept {value}" + ); + } + } + + #[test] + fn canonical_preserves_undecodable_policy_values() { + for name in POLICY_HEADERS { + let mut first = fetched(&[]); + let mut second = fetched(&[]); + first.headers.insert( + *name, + reqwest::header::HeaderValue::from_bytes(b"policy=\xe9") + .expect("should accept raw header"), + ); + second.headers.insert( + *name, + reqwest::header::HeaderValue::from_bytes(b"policy=\xe8") + .expect("should accept raw header"), + ); + assert_ne!( + first.canonical(), + second.canonical(), + "should compare raw {name}" + ); + second.headers.insert( + *name, + reqwest::header::HeaderValue::from_bytes(b" \tpolicy=\xe9 \t") + .expect("should accept padded header"), + ); + assert_eq!( + first.canonical(), + second.canonical(), + "should trim whitespace for {name}" + ); + } + } + + #[test] + fn absent_cache_control_fails_freshness() { + assert!( + !freshness_verdict(&fetched(&[])).passed, + "readthrough would store this on a platform default where the template cache \ + declines it" + ); + } + + #[test] + fn positive_max_age_passes_freshness() { + assert!(freshness_verdict(&fetched(&[("cache-control", "public, max-age=300")])).passed); + assert!(freshness_verdict(&fetched(&[("surrogate-control", "max-age=60")])).passed); + } + + #[test] + fn zero_max_age_is_not_positive_freshness() { + assert!(!freshness_verdict(&fetched(&[("cache-control", "max-age=0")])).passed); + } + + #[test] + fn private_or_no_store_fails_freshness_even_with_a_max_age() { + assert!( + !freshness_verdict(&fetched(&[("cache-control", "private, max-age=300")])).passed, + "an origin that marks HTML private must not be declared shareable" + ); + assert!(!freshness_verdict(&fetched(&[("cache-control", "no-store, max-age=300")])).passed); + } + + #[test] + fn set_cookie_fails_its_verdict() { + assert!(set_cookie_verdict(&fetched(&[])).passed); + assert!( + !set_cookie_verdict(&fetched(&[("set-cookie", "sid=1")])).passed, + "a cached Set-Cookie is replayed to every later cookieless reader" + ); + } + + #[test] + fn csp_nonce_fails_its_verdict() { + assert!( + csp_nonce_verdict(&fetched(&[( + "content-security-policy", + "default-src 'self'" + )])) + .passed + ); + assert!( + !csp_nonce_verdict(&fetched(&[( + "content-security-policy", + "script-src 'nonce-abc123'" + )])) + .passed + ); + } + + #[test] + fn a_varying_axis_must_be_declared_in_vary() { + let axes = vec![failing_axis("user-agent")]; + assert!( + !vary_coverage_verdict(&fetched(&[]), &axes).passed, + "varying on User-Agent without declaring it is cross-served" + ); + assert!( + vary_coverage_verdict(&fetched(&[("vary", "User-Agent")]), &axes).passed, + "a declared axis is keyed by the platform cache and is therefore safe" + ); + } + + #[test] + fn vary_star_refuses_sharing() { + let axes = vec![failing_axis("user-agent"), failing_axis("cookie")]; + assert!( + !vary_coverage_verdict(&fetched(&[("vary", "*")]), &axes).passed, + "should refuse wildcard Vary" + ); + } + + #[test] + fn self_identity_failure_is_not_blamed_on_vary() { + let axes = vec![failing_axis("self-identity")]; + assert!( + vary_coverage_verdict(&fetched(&[]), &axes).passed, + "an unstable origin is a self-identity failure; Vary cannot express it and the \ + operator must not be sent looking for a header" + ); + } + + #[test] + fn a_stable_body_with_a_different_policy_header_is_a_difference() { + // Fastly stores response headers with the body, so a per-audience CSP is + // cross-served exactly as a per-audience document would be. + let strict = fetched(&[("content-security-policy", "default-src 'self'")]); + let weak = fetched(&[("content-security-policy", "default-src *")]); + assert_eq!( + strict.body, weak.body, + "the bodies are identical on purpose" + ); + assert!( + first_difference(&strict.canonical(), &weak.canonical()).is_some(), + "a weaker policy served to one audience must not be storable for another" + ); + } + + #[test] + fn a_volatile_header_is_not_a_difference() { + // A request id or trace header changes on every response. Failing an origin for + // one would teach operators to rerun the probe until it passes. + let first = fetched(&[ + ("x-request-id", "a"), + ("date", "Mon, 01 Jan 2035 00:00:00 GMT"), + ]); + let second = fetched(&[ + ("x-request-id", "b"), + ("date", "Mon, 01 Jan 2035 00:00:01 GMT"), + ]); + assert_eq!( + first_difference(&first.canonical(), &second.canonical()), + None + ); + } + + #[test] + fn canonical_header_order_does_not_depend_on_the_wire_order() { + let one = fetched(&[ + ("referrer-policy", "no-referrer"), + ("x-frame-options", "DENY"), + ]); + let other = fetched(&[ + ("x-frame-options", "DENY"), + ("referrer-policy", "no-referrer"), + ]); + assert_eq!(first_difference(&one.canonical(), &other.canonical()), None); + } + + #[test] + fn the_bot_and_prefetch_axes_are_covered_by_the_headers_they_send() { + // Each axis is named for the classification it tests, not for the header it + // varies, so `Vary` coverage has to be mapped rather than matched by name. + assert!(vary_covers_axis(&["user-agent"], "bot")); + assert!(!vary_covers_axis(&["bot"], "bot")); + assert!(vary_covers_axis(&["sec-purpose", "purpose"], "prefetch")); + assert!(!vary_covers_axis(&["sec-purpose"], "prefetch")); + assert!(!vary_covers_axis(&["purpose"], "prefetch")); + } + + #[test] + fn cookie_header_carries_the_cookies_a_repeat_visitor_has() { + let header = cookie_header(&["publisher_session=1".to_owned()], None); + assert!(header.contains("ts-ec="), "TS sets its own identity cookie"); + assert!(header.contains("publisher_session=1")); + } + + #[test] + fn the_cookie_arm_keeps_the_admission_cookie() { + // Otherwise the cookie axis would vary two things at once: the added cookies, and + // whether the request is admitted past the bot wall at all. + let header = cookie_header(&[], Some("datadome=abc")); + assert!(header.contains("datadome=abc")); + assert!(header.contains("ts-ec=")); + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/report.rs b/crates/trusted-server-cli/src/commands/origin/report.rs new file mode 100644 index 000000000..4a47b368f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/report.rs @@ -0,0 +1,374 @@ +//! The probe's result model and its two renderings. +//! +//! Every axis and every verdict is **blocking**. That is not a style choice: the origin +//! readthrough gate is decided before the origin responds, and no post-response hook is +//! reachable on the Fastly adapter, so none of the template cache's response-side refusals +//! can be applied to the readthrough path. This probe is the only control, which is why a +//! failure exits non-zero rather than printing a warning. + +use serde::{Deserialize, Serialize}; + +/// How far apart two responses were, when they differed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Difference { + /// Byte offset of the first divergence. + pub offset: usize, + /// Short, escaped context from the first arm. + pub left: String, + /// Short, escaped context from the second arm. + pub right: String, +} + +/// One comparison between two fetches that differ in exactly one request signal. +/// +/// The two arms are compared as a cache would store them: the response's policy headers +/// followed by its body. A byte offset in [`Difference`] is therefore an offset into that +/// representation, not into the HTML. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AxisResult { + /// Axis name. Usually the request header this axis varies; the `bot` and `prefetch` + /// axes are named for the request class they test instead. + pub name: String, + /// What the two arms varied. + pub description: String, + /// `None` when the arms matched. + pub difference: Option, + /// Whether every sample declares this signal in `Vary` and that permits variation. + /// + /// Always false for self-identity, cookie, and decoded encoding comparisons: + /// the template cache requires those bodies to match regardless of `Vary`. + #[serde(default)] + pub covered_by_vary: bool, +} + +impl AxisResult { + /// Whether the two arms returned different bytes, before asking whether that is safe. + #[must_use] + pub fn differs(&self) -> bool { + self.difference.is_some() + } + + /// Whether this axis is safe. + /// + /// Differences pass only when [`Self::covered_by_vary`] permits them. Cookie, + /// self-identity, and decoded encoding differences always fail. + #[must_use] + pub fn passed(&self) -> bool { + !self.differs() || self.covered_by_vary + } +} + +/// One check on the origin's response headers. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VerdictResult { + /// Verdict name, as printed. + pub name: String, + /// Whether the origin satisfied it. + pub passed: bool, + /// What was observed, pass or fail. + pub detail: String, +} + +/// Everything the probe learned about one URL. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UrlReport { + /// The URL probed. + pub url: String, + /// Comparison axes, in the order they ran. + pub axes: Vec, + /// Response-header verdicts. + pub verdicts: Vec, +} + +impl UrlReport { + /// Whether every axis and verdict passed. + #[must_use] + pub fn passed(&self) -> bool { + self.axes.iter().all(AxisResult::passed) && self.verdicts.iter().all(|v| v.passed) + } +} + +/// The whole probe run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProbeReport { + /// One entry per `--url`. + pub urls: Vec, +} + +impl ProbeReport { + /// Whether the origin may be declared shareable on the evidence gathered. + /// + /// A single failing axis or verdict on a single URL is enough to say no. + #[must_use] + pub fn passed(&self) -> bool { + !self.urls.is_empty() && self.urls.iter().all(UrlReport::passed) + } + + /// Render for a human reader. + #[must_use] + pub fn render_text(&self) -> String { + let mut out = String::new(); + for url in &self.urls { + out.push_str(&format!("{}\n", url.url)); + for axis in &url.axes { + match &axis.difference { + None => { + out.push_str(&format!(" PASS {:<16} {}\n", axis.name, axis.description)) + } + Some(_) if axis.covered_by_vary => out.push_str(&format!( + " PASS {:<16} {} — differs, but the origin declares it in \ + Vary, so the cache keys on it\n", + axis.name, axis.description + )), + Some(difference) => { + out.push_str(&format!( + " FAIL {:<16} {} — differs at byte {}\n", + axis.name, axis.description, difference.offset + )); + out.push_str(&format!(" a: {}\n", difference.left)); + out.push_str(&format!(" b: {}\n", difference.right)); + } + } + } + for verdict in &url.verdicts { + let label = if verdict.passed { "PASS" } else { "FAIL" }; + out.push_str(&format!( + " {label} {:<16} {}\n", + verdict.name, verdict.detail + )); + } + out.push('\n'); + } + + out.push_str(if self.passed() { + "VERDICT: shareable on the URLs sampled.\n" + } else { + "VERDICT: NOT shareable. Do not enable origin_readthrough_enabled or origin_is_cookie_independent.\n" + }); + out.push_str(LIMITS); + out + } +} + +/// Printed on every run, pass or fail. +/// +/// An operator who reads only a green verdict will over-generalize it, and both of these +/// limits are invisible from the output itself. +pub const LIMITS: &str = "\nLimits of this result:\n \ + - Runs from one client address, so origin personalization keyed on the forwarded\n \ + client IP (geo, rate-class) is undetectable here.\n \ + - Covers the URLs sampled, not the origin as a whole.\n \ + - Sends synthetic cookies. An origin that personalizes only for a genuine\n \ + authenticated session shows no difference unless you supply that session's\n \ + cookies in TRUSTED_SERVER_PROBE_COOKIES.\n \ + - Compares the body and the policy headers a cache stores with it, not every\n \ + response header. A difference in a field outside that set is not reported.\n \ + - Varies only the signals it has axes for. Accept-Language, Referer and client\n \ + hints are never varied, so locale-based personalization would not be seen.\n \ + - Compares a handful of back-to-back requests, so variation on a slower cycle\n \ + (an hourly rotation, a low-frequency experiment bucket) can fall between them.\n"; + +/// First byte at which two representations diverge, with a short escaped window from each. +/// +/// Returns `None` when they are identical. +#[must_use] +pub fn first_difference(left: &[u8], right: &[u8]) -> Option { + if left == right { + return None; + } + let offset = left + .iter() + .zip(right.iter()) + .position(|(a, b)| a != b) + .unwrap_or_else(|| left.len().min(right.len())); + Some(Difference { + offset, + left: context_window(left, offset), + right: context_window(right, offset), + }) +} + +/// A bounded, escaped excerpt starting at `offset`. +/// +/// Bounded because this is publisher HTML: it can be large, and it can contain bytes that +/// would corrupt a terminal. +fn context_window(bytes: &[u8], offset: usize) -> String { + const WINDOW: usize = 48; + let end = bytes.len().min(offset + WINDOW); + let slice = bytes.get(offset..end).unwrap_or_default(); + let mut rendered = String::with_capacity(slice.len()); + for &byte in slice { + match byte { + b'\n' => rendered.push_str("\\n"), + b'\r' => rendered.push_str("\\r"), + b'\t' => rendered.push_str("\\t"), + 0x20..=0x7e => rendered.push(byte as char), + _ => rendered.push_str(&format!("\\x{byte:02x}")), + } + } + if end < bytes.len() { + rendered.push('…'); + } + if rendered.is_empty() { + "".to_owned() + } else { + rendered + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn axis(name: &str, difference: Option) -> AxisResult { + AxisResult { + name: name.to_owned(), + description: "test".to_owned(), + difference, + covered_by_vary: false, + } + } + + fn verdict(name: &str, passed: bool) -> VerdictResult { + VerdictResult { + name: name.to_owned(), + passed, + detail: "test".to_owned(), + } + } + + #[test] + fn identical_bodies_have_no_difference() { + assert_eq!( + first_difference(b"ok", b"ok"), + None + ); + } + + #[test] + fn difference_reports_the_first_diverging_byte() { + let difference = first_difference(b"aaa", b"bbb") + .expect("should report a difference"); + assert_eq!(difference.offset, 6, "divergence starts after ``"); + assert!(difference.left.starts_with("aaa")); + assert!(difference.right.starts_with("bbb")); + } + + #[test] + fn difference_handles_one_body_being_a_prefix_of_the_other() { + let difference = + first_difference(b"", b"more").expect("should report a difference"); + assert_eq!(difference.offset, 6); + assert_eq!( + difference.left, "", + "a truncated arm must say so rather than render an empty window" + ); + } + + #[test] + fn context_window_escapes_bytes_that_would_corrupt_a_terminal() { + let difference = first_difference(b"a\x00\x01", b"b\x00\x01").expect("should differ"); + assert_eq!(difference.left, "a\\x00\\x01"); + } + + #[test] + fn context_window_is_bounded() { + let long = vec![b'x'; 500]; + let mut other = long.clone(); + other[0] = b'y'; + let difference = first_difference(&long, &other).expect("should differ"); + assert!( + difference.left.chars().count() <= 49, + "a 500-byte body must not dump 500 bytes into the terminal" + ); + assert!( + difference.left.ends_with('…'), + "truncation should be visible" + ); + } + + #[test] + fn one_failing_axis_fails_the_whole_report() { + let report = ProbeReport { + urls: vec![UrlReport { + url: "https://example.com/a".to_owned(), + axes: vec![ + axis("cookie", None), + axis( + "user-agent", + Some(Difference { + offset: 0, + left: "a".to_owned(), + right: "b".to_owned(), + }), + ), + ], + verdicts: vec![verdict("freshness", true)], + }], + }; + + assert!( + !report.passed(), + "any axis failing must fail the run; the verdicts are the only control on this path" + ); + } + + #[test] + fn one_failing_verdict_fails_the_whole_report() { + let report = ProbeReport { + urls: vec![UrlReport { + url: "https://example.com/a".to_owned(), + axes: vec![axis("cookie", None)], + verdicts: vec![verdict("freshness", true), verdict("set-cookie", false)], + }], + }; + + assert!(!report.passed()); + } + + #[test] + fn one_failing_url_fails_a_multi_url_run() { + let good = UrlReport { + url: "https://example.com/a".to_owned(), + axes: vec![axis("cookie", None)], + verdicts: vec![verdict("freshness", true)], + }; + let mut bad = good.clone(); + bad.url = "https://example.com/b".to_owned(); + bad.verdicts = vec![verdict("freshness", false)]; + + assert!( + !ProbeReport { + urls: vec![good, bad] + } + .passed(), + "a clean result on one URL says nothing about another" + ); + } + + #[test] + fn an_empty_run_is_not_a_pass() { + assert!( + !ProbeReport { urls: Vec::new() }.passed(), + "probing nothing must not read as evidence of shareability" + ); + } + + #[test] + fn rendered_text_always_states_the_limits() { + let report = ProbeReport { + urls: vec![UrlReport { + url: "https://example.com/a".to_owned(), + axes: vec![axis("cookie", None)], + verdicts: vec![verdict("freshness", true)], + }], + }; + let rendered = report.render_text(); + + assert!(rendered.contains("VERDICT: shareable")); + assert!( + rendered.contains("one client address"), + "a green verdict is the most likely to be over-generalized, so the limits print too" + ); + } +} diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index a3352ba93..74045ed7a 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -8,6 +8,10 @@ mod error; mod prebid_bundle; #[cfg(not(target_arch = "wasm32"))] mod run; +#[cfg(not(target_arch = "wasm32"))] +mod tls; +#[cfg(not(target_arch = "wasm32"))] +mod url_guard; #[cfg(not(target_arch = "wasm32"))] pub use run::{RunOutcome, run_from_env}; diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 44ddbafe7..e06bb998c 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -29,6 +29,9 @@ enum Command { Auth(AuthArgs), /// Build the project for a target adapter. Build(BuildArgs), + /// Shared template cache commands. + #[command(subcommand)] + Cache(crate::commands::cache::CacheCommand), /// Trusted Server app-config commands. #[command(subcommand)] Config(ConfigCommand), @@ -47,6 +50,9 @@ enum Command { /// Local developer tools (e.g. the macOS-only production-hostname proxy). #[command(subcommand)] Dev(crate::commands::dev::DevCommand), + /// Questions about a publisher origin's behaviour. + #[command(subcommand)] + Origin(crate::commands::origin::OriginCommand), } #[derive(Debug, Subcommand)] @@ -160,6 +166,16 @@ fn dispatch(args: Args) -> Result { Command::Rollback(args) => edgezero_cli::run_rollback(&args).map(|()| RunOutcome::Success), Command::Serve(args) => edgezero_cli::run_serve(&args).map(|()| RunOutcome::Success), Command::Dev(command) => crate::commands::dev::run(command).map(|()| RunOutcome::Success), + Command::Cache(command) => { + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + crate::commands::cache::run(command, &mut out).map(|()| RunOutcome::Success) + } + Command::Origin(command) => { + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + crate::commands::origin::run(command, &mut out).map(|()| RunOutcome::Success) + } } } diff --git a/crates/trusted-server-cli/src/tls.rs b/crates/trusted-server-cli/src/tls.rs new file mode 100644 index 000000000..873d3bc7c --- /dev/null +++ b/crates/trusted-server-cli/src/tls.rs @@ -0,0 +1,19 @@ +//! The rustls crypto provider this CLI's HTTPS clients depend on. + +/// Install the default rustls crypto provider, once per process. +/// +/// **Call this before building any `reqwest` client.** Without it the first HTTPS request +/// panics with "No provider set" — at runtime, on the operator's machine, not at compile +/// time here. +/// +/// This crate's `reqwest` is built with a `-no-provider` rustls feature on purpose: it +/// already links `aws-lc-rs` through `reqwest` 0.13, and letting `reqwest` 0.12 pull `ring` +/// as well would compile two providers, which makes rustls's default ambiguous and panics +/// the dev proxy. The cost of that choice is that somebody must install the default, and +/// this is the one place that does. +/// +/// Idempotent: a second call returns `Err` because one is already installed, which is not +/// a failure. +pub(crate) fn install_crypto_provider() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); +} diff --git a/crates/trusted-server-cli/src/url_guard.rs b/crates/trusted-server-cli/src/url_guard.rs new file mode 100644 index 000000000..240f10d54 --- /dev/null +++ b/crates/trusted-server-cli/src/url_guard.rs @@ -0,0 +1,85 @@ +//! Transport checks shared by the commands that send an operator's credentials to a URL. +//! +//! One implementation rather than one per command: a second copy is a second place for the +//! loopback exemption to drift, and every command that gets this wrong sends a session +//! cookie or an admin password over the wire in the clear. + +use crate::error::{CliResult, cli_error}; + +/// Whether the URL names a loopback host a developer runs a service on. +fn is_loopback(url: &reqwest::Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(address)) => address.is_loopback(), + Some(url::Host::Ipv6(address)) => address.is_loopback(), + Some(url::Host::Domain(name)) => name.eq_ignore_ascii_case("localhost"), + None => false, + } +} + +/// Refuse a URL that would carry a credential in the clear, or inside its own userinfo. +/// +/// `flag` names the option being validated, so the message points at what the operator +/// typed rather than at an internal value. +/// +/// # Errors +/// +/// Returns an error when the scheme is not HTTPS and the host is not loopback, or when the +/// URL embeds a username or password. +pub(crate) fn require_credential_safe_transport(url: &reqwest::Url, flag: &str) -> CliResult<()> { + if url.scheme() != "https" && !(url.scheme() == "http" && is_loopback(url)) { + return cli_error(format!( + "{flag} requires HTTPS to protect the credentials sent with the request; HTTP is \ + allowed only for loopback development services" + )); + } + if !url.username().is_empty() || url.password().is_some() { + return cli_error(format!( + "{flag} must not embed credentials in the URL; userinfo is logged by proxies and \ + is sent before any transport check can protect it" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check(url: &str) -> CliResult<()> { + let parsed = reqwest::Url::parse(url).expect("should parse the test URL"); + require_credential_safe_transport(&parsed, "--url") + } + + #[test] + fn https_is_accepted() { + assert!(check("https://example.com/article").is_ok()); + } + + #[test] + fn plain_http_is_refused_off_loopback() { + assert!( + check("http://example.com/article").is_err(), + "a probe cookie sent over HTTP is readable by every hop in between" + ); + assert!( + check("http://192.0.2.10:8080/article").is_err(), + "a private-range address is still not loopback" + ); + } + + #[test] + fn http_is_allowed_only_on_loopback() { + assert!(check("http://127.0.0.1:8080/article").is_ok()); + assert!(check("http://localhost:8080/article").is_ok()); + assert!(check("http://[::1]:8080/article").is_ok()); + } + + #[test] + fn userinfo_is_refused() { + assert!( + check("https://user:example-password@example.com/").is_err(), + "credentials in the URL reach proxy logs and shell history" + ); + assert!(check("https://user@example.com/").is_err()); + } +} diff --git a/crates/trusted-server-cli/tests/cache_purge.rs b/crates/trusted-server-cli/tests/cache_purge.rs new file mode 100644 index 000000000..9db1938ef --- /dev/null +++ b/crates/trusted-server-cli/tests/cache_purge.rs @@ -0,0 +1,276 @@ +//! Tests for `ts cache purge` against a local fixture standing in for a deployed service. +//! +//! Run with `./scripts/test-cli.sh`. + +mod support_origin; + +use support_origin::{FixtureResponse, FixtureServer}; +use trusted_server_cli::commands::cache::{CacheCommand, PurgeArgs, run}; + +/// Serializes the environment across this binary's tests. +/// +/// The CLI reads the password from the environment on purpose, so exercising that path +/// means writing to it — and the test harness runs these in parallel threads of one +/// process, where one test clearing the variable races another that just set it. +static ENVIRONMENT: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +#[test] +fn plaintext_remote_services_are_rejected_before_sending_credentials() { + for service in [ + "http://edge.example.com", + "http://localhost.example.com", + "ftp://127.0.0.1", + ] { + let mut out = Vec::new(); + let error = with_password(Some("example-password"), || { + run( + CacheCommand::Purge(PurgeArgs { + service: service.to_owned(), + all: true, + page: None, + username: "admin".to_owned(), + }), + &mut out, + ) + }) + .expect_err("should refuse insecure remote transport"); + assert!( + error.to_string().contains("HTTPS"), + "should explain the transport requirement: {error}" + ); + assert!(out.is_empty(), "should not report a successful purge"); + } +} + +/// Set the admin password for one call and clear it afterwards. +fn with_password(password: Option<&str>, body: impl FnOnce() -> T) -> T { + // A panicking test poisons the lock; the data is `()`, so recovering it loses nothing + // and keeps one failure from cascading into every other test in the file. + let _guard = ENVIRONMENT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + // SAFETY: the guard above makes this the only thread touching the environment. + unsafe { + match password { + Some(value) => std::env::set_var("TRUSTED_SERVER_ADMIN_PASSWORD", value), + None => std::env::remove_var("TRUSTED_SERVER_ADMIN_PASSWORD"), + } + } + let result = body(); + // SAFETY: as above; still holding the guard. + unsafe { + std::env::remove_var("TRUSTED_SERVER_ADMIN_PASSWORD"); + } + result +} + +fn args(server: &FixtureServer, all: bool, page: Option<&str>) -> PurgeArgs { + PurgeArgs { + service: server.url(""), + all, + page: page.map(str::to_owned), + username: "admin".to_owned(), + } +} + +#[test] +fn a_successful_purge_prints_the_services_answer_and_exits_zero() { + let server = FixtureServer::start(|_request| { + FixtureResponse::html(r#"{"purged":true,"scope":"all"}"#) + .with_header("content-type", "application/json") + }); + + let mut out = Vec::new(); + let outcome = with_password(Some("admin-pass"), || { + run(CacheCommand::Purge(args(&server, true, None)), &mut out) + }); + + assert!(outcome.is_ok(), "a 200 from the service must exit zero"); + let rendered = String::from_utf8(out).expect("output should be UTF-8"); + assert!( + rendered.contains(r#""purged":true"#), + "the service's own answer is what an operator needs, got: {rendered}" + ); +} + +#[test] +fn the_request_carries_credentials_json_and_the_expected_body() { + // Pins the wire format against the endpoint's guards: it requires POST, rejects any + // Content-Type but application/json, and authenticates on ^/_ts/admin. + let server = FixtureServer::start(|request| { + FixtureResponse::html(format!( + r#"{{"purged":true,"scope":"all","method":"{}","auth":{},"type":"{}","path":"{}"}}"#, + request.method, + request.header("authorization").is_some(), + request.header("content-type").unwrap_or("none"), + request.path + )) + }); + + let mut out = Vec::new(); + with_password(Some("admin-pass"), || { + run(CacheCommand::Purge(args(&server, true, None)), &mut out).expect("should succeed") + }); + + let rendered = String::from_utf8(out).expect("output should be UTF-8"); + assert!(rendered.contains(r#""method":"POST""#), "got: {rendered}"); + assert!(rendered.contains(r#""auth":true"#), "got: {rendered}"); + assert!( + rendered.contains(r#""type":"application/json""#), + "got: {rendered}" + ); + assert!( + rendered.contains(r#""path":"/_ts/admin/cache/purge""#), + "got: {rendered}" + ); +} + +#[test] +fn a_missing_password_fails_before_the_service_is_touched() { + let server = FixtureServer::start(|_request| FixtureResponse::html("{}")); + + let mut out = Vec::new(); + let outcome = with_password(None, || { + run(CacheCommand::Purge(args(&server, true, None)), &mut out) + }); + + assert!(outcome.is_err()); + assert_eq!( + server.request_count(), + 0, + "a purge must not be attempted without a credential" + ); +} + +#[test] +fn a_rejected_purge_exits_non_zero_and_explains_the_status() { + for (status, expected_hint) in [ + (401u16, "admin username"), + (404, "predate the purge endpoint"), + (501, "cannot purge"), + ] { + let server = + FixtureServer::start(move |_request| FixtureResponse::html("{}").with_status(status)); + + let mut out = Vec::new(); + let outcome = with_password(Some("admin-pass"), || { + run(CacheCommand::Purge(args(&server, true, None)), &mut out) + }); + + let error = outcome.expect_err("a refused purge must not exit zero"); + let message = error.to_string(); + assert!( + message.contains(expected_hint), + "a {status} must say what to do next, got: {message}" + ); + } +} + +#[test] +fn a_page_purge_sends_the_url_the_operator_typed() { + let server = FixtureServer::start(|request| { + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should parse purge body"); + assert_eq!( + body, + serde_json::json!({"scope": "url", "url": "https://example.com/article"}), + "should send the requested reader URL" + ); + FixtureResponse::html( + serde_json::json!({"purged": true, "scope": "url", "surrogate_key": "example-key"}) + .to_string(), + ) + }); + + let mut out = Vec::new(); + with_password(Some("admin-pass"), || { + run( + CacheCommand::Purge(args(&server, false, Some("https://example.com/article"))), + &mut out, + ) + .expect("should succeed") + }); + + assert_eq!(server.request_count(), 1); +} + +#[test] +fn redirects_cannot_turn_a_login_page_into_a_successful_purge() { + let server = FixtureServer::start(|request| { + if request.path == "/_ts/admin/cache/purge" { + FixtureResponse::html("") + .with_status(302) + .with_header("location", "/login") + } else { + FixtureResponse::html("login") + } + }); + let mut out = Vec::new(); + + let outcome = with_password(Some("admin-pass"), || { + run(CacheCommand::Purge(args(&server, true, None)), &mut out) + }); + + assert!(outcome.is_err(), "should refuse a redirected purge"); + assert!(out.is_empty(), "should print no success response"); + assert_eq!(server.request_count(), 1, "should not follow the redirect"); +} + +#[test] +fn success_requires_a_valid_acknowledgment_for_the_requested_scope() { + for body in [ + "login".to_owned(), + serde_json::json!({"purged": false, "scope": "all"}).to_string(), + serde_json::json!({"purged": true, "scope": "url"}).to_string(), + serde_json::json!({"scope": "all"}).to_string(), + ] { + let server = FixtureServer::start(move |_request| FixtureResponse::html(body.clone())); + let mut out = Vec::new(); + + let outcome = with_password(Some("admin-pass"), || { + run(CacheCommand::Purge(args(&server, true, None)), &mut out) + }); + + assert!( + outcome.is_err(), + "should refuse a missing or mismatched purge acknowledgment" + ); + assert!(out.is_empty(), "should print no success response"); + } +} + +#[test] +fn a_url_purge_requires_a_surrogate_key_in_its_acknowledgment() { + for key in [ + serde_json::Value::Null, + serde_json::json!(""), + serde_json::json!(" "), + ] { + let server = FixtureServer::start(move |_request| { + FixtureResponse::html( + serde_json::json!({ + "purged": true, "scope": "url", "surrogate_key": key + }) + .to_string(), + ) + }); + let mut out = Vec::new(); + + let outcome = with_password(Some("admin-pass"), || { + run( + CacheCommand::Purge(args(&server, false, Some("https://example.com/article"))), + &mut out, + ) + }); + + assert!( + outcome.is_err(), + "should require the service to name the purged URL key" + ); + assert!( + out.is_empty(), + "should not print an incomplete acknowledgment as success" + ); + } +} diff --git a/crates/trusted-server-cli/tests/origin_probe.rs b/crates/trusted-server-cli/tests/origin_probe.rs new file mode 100644 index 000000000..7de11fa73 --- /dev/null +++ b/crates/trusted-server-cli/tests/origin_probe.rs @@ -0,0 +1,1446 @@ +//! Tests for `ts origin probe-shareability` against a local fixture origin. +//! +//! Run with: +//! `cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target ` +//! or `./scripts/test-cli.sh`. + +mod support_origin; + +use std::io::Write as _; + +use support_origin::{FixtureResponse, FixtureServer}; + +fn fetch(url: &str) -> String { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("should build a Tokio runtime for the fixture fetch"); + runtime.block_on(async { + reqwest::get(url) + .await + .expect("should reach the fixture origin") + .text() + .await + .expect("should read the fixture body") + }) +} + +#[test] +fn fixture_server_answers_repeated_requests() { + let server = FixtureServer::start(|_request| FixtureResponse::html("")); + + for attempt in 0..3 { + let body = fetch(&server.url("/")); + assert_eq!( + body, "", + "every request must be answered, not just the first (attempt {attempt})" + ); + } + + assert_eq!( + server.request_count(), + 3, + "the fixture should have counted every request it served" + ); +} + +#[test] +fn fixture_server_can_vary_its_answer_per_request() { + // The self-identity axis needs an origin that is *not* stable against itself, so the + // fixture has to be able to differ across requests on demand. + let server = FixtureServer::start(|request| { + FixtureResponse::html(format!("{}", request.request_index)) + }); + + assert_ne!( + fetch(&server.url("/")), + fetch(&server.url("/")), + "a fixture that cannot vary per request cannot exercise the self-identity axis" + ); +} + +#[test] +fn fixture_server_sees_request_headers_and_cookies() { + let server = FixtureServer::start(|request| { + let ec = if request.has_cookie("ts-ec") { + "with-ec" + } else { + "no-ec" + }; + let agent = request.header("user-agent").unwrap_or("none").to_owned(); + FixtureResponse::html(format!("{ec}|{agent}")) + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("should build a Tokio runtime"); + let body = runtime.block_on(async { + reqwest::Client::new() + .get(server.url("/")) + .header("cookie", "ts-ec=abc; other=1") + .header("user-agent", "FictionalBrowser/1.0") + .send() + .await + .expect("should reach the fixture origin") + .text() + .await + .expect("should read the fixture body") + }); + + assert_eq!( + body, "with-ec|FictionalBrowser/1.0", + "the cookie and user-agent axes both depend on the fixture seeing request headers" + ); +} + +// --------------------------------------------------------------------------- +// The probe itself, driven against the fixture origin. +// --------------------------------------------------------------------------- + +use trusted_server_cli::commands::origin::{ + OriginCommand, ProbeShareabilityArgs, report::ProbeReport, run, +}; + +/// A fixture that is shareable on every axis and verdict. +fn shareable(body: &'static str) -> impl Fn(&support_origin::FixtureRequest) -> FixtureResponse { + move |_request| FixtureResponse::html(body).with_header("cache-control", "public, max-age=300") +} + +fn probe(server: &FixtureServer, args: ProbeShareabilityArgs) -> (bool, ProbeReport) { + let _ = server; + let mut out = Vec::new(); + let outcome = run(OriginCommand::ProbeShareability(args), &mut out); + let rendered = String::from_utf8(out).expect("probe output should be UTF-8"); + let report: ProbeReport = + serde_json::from_str(rendered.trim()).expect("probe should emit parseable JSON"); + (outcome.is_ok(), report) +} + +fn json_args(server: &FixtureServer) -> ProbeShareabilityArgs { + ProbeShareabilityArgs { + url: vec![server.url("/article")], + repeat: 1, + cookie: Vec::new(), + vary_header: Vec::new(), + admission_cookie: None, + json: true, + } +} + +fn axis<'a>( + report: &'a ProbeReport, + name: &str, +) -> &'a trusted_server_cli::commands::origin::report::AxisResult { + let found = report.urls[0].axes.iter().find(|axis| axis.name == name); + assert!(found.is_some(), "report should contain the {name} axis"); + found.expect("should be present, asserted above") +} + +fn verdict<'a>( + report: &'a ProbeReport, + name: &str, +) -> &'a trusted_server_cli::commands::origin::report::VerdictResult { + let found = report.urls[0] + .verdicts + .iter() + .find(|verdict| verdict.name == name); + assert!(found.is_some(), "report should contain the {name} verdict"); + found.expect("should be present, asserted above") +} + +#[test] +fn a_shareable_origin_passes_every_axis_and_verdict() { + let server = FixtureServer::start(shareable("stable")); + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + ok, + "a stable, cookie-independent, freshness-declaring origin must pass: {}", + report.render_text() + ); + assert!(report.passed()); +} + +#[test] +fn an_unstable_origin_fails_self_identity_and_exits_non_zero() { + let server = FixtureServer::start(|request| { + // A per-request timestamp or CSRF nonce looks like this. + FixtureResponse::html(format!("{}", request.request_index)) + .with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "an unstable origin must exit non-zero"); + assert!(!axis(&report, "self-identity").passed()); +} + +#[test] +fn a_cookie_personalized_origin_fails_the_cookie_axis() { + let server = FixtureServer::start(|request| { + let body = if request.has_cookie("ts-ec") { + "signed in" + } else { + "anonymous" + }; + FixtureResponse::html(body).with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!(!axis(&report, "cookie").passed()); + assert!( + axis(&report, "self-identity").passed(), + "cookie personalization must not be misreported as instability" + ); +} + +#[test] +fn a_user_agent_varying_origin_fails_unless_it_declares_vary() { + let undeclared = FixtureServer::start(|request| { + let mobile = request + .header("user-agent") + .is_some_and(|agent| agent.contains("Phone")); + FixtureResponse::html(if mobile { + "m" + } else { + "d" + }) + .with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&undeclared, json_args(&undeclared)); + assert!(!ok); + assert!(!axis(&report, "user-agent").passed()); + assert!( + !verdict(&report, "vary-coverage").passed, + "an undeclared varying axis is exactly what gets cross-served" + ); + + let declared = FixtureServer::start(|request| { + let mobile = request + .header("user-agent") + .is_some_and(|agent| agent.contains("Phone")); + FixtureResponse::html(if mobile { + "m" + } else { + "d" + }) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "User-Agent") + }); + let (_, declared_report) = probe(&declared, json_args(&declared)); + assert!( + verdict(&declared_report, "vary-coverage").passed, + "a declared axis is keyed by the platform cache and is therefore safe" + ); +} + +#[test] +fn a_bot_varying_origin_fails_the_bot_axis() { + // Bots are excluded from the ad stack but not from shareability, so a crawler or + // challenge document an origin serves without Vary can be stored and then handed to a + // human navigation. + let server = FixtureServer::start(|request| { + let bot = request + .header("user-agent") + .is_some_and(|agent| agent.contains("Googlebot")); + FixtureResponse::html(if bot { + "crawler document" + } else { + "reader document" + }) + .with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "a crawler-specific document must not be cross-served"); + assert!(!axis(&report, "bot").passed()); + assert!( + !verdict(&report, "vary-coverage").passed, + "the undeclared signal is what gets cross-served" + ); +} + +#[test] +fn a_prefetch_varying_origin_fails_the_prefetch_axis() { + let server = FixtureServer::start(|request| { + let prefetch = request + .header("sec-purpose") + .is_some_and(|purpose| purpose.contains("prefetch")); + FixtureResponse::html(if prefetch { + "prefetch shell" + } else { + "reader document" + }) + .with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "a prefetch-specific document must not be cross-served"); + assert!(!axis(&report, "prefetch").passed()); +} + +#[test] +fn a_declared_vary_still_excuses_the_bot_axis() { + // The axis is named for the classification; the cache keys on the header it varied. + let server = FixtureServer::start(|request| { + let bot = request + .header("user-agent") + .is_some_and(|agent| agent.contains("Googlebot")); + FixtureResponse::html(if bot { + "crawler document" + } else { + "reader document" + }) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "User-Agent") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + ok, + "an origin that declares User-Agent is keyed on it: {}", + report.render_text() + ); +} + +#[test] +fn a_stable_body_with_a_varying_policy_header_is_not_shareable() { + // Fastly stores response headers with the body. A per-audience CSP is cross-served + // exactly as a per-audience document would be: the weaker policy removes a browser + // protection for readers the origin meant to protect. + let server = FixtureServer::start(|request| { + let mobile = request + .header("user-agent") + .is_some_and(|agent| agent.contains("iPhone")); + FixtureResponse::html("one document for everyone") + .with_header("cache-control", "public, max-age=300") + .with_header( + "content-security-policy", + if mobile { + "default-src *" + } else { + "default-src 'self'" + }, + ) + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + !ok, + "identical HTML is not identical cached representations" + ); + assert!( + !axis(&report, "user-agent").passed(), + "the axis compares what the cache stores, not just the body" + ); +} + +#[test] +fn a_plain_http_url_is_refused_before_any_cookie_is_sent() { + // The probe carries publisher session cookies. Sending them to a non-loopback origin + // over HTTP puts them on the wire in the clear. + let mut args = json_args(&FixtureServer::start(shareable("stable"))); + args.url = vec!["http://origin.example.com/article".to_owned()]; + + let mut out = Vec::new(); + let outcome = run(OriginCommand::ProbeShareability(args), &mut out); + + let error = outcome.expect_err("plain HTTP must be refused"); + assert!( + error.contains("HTTPS"), + "the error must name the requirement, got: {error}" + ); +} + +#[test] +fn a_url_carrying_credentials_is_refused() { + let mut args = json_args(&FixtureServer::start(shareable("stable"))); + args.url = vec!["https://reader:example-password@origin.example.com/article".to_owned()]; + + let mut out = Vec::new(); + let outcome = run(OriginCommand::ProbeShareability(args), &mut out); + + assert!( + outcome.is_err(), + "userinfo reaches proxy logs and shell history" + ); +} + +#[test] +fn an_rsc_varying_origin_fails_the_rsc_axis() { + // RSC fetches already flow through the readthrough cache while HTML navigations are + // passed, so this is the axis specific to removing the bypass. + let server = FixtureServer::start(|request| { + let body = if request.header("rsc").is_some() { + "flight payload" + } else { + "document" + }; + FixtureResponse::html(body).with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!(!axis(&report, "rsc").passed()); +} + +#[test] +fn gzip_and_identity_are_compared_after_decoding() { + let server = FixtureServer::start(|request| { + let body = "same document either way"; + let wants_gzip = request + .header("accept-encoding") + .is_some_and(|value| value.contains("gzip")); + if wants_gzip { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(body.as_bytes()) + .expect("should gzip the fixture body"); + let compressed = encoder.finish().expect("should finish gzipping"); + FixtureResponse::html("") + .with_header("content-encoding", "gzip") + .with_header("cache-control", "public, max-age=300") + .with_body(compressed) + } else { + FixtureResponse::html(body).with_header("cache-control", "public, max-age=300") + } + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + ok, + "compressed and identity arms carry the same document, so this must pass: {}", + report.render_text() + ); + assert!(axis(&report, "accept-encoding").passed()); +} + +#[test] +fn an_origin_without_freshness_fails_its_verdict() { + let server = FixtureServer::start(|_request| FixtureResponse::html("stable")); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!( + !verdict(&report, "freshness").passed, + "readthrough would store this on a platform default where the template cache declines it" + ); +} + +#[test] +fn a_private_origin_fails_freshness_even_with_a_max_age() { + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "private, max-age=300") + }); + let (ok, _) = probe(&server, json_args(&server)); + assert!( + !ok, + "an origin that marks HTML private must not be declared shareable" + ); +} + +#[test] +fn a_set_cookie_response_fails_its_verdict() { + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("set-cookie", "sid=abc123; Path=/") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!( + !verdict(&report, "set-cookie").passed, + "a cached Set-Cookie is replayed to every later cookieless reader" + ); +} + +#[test] +fn a_csp_nonce_response_fails_its_verdict() { + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("content-security-policy", "script-src 'nonce-r4nd0m'") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!(!verdict(&report, "csp-nonce").passed); +} + +#[test] +fn human_output_states_the_limits_and_the_verdict() { + let server = FixtureServer::start(shareable("stable")); + let mut args = json_args(&server); + args.json = false; + + let mut out = Vec::new(); + let outcome = run(OriginCommand::ProbeShareability(args), &mut out); + let rendered = String::from_utf8(out).expect("probe output should be UTF-8"); + + assert!(outcome.is_ok()); + assert!(rendered.contains("VERDICT: shareable")); + assert!( + rendered.contains("one client address"), + "IP-keyed personalization is invisible to this tool and the output must say so" + ); +} + +#[test] +fn a_malformed_cookie_argument_is_rejected_before_any_fetch() { + let server = FixtureServer::start(shareable("stable")); + let mut args = json_args(&server); + args.cookie = vec!["not-a-pair".to_owned()]; + + let mut out = Vec::new(); + let outcome = run(OriginCommand::ProbeShareability(args), &mut out); + + assert!(outcome.is_err()); + assert_eq!( + server.request_count(), + 0, + "argument validation should happen before the origin is touched" + ); +} + +#[test] +fn an_axis_override_replaces_the_default_header_rather_than_appending_to_it() { + // `RequestBuilder::header` appends. If an arm's override is layered on top of the + // default, the origin receives the header twice, and one that reads the first instance + // never sees the override — the axis then compares two identical responses and passes + // an origin it never actually varied. + let server = FixtureServer::start(|request| { + FixtureResponse::html(format!( + "ua={} ae={}", + request.header_count("user-agent"), + request.header_count("accept-encoding") + )) + .with_header("cache-control", "public, max-age=300") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + ok, + "every arm must send exactly one user-agent and one accept-encoding: {}", + report.render_text() + ); +} + +#[test] +fn a_later_private_directive_is_not_hidden_by_an_earlier_permissive_one() { + // A proxy in front of the origin can append its own Cache-Control rather than + // replacing the origin's. Judging only the first instance would store a private + // response in a shared cache. + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("cache-control", "private") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "a private response must not be declared shareable"); + assert!( + !verdict(&report, "freshness").passed, + "the freshness verdict must read every Cache-Control instance, not just the first" + ); +} + +#[test] +fn a_response_served_from_a_fronting_cache_is_not_declared_shareable() { + // Every axis compares two fetches. A cache in front of the origin can answer both from + // one object, so the axes agree and say nothing about the origin behind it. + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("age", "42") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "a cached answer is not evidence about the origin"); + assert!(!verdict(&report, "fronting-cache").passed); + assert!( + report.urls[0] + .axes + .iter() + .all(trusted_server_cli::commands::origin::report::AxisResult::passed), + "the axes agreeing is exactly the symptom, so the verdict must be what fails" + ); +} + +#[test] +fn a_vendor_cache_hit_header_is_caught_even_without_an_age() { + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("x-cache", "HIT") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!(!verdict(&report, "fronting-cache").passed); +} + +#[test] +fn an_age_of_zero_cannot_establish_origin_shareability() { + // Fresh cache hits can mask origin personalization within the first second. + let server = FixtureServer::start(|_request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("age", "0") + }); + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "should reject a fresh cached response"); + assert!(!verdict(&report, "fronting-cache").passed); +} + +#[test] +fn a_bot_wall_aborts_the_probe_instead_of_judging_the_challenge_page() { + // Measured against a real protected origin: the baseline arm was answered with a + // challenge page, and every verdict then described that page rather than the origin — + // a confident FAIL on an origin that sends `max-age=60` with no `Set-Cookie`. + let server = FixtureServer::start(|request| { + if request + .header("cookie") + .is_some_and(|c| c.contains("admit=1")) + { + FixtureResponse::html("real content") + .with_header("cache-control", "public, max-age=300") + } else { + FixtureResponse::html("are you a robot").with_status(403) + } + }); + + let mut out = Vec::new(); + let outcome = run( + OriginCommand::ProbeShareability(json_args(&server)), + &mut out, + ); + + let error = outcome.expect_err("a challenge page must not be judged"); + let message = error.to_string(); + assert!( + message.contains("403") && message.contains("TRUSTED_SERVER_PROBE_ADMISSION_COOKIE"), + "the error must name the status and how to get past it, got: {message}" + ); +} + +#[test] +fn an_admission_cookie_lets_the_probe_reach_real_content() { + let server = FixtureServer::start(|request| { + if request + .header("cookie") + .is_some_and(|c| c.contains("admit=1")) + { + FixtureResponse::html("real content") + .with_header("cache-control", "public, max-age=300") + } else { + FixtureResponse::html("are you a robot").with_status(403) + } + }); + + let mut args = json_args(&server); + args.admission_cookie = Some("admit=1".to_owned()); + let (ok, report) = probe(&server, args); + + assert!( + !ok, + "should keep admission-only diagnostics from passing the gate" + ); + assert!( + verdict(&report, "status").passed, + "should reach real content" + ); + assert!(!verdict(&report, "cookieless-coverage").passed); + assert!(report.render_text().contains("cookieless")); +} + +#[test] +fn decoded_encoding_differences_fail_even_when_vary_declares_encoding() { + let server = FixtureServer::start(|request| { + let mut response = FixtureResponse::html("identity document") + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "Accept-Encoding"); + if request.header("accept-encoding") == Some("gzip") { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(b"different document") + .expect("should gzip the variant"); + response = response + .with_header("content-encoding", "gzip") + .with_body(encoder.finish().expect("should finish gzip")); + } + response + }); + + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + !ok, + "should reject different decoded documents for the template cache" + ); + assert!( + !axis(&report, "accept-encoding").passed(), + "should keep encoding differences blocking" + ); +} + +#[test] +fn an_undeclared_varying_axis_still_fails() { + // The safety half: the same variance without the declaration is what gets cross-served. + let server = FixtureServer::start(|request| { + let gzip = request + .header("accept-encoding") + .is_some_and(|value| value.contains("gzip")); + FixtureResponse::html(if gzip { + "compressed variant" + } else { + "identity variant" + }) + .with_header("cache-control", "public, max-age=300") + }); + + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok); + assert!(!axis(&report, "accept-encoding").passed()); + assert!(!verdict(&report, "vary-coverage").passed); +} + +#[test] +fn vary_cookie_does_not_excuse_the_cookie_axis() { + // `Vary: Cookie` is keyed by a conforming cache, so it is not unsafe — but this axis + // answers "does the origin ignore cookies", and the origin is saying it does not. + // Passing would print a green verdict whose closing line says not to enable the flag, + // and would contradict the template cache, which refuses `Vary: Cookie` at runtime. + let server = FixtureServer::start(|request| { + let signed_in = request.has_cookie("ts-ec"); + FixtureResponse::html(if signed_in { + "signed in" + } else { + "anonymous" + }) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "Cookie") + }); + + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + !ok, + "a cookie-varying origin must not read as cookie-independent" + ); + assert!(!axis(&report, "cookie").passed()); +} + +#[test] +fn every_sample_is_checked_for_unsafe_headers() { + for (header, value, expected_verdict) in [ + ("x-cache", "HIT", "fronting-cache"), + ("age", "0", "fronting-cache"), + ("age", "invalid", "fronting-cache"), + ("set-cookie", "session=example-session", "set-cookie"), + ("cache-control", "private", "freshness"), + ( + "content-security-policy", + "script-src 'nonce-example'", + "csp-nonce", + ), + ] { + for on_repeat in [true, false] { + let server = FixtureServer::start(move |request| { + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300"); + let unsafe_sample = if on_repeat { + request.request_index == 1 + } else { + request.header("accept-encoding") == Some("gzip") + }; + if unsafe_sample { + response.with_header(header, value) + } else { + response + } + }); + + let (ok, report) = probe(&server, json_args(&server)); + + assert!( + !ok, + "should reject {header} on a later sample (repeat={on_repeat})" + ); + assert!( + !verdict(&report, expected_verdict).passed, + "should report the unsafe header" + ); + } + } +} + +#[test] +fn a_non_success_variant_cannot_pass_with_an_identical_body() { + let server = FixtureServer::start(|request| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_status(if request.header("rsc").is_some() { + 403 + } else { + 200 + }) + }); + let mut out = Vec::new(); + + let outcome = run( + OriginCommand::ProbeShareability(json_args(&server)), + &mut out, + ); + + assert!( + outcome.is_err(), + "should reject a non-200 variant even when its body matches" + ); +} + +#[test] +fn configured_headers_are_not_excused_by_vary_rsc() { + let server = FixtureServer::start(|request| { + FixtureResponse::html(if request.header("x-layout").is_some() { + "alternate" + } else { + "default" + }) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "rsc") + }); + let mut args = json_args(&server); + args.vary_header = vec!["x-layout".to_owned()]; + + let (ok, report) = probe(&server, args); + + assert!( + !ok, + "should reject an undeclared custom signal even with Vary: rsc" + ); + assert!( + !axis(&report, "x-layout").passed(), + "should identify the actual varying header" + ); +} + +#[test] +fn a_variant_must_declare_its_own_vary_coverage() { + let server = FixtureServer::start(|request| { + if request.header("rsc").is_some() { + FixtureResponse::html("flight") + .with_header("cache-control", "public, max-age=300") + } else { + FixtureResponse::html("document") + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "rsc") + } + }); + + let (ok, _) = probe(&server, json_args(&server)); + + assert!(!ok, "should require Vary coverage on both representations"); +} + +#[test] +fn the_mobile_axis_reaches_a_recognizable_mobile_browser_variant() { + let server = FixtureServer::start(|request| { + let mobile = request + .header("user-agent") + .is_some_and(|agent| agent.contains("iPhone") || agent.contains("Android")); + FixtureResponse::html(if mobile { + "mobile" + } else { + "desktop" + }) + .with_header("cache-control", "public, max-age=300") + }); + + let (ok, report) = probe(&server, json_args(&server)); + + assert!(!ok, "should discover undeclared mobile document variation"); + assert!( + !axis(&report, "user-agent").passed(), + "should test a recognizable mobile browser" + ); +} + +#[test] +fn declared_custom_signals_are_probed_independently_without_duplicate_axes() { + let server = FixtureServer::start(|request| { + FixtureResponse::html(format!( + "rsc={} layout={}", + request.header("rsc").unwrap_or("absent"), + request.header("x-layout").unwrap_or("absent") + )) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "rsc, x-layout") + }); + let mut args = json_args(&server); + args.vary_header = [ + "X-Layout", + "x-layout", + "RSC", + "Cookie", + "Accept-Encoding", + "User-Agent", + ] + .map(str::to_owned) + .to_vec(); + + let (ok, report) = probe(&server, args); + + assert!( + ok, + "should accept independently declared signals: {}", + report.render_text() + ); + assert!( + axis(&report, "rsc").differs(), + "should vary RSC independently" + ); + assert!( + axis(&report, "x-layout").differs(), + "should vary the configured signal independently" + ); + assert_eq!( + server.request_count(), + 11, + "should sample each signal independently and the configured header with RSC" + ); +} + +#[test] +fn unsafe_headers_are_still_checked_after_self_identity_first_differs() { + let server = FixtureServer::start(|request| { + let response = FixtureResponse::html(format!("{}", request.request_index)) + .with_header("cache-control", "public, max-age=300"); + if request.request_index == 2 { + response.with_header("set-cookie", "session=example-session") + } else { + response + } + }); + let mut args = json_args(&server); + args.repeat = 3; + + let (ok, report) = probe(&server, args); + + assert!(!ok, "should reject unstable responses"); + assert!( + !axis(&report, "self-identity").passed(), + "should retain the first body difference" + ); + assert!( + !verdict(&report, "set-cookie").passed, + "should inspect headers after the first mismatch" + ); + assert_eq!( + server.request_count(), + 11, + "should complete every requested sample" + ); +} + +#[test] +fn configured_headers_are_also_compared_with_rsc_held_constant() { + let server = FixtureServer::start(|request| { + let variant = request.header("rsc").is_some() && request.header("x-layout").is_some(); + FixtureResponse::html(if variant { + "alternate flight" + } else { + "default" + }) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "rsc") + }); + let mut args = json_args(&server); + args.vary_header = vec!["x-layout".to_owned()]; + + let (ok, report) = probe(&server, args); + + assert!( + !ok, + "should discover undeclared variation within an RSC representation" + ); + assert!( + !axis(&report, "x-layout").passed(), + "should attribute the difference to the configured header" + ); +} + +#[test] +fn admission_cookie_cannot_hide_first_visitor_session_issuance() { + let server = FixtureServer::start(|request| { + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300"); + if request.header("cookie").is_none() { + response.with_header("set-cookie", "session=example-session") + } else { + response + } + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!(!ok, "should reject first-visitor session issuance"); + assert!(!verdict(&report, "set-cookie").passed); + + let mut args = json_args(&server); + args.admission_cookie = Some("session=existing-reader".to_owned()); + let (ok, report) = probe(&server, args); + assert!(!ok, "should not certify unobserved cookieless responses"); + assert!(!verdict(&report, "cookieless-coverage").passed); + assert!( + !report.passed(), + "should fail the machine-readable gate too" + ); +} + +#[test] +fn navigation_negotiation_cannot_hide_session_cookies_behind_json() { + let server = FixtureServer::start(|request| { + let navigation = request + .header("accept") + .is_some_and(|value| value.contains("text/html")) + && request.header("sec-fetch-mode") == Some("navigate") + && request.header("sec-fetch-dest") == Some("document"); + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "Accept"); + if navigation { + response.with_header("set-cookie", "session=example; Path=/") + } else { + response + .without_header("content-type") + .with_header("content-type", "application/json") + .with_body(serde_json::json!({"stable": true}).to_string()) + } + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!(!ok, "should reject a navigation that sets session cookies"); + assert!( + !verdict(&report, "set-cookie").passed, + "should inspect HTML navigation cookies" + ); +} + +#[test] +fn non_html_navigation_responses_cannot_be_certified() { + for content_type in [ + None, + Some("application/json"), + Some("text/plain"), + Some("text/html-invalid"), + ] { + let server = FixtureServer::start(move |_| { + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .without_header("content-type"); + match content_type { + Some(value) => response.with_header("content-type", value), + None => response, + } + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!( + !ok, + "should reject unexpected navigation representation {content_type:?}" + ); + assert!( + !verdict(&report, "content-type").passed, + "should report representation failure" + ); + } +} + +#[test] +fn revalidation_directives_override_positive_freshness() { + for (name, value) in [ + ("cache-control", "public, max-age=300, no-cache"), + ("cache-control", "No-Cache=\"Set-Cookie\""), + ("surrogate-control", "max-age=300, no-cache"), + ("pragma", "no-cache"), + ] { + let server = FixtureServer::start(move |_| { + FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header(name, value) + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!(!ok, "should reject required revalidation from {name}"); + assert!( + !verdict(&report, "freshness").passed, + "should refuse freshness despite positive max-age" + ); + } +} + +#[test] +fn wildcard_vary_refuses_both_stable_and_varying_responses() { + for varying in [false, true] { + let server = FixtureServer::start(move |request| { + let body = if varying { + request.header("user-agent").unwrap_or("none") + } else { + "stable" + }; + FixtureResponse::html(format!("{body}")) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "*") + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!(!ok, "should refuse Vary wildcard even with stable bytes"); + assert!( + !verdict(&report, "vary-coverage").passed, + "should explain wildcard refusal" + ); + assert!( + !axis(&report, "user-agent").covered_by_vary, + "should not treat wildcard as axis coverage" + ); + } +} + +#[test] +fn undecodable_safety_headers_fail_closed_on_every_sample() { + for name in [ + "set-cookie", + "cache-control", + "surrogate-control", + "pragma", + "vary", + "content-security-policy", + "age", + "content-type", + "x-cache", + ] { + for sample in [0, 1, 3] { + let server = FixtureServer::start(move |request| { + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300"); + if request.request_index == sample { + response.with_raw_header(name, b"session=example; extension=caf\xe9") + } else { + response + } + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!( + !ok, + "should refuse undecodable {name} on sample {sample}: {}", + report.render_text() + ); + if name == "set-cookie" { + assert!( + !verdict(&report, "set-cookie").passed, + "should preserve cookie presence" + ); + } + } + } +} + +#[test] +fn rsc_uses_a_fetch_profile_and_accepts_flight_with_declared_rsc_variation() { + let server = FixtureServer::start(|request| { + let is_fetch = request.header("sec-fetch-mode") == Some("cors"); + for name in [ + "accept", + "sec-fetch-mode", + "sec-fetch-dest", + "sec-fetch-site", + ] { + assert_eq!(request.header_count(name), 1, "should send one {name}"); + } + if is_fetch { + assert_eq!( + request.header("accept"), + Some("*/*"), + "should request a browser fetch representation" + ); + assert_eq!( + request.header("sec-fetch-dest"), + Some("empty"), + "should use the fetch destination" + ); + assert_eq!( + request.header("sec-fetch-site"), + Some("same-origin"), + "should describe the same-origin fetch" + ); + assert!( + request.header("sec-fetch-user").is_none(), + "should not label fetches as user navigations" + ); + } else { + assert_eq!( + request.header("sec-fetch-mode"), + Some("navigate"), + "should use navigation metadata" + ); + assert_eq!( + request.header("sec-fetch-user"), + Some("?1"), + "should represent user navigation" + ); + } + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "rsc"); + if request.header("rsc").is_some() { + assert!( + is_fetch, + "should hold the fetch profile for every RSC variant" + ); + response + .without_header("content-type") + .with_header("content-type", "text/x-component") + .with_body("0:example-flight") + } else { + response + } + }); + let mut args = json_args(&server); + args.vary_header = vec!["x-layout".to_owned()]; + let (ok, report) = probe(&server, args); + assert!( + ok, + "should allow independently covered RSC variation: {}", + report.render_text() + ); +} + +#[test] +fn vary_rsc_cannot_excuse_an_accept_negotiated_difference() { + let server = FixtureServer::start(|request| { + let body = if request.header("accept") == Some("*/*") { + "fetch" + } else { + "navigation" + }; + FixtureResponse::html(format!("{body}")) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "rsc") + }); + let (ok, report) = probe(&server, json_args(&server)); + assert!(!ok, "should not attribute Accept variation to RSC"); + assert!( + !axis(&report, "fetch-profile").passed(), + "should retain the independently varied profile" + ); + assert!( + !axis(&report, "rsc").differs(), + "should hold the fetch profile constant while toggling RSC" + ); +} + +#[test] +fn plain_fetch_cannot_claim_the_flight_exemption() { + let server = FixtureServer::start(|request| { + let response = FixtureResponse::html("stable") + .with_header("cache-control", "public, max-age=300"); + if request.header("sec-fetch-mode") == Some("cors") { + response + .without_header("content-type") + .with_header("content-type", "text/x-component") + } else { + response + } + }); + let (_, report) = probe(&server, json_args(&server)); + assert!( + !verdict(&report, "content-type").passed, + "should refuse flight without an RSC request" + ); +} + +#[test] +fn legacy_prefetch_variation_is_exercised() { + let server = FixtureServer::start(|request| { + FixtureResponse::html(if request.header("purpose") == Some("prefetch") { + "prefetch" + } else { + "navigation" + }) + .with_header("cache-control", "public, max-age=300") + }); + let (_, report) = probe(&server, json_args(&server)); + assert!( + !axis(&report, "prefetch").passed(), + "should detect legacy prefetch variation" + ); +} + +#[test] +fn representative_header_values_detect_variants_hidden_by_the_default_sample() { + for rsc_only in [false, true] { + for declared in [false, true] { + let server = FixtureServer::start(move |request| { + let value = request.header("x-exp-variant").unwrap_or("default"); + let body = if !rsc_only || request.header("rsc").is_some() { + match value { + "A" => "variant A", + "B" => "variant B", + _ => "default", + } + } else { + "default" + }; + let response = + FixtureResponse::html(body).with_header("cache-control", "public, max-age=300"); + if declared { + response.with_header("vary", "x-exp-variant") + } else { + response + } + }); + let mut args = json_args(&server); + args.vary_header = vec![ + "x-exp-variant".to_owned(), + "X-Exp-Variant=A".to_owned(), + "x-exp-variant=B".to_owned(), + ]; + + let (ok, report) = probe(&server, args); + + assert_eq!( + ok, declared, + "should require Vary for real categorical values" + ); + assert!( + axis(&report, "x-exp-variant").differs(), + "should detect A/B despite absent/1 matching" + ); + assert_eq!( + verdict(&report, "vary-coverage").passed, + declared, + "should attribute variation to the custom header" + ); + assert_eq!( + server.request_count(), + 15, + "should sample every supplied value with and without RSC" + ); + } + } +} + +#[test] +fn representative_values_for_builtin_headers_are_not_skipped() { + let server = FixtureServer::start(|request| { + FixtureResponse::html( + if request.header("user-agent") == Some("ExampleBrowser/1.0") { + "alternate" + } else { + "default" + }, + ) + .with_header("cache-control", "public, max-age=300") + }); + let mut args = json_args(&server); + args.vary_header = vec!["user-agent=ExampleBrowser/1.0".to_owned()]; + + let (ok, report) = probe(&server, args); + + assert!(!ok, "should inspect explicit values even for built-in axes"); + assert!( + axis(&report, "user-agent").differs(), + "should merge explicit evidence into the existing axis" + ); +} + +#[test] +fn surrogate_freshness_takes_precedence_over_cache_control() { + for (surrogate, cache_control, expected) in [ + ("max-age=0", "public, max-age=300", false), + ("max-age=0", "public, s-maxage=300", false), + ("max-age=invalid", "public, max-age=300", false), + ("MAX-AGE = \"0\"", "public, max-age=300", false), + ("max-age=60", "public, max-age=0, s-maxage=0", true), + ( + "stale-while-revalidate=60", + "public, s-maxage=60, max-age=0", + true, + ), + ("s-maxage=60", "public, max-age=0", false), + ("max-age=60, max-age=0", "public, max-age=300", false), + ] { + let server = FixtureServer::start(move |_| { + FixtureResponse::html("stable") + .with_header("cache-control", cache_control) + .with_header("surrogate-control", surrogate) + }); + + let (ok, report) = probe(&server, json_args(&server)); + + assert_eq!( + ok, expected, + "should use authoritative freshness for {surrogate:?} over {cache_control:?}" + ); + assert_eq!( + verdict(&report, "freshness").passed, + expected, + "should report the edge freshness decision" + ); + } +} + +#[test] +fn every_representative_value_is_checked_after_a_difference_is_found() { + let server = FixtureServer::start(|request| { + let value = request.header("x-exp-variant").unwrap_or("default"); + let response = FixtureResponse::html(format!("{value}")) + .with_header("cache-control", "public, max-age=300") + .with_header("vary", "x-exp-variant"); + if value == "B" { + response.with_header("set-cookie", "session=example") + } else { + response + } + }); + let mut args = json_args(&server); + args.vary_header = vec!["x-exp-variant=A".to_owned(), "x-exp-variant=B".to_owned()]; + + let (ok, report) = probe(&server, args); + + assert!(!ok, "should reject unsafe headers on later representatives"); + assert!( + !verdict(&report, "set-cookie").passed, + "should inspect all values despite an earlier difference" + ); +} + +#[test] +fn invalid_representative_headers_are_rejected_before_fetching() { + for header in [ + "bad name=A", + "x-layout=bad\nvalue", + "bot=A", + "prefetch=A", + "fetch-profile=A", + "self-identity=A", + ] { + let server = FixtureServer::start(shareable("stable")); + let mut args = json_args(&server); + args.vary_header = vec![header.to_owned()]; + let mut out = Vec::new(); + + let result = run(OriginCommand::ProbeShareability(args), &mut out); + + assert!( + result.is_err(), + "should reject malformed or reserved header input" + ); + assert_eq!( + server.request_count(), + 0, + "should validate before any network requests" + ); + } +} diff --git a/crates/trusted-server-cli/tests/support_origin/mod.rs b/crates/trusted-server-cli/tests/support_origin/mod.rs new file mode 100644 index 000000000..96619cf10 --- /dev/null +++ b/crates/trusted-server-cli/tests/support_origin/mod.rs @@ -0,0 +1,291 @@ +//! A portable HTTP fixture origin for the shareability probe tests. +//! +//! Deliberately **not** built on the `support` module next door: that one is +//! `tokio` + `tokio-rustls` + the dev proxy, all of which are scoped to macOS in +//! `Cargo.toml`, and the probe has to be testable on Linux CI as well. This is plain +//! `std::net` and `std::thread`, so it builds anywhere the CLI does. +//! +//! **It loop-accepts.** A single-accept fixture caused a CI flake in this repo before +//! (fixed in PR #823): clients open more sockets than they send requests on. The probe +//! opens one connection per arm and per `--repeat`, so a one-shot server would hang the +//! second fetch rather than fail it. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::io::{BufRead as _, BufReader, Read as _, Write as _}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::thread::JoinHandle; + +/// One request as the fixture saw it. +pub struct FixtureRequest { + /// Request method, uppercased as sent. + pub method: String, + /// Request target, for example `/article`. + pub path: String, + /// Request body, empty when none was sent. + pub body: Vec, + /// Header names lowercased; every instance kept, in arrival order. + pub headers: HashMap>, + /// How many requests this server had already answered, starting at 0. + pub request_index: u64, +} + +impl FixtureRequest { + /// First value of a header, matched case-insensitively. + /// + /// First and not last on purpose: a duplicated request header is resolved differently + /// by different origins, and the ones that read the first instance are the ones a + /// probe arm can fail to reach. Modelling that here keeps the axis tests honest. + #[must_use] + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .get(&name.to_ascii_lowercase()) + .and_then(|values| values.first()) + .map(String::as_str) + } + + /// How many times a header was sent, matched case-insensitively. + #[must_use] + pub fn header_count(&self, name: &str) -> usize { + self.headers + .get(&name.to_ascii_lowercase()) + .map_or(0, Vec::len) + } + + /// Whether a cookie with this name was sent. + #[must_use] + pub fn has_cookie(&self, name: &str) -> bool { + self.header("cookie").is_some_and(|cookies| { + cookies + .split(';') + .filter_map(|pair| pair.split('=').next()) + .any(|candidate| candidate.trim() == name) + }) + } +} + +/// What the fixture should answer with. +pub struct FixtureResponse { + status: u16, + headers: Vec<(String, Vec)>, + body: Vec, +} + +impl FixtureResponse { + /// A `200` HTML response. + #[must_use] + pub fn html(body: impl Into>) -> Self { + Self { + status: 200, + headers: vec![("content-type".to_owned(), b"text/html".to_vec())], + body: body.into(), + } + } + + /// Replace the status. + #[must_use] + pub fn with_status(mut self, status: u16) -> Self { + self.status = status; + self + } + + /// Append a response header. Repeatable. + #[must_use] + pub fn with_header(mut self, name: &str, value: &str) -> Self { + self.headers + .push((name.to_owned(), value.as_bytes().to_vec())); + self + } + + /// Append a header containing bytes that cannot be represented as text. + #[must_use] + pub fn with_raw_header(mut self, name: &str, value: &[u8]) -> Self { + self.headers.push((name.to_owned(), value.to_vec())); + self + } + + /// Remove every instance of a response header. + #[must_use] + pub fn without_header(mut self, name: &str) -> Self { + self.headers + .retain(|(existing, _)| !existing.eq_ignore_ascii_case(name)); + self + } + + /// Replace the body without touching headers, for encoding arms. + #[must_use] + pub fn with_body(mut self, body: impl Into>) -> Self { + self.body = body.into(); + self + } +} + +/// A fixture origin bound to an ephemeral loopback port. +/// +/// Stops when dropped. +pub struct FixtureServer { + addr: SocketAddr, + shutdown: Arc, + requests: Arc, + worker: Option>, +} + +impl FixtureServer { + /// Start answering every connection from `handler`. + /// + /// # Panics + /// + /// Panics if the loopback listener cannot be bound, which in a test means the + /// environment is unusable rather than the code under test being wrong. + pub fn start(handler: H) -> Self + where + H: Fn(&FixtureRequest) -> FixtureResponse + Send + Sync + 'static, + { + let listener = + TcpListener::bind("127.0.0.1:0").expect("should bind a loopback fixture listener"); + let addr = listener + .local_addr() + .expect("should read the fixture listener address"); + // Non-blocking accept plus a short sleep, so the worker notices the shutdown flag + // instead of parking forever in accept() after the last request. + listener + .set_nonblocking(true) + .expect("should set the fixture listener non-blocking"); + + let shutdown = Arc::new(AtomicBool::new(false)); + let requests = Arc::new(AtomicU64::new(0)); + let worker = { + let shutdown = Arc::clone(&shutdown); + let requests = Arc::clone(&requests); + let handler = Arc::new(handler); + std::thread::spawn(move || { + while !shutdown.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + let index = requests.fetch_add(1, Ordering::SeqCst); + // One connection at a time is enough: the probe is sequential, + // and serving inline keeps request_index deterministic. + serve_one(stream, handler.as_ref(), index); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + Err(_) => break, + } + } + }) + }; + + Self { + addr, + shutdown, + requests, + worker: Some(worker), + } + } + + /// Absolute URL for a path on this fixture. + #[must_use] + pub fn url(&self, path: &str) -> String { + format!("http://{}{path}", self.addr) + } + + /// How many requests have been answered. + #[must_use] + pub fn request_count(&self) -> u64 { + self.requests.load(Ordering::SeqCst) + } +} + +impl Drop for FixtureServer { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::SeqCst); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn serve_one(mut stream: TcpStream, handler: &H, request_index: u64) +where + H: Fn(&FixtureRequest) -> FixtureResponse, +{ + stream + .set_nonblocking(false) + .expect("should set the accepted fixture stream blocking"); + let Some(request) = read_request(&stream, request_index) else { + return; + }; + let response = handler(&request); + + let mut out = Vec::new(); + let reason = if response.status == 200 { + "OK" + } else { + "Status" + }; + out.extend_from_slice(format!("HTTP/1.1 {} {reason}\r\n", response.status).as_bytes()); + for (name, value) in &response.headers { + out.extend_from_slice(format!("{name}: ").as_bytes()); + out.extend_from_slice(value); + out.extend_from_slice(b"\r\n"); + } + out.extend_from_slice(format!("content-length: {}\r\n", response.body.len()).as_bytes()); + // No keep-alive: one request per connection keeps the fixture trivial, and the probe + // opens a fresh connection per arm anyway. + out.extend_from_slice(b"connection: close\r\n\r\n"); + out.extend_from_slice(&response.body); + + let _ = stream.write_all(&out); + let _ = stream.flush(); +} + +fn read_request(stream: &TcpStream, request_index: u64) -> Option { + let mut reader = BufReader::new(stream); + let mut request_line = String::new(); + if reader.read_line(&mut request_line).ok()? == 0 { + return None; + } + let mut request_parts = request_line.split_whitespace(); + let method = request_parts.next()?.to_owned(); + let path = request_parts.next()?.to_owned(); + + let mut headers = HashMap::new(); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).ok()? == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + let name = name.trim().to_ascii_lowercase(); + let value = value.trim().to_owned(); + if name == "content-length" { + content_length = value.parse().unwrap_or(0); + } + headers.entry(name).or_insert_with(Vec::new).push(value); + } + } + + // Read any body, which also drains it so the client is not left writing into a closed + // socket. + let mut body = vec![0u8; content_length]; + if content_length > 0 && reader.read_exact(&mut body).is_err() { + body.clear(); + } + + Some(FixtureRequest { + method, + path, + body, + headers, + request_index, + }) +} diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index 9b639025a..c6e37d02e 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -121,6 +121,16 @@ pub struct AuctionObservationContext { pub consent_present: bool, /// Requested slot count for this candidate. pub slot_count: u16, + /// Whether this request's origin response *would be* eligible to share. + /// + /// Records a predicate, not a cache hit. Ad-serving requests still bypass the + /// platform cache unless `origin_readthrough_enabled` is set, so this measures + /// the gate's potential reach before enablement as well as its eligibility afterwards. + /// + /// `None` on sources that do not make the decision, which is not the same as + /// `Some(false)` — a dashboard that reads absence as "not shareable" will be wrong for + /// every `/auction` row. + pub origin_cache_shareable: Option, started_at: Instant, } @@ -187,10 +197,19 @@ impl AuctionObservationContext { gdpr_applies: consent.gdpr_applies, consent_present: !consent.is_empty(), slot_count, + origin_cache_shareable: None, started_at: Instant::now(), } } + /// Record whether this request's origin response would be eligible for readthrough. + /// + /// A predicate about the request, recorded whether or not `origin_readthrough_enabled` + /// is set — not an outcome of the gate. + pub fn set_origin_cache_shareable(&mut self, shareable: bool) { + self.origin_cache_shareable = Some(shareable); + } + /// Return elapsed milliseconds since the observation was created. #[must_use] pub fn elapsed_ms(&self) -> u64 { @@ -212,6 +231,7 @@ impl AuctionObservationContext { gdpr_applies: false, consent_present: false, slot_count, + origin_cache_shareable: None, started_at: Instant::now(), } } @@ -341,6 +361,11 @@ pub struct AuctionEventRow { pub ad_domain: Option, /// Creative/ad ID. pub ad_id: Option, + /// `0` or `1`; absent when this source does not make the readthrough decision. + /// + /// Absent is not the same as `0` — see + /// [`AuctionObservationContext::origin_cache_shareable`]. + pub origin_cache_shareable: Option, } impl AuctionEventRow { @@ -379,6 +404,7 @@ impl AuctionEventRow { is_win: None, ad_domain: None, ad_id: None, + origin_cache_shareable: observation.origin_cache_shareable.map(u8::from), } } } @@ -957,6 +983,69 @@ mod tests { use super::*; + /// A plain observation context, for tests that only care about the fields they set. + /// + /// Wraps the existing [`AuctionObservationContext::for_test`] so the cache-field tests and + /// the row tests build it identically. + fn test_observation() -> AuctionObservationContext { + AuctionObservationContext::for_test(AuctionSource::InitialNavigation, "/article", 1) + } + + #[test] + fn summary_row_carries_cache_outcomes_from_the_observation() { + let mut observation = test_observation(); + observation.set_origin_cache_shareable(false); + + let mut rows = Vec::new(); + push_summary( + &mut rows, + &observation, + "2026-09-15 00:00:00.000", + AuctionTerminalStatus::Completed, + None, + 12, + 1, + ); + + let row = rows.first().expect("should emit one summary row"); + assert_eq!(row.origin_cache_shareable, Some(0)); + } + + #[test] + fn rows_omit_cache_outcomes_when_the_observation_has_none() { + let observation = test_observation(); + let mut rows = Vec::new(); + push_summary( + &mut rows, + &observation, + "2026-09-15 00:00:00.000", + AuctionTerminalStatus::Completed, + None, + 12, + 1, + ); + + let row = rows.first().expect("should emit one summary row"); + assert_eq!( + row.origin_cache_shareable, None, + "an unmeasured source must be distinguishable from a measured miss" + ); + } + + #[test] + fn observation_cache_fields_default_to_absent_and_round_trip() { + let mut observation = test_observation(); + + assert_eq!( + observation.origin_cache_shareable, None, + "a freshly built observation should not claim a cache outcome" + ); + + observation.set_origin_cache_shareable(true); + + assert_eq!(observation.origin_cache_shareable, Some(true)); + } + fn test_request(id: &str) -> AuctionRequest { AuctionRequest { id: id.to_owned(), diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs index f5e45bbd3..9bbd40b69 100644 --- a/crates/trusted-server-core/src/auth.rs +++ b/crates/trusted-server-core/src/auth.rs @@ -124,6 +124,19 @@ pub fn enforce_basic_auth( } } +/// Username from a basic-auth request, for audit logging. +/// +/// Returns the username alone. The password is a shared static secret and must never reach +/// a log line. +/// +/// This parses a header; it verifies nothing. Call it only on a request +/// [`enforce_basic_auth`] has already accepted, where the username identifies which +/// operator credential was used. +#[must_use] +pub fn authenticated_username(req: &Request) -> Option { + extract_credentials(req).map(|(username, _password, _digest)| username) +} + fn extract_credentials(req: &Request) -> Option<(String, String, [u8; 32])> { let mut header_values = req.headers().get_all(header::AUTHORIZATION).iter(); let header_value = header_values.next()?; diff --git a/crates/trusted-server-core/src/cache_purge.rs b/crates/trusted-server-core/src/cache_purge.rs new file mode 100644 index 000000000..f77f43e4d --- /dev/null +++ b/crates/trusted-server-core/src/cache_purge.rs @@ -0,0 +1,373 @@ +//! The operator-facing purge endpoint for templates and tagged origin responses. +//! +//! `POST /_ts/admin/cache/purge` with `{"scope":"all"}` or +//! `{"scope":"url","url":"https://example.com/page"}`. +//! +//! # Why every method is registered, not just `POST` +//! +//! A named route only claims the methods it lists. A method it does not claim falls through +//! to the publisher, and `enforce_basic_auth` leaves the `Authorization` header in place, so +//! a `GET` to this path would authenticate and then ship the shared admin credential to the +//! publisher origin. The route therefore claims every method and this handler answers the +//! non-`POST` ones with 405 itself. +//! +//! # Why `Content-Type` is enforced exactly +//! +//! Browsers attach basic-auth credentials automatically. A cross-origin form POST with +//! `enctype="text/plain"` is not preflighted, so requiring `POST` alone does not stop CSRF; +//! requiring a `Content-Type` that a form cannot produce does. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{Method, Request, Response, StatusCode, header}; +use serde::{Deserialize, Serialize}; + +use crate::error::TrustedServerError; +use crate::http_util::enforce_max_body_size; +use crate::platform::{RuntimeServices, reader_url_surrogate_key}; + +/// Purge bodies name a scope and, for a URL purge, one URL. Nothing here is unbounded. +const PURGE_MAX_BODY_BYTES: usize = 4096; + +/// The request body, before its scope is validated. +/// +/// Deserialized as a flat struct rather than an internally-tagged enum on purpose. +/// `deny_unknown_fields` does not reach the unit variant of such an enum, so +/// `{"scope":"all","url":"…"}` would parse as a full flush — an operator who typed the +/// wrong scope while intending to purge one page would empty the whole cache and be told +/// it succeeded. Validating the pair by hand makes that combination an error. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PurgeBody { + scope: String, + url: Option, +} + +/// What an operator asked to purge. +#[derive(Debug, PartialEq, Eq)] +enum PurgeRequest { + /// Every template and tagged origin response this service has cached. + All, + /// One reader-facing URL, as a reader would type it; canonicalized before hashing. + Url(String), +} + +impl PurgeRequest { + /// Parse and validate a purge body. + /// + /// # Errors + /// + /// Returns a message naming the problem when the JSON is malformed, the scope is not + /// one of the two documented values, or the scope and `url` field disagree. + fn parse(bytes: &[u8]) -> Result { + let body: PurgeBody = + serde_json::from_slice(bytes).map_err(|error| format!("invalid JSON: {error}"))?; + + match (body.scope.as_str(), body.url) { + ("all", None) => Ok(Self::All), + ("all", Some(_)) => Err( + "scope \"all\" takes no url; did you mean {\"scope\":\"url\",\"url\":…}?" + .to_owned(), + ), + // Parsed, not merely non-empty. `reader_url_surrogate_key` hashes whatever it + // is given, so a path, a scheme-less host, or an `ftp://` URL would be + // acknowledged as purged under a key nothing was ever stored with — the exact + // silent no-op that key's own documentation calls the failure that matters. + // A CMS webhook sending paths would purge nothing and never find out. + ("url", Some(url)) + if url::Url::parse(&url).is_ok_and(|parsed| { + matches!(parsed.scheme(), "http" | "https") && parsed.host_str().is_some() + }) => + { + Ok(Self::Url(url)) + } + ("url", _) => Err( + "scope \"url\" requires an absolute http(s) url, as a reader addresses the page" + .to_owned(), + ), + (other, _) => Err(format!( + "unknown scope {other:?}; expected \"all\" or \"url\"" + )), + } + } +} + +/// What the purge did, in terms an operator mid-incident can act on. +#[derive(Debug, Serialize)] +struct PurgeResponse { + purged: bool, + scope: &'static str, + /// Present on a URL purge, so an operator can confirm which key was hit. + #[serde(skip_serializing_if = "Option::is_none")] + surrogate_key: Option, + message: &'static str, +} + +/// Handle a purge request. +/// +/// # Errors +/// +/// Returns an error when the body cannot be read, exceeds [`PURGE_MAX_BODY_BYTES`], is not +/// valid JSON for a known scope, or when the platform's purge fails. +pub async fn handle_cache_purge( + services: &RuntimeServices, + req: Request, + authenticated_principal: Option<&str>, +) -> Result, Report> { + if req.method() != Method::POST { + return Ok(method_not_allowed()); + } + + // Checked before the body is read: a request that cannot be a legitimate API call + // should not have its payload parsed at all. + let content_type = req + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if !is_exactly_json(content_type) { + return Ok(json_response( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + r#"{"purged":false,"error":"Content-Type must be application/json"}"#, + )); + } + + let body = req.into_body(); + if body.is_stream() { + return Err(Report::new(TrustedServerError::BadRequest { + message: "cache-purge request body must be buffered, not streamed".into(), + })); + } + let bytes = body.into_bytes().unwrap_or_default(); + enforce_max_body_size(&bytes, PURGE_MAX_BODY_BYTES, "cache-purge")?; + + let request = + PurgeRequest::parse(&bytes).map_err(|message| TrustedServerError::BadRequest { + message: format!("invalid cache-purge request: {message}"), + })?; + + match request { + PurgeRequest::All => { + // An unbounded flush behind one shared static credential is also an + // origin-stampede lever, and there is no rate-limit primitive on this path, so + // who used it is recorded every time. + log::warn!( + "Cache purge: ALL templates purged by {}", + authenticated_principal.unwrap_or("") + ); + services + .template_cache() + .purge_all() + .await + .map_err(|error| TrustedServerError::Configuration { + message: format!("cache purge failed: {error}"), + })?; + Ok(purge_ok(&PurgeResponse { + purged: true, + scope: "all", + surrogate_key: None, + // One surrogate-key call, so there is no partial state to reason about: + // an error means nothing was purged and the call is safe to retry. + message: "All templates purged. Purge is idempotent and safe to repeat.", + })) + } + PurgeRequest::Url(url) => { + // The reader-facing key, so callers pass the URL a reader would see and never + // have to replay this service's origin rewriting. + let surrogate_key = reader_url_surrogate_key(&url); + log::info!( + "Cache purge: url {url} (key {surrogate_key}) by {}", + authenticated_principal.unwrap_or("") + ); + services + .template_cache() + .purge_url_surrogate_key(&surrogate_key) + .await + .map_err(|error| TrustedServerError::Configuration { + message: format!("cache purge failed: {error}"), + })?; + Ok(purge_ok(&PurgeResponse { + purged: true, + scope: "url", + surrogate_key: Some(surrogate_key), + message: "URL purged. Purge is idempotent and safe to repeat.", + })) + } + } +} + +/// Exactly `application/json`, with optional parameters and whitespace. +/// +/// Deliberately strict: accepting `application/json`-ish media types would readmit the +/// form-post CSRF shape this check exists to close. +fn is_exactly_json(content_type: &str) -> bool { + content_type + .split(';') + .next() + .unwrap_or_default() + .trim() + .eq_ignore_ascii_case("application/json") +} + +fn method_not_allowed() -> Response { + let mut response = json_response( + StatusCode::METHOD_NOT_ALLOWED, + r#"{"purged":false,"error":"cache purge requires POST"}"#, + ); + response + .headers_mut() + .insert(header::ALLOW, header::HeaderValue::from_static("POST")); + response +} + +fn purge_ok(payload: &PurgeResponse) -> Response { + let body = serde_json::to_string(payload) + .unwrap_or_else(|_| r#"{"purged":true,"scope":"unknown"}"#.to_owned()); + json_response(StatusCode::OK, &body) +} + +/// Purge answers are per-operator and per-moment; nothing may store one. +fn json_response(status: StatusCode, body: &str) -> Response { + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "application/json") + .header(header::CACHE_CONTROL, "private, no-store") + .body(EdgeBody::from(body.to_owned())) + .expect("should build a cache-purge response") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_type_must_be_exactly_json() { + assert!(is_exactly_json("application/json")); + assert!(is_exactly_json("application/json; charset=utf-8")); + assert!(is_exactly_json(" APPLICATION/JSON ")); + } + + #[test] + fn form_postable_content_types_are_refused() { + // These three are the only types a cross-origin form can send, and a form POST + // carries basic-auth credentials without a preflight. + for content_type in [ + "text/plain", + "application/x-www-form-urlencoded", + "multipart/form-data", + "", + ] { + assert!( + !is_exactly_json(content_type), + "{content_type} must not be accepted" + ); + } + } + + #[test] + fn json_lookalike_types_are_refused() { + for content_type in [ + "application/jsonrequest", + "text/json", + "application/ld+json", + ] { + assert!( + !is_exactly_json(content_type), + "{content_type} must not be accepted" + ); + } + } + + #[test] + fn a_non_post_answer_names_the_method_it_accepts() { + let response = method_not_allowed(); + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); + assert_eq!( + response + .headers() + .get(header::ALLOW) + .and_then(|value| value.to_str().ok()), + Some("POST") + ); + } + + #[test] + fn every_answer_forbids_storage() { + for response in [method_not_allowed(), json_response(StatusCode::OK, "{}")] { + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "a purge answer must never be stored" + ); + } + } + + #[test] + fn scope_parsing_accepts_both_documented_shapes() { + assert_eq!( + PurgeRequest::parse(br#"{"scope":"all"}"#).expect("should parse"), + PurgeRequest::All + ); + assert_eq!( + PurgeRequest::parse(br#"{"scope":"url","url":"https://example.com/a"}"#) + .expect("should parse"), + PurgeRequest::Url("https://example.com/a".to_owned()) + ); + } + + #[test] + fn a_url_alongside_scope_all_is_refused_rather_than_flushing_everything() { + // The dangerous typo: an operator means to purge one page, mistypes the scope, and + // would otherwise be told a full flush succeeded. + let error = PurgeRequest::parse(br#"{"scope":"all","url":"https://example.com/a"}"#) + .expect_err("should refuse"); + assert!( + error.contains("takes no url"), + "the error must point at the confusion, got: {error}" + ); + } + + #[test] + fn an_unknown_or_malformed_scope_is_refused() { + // A typo must not silently widen into a full flush. + for body in [ + &br#"{"scope":"everything"}"#[..], + br#"{"scope":"url"}"#, + br#"{"scope":"url","url":" "}"#, + // Hashed as given, these would each be acknowledged as purged under a key + // that can never match a stored object. + br#"{"scope":"url","url":"/article"}"#, + br#"{"scope":"url","url":"example.com/article"}"#, + br#"{"scope":"url","url":"ftp://example.com/article"}"#, + br#"{"scope":"all","url":"https://example.com/a"}"#, + br#"{}"#, + br#"{"scope":"ALL"}"#, + br#"{"scope":"all","extra":1}"#, + br#"not json"#, + ] { + assert!( + PurgeRequest::parse(body).is_err(), + "{} must not parse", + String::from_utf8_lossy(body) + ); + } + } + + #[test] + fn a_reader_facing_url_still_parses() { + for body in [ + &br#"{"scope":"url","url":"https://ts.example.com/article"}"#[..], + br#"{"scope":"url","url":"http://ts.example.com/article?a=1"}"#, + br#"{"scope":"url","url":"https://ts.example.com"}"#, + ] { + assert!( + PurgeRequest::parse(body).is_ok(), + "{} is a URL a reader can address and must still purge", + String::from_utf8_lossy(body) + ); + } + } +} diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index f44ba1d4a..8a20c7296 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -195,7 +195,9 @@ pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> /// `` and the root document is therefore uncacheable. `Esi` stores a /// request-neutral shared template and fills its per-request byte seam at the edge. /// -/// Spike-only, for the #1009 ESI validation. Remove with the spike. +/// Defaults to `Inline`. `Esi` is opt-in per deployment and is verified by the +/// `template-cache-local-test.sh` harness plus the rendered-document byte-identity +/// tests; it is not a trial mode, but it is also not the default. /// /// # Why the template must be request-neutral /// @@ -303,9 +305,10 @@ pub struct CreativeOpportunitiesConfig { /// `Option` rather than a bare enum, and `skip_serializing_if`, deliberately: /// these structs use `deny_unknown_fields`, so a pushed key makes an older /// binary fail configuration load. Keeping it absent when unset means a - /// deployment that never sets it stays rollback-compatible. + /// deployment that never sets it stays rollback-compatible. That reasoning applies + /// to every optional field in this struct. /// - /// Spike-only. See [`AssemblyMode`]. + /// See [`AssemblyMode`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub assembly_mode: Option, /// Request headers the origin varies on, which the shared-template cache key must @@ -322,7 +325,7 @@ pub struct CreativeOpportunitiesConfig { /// deployment that has not stated what its origin varies on from gaining a shared /// cache by omission. /// - /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + /// Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. #[serde(default, skip_serializing_if = "Option::is_none")] pub template_cache_vary: Option>, /// Maximum time a reader-neutral transformed template may remain in the shared template cache. @@ -332,7 +335,7 @@ pub struct CreativeOpportunitiesConfig { /// the origin's remaining edge freshness and this value. Defaults to 60 seconds /// and may be configured from 1 second through 1 day. /// - /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + /// Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. #[serde(default, skip_serializing_if = "Option::is_none")] pub template_cache_max_age_seconds: Option, /// Named bounded cookie variants whose raw values enter the shared-template key. @@ -355,11 +358,43 @@ pub struct CreativeOpportunitiesConfig { /// its own identity cookie, so most repeat visitors bypass unless the operator can /// safely assert independence. No identity or consent cookie is implicitly exempt. /// - /// Named bypass cookies always disqualify, and origin `Vary: Cookie` always refuses - /// storage regardless of this assertion or the configured key. The origin must - /// still authorize positive shared freshness and pass every other response guard. + /// Named bypass cookies always disqualify. On the template cache path, an origin's + /// `Vary: Cookie` still refuses storage regardless of this assertion or the key. + /// Readthrough performs no response-side check: when + /// [`Self::origin_readthrough_enabled`] is also `true`, otherwise eligible requests + /// with unlisted cookies rely on this assertion alone. Configured key-cookie + /// dimensions disable readthrough because its key does not include their values. + /// Verify the cookie axis with `ts origin probe-shareability` before enabling both + /// flags; see the configuration guide's template-cache section. #[serde(default, skip_serializing_if = "Option::is_none")] pub origin_is_cookie_independent: Option, + /// Whether this origin's responses may be held in the platform's shared readthrough + /// cache. + /// + /// Unset or `false` preserves the existing policy: ad-serving requests bypass the + /// cache, while other publisher requests retain the platform default. Setting `true` + /// uses request shareability instead: eligible ad-serving requests may use the cache, + /// and ineligible non-ad requests bypass it. + /// + /// **This flag is the whole opt-in.** Unlike + /// [`Self::origin_is_cookie_independent`], which only ever applies to cookie-bearing + /// requests, readthrough admits *cookieless* requests — first-time visitors, which is + /// exactly when an origin issues a session cookie. There is no response-side guard on + /// this path: the decision is made before the origin replies, and no post-response + /// hook is reachable on the Fastly adapter. Safety therefore rests on the origin's own + /// `Cache-Control` plus an operator's verification, so enabling it must be a + /// deliberate act rather than a consequence of deploying. + /// + /// Configured [`Self::template_cache_key_cookies`] disable readthrough because + /// its key does not include their values; template variants remain eligible. + /// Verify with `ts origin probe-shareability` before setting this. + /// + /// Setting this back to `false` restores the existing ad-stack bypass policy, not + /// a global cache bypass. Purge already-stored tagged objects with `ts cache purge` + /// (`--all` or the exact reader-facing `--page` URL). Objects stored by versions + /// without readthrough tags must still expire on the origin's TTL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_readthrough_enabled: Option, /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, @@ -381,6 +416,15 @@ impl CreativeOpportunitiesConfig { self.origin_is_cookie_independent.unwrap_or(false) } + /// Whether the origin's responses may enter the platform's shared readthrough cache. + /// + /// Defaults to `false`: the conservative reading, and the one that preserves today's + /// shipped behavior for every deployment that does not ask for the change. + #[must_use] + pub fn origin_readthrough_enabled(&self) -> bool { + self.origin_readthrough_enabled.unwrap_or(false) + } + /// Exact cookie names included as bounded shared-template key dimensions. #[must_use] pub fn template_cache_key_cookies(&self) -> &[String] { @@ -1720,6 +1764,7 @@ mod tests { template_cache_key_cookies: None, template_cache_bypass_cookies: None, origin_is_cookie_independent: None, + origin_readthrough_enabled: None, section_segment: None, slot: vec![slot], } @@ -2124,6 +2169,7 @@ mod tests { template_cache_key_cookies: None, template_cache_bypass_cookies: None, origin_is_cookie_independent: None, + origin_readthrough_enabled: None, section_segment: None, slot: Vec::new(), }; @@ -2694,7 +2740,7 @@ mod tests { assert_eq!( config.template_cache_max_age(), std::time::Duration::from_secs(60), - "an absent ceiling must preserve the spike's existing lifetime" + "an absent ceiling must preserve the existing lifetime" ); let serialized = toml::to_string(&config).expect("should serialize"); diff --git a/crates/trusted-server-core/src/integrations/didomi.rs b/crates/trusted-server-core/src/integrations/didomi.rs index d1fcb7fb5..e1e7c87e8 100644 --- a/crates/trusted-server-core/src/integrations/didomi.rs +++ b/crates/trusted-server-core/src/integrations/didomi.rs @@ -554,6 +554,7 @@ mod tests { use super::*; use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; + use crate::platform::PlatformCacheIntent; use crate::platform::test_support::{ NoopConfigStore, NoopSecretStore, StubBackend, StubHttpClient, build_services_with_http_client, @@ -1057,8 +1058,8 @@ mod tests { .expect("should proxy API request"); assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![true], + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], "should bypass the platform cache for API requests" ); assert_eq!( @@ -1116,8 +1117,8 @@ mod tests { .expect("should proxy SDK request"); assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![false], + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], "should retain normal platform caching for canonical SDK loaders" ); for (name, expected) in [ diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 000c54c35..249f741ab 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -36,6 +36,7 @@ pub mod auction; pub mod auction_config_types; pub mod auth; pub mod cache_policy; +pub mod cache_purge; pub mod config; pub mod config_payload; pub mod consent; diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index bd09cd215..b47abaebb 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -7,6 +7,47 @@ use error_stack::Report; use super::PlatformError; use super::image_optimizer::PlatformImageOptimizerOptions; +/// What the caller wants the platform's intermediary cache to do with this request. +/// +/// One enum rather than independent flags because on Fastly they are *not* independent: +/// `set_surrogate_key` and `set_ttl` each "override any previous `Request::set_pass` call" +/// (`fastly-0.12.1/src/http/request.rs:2462`, `:2381`). A pair of booleans could express +/// "bypass the cache, and tag it for purge", which on Fastly silently means "do not +/// bypass" — the opposite of how it reads. This type makes that combination unspellable. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum PlatformCacheIntent { + /// Let the platform apply its default behavior, honoring origin freshness. + #[default] + Default, + /// Do not use the intermediary cache for this request. + Bypass, + /// Allow caching, tagged with a surrogate key so it can be purged. + Shared { + /// Key attached to the stored object. + surrogate_key: String, + }, +} + +impl PlatformCacheIntent { + /// Whether this intent asks the platform to skip its cache entirely. + #[must_use] + pub fn is_bypass(&self) -> bool { + matches!(self, Self::Bypass) + } + + /// The surrogate key to tag the stored object with, when there is one. + /// + /// `None` for both [`Self::Default`] and [`Self::Bypass`]: attaching a key to a + /// bypassed request would reverse the bypass on Fastly. + #[must_use] + pub fn surrogate_key(&self) -> Option<&str> { + match self { + Self::Shared { surrogate_key } => Some(surrogate_key), + Self::Default | Self::Bypass => None, + } + } +} + /// Outbound HTTP request paired with a pre-resolved backend name. /// /// Uses `EdgeZero`'s neutral [`EdgeRequest`] type so adapters share one @@ -23,12 +64,11 @@ pub struct PlatformHttpRequest { /// Adapters that cannot attach this metadata to their send path should /// return an error rather than silently dropping transformations. pub image_optimizer: Option, - /// Whether the platform's intermediary response cache must be bypassed. + /// What the platform's intermediary response cache should do with this request. /// - /// Adapters without an intermediary outbound cache may treat this as already - /// satisfied. The option defaults to `false` so existing call sites preserve - /// their current cache behavior. - pub bypass_cache: bool, + /// Adapters without an intermediary outbound cache may ignore it. Defaults to + /// [`PlatformCacheIntent::Default`] so existing call sites preserve their behavior. + pub cache_intent: PlatformCacheIntent, /// Whether the response body should stay streaming in the platform response. /// /// Adapters that cannot preserve streaming response bodies should return an @@ -44,7 +84,7 @@ impl PlatformHttpRequest { request, backend_name: backend_name.into(), image_optimizer: None, - bypass_cache: false, + cache_intent: PlatformCacheIntent::Default, stream_response: false, } } @@ -63,7 +103,19 @@ impl PlatformHttpRequest { /// Bypass the platform's intermediary response cache for this request. #[must_use] pub fn with_cache_bypass(mut self) -> Self { - self.bypass_cache = true; + self.cache_intent = PlatformCacheIntent::Bypass; + self + } + + /// Allow the platform to cache this response, tagged for purge. + /// + /// Replaces any prior bypass rather than combining with it, because the two cannot + /// both hold: see [`PlatformCacheIntent`]. + #[must_use] + pub fn with_shared_cache(mut self, surrogate_key: impl Into) -> Self { + self.cache_intent = PlatformCacheIntent::Shared { + surrogate_key: surrogate_key.into(), + }; self } @@ -396,8 +448,9 @@ mod tests { "stub-backend", ); - assert!( - !request.bypass_cache, + assert_eq!( + request.cache_intent, + PlatformCacheIntent::Default, "should preserve existing cache behavior by default" ); } @@ -412,12 +465,61 @@ mod tests { ) .with_cache_bypass(); - assert!( - request.bypass_cache, + assert_eq!( + request.cache_intent, + PlatformCacheIntent::Bypass, "should enable intermediary cache bypass" ); } + #[test] + fn cache_intent_cannot_request_bypass_and_a_surrogate_key_at_once() { + // The whole reason this is an enum. On Fastly a surrogate key reverses a prior + // set_pass, so "bypass, and tag for purge" would silently mean "do not bypass". + let bypass = PlatformCacheIntent::Bypass; + let shared = PlatformCacheIntent::Shared { + surrogate_key: "ts-origin".to_owned(), + }; + + assert!(bypass.is_bypass()); + assert!(bypass.surrogate_key().is_none()); + assert!(!shared.is_bypass()); + assert_eq!(shared.surrogate_key(), Some("ts-origin")); + assert!(PlatformCacheIntent::Default.surrogate_key().is_none()); + assert!(!PlatformCacheIntent::Default.is_bypass()); + } + + #[test] + fn the_last_cache_builder_call_wins_rather_than_combining() { + // Builders replace rather than accumulate: a request cannot end up asking for + // both, whichever order a caller writes them in. + let base = || { + PlatformHttpRequest::new( + request_builder() + .body(Body::empty()) + .expect("should build request"), + "stub-backend", + ) + }; + + assert_eq!( + base() + .with_cache_bypass() + .with_shared_cache("ts-origin") + .cache_intent, + PlatformCacheIntent::Shared { + surrogate_key: "ts-origin".to_owned() + } + ); + assert_eq!( + base() + .with_shared_cache("ts-origin") + .with_cache_bypass() + .cache_intent, + PlatformCacheIntent::Bypass + ); + } + // --------------------------------------------------------------------------- // Error-correlation interim scope (before EdgeZero #213) // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 1ea37f339..63f5c2a94 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -53,8 +53,8 @@ pub use backend_naming::{ pub use edgezero_core::key_value_store::{KvError, KvHandle, KvStore as PlatformKvStore}; pub use error::PlatformError; pub use http::{ - PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, UnavailableHttpClient, + PlatformCacheIntent, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, UnavailableHttpClient, }; pub use image_optimizer::{ PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, @@ -70,7 +70,7 @@ pub use template_cache::{ TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, TEMPLATE_SCHEMA_VERSION, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateCookieValue, TemplateEntry, TemplateMetadata, TemplateMetadataEncodeError, - UnavailableTemplateCache, VaryHeaderValues, VarySpec, + UnavailableTemplateCache, VaryHeaderValues, VarySpec, reader_url_surrogate_key, }; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 574f8d4a2..440e5a7b8 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -1,21 +1,23 @@ -//! The shared transformed-template cache for the #1009 ESI validation spike. +//! The shared transformed-template cache. //! //! Three caches are in play and conflating them is what produced the original wrong //! conclusion in the design doc, so this module names which one it is: //! -//! | Cache | Contents | Owner | -//! | ----- | --------------------------------- | ------------------------------ | -//! | C1 | raw origin bytes | Fastly read-through. Not this. | -//! | Template cache | post-`lol_html`, pre-assembly | **This module.** | -//! | Final response | final per-user assembled response | **Must never exist.** | +//! | Cache | Contents | Owner | +//! | -------------------- | --------------------------------- | ------------------------------ | +//! | Origin readthrough | raw origin bytes | The platform. Not this module. | +//! | Template cache | post-`lol_html`, pre-assembly | **This module.** | +//! | Assembled response | final per-user assembled response | **Must never exist.** | +//! +//! The origin readthrough cache is gated separately by +//! `creative_opportunities.origin_readthrough_enabled` (issue #852). The two are +//! independent: one can be on while the other is off. //! //! The template cache holds a *shared template*: no per-user bytes, and no decisions that depend on //! the request. What may and may not live in it is //! [§6.7 of the design doc](../../../../docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md), //! and the invariant is enforced by the rendered-document byte-identity tests in //! `publisher`. -//! -//! Spike-only. Remove with the spike. use core::fmt; use std::collections::HashSet; @@ -34,7 +36,8 @@ use crate::creative_opportunities::AssemblyMode; /// | 2 | Marker became the inert comment ``; the seam hands slots to `scheduleInitialAdInit` instead of assigning them | /// | 3 | Marker became ``; canonical collision-safe key, explicit origin freshness, and complete repeated document-policy metadata | /// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | -pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; +/// | 5 | Key gained `request_path`, so entries from version 4 hash differently and must not be read | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 5; /// Surrogate key attached to every template so an incident can purge the template cache globally. pub const TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; @@ -57,6 +60,12 @@ pub struct TemplateCacheKey { pub request_host: String, /// See [`Self::request_host`]. pub request_scheme: String, + /// Path and query as the **reader** addressed them, before origin rewriting. + /// + /// Distinct from [`Self::url`], which is the rewritten origin target. Kept so a purge + /// caller holding only the page address can derive the same key core attached at + /// insert — see [`Self::reader_url_surrogate_key`]. + pub request_path: String, /// Publisher origin identity, including the outbound Host override. Two virtual /// hosts can share a connection target while producing unrelated documents. pub origin_identity: String, @@ -102,6 +111,7 @@ impl TemplateCacheKey { ); push(&mut canonical, self.request_scheme.as_bytes()); push(&mut canonical, self.request_host.as_bytes()); + push(&mut canonical, self.request_path.as_bytes()); push(&mut canonical, self.origin_identity.as_bytes()); push(&mut canonical, self.url.as_bytes()); push(&mut canonical, self.template_fingerprint.as_bytes()); @@ -152,23 +162,107 @@ impl TemplateCacheKey { /// Surrogate keys to attach at insert, for purge-based rollback. /// /// `ts-template` purges every template at once, which is the rollback lever. - /// The per-URL key allows targeted invalidation. Both are needed: the broad one - /// for an incident, the narrow one for ordinary invalidation. + /// The per-URL keys allow targeted invalidation: one derived from the origin target + /// URI for core's own eviction of a bad entry, one derived from the reader-facing URL + /// for an operator or a CMS that only knows the page address. #[must_use] pub fn surrogate_keys(&self) -> Vec { vec![ TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY.to_string(), self.url_surrogate_key(), + self.reader_url_surrogate_key(), ] } - /// Surrogate key for every variant of this publisher URL. + /// Surrogate key for every variant of this publisher URL, as the origin saw it. /// /// Used to evict a malformed object without flushing unrelated article templates. + /// Keyed on [`Self::url`], which is the **origin-rewritten** target URI — not the + /// address a reader or an operator would type. #[must_use] pub fn url_surrogate_key(&self) -> String { format!("ts-template-url-{}", digest_hex(self.url.as_bytes())) } + + /// Surrogate key for this page as a reader addresses it. + /// + /// Purge callers know the page URL, not the origin the request was rewritten to, and + /// reconstructing the latter from the former would mean reimplementing the publisher + /// path's rewrite in every caller. When that reimplementation drifts it does not + /// fail — it produces a well-formed key that matches nothing, so a purge returns + /// success and invalidates nothing. Keying on the reader-facing URL removes the + /// reimplementation instead of trying to keep it in step. + #[must_use] + pub fn reader_url_surrogate_key(&self) -> String { + reader_url_surrogate_key(&format!( + "{}://{}{}", + self.request_scheme, self.request_host, self.request_path + )) + } +} + +/// Surrogate key for a reader-facing URL, as an operator would type it. +/// +/// A free function because both halves of the purge path need it and neither can build a +/// whole [`TemplateCacheKey`]: the endpoint and the CLI have a URL, not an origin identity, +/// a template fingerprint, or the origin's `Vary` values. +/// +/// # Canonicalization +/// +/// The digest is over exact bytes, so spellings that name the same page must be reduced to +/// one form or a purge silently misses. Normalized: scheme and host case, a default port, +/// one trailing slash, an empty query, and the *order* of the query parameters. **Not** +/// normalized: the parameters themselves, since a different query is a different page. +/// HTTP and HTTPS remain distinct: purge with the reader-facing scheme, host, and port. +/// A successful purge acknowledges invalidation of that key, not that an object existed. +/// +/// Parameter order is normalized because a reader reaching `?a=1&b=2` and one reaching +/// `?b=2&a=1` are on the same page, and both orderings can be cached as separate entries. +/// Leaving them as separate purge handles would let a purge report success while a stale +/// entry for the other ordering survived — the failure direction that matters here. +/// +/// A URL that cannot be parsed is hashed as given. An operator typo then purges nothing, +/// which is the same outcome as a correct URL that was never cached, and is preferable to +/// failing the command. The same fallback applies to reader URLs composed during cache +/// insertion: a malformed Host can produce a raw key that a corrected `--page` URL cannot +/// target. Such objects remain reachable through `--all` via the `ts-template` tag. +#[must_use] +pub fn reader_url_surrogate_key(url: &str) -> String { + format!( + "ts-template-readerurl-{}", + digest_hex(canonical_reader_url(url).as_bytes()) + ) +} + +fn canonical_reader_url(url: &str) -> String { + let Ok(parsed) = url::Url::parse(url) else { + return url.to_owned(); + }; + let scheme = parsed.scheme().to_ascii_lowercase(); + let host = parsed.host_str().unwrap_or_default().to_ascii_lowercase(); + let port = match parsed.port() { + // `Url::port` already returns `None` for the scheme's default, so anything left + // is meaningful. + Some(port) => format!(":{port}"), + None => String::new(), + }; + let path = parsed.path().trim_end_matches('/'); + let query = match parsed.query() { + Some(query) if !query.is_empty() => { + // Sorted on the raw pairs rather than decoded ones, so percent-encoded values + // stay byte-exact. Empty pairs are dropped, which folds `?a=1&&b=2` onto + // `?a=1&b=2`; that over-purges by one spelling, which is the safe direction. + let mut pairs: Vec<&str> = query.split('&').filter(|pair| !pair.is_empty()).collect(); + pairs.sort_unstable(); + if pairs.is_empty() { + String::new() + } else { + format!("?{}", pairs.join("&")) + } + } + _ => String::new(), + }; + format!("{scheme}://{host}{port}{path}{query}") } fn digest_hex(bytes: &[u8]) -> String { @@ -243,8 +337,8 @@ pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ /// Without this carve-out, the ordinary declaration sent by any compressing origin reads /// as an uncovered gap and disqualifies the response, so **the template cache would never store anything /// against a real origin** unless the operator redundantly listed a header the transform -/// already normalizes. Found by review before it could make the spike measure a hit rate -/// of approximately zero and read that as a result. +/// already normalizes. Found by review before it could drive the hit rate to +/// approximately zero and have that read as a measurement rather than a bug. const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; /// Request headers to include in the cache key, and where the list comes from. @@ -265,10 +359,17 @@ const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; /// 3. **Store the list alongside** and re-key on mismatch. Same cost as (2) plus /// complexity. /// -/// (1) is chosen for the spike because Step A already measured the origin's actual -/// `Vary`, the origin response is checked for drift before storage, and the configured -/// template-cache ceiling bounds how long a newly introduced mismatch can survive. -/// **This is a spike-grade choice, not a production one** — see the drift guard below. +/// (1) is chosen, and it holds for production because the drift it risks **fails +/// closed**: before storage the origin's actual `Vary` is compared against this list, and +/// an uncovered name refuses the template +/// (`TemplateCacheBypassReason::VaryNotCovered`, which carries the offending names so a +/// stale config is identifiable from one log line). A configuration that falls behind the +/// origin therefore costs hit rate, not correctness — the cache stops storing rather than +/// serving the wrong representation. The configured ceiling additionally bounds how long +/// an already-stored entry can outlive a change. +/// +/// Revisit (2) if operators find the configured list burdensome in practice; the reason to +/// prefer (1) is the extra lookup on every request, not a limitation of the guard. #[derive(Debug, Clone, PartialEq, Eq)] pub struct VarySpec { /// Header names, lowercased, in a fixed order. @@ -715,6 +816,14 @@ pub trait PlatformTemplateCache: Send + Sync { /// Purge every cached variant for one publisher URL. async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; + /// Purge one already-derived surrogate key. + /// + /// Exists because [`Self::purge_url`] needs a whole [`TemplateCacheKey`], which a + /// purge caller cannot build: an operator or a CMS webhook has a URL, not an origin + /// identity, a template fingerprint, or the origin's `Vary` values. Pair it with + /// [`reader_url_surrogate_key`]. + async fn purge_url_surrogate_key(&self, key: &str) -> Result<(), TemplateCacheError>; + /// Purge every stored template. The rollback lever. async fn purge_all(&self) -> Result<(), TemplateCacheError>; } @@ -762,6 +871,13 @@ impl PlatformTemplateCache for UnavailableTemplateCache { Err(TemplateCacheError::Unsupported) } + /// Reports unsupported rather than succeeding. A purge surface that silently does + /// nothing is worse than one that refuses: an operator mid-incident would read the + /// success and stop looking. + async fn purge_url_surrogate_key(&self, _key: &str) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + async fn purge_all(&self) -> Result<(), TemplateCacheError> { Err(TemplateCacheError::Unsupported) } @@ -778,6 +894,7 @@ mod tests { url: "https://example.com/news/article".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/news/article".to_string(), origin_identity: "https://origin.example.com\0origin.example.com".to_string(), assembly_mode: AssemblyMode::Esi, vary_values: vec![VaryHeaderValues { @@ -808,6 +925,15 @@ mod tests { } } + #[test] + fn reader_url_purges_are_scheme_specific() { + assert_ne!( + reader_url_surrogate_key("http://example.com/a"), + reader_url_surrogate_key("https://example.com/a"), + "should preserve the reader URL scheme in purge identity" + ); + } + #[test] fn dropping_an_unfulfilled_reservation_cancels_exactly_once() { let cancellations = Arc::new(AtomicUsize::new(0)); @@ -923,9 +1049,9 @@ mod tests { let rendered = key().to_cache_key(); assert_eq!( rendered, - "ts-template-cache-v4-54431eb4ea82644d6378717a8c3f18302fafbf739e684598da79e392b16900a6" + "ts-template-cache-v5-499cb43a3160fe173ffa53ea0b999658c8f2c50fbe23818ab451a59f7dc040da" ); - assert!(rendered.starts_with("ts-template-cache-v4-")); + assert!(rendered.starts_with("ts-template-cache-v5-")); assert_eq!(rendered.len(), 85); for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { assert!( @@ -1049,17 +1175,187 @@ mod tests { keys.contains(&TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY.to_string()), "a global purge lever is what makes rollback possible" ); - assert_eq!(keys.len(), 2, "global plus per-URL"); + assert_eq!( + keys.len(), + 3, + "global, origin-derived per-URL, and reader-facing per-URL" + ); + for key in keys.iter().skip(1) { + assert!( + !key.contains(char::is_whitespace), + "surrogate keys are space-delimited; whitespace would purge more than \ + intended, got {key:?}" + ); + assert!( + !key.contains('/') && !key.contains(':'), + "URL punctuation must be reduced, got {key:?}" + ); + } + } + + #[test] + fn an_adapter_without_a_template_cache_refuses_a_surrogate_purge() { + let outcome = futures::executor::block_on( + UnavailableTemplateCache.purge_url_surrogate_key("ts-template-readerurl-abc"), + ); + assert!( - !keys[1].contains(char::is_whitespace), - "surrogate keys are space-delimited; whitespace would purge more than \ - intended, got {:?}", - keys[1] + matches!(outcome, Err(TemplateCacheError::Unsupported)), + "a purge surface that silently does nothing is worse than one that refuses: \ + an operator mid-incident would read the success and stop looking" + ); + } + + #[test] + fn the_reader_facing_key_ignores_origin_rewriting() { + // `url` is the origin-rewritten target URI, not what an operator types. Two + // publisher hosts behind one reader-facing page must purge together, and core's + // own per-URL key must stay distinct so it can still evict one bad object. + let mut a = key(); + let mut b = key(); + a.url = "https://origin-one.internal.example/article".to_string(); + b.url = "https://origin-two.internal.example/article".to_string(); + + assert_eq!( + a.reader_url_surrogate_key(), + b.reader_url_surrogate_key(), + "the reader-facing key must not depend on which origin served the page" + ); + assert_ne!( + a.url_surrogate_key(), + b.url_surrogate_key(), + "the origin-derived key must stay distinct; core uses it to evict one entry" ); + } + + #[test] + fn the_reader_facing_key_uses_a_distinct_namespace() { + let keys = key().surrogate_keys(); + assert_eq!(keys.len(), 3, "global, origin-derived, and reader-facing"); assert!( - !keys[1].contains('/') && !keys[1].contains(':'), - "URL punctuation must be reduced, got {:?}", - keys[1] + keys.contains(&key().reader_url_surrogate_key()), + "the reader-facing key must be attached at insert or a purge cannot find it" + ); + assert_ne!( + key().url_surrogate_key(), + key().reader_url_surrogate_key(), + "distinct prefixes keep a staging host whose edge URL equals the origin URL \ + from aliasing the two derivations" + ); + } + + #[test] + fn reader_url_canonicalization_is_stable_across_operator_spellings() { + for (a, b) in [ + ( + "https://example.com/article", + "https://example.com/article/", + ), + ("https://Example.COM/article", "https://example.com/article"), + ( + "https://example.com:443/article", + "https://example.com/article", + ), + ( + "http://example.com:80/article", + "http://example.com/article", + ), + ( + "https://example.com/article?", + "https://example.com/article", + ), + ( + "https://example.com/article?a=1&b=2", + "https://example.com/article?b=2&a=1", + ), + ] { + assert_eq!( + reader_url_surrogate_key(a), + reader_url_surrogate_key(b), + "{a} and {b} name the same page and must purge together" + ); + } + } + + #[test] + fn reader_url_canonicalization_keeps_meaningful_differences() { + // A query selects a different page, so it must not be normalized away. + assert_ne!( + reader_url_surrogate_key("https://example.com/a?page=1"), + reader_url_surrogate_key("https://example.com/a?page=2"), + ); + assert_ne!( + reader_url_surrogate_key("https://example.com/a"), + reader_url_surrogate_key("https://example.com/b"), + ); + assert_ne!( + reader_url_surrogate_key("https://example.com/a"), + reader_url_surrogate_key("https://other.example/a"), + "two hosts are two pages" + ); + } + + #[test] + fn query_parameter_order_does_not_split_one_page_into_two_purge_handles() { + // Both orderings can be cached as separate entries, because the cache key is built + // from the path exactly as the reader sent it. They must still share one purge + // handle, or purging the ordering the operator happened to type would leave the + // other serving stale content while reporting success. + for (a, b) in [ + ( + "https://example.com/a?one=1&two=2&three=3", + "https://example.com/a?three=3&one=1&two=2", + ), + // A repeated parameter is order-independent the same way. + ( + "https://example.com/a?tag=x&tag=y", + "https://example.com/a?tag=y&tag=x", + ), + // A valueless parameter still sorts. + ( + "https://example.com/a?debug&page=2", + "https://example.com/a?page=2&debug", + ), + ] { + assert_eq!( + reader_url_surrogate_key(a), + reader_url_surrogate_key(b), + "{a} and {b} are the same page in a different spelling" + ); + } + } + + #[test] + fn sorting_the_query_does_not_merge_pages_that_genuinely_differ() { + // Sorting must collapse orderings only. If it also collapsed differing values or + // a dropped parameter, a purge would reach entries it was never asked to touch. + assert_ne!( + reader_url_surrogate_key("https://example.com/a?x=1&y=2"), + reader_url_surrogate_key("https://example.com/a?x=2&y=1"), + "swapping which parameter holds which value is a different page" + ); + assert_ne!( + reader_url_surrogate_key("https://example.com/a?x=1&y=2"), + reader_url_surrogate_key("https://example.com/a?x=1"), + "dropping a parameter is a different page" + ); + assert_ne!( + reader_url_surrogate_key("https://example.com/a?tag=x&tag=y"), + reader_url_surrogate_key("https://example.com/a?tag=x"), + "a repeated parameter must not be deduplicated into a single one" + ); + } + + #[test] + fn a_reader_url_that_cannot_be_parsed_still_yields_a_stable_key() { + // An operator typo must not panic the CLI; it should simply purge nothing. + assert_eq!( + reader_url_surrogate_key("not a url"), + reader_url_surrogate_key("not a url"), + ); + assert_ne!( + reader_url_surrogate_key("not a url"), + reader_url_surrogate_key("https://example.com/a"), ); } @@ -1154,8 +1450,8 @@ mod tests { fn a_key_field_counts_as_coverage_without_being_configured() { // The failure this prevents is silent and total: every compressing origin sends // `Vary: Accept-Encoding`, so treating it as a gap means the cache never stores - // anything, and a spike measuring hit rate would report ~0 and look like a - // finding rather than a bug. + // anything, and the resulting ~0 hit rate would read as a finding rather than a + // bug. let spec = VarySpec::new([]); assert!( diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 70eb55a9a..17813c77d 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -9,10 +9,11 @@ use error_stack::{Report, ResultExt as _}; use rand::rngs::OsRng; use super::{ - ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, - PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerOptions, - PlatformImageOptimizerParams, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, + ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, + PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, + StoreName, }; use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; @@ -254,7 +255,7 @@ pub(crate) struct StubHttpClient { streaming_responses_supported: std::sync::atomic::AtomicBool, pending_streaming_responses_supported: std::sync::atomic::AtomicBool, image_optimizer_options: Mutex>>, - cache_bypass_flags: Mutex>, + cache_intents: Mutex>, stream_response_flags: Mutex>, request_methods: Mutex>, request_uris: Mutex>, @@ -286,7 +287,7 @@ impl StubHttpClient { streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), pending_streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), image_optimizer_options: Mutex::new(Vec::new()), - cache_bypass_flags: Mutex::new(Vec::new()), + cache_intents: Mutex::new(Vec::new()), stream_response_flags: Mutex::new(Vec::new()), request_methods: Mutex::new(Vec::new()), request_uris: Mutex::new(Vec::new()), @@ -446,11 +447,11 @@ impl StubHttpClient { .clone() } - /// Return cache-bypass flags captured per `send` or `send_async` call, in order. - pub(crate) fn recorded_cache_bypass_flags(&self) -> Vec { - self.cache_bypass_flags + /// Return the cache intent captured per `send` or `send_async` call, in order. + pub(crate) fn recorded_cache_intents(&self) -> Vec { + self.cache_intents .lock() - .expect("should lock cache bypass flags") + .expect("should lock cache intents") .clone() } @@ -528,10 +529,10 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock image optimizer options") .push(request.image_optimizer.clone()); - self.cache_bypass_flags + self.cache_intents .lock() - .expect("should lock cache bypass flags") - .push(request.bypass_cache); + .expect("should lock cache intents") + .push(request.cache_intent.clone()); self.stream_response_flags .lock() .expect("should lock stream response flags") @@ -627,10 +628,10 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock calls") .push(backend_name.clone()); - self.cache_bypass_flags + self.cache_intents .lock() - .expect("should lock cache bypass flags") - .push(request.bypass_cache); + .expect("should lock cache intents") + .push(request.cache_intent.clone()); self.stream_response_flags .lock() .expect("should lock stream response flags") @@ -1113,9 +1114,9 @@ mod tests { "should record the backend name" ); assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![false], - "should record the default cache-bypass flag" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should record the default cache intent" ); } @@ -1204,9 +1205,9 @@ mod tests { "should record both send_async calls in order" ); assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![false, true], - "should record both send_async cache-bypass flags in order" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default, PlatformCacheIntent::Bypass], + "should record both send_async cache intents in order" ); } diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 7a3d09334..3570bdc10 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -170,7 +170,7 @@ pub struct RuntimeServices { pub(crate) kv_store: Arc, /// Shared transformed-template cache. Defaults to /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming - /// per request rather than failing. Spike-only; see + /// per request rather than failing. See /// [`crate::platform::template_cache`]. pub(crate) template_cache: Arc, /// Platform-specific cold-response template assembler. @@ -233,7 +233,7 @@ impl RuntimeServices { &*self.kv_store } - /// The shared transformed-template cache. Spike-only. + /// The shared transformed-template cache. #[must_use] pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { &*self.template_cache @@ -296,8 +296,6 @@ impl RuntimeServices { } /// Returns a clone of this instance with the template cache replaced. - /// - /// Spike-only (#1009). #[must_use] pub fn with_template_cache(self, cache: Arc) -> Self { Self { @@ -374,7 +372,7 @@ impl RuntimeServicesBuilder { self } - /// Set the shared transformed-template cache. Spike-only. + /// Set the shared transformed-template cache. #[must_use] pub fn template_cache(mut self, cache: Arc) -> Self { self.template_cache = Some(cache); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6f89edcd0..ddadffa66 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -70,8 +70,9 @@ use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; use crate::platform::{ - GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, - contains_publisher_esi_directive, + GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, + TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, VarySpec, contains_publisher_esi_directive, + reader_url_surrogate_key, }; use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::response_privacy::{ @@ -1567,7 +1568,7 @@ pub enum PublisherResponse { /// byte, which measured ~100x worse TTFB than doing nothing. The finalizer owns the /// `Arc`s a `'static` stream needs. /// - /// Spike-only, for the #1009 ESI validation. + /// Used only by the shared-template assembly modes, which are opt-in per deployment. AssembleTemplate { /// Response with every header already set. `Content-Length` must stay absent: /// the assembled length is unknown until bids resolve. @@ -1687,7 +1688,7 @@ pub struct OwnedProcessResponseParams { /// presence *is* the decision — there is no second place that could disagree with /// the gate, and no way to reach the store without having passed it. /// - /// Spike-only, for the #1009 ESI validation. + /// Used only by the shared-template assembly modes, which are opt-in per deployment. pub(crate) template_cache_key: Option, /// Slot definitions for the `` seam under a shared mode, as JSON. /// @@ -1832,12 +1833,12 @@ pub async fn buffer_publisher_response_async( // `process_response_streaming_async`; inline transforms retain the origin // coding. This avoids recompressing and immediately decoding a full document. let bytes = output.into_inner(); - // Cache taxonomy for this path: C1 is the raw origin/read-through cache, - // the template cache stores processed reader-neutral HTML, and C3 would be - // a forbidden cache of the final per-user assembled response. + // The origin readthrough cache holds raw origin bytes; the template cache + // stores processed reader-neutral HTML. Caching the final per-user + // assembled response is forbidden. // Store first, assemble second — never the reverse. The stored bytes are // shared between visitors; the assembled ones carry this visitor's bids. - // Swapping these two lines would create the forbidden C3 leak. + // Swapping these two lines would leak an assembled response into the cache. // Read before the store: `store_template_if_authorized` *takes* the key so a // request cannot store twice, which would leave nothing for assembly to gate // on. @@ -2259,7 +2260,7 @@ impl core::error::Error for SeamError {} /// the publisher path stamps `private, no-store` and strips validators. Omitting it /// here does not fall back to a safe default — it emits HTML with no `Cache-Control` at /// all, which is heuristically cacheable by browsers and intermediaries. That is a -/// forbidden C3 cache of a final per-user assembled response. +/// forbidden cache of a final per-user assembled response. /// /// Asserting the absence of `public`/`s-maxage`/`Surrogate-Control` would not have /// caught it. Nothing was present to forbid. @@ -2269,7 +2270,7 @@ impl core::error::Error for SeamError {} /// Returns an error if the stored metadata cannot be rendered as header values, which /// would mean a corrupt entry. /// -/// Spike-only, for the #1009 ESI validation. +/// Used only by the shared-template assembly modes, which are opt-in per deployment. fn build_cached_template_response( entry: &crate::platform::TemplateEntry, reader_compression: Compression, @@ -2323,7 +2324,7 @@ fn build_cached_template_response( /// service, not a broken one, and the whole point of the template cache is that the response is /// reproducible without it. /// -/// Spike-only, for the #1009 ESI validation. +/// Used only by the shared-template assembly modes, which are opt-in per deployment. async fn store_template_if_authorized( params: &mut OwnedProcessResponseParams, bytes: &[u8], @@ -2401,10 +2402,10 @@ pub async fn publisher_response_into_streaming_response( // Deliberately keyed on the store authorization rather than on the assembly mode: // a shared-mode response the gate rejected has nothing to store, so it keeps // streaming. `Inline` — the shipped path — never reaches this branch at all, which - // is the point. The spike cannot regress production latency by construction. + // is the point: the shared-template path cannot regress the default by construction. // // The cost is that a template cache *miss* buffers. That is the right trade: misses are already - // paying an origin fetch and a full transform, and what the spike measures is the + // paying an origin fetch and a full transform, and the case that matters is the // hit, where there is no origin fetch to stream from in the first place. if matches!( &publisher_response, @@ -4300,6 +4301,108 @@ pub struct AuctionDispatch<'a> { pub registry: Option<&'a PartnerRegistry>, } +/// Request-side conditions that decide whether this request's origin response may be +/// shared between readers. +/// +/// Necessary for both the origin readthrough cache and the template cache, which is why it +/// is one type rather than two parallel expressions that must be kept in step. Keeping the +/// template-only conditions out of it is deliberate: the assembly mode and the reader's +/// encoding support say whether *this pipeline* can assemble a shared template, not whether +/// the origin's bytes may be shared at all. +#[derive(Debug, Clone, Copy)] +pub(crate) struct SharedRequestInputs { + /// The method admits a shared representation. Only `GET` does. + pub(crate) method_is_cacheable: bool, + /// A request host was resolved. The post-processed output is host-dependent. + pub(crate) host_present: bool, + /// The request carried an `Authorization` value that did not pass edge auth unchanged. + pub(crate) authorization_disqualifies: bool, + /// Cookie policy refuses this cache's key. Readthrough additionally refuses + /// configured cookie dimensions that only the template key can represent. + pub(crate) cookie_disqualifies: bool, + /// Request cache semantics, diagnostics, or an integration require a fresh origin + /// response for this reader specifically. + pub(crate) request_requires_origin: bool, +} + +/// Whether this request's origin response may be shared between readers at all. +pub(crate) fn origin_response_is_shareable(inputs: SharedRequestInputs) -> bool { + inputs.method_is_cacheable + && inputs.host_present + && !inputs.authorization_disqualifies + && !inputs.cookie_disqualifies + && !inputs.request_requires_origin +} + +/// Apply the origin fetch's cache intent. +/// +/// Both publisher-origin fetch paths call this rather than deciding for themselves. They +/// are alternatives for the same fetch — one inside the EC-preload fan-out, one in the +/// branch taken when that did not fire — so a condition written twice could drift and make +/// readthrough eligibility depend on whether EC preload happened, which is not a property +/// of the origin response at all. +fn apply_origin_cache_intent( + request: PlatformHttpRequest, + readthrough_enabled: bool, + origin_response_is_shareable: bool, + should_run_ad_stack: bool, + request_is_document: bool, + reader_url: &str, +) -> PlatformHttpRequest { + // With the opt-in disabled, preserve the existing policy: ad-serving requests + // bypass and other publisher requests use the platform default. Enabling the flag + // replaces that policy with request shareability, both widening eligible ad traffic + // and tightening non-ad *document* traffic that carries disqualifying reader state. + // + // Documents only. Subresources keep the platform default, because the readthrough + // gate answers a question about pages: whether the origin's HTML may be shared + // between readers. Judging them on it would bypass the edge cache for every + // cookie-bearing or conditional asset request — browsers send first-party cookies on + // subresources, so that is most repeat-visitor asset traffic — and would tag every + // cached asset with `ts-template`, turning the template rollback lever into an + // origin-wide asset flush. + let readthrough_applies = readthrough_enabled && request_is_document; + let bypass = if readthrough_applies { + !origin_response_is_shareable + } else { + should_run_ad_stack + }; + if bypass { + request.with_cache_bypass() + } else if readthrough_applies { + // The same reader-facing key and all-scope key used by the purge endpoint. + // Fastly accepts multiple space-separated surrogate keys without changing TTL. + request.with_shared_cache(format!( + "{TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY} {}", + reader_url_surrogate_key(reader_url), + )) + } else { + request + } +} + +/// Whether this request may additionally use a shared *template*. +/// +/// The extra conditions say whether this pipeline can assemble one, and whether this +/// reader may be served one — not whether the origin's bytes may be shared, which is +/// [`origin_response_is_shareable`]. +/// +/// `reader_requires_origin` is read from the request *before* conditional and range +/// headers are stripped. A reader who asked for a range or a revalidation must reach the +/// origin, whatever is subsequently asked on their behalf; stripping changes what the +/// origin is asked, not what the reader wanted. +pub(crate) fn request_can_use_shared_template( + inputs: SharedRequestInputs, + assembly_mode_is_esi: bool, + reader_supports_assembly: bool, + reader_requires_origin: bool, +) -> bool { + origin_response_is_shareable(inputs) + && assembly_mode_is_esi + && reader_supports_assembly + && !reader_requires_origin +} + /// Proxies requests to the publisher's origin server. /// /// Returns a [`PublisherResponse`] indicating how the response should be sent: @@ -4551,7 +4654,11 @@ pub async fn handle_publisher_request( let datadome_suppression_requires_origin = suppress_datadome_client_side_tag; let datadome_suppression_requires_full_body = suppress_datadome_client_side_tag && is_html_document_request(&req); - let request_requires_origin = request_bypasses_template_cache(req.headers()) + // The reader's own request semantics, read before any stripping. A reader who asked + // for a range or a conditional response must not be handed a full document + // synthesized from a template shared with other readers, whatever the origin is then + // asked for on their behalf. + let reader_requires_origin = request_bypasses_template_cache(req.headers()) || gpt_diagnostics.requires_private_no_store() || datadome_suppression_requires_origin; let reader_compression = negotiate_reader_compression(req.headers()); @@ -4568,14 +4675,52 @@ pub async fn handle_publisher_request( strip_conditional_and_range_headers(&mut req); } + // Computed *after* the strip above, deliberately. These conditions ask whether the + // origin's response can be shared, and the origin only ever sees the request as it + // stands here. Judging the pre-strip headers marked a repeat visitor's `If-None-Match` + // navigation unshareable even though the origin was about to be asked an + // unconditional question and return a full document — losing readthrough for exactly + // the repeat-visit population this work targets, with nothing gained. + // + // The strip removes four headers; this predicate tests six plus `Cache-Control` + // request directives, so `If-Match`, `If-Unmodified-Since` and a `no-store` reader + // still disqualify, stripped or not. + let request_requires_origin = request_bypasses_template_cache(req.headers()) + || gpt_diagnostics.requires_private_no_store() + || datadome_suppression_requires_origin; + let method_is_cacheable = req.method() == Method::GET; - let request_can_use_shared_template = method_is_cacheable - && matches!(assembly_mode, AssemblyMode::Esi) - && !request_host.is_empty() - && !authorization_disqualifies - && !cookie_disqualifies - && !request_requires_origin - && reader_supports_assembly; + // Read while the request is still in hand: the readthrough policy below applies to + // documents only, and the origin send consumes these headers. + let request_is_document = is_html_document_request(&req); + let shared_request_inputs = SharedRequestInputs { + method_is_cacheable, + host_present: !request_host.is_empty(), + authorization_disqualifies, + cookie_disqualifies, + request_requires_origin, + }; + // Cookie variants belong to the template key only. The platform readthrough key + // cannot distinguish them, including an absent configured cookie, so these + // deployments must fetch origin bytes while retaining per-variant templates. + let origin_response_is_shareable = origin_response_is_shareable(SharedRequestInputs { + cookie_disqualifies: cookie_disqualifies || !key_cookie_names.is_empty(), + ..shared_request_inputs + }); + // Readthrough is off unless an operator turns it on. Unlike the cookie flag, which + // only ever applies to cookie-bearing requests, this gate would otherwise admit every + // cookieless request the moment this code deploys — and cookieless first-time visitors + // are exactly the readers an origin issues a session cookie to. + let origin_readthrough_enabled = settings + .creative_opportunities + .as_ref() + .is_some_and(CreativeOpportunitiesConfig::origin_readthrough_enabled); + let request_can_use_shared_template = request_can_use_shared_template( + shared_request_inputs, + matches!(assembly_mode, AssemblyMode::Esi), + reader_supports_assembly, + reader_requires_origin, + ); // Only advertise encodings the rewrite pipeline can decode and re-encode. This // remains unconditional when template cache negotiation fails: that request bypasses shared @@ -4611,14 +4756,31 @@ pub async fn handle_publisher_request( req.method() ); } - if request_requires_origin && matches!(assembly_mode, AssemblyMode::Esi) { + if reader_requires_origin && matches!(assembly_mode, AssemblyMode::Esi) { log::debug!("template_cache bypass: request cache semantics or diagnostics require origin"); } + // Capture the reader URL before origin rewriting, including its query. Inline + // assembly has no template key, but readthrough still needs both purge scopes. + let readthrough_reader_url = + if origin_readthrough_enabled && origin_response_is_shareable && request_is_document { + let path = req.uri().path_and_query().map_or("/", |path| path.as_str()); + format!("{request_scheme}://{request_host}{path}") + } else { + String::new() + }; let template_cache_key = request_can_use_shared_template.then(|| crate::platform::TemplateCacheKey { url: target_uri.to_string(), request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), + // Read here, before `rewrite_origin_request` below replaces the URI with the + // origin target. Path *and* query: a different query is a different page, and + // a purge caller types the whole address. + request_path: req + .uri() + .path_and_query() + .map(|path_and_query| path_and_query.as_str().to_owned()) + .unwrap_or_else(|| "/".to_owned()), origin_identity: format!("{}\0{}", settings.publisher.origin_url, origin_host_header), assembly_mode, vary_values: settings @@ -4677,9 +4839,16 @@ pub async fn handle_publisher_request( })?; let mut platform_request = PlatformHttpRequest::new(origin_req, backend_name.clone()).with_stream_response(); - if should_run_ad_stack { - platform_request = platform_request.with_cache_bypass(); - } + // Apply the opt-in shareability policy, or preserve the existing ad-stack + // bypass policy when readthrough is disabled. + platform_request = apply_origin_cache_intent( + platform_request, + origin_readthrough_enabled, + origin_response_is_shareable, + should_run_ad_stack, + request_is_document, + &readthrough_reader_url, + ); pending_origin = Some( services .http_client() @@ -4724,7 +4893,7 @@ pub async fn handle_publisher_request( .headers() .get("user-agent") .and_then(|value| value.to_str().ok()); - let observation = AuctionObservationContext::from_parts( + let mut observation = AuctionObservationContext::from_parts( AuctionSource::InitialNavigation, &settings.publisher.domain, &request_path, @@ -4732,6 +4901,9 @@ pub async fn handle_publisher_request( user_agent, ec_context, ); + // Written on the value, before it is moved into `auction_observation` below. Sites + // after that move reach it through `auction_observation.as_mut()` instead. + observation.set_origin_cache_shareable(origin_response_is_shareable && request_is_document); if should_run_auction { let slots_ctx = MatchedSlotsContext { @@ -4978,9 +5150,16 @@ pub async fn handle_publisher_request( if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); } - if should_run_ad_stack { - platform_request = platform_request.with_cache_bypass(); - } + // Apply the opt-in shareability policy, or preserve the existing ad-stack + // bypass policy when readthrough is disabled. + platform_request = apply_origin_cache_intent( + platform_request, + origin_readthrough_enabled, + origin_response_is_shareable, + should_run_ad_stack, + request_is_document, + &readthrough_reader_url, + ); services.http_client().send(platform_request).await }; let mut response = match origin_result { @@ -5934,7 +6113,7 @@ fn match_renderable_slots( /// rejects nothing on its own. Every safety condition is the caller's to enforce, /// so they are enumerated here rather than left implicit. /// -/// Spike-only, for the #1009 ESI validation. +/// Used only by the shared-template assembly modes, which are opt-in per deployment. #[derive(Debug, Clone, PartialEq, Eq, derive_more::Display)] pub(crate) enum TemplateCacheBypassReason { /// Not a shared-template mode; there is no template cache object to write. @@ -6120,8 +6299,8 @@ impl TemplateCachePolicy { /// the most serious one that applies. /// /// See `docs/superpowers/archive/2026-08-08-esi-cacheable-root-validation-design.md` -/// §6.6 for why the C1 raw-origin/read-through cache, the reader-neutral template cache, and -/// the forbidden C3 final assembled-response cache are distinct. +/// §6.6 for why the raw origin readthrough cache, the reader-neutral template cache, and +/// the forbidden final assembled-response cache are distinct. #[cfg(test)] pub(crate) fn template_cache_bypass_reason( mode: AssemblyMode, @@ -6232,7 +6411,7 @@ fn surrogate_control_freshness( match name.as_str() { // Deliberately not `cache_policy::cache_control_headers_are_private_or_no_store`: // this gate additionally treats `no-cache` as non-shareable, because "revalidate - // before reuse" is correct for an HTTP cache and too permissive for a spike-owned + // before reuse" is correct for an HTTP cache and too permissive for a TS-owned // one. Consolidating the two would loosen this gate rather than tidy it. "private" | "no-store" | "no-cache" => { return Err(TemplateCacheBypassReason::OriginNotShareable); @@ -7078,6 +7257,145 @@ mod tests { use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use crate::auction::types::AuctionResponse; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::PlatformCacheIntent; + + /// Every shared condition passing, as the base for single-condition negations. + fn all_shareable() -> SharedRequestInputs { + SharedRequestInputs { + method_is_cacheable: true, + host_present: true, + authorization_disqualifies: false, + cookie_disqualifies: false, + request_requires_origin: false, + } + } + + #[test] + fn template_eligibility_implies_origin_shareability() { + for bits in 0u16..256 { + let inputs = SharedRequestInputs { + method_is_cacheable: bits & 1 != 0, + host_present: bits & 2 != 0, + authorization_disqualifies: bits & 4 != 0, + cookie_disqualifies: bits & 8 != 0, + request_requires_origin: bits & 16 != 0, + }; + let is_esi = bits & 32 != 0; + let reader_supports_assembly = bits & 64 != 0; + let reader_requires_origin = bits & 128 != 0; + + let shareable = origin_response_is_shareable(inputs); + let template = request_can_use_shared_template( + inputs, + is_esi, + reader_supports_assembly, + reader_requires_origin, + ); + + assert!( + !template || shareable, + "template eligibility must imply origin shareability, input bits {bits}" + ); + assert_eq!( + template, + shareable && is_esi && reader_supports_assembly && !reader_requires_origin, + "template eligibility must be the shared base plus the three template \ + conditions, input bits {bits}" + ); + } + } + + #[test] + fn every_shared_input_is_necessary_for_shareability() { + assert!( + origin_response_is_shareable(all_shareable()), + "should be shareable when every condition passes" + ); + + for (label, broken) in [ + ( + "method", + SharedRequestInputs { + method_is_cacheable: false, + ..all_shareable() + }, + ), + ( + "host", + SharedRequestInputs { + host_present: false, + ..all_shareable() + }, + ), + ( + "authorization", + SharedRequestInputs { + authorization_disqualifies: true, + ..all_shareable() + }, + ), + ( + "cookie", + SharedRequestInputs { + cookie_disqualifies: true, + ..all_shareable() + }, + ), + ( + "requires-origin", + SharedRequestInputs { + request_requires_origin: true, + ..all_shareable() + }, + ), + ] { + assert!( + !origin_response_is_shareable(broken), + "dropping the {label} condition must make the request unshareable" + ); + } + } + + /// The two conditions that make template caching stricter than plain shareability. + /// + /// Pinned against hardcoded expectations rather than against the predicate's own + /// formula. `template_eligibility_implies_origin_shareability` compares the function + /// with a restatement of its body, so it catches a wrong combinator but would not + /// notice either of these terms being dropped — both sides of that equality would drop + /// it together. + #[test] + fn esi_mode_and_reader_support_are_each_necessary_for_template_eligibility() { + assert!( + request_can_use_shared_template(all_shareable(), true, true, false), + "should be eligible when every condition passes" + ); + + assert!( + !request_can_use_shared_template(all_shareable(), false, true, false), + "a shared template is assembled by ESI, so a non-ESI request must not read one" + ); + assert!( + !request_can_use_shared_template(all_shareable(), true, false, false), + "a reader that cannot assemble the seam must not be served an unassembled template" + ); + assert!( + !request_can_use_shared_template( + SharedRequestInputs { + cookie_disqualifies: true, + ..all_shareable() + }, + true, + true, + false + ), + "template eligibility must never outlive origin shareability" + ); + assert!( + !request_can_use_shared_template(all_shareable(), true, true, true), + "a reader who asked for a range or a revalidation must reach the origin, not be \ + served a document synthesized from a template shared with other readers" + ); + } #[test] fn request_head_snapshot_preserves_downstream_shape_without_body() { @@ -7259,7 +7577,7 @@ mod tests { lookups: usize, http_calls_at_lookup: usize, stream_flags: Vec, - cache_bypass_flags: Vec, + cache_intents: Vec, body_is_stream: bool, request_rewritten: bool, response_is_private: bool, @@ -7516,7 +7834,7 @@ mod tests { lookups: lookups.load(Ordering::SeqCst), http_calls_at_lookup: http_calls_at_lookup.load(Ordering::SeqCst), stream_flags: http.recorded_stream_response_flags(), - cache_bypass_flags: http.recorded_cache_bypass_flags(), + cache_intents: http.recorded_cache_intents(), body_is_stream, request_rewritten, response_is_private, @@ -7702,8 +8020,8 @@ mod tests { ); assert_eq!(outcome.stream_flags, vec![true]); assert_eq!( - outcome.cache_bypass_flags, - vec![true], + outcome.cache_intents, + vec![PlatformCacheIntent::Bypass], "pending publisher origin request should bypass platform caching" ); assert!( @@ -8988,8 +9306,11 @@ mod tests { fn shared_template_ad_seam_is_readable_and_versioned() { assert_eq!( (crate::platform::TEMPLATE_SCHEMA_VERSION, AD_ASSEMBLY_SEAM,), - (4, ""), - "the readable seam and its cache schema must move together" + (5, ""), + "changing the seam must bump the cache schema, or a deploy assembles \ + against a marker that moved. The converse does not hold — the schema \ + also moves when the cache key's shape changes, as it did for v5 — so \ + updating this pin with an unchanged seam is legitimate." ); assert_eq!( body_close_injection(AssemblyMode::Esi, false), @@ -9237,7 +9558,7 @@ mod tests { mod page_bids_format_tests { //! Page-bids is a JSON API. The old executable fragment was part of the removed - //! parser-based spike and must not remain as an accidental public surface. + //! parser-based path and must not remain as an accidental public surface. use super::*; @@ -9346,6 +9667,13 @@ mod tests { Ok(()) } + async fn purge_url_surrogate_key( + &self, + _key: &str, + ) -> Result<(), crate::platform::TemplateCacheError> { + Ok(()) + } + async fn purge_all(&self) -> Result<(), crate::platform::TemplateCacheError> { Ok(()) } @@ -9365,6 +9693,7 @@ mod tests { url: "https://example.com/page".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/page".to_string(), origin_identity: "https://origin.example.com\0origin.example.com".to_string(), assembly_mode: AssemblyMode::Esi, vary_values: vec![], @@ -9492,6 +9821,10 @@ mod tests { /// Force the lookup transaction to fail, for the fail-open + telemetry /// contract. A backend outage must never become a publisher outage. fail_lookup: AtomicBool, + /// Surrogate keys a purge asked for. This double stores by cache key, so it + /// cannot resolve a surrogate key to entries the way the platform does — + /// recording the request is what a test can assert on. + purged_surrogate_keys: Arc>>, } struct MemoryTemplateReservation { @@ -9677,6 +10010,21 @@ mod tests { Ok(()) } + /// Records the key so a test can assert what a purge asked for. + /// + /// This double stores by cache key, not by surrogate key, so it cannot + /// resolve one to the other the way the platform does. + async fn purge_url_surrogate_key( + &self, + key: &str, + ) -> Result<(), crate::platform::TemplateCacheError> { + self.purged_surrogate_keys + .lock() + .expect("should lock purged surrogate keys") + .push(key.to_owned()); + Ok(()) + } + async fn purge_all(&self) -> Result<(), crate::platform::TemplateCacheError> { self.entries.lock().expect("should lock entries").clear(); Ok(()) @@ -9711,6 +10059,20 @@ mod tests { settings } + /// Settings with the readthrough opt-in turned on. + /// + /// A separate helper rather than a default, because the flag being off by default + /// is the property that keeps this change inert until an operator asks for it. + fn settings_with_readthrough_enabled(mode: &str) -> Settings { + let mut settings = settings_with_mode(mode); + settings + .creative_opportunities + .as_mut() + .expect("settings_with_mode should configure creative opportunities") + .origin_readthrough_enabled = Some(true); + settings + } + fn settings_with_mode_and_template_cache_max_age(mode: &str, seconds: u32) -> Settings { let mut settings = settings_with_mode(mode); settings @@ -9745,6 +10107,55 @@ mod tests { services(http_client, cache).with_template_assembler(assembler) } + /// A template cache **and** a telemetry sink. + /// + /// Neither existing builder wires both — `services` above sets the cache and no + /// sink, and `services_with_telemetry` in the SSAT module sets the sink and no + /// cache. Cache-outcome telemetry cannot be asserted end to end without both. + fn services_with_cache_and_telemetry( + http_client: Arc, + cache: Arc, + telemetry_sink: Arc, + ) -> RuntimeServices { + let telemetry_sink: Arc = + telemetry_sink; + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .template_cache(cache) + .auction_telemetry_sink(telemetry_sink) + .build() + } + + /// The most recent `summary` row the sink recorded. + /// + /// `RecordingTelemetrySink` exposes no accessor, so this reads the field directly. + fn last_summary_row( + sink: &RecordingTelemetrySink, + ) -> Option { + sink.batches + .lock() + .expect("should lock recorded telemetry batches") + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .rfind(|row| row.event_kind == "summary") + .cloned() + } + + fn navigation_request_with_cookie(cookie: &str) -> Request { + let mut request = navigation_request(); + request.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(cookie).expect("should build a cookie header"), + ); + request + } + /// Shareable HTML: no `Set-Cookie`, no `Vary`, a public `Cache-Control`. Every /// condition the gate checks is satisfied, so a bypass here would be a bug in /// the wiring rather than in the fixture. @@ -10103,95 +10514,640 @@ mod tests { } #[tokio::test] - async fn a_second_request_is_served_from_the_cache_without_touching_the_origin() { + async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - - // Only one origin response is queued. If the second request reached the - // origin it would find the queue empty, so this fixture is itself part of - // the assertion. queue_shareable_html(&stub); - let cold = run(&settings, &services, navigation_request()).await; - assert_eq!( - cold.headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("miss-stored"), - "a successful cold fill must be distinguishable from a bypass" - ); - assert!( - cold.headers() - .get(header::HeaderName::from_static("x-ts-c2-cache")) - .is_none(), - "the retired template cache diagnostic must be absent from cold responses" - ); - let first = body_of(cold).await; - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "the cold request must fetch the origin" - ); + let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; - assert_eq!( - warm.headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("hit") - ); assert!( - warm.headers() - .get(header::HeaderName::from_static("x-ts-c2-cache")) - .is_none(), - "the retired template cache diagnostic must be absent from warm responses" - ); - let second = body_of(warm).await; - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "the warm request must not fetch the origin — that saving is the point" - ); - assert_eq!( - second, first, - "the cached template must be byte-identical to what was stored" + last_summary_row(&sink).is_some(), + "the harness must emit a summary row, or every assertion built on it is vacuous" ); } + /// Drive `handle_publisher_request` through the **EC-preload** fetch path. + /// + /// `run_with_orchestrator` passes `kv: None` and no EC id, so + /// `should_preload_ec_snapshot` is false for every other test in this file and the + /// first of the two origin-fetch call sites is never exercised. Supplying both + /// reaches it. + async fn run_through_ec_preload( + settings: &Arc, + services: &RuntimeServices, + request: Request, + ) { + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let kv = crate::ec::kv::KvIdentityGraph::in_memory("test-store"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(Some("test-ec-id".to_owned()), consent); + + let _ = handle_publisher_request( + settings, + services, + Some(&kv), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + } + #[tokio::test] - async fn a_pass_through_authorization_never_reaches_the_shared_template() { + async fn the_ec_preload_fetch_path_applies_the_same_cache_gate() { + // The call site a naive revert missed. Reverting only this one previously + // passed all 2,697 tests; this is the behavioral cover for it. let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + stub.set_pending_streaming_responses_supported(true); + let settings = Arc::new(settings_with_readthrough_enabled("inline")); queue_shareable_html(&stub); - let mut request = navigation_request(); - request.headers_mut().insert( - header::AUTHORIZATION, - HeaderValue::from_static("Basic dXNlcjpwYXNz"), - ); + run_through_ec_preload(&settings, &services, navigation_request()).await; - let response = run(&settings, &services, request).await; - assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request"), - "a credential TS did not terminate is bound for the origin, so its response must never be shared" - ); assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), - 0, - "an unterminated authorization must bypass before the lookup, not after it" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + crate::platform::reader_url_surrogate_key("http://ts.example.com/article") + ), + }], + "the EC-preload path must honor the gate, not decide for itself" ); } #[tokio::test] - async fn repeated_authorization_never_reaches_the_shared_template() { + async fn the_ec_preload_fetch_path_still_bypasses_when_readthrough_is_off() { + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + stub.set_pending_streaming_responses_supported(true); + let settings = Arc::new(settings_with_mode("inline")); + queue_shareable_html(&stub); + + run_through_ec_preload(&settings, &services, navigation_request()).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "the opt-in must gate both fetch paths, not only the non-preload one" + ); + } + + #[tokio::test] + async fn disabled_readthrough_preserves_non_ad_origin_caching() { + for without_creative_config in [false, true] { + let stub = Arc::new(StubHttpClient::new()); + let services = + services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + let mut settings = settings_with_mode("inline"); + if without_creative_config { + settings.creative_opportunities = None; + } + let settings = Arc::new(settings); + stub.push_response_with_headers( + 200, + b"body {}".to_vec(), + vec![ + ("content-type", "text/css"), + ("cache-control", "public, max-age=300"), + ], + ); + let request = HttpRequest::builder() + .uri("https://ts.example.com/style.css") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::empty()) + .expect("should build an asset request"); + + let _ = run(&settings, &services, request).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve existing subresource caching with unchanged settings" + ); + } + } + + #[tokio::test] + async fn enabled_readthrough_still_preserves_subresource_caching() { + // Browsers send first-party cookies on subresources, so judging assets on + // shareability would bypass the edge cache for most repeat-visitor asset + // traffic — and tag the rest with `ts-template`, so the template rollback + // purge would flush every cached asset at once. + for cookie in [None, Some("ts-ec=abc")] { + let stub = Arc::new(StubHttpClient::new()); + let services = + services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + let settings = Arc::new(settings_with_readthrough_enabled("inline")); + stub.push_response_with_headers( + 200, + b"body {}".to_vec(), + vec![ + ("content-type", "text/css"), + ("cache-control", "public, max-age=300"), + ], + ); + let mut request = HttpRequest::builder() + .uri("https://ts.example.com/style.css") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "style") + .body(EdgeBody::empty()) + .expect("should build an asset request"); + if let Some(cookie) = cookie { + request.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(cookie).expect("should build a cookie header"), + ); + } + + let _ = run(&settings, &services, request).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "the opt-in governs documents; a subresource keeps the platform default \ + (cookie={cookie:?})" + ); + } + } + + #[tokio::test] + async fn disabled_readthrough_preserves_non_ad_preload_caching() { + let stub = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + stub.set_pending_streaming_responses_supported(true); + let settings = Arc::new(settings_with_mode("inline")); + queue_shareable_html(&stub); + + run_through_ec_preload(&settings, &services, prefetch_navigation_request()).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve existing non-ad caching through EC preload" + ); + } + + #[test] + fn cache_intent_matches_the_explicit_opt_in_policy() { + let shared = PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + reader_url_surrogate_key("https://example.com/article") + ), + }; + for (enabled, shareable, ad_stack, document, expected) in [ + (false, false, false, true, PlatformCacheIntent::Default), + (false, false, true, true, PlatformCacheIntent::Bypass), + (false, true, false, true, PlatformCacheIntent::Default), + (false, true, true, true, PlatformCacheIntent::Bypass), + (true, false, false, true, PlatformCacheIntent::Bypass), + (true, false, true, true, PlatformCacheIntent::Bypass), + (true, true, false, true, shared.clone()), + (true, true, true, true, shared), + // A subresource keeps the platform default whatever the gate says, so + // enabling readthrough never bypasses an asset fetch or tags it for the + // template rollback purge. + (true, false, false, false, PlatformCacheIntent::Default), + (true, true, false, false, PlatformCacheIntent::Default), + ] { + let request = PlatformHttpRequest::new( + HttpRequest::builder() + .body(EdgeBody::empty()) + .expect("should build request"), + "backend", + ); + assert_eq!( + apply_origin_cache_intent( + request, + enabled, + shareable, + ad_stack, + document, + "https://example.com/article" + ) + .cache_intent, + expected, + "should honor the explicit policy (enabled={enabled}, shareable={shareable}, ad_stack={ad_stack}, document={document})" + ); + } + } + + #[tokio::test] + async fn readthrough_tags_the_reader_url_before_origin_rewriting_in_inline_mode() { + for preload in [false, true] { + let stub = Arc::new(StubHttpClient::new()); + let services = + services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + stub.set_pending_streaming_responses_supported(preload); + let settings = Arc::new(settings_with_readthrough_enabled("inline")); + queue_shareable_html(&stub); + let mut request = navigation_request(); + request + .headers_mut() + .insert("x-forwarded-proto", HeaderValue::from_static("https")); + *request.uri_mut() = "https://ts.example.com/article?b=2&a=1" + .parse() + .expect("should parse reader URI"); + if preload { + run_through_ec_preload(&settings, &services, request).await; + } else { + let _ = run(&settings, &services, request).await; + } + let page_key = crate::platform::reader_url_surrogate_key( + "https://ts.example.com/article?a=1&b=2", + ); + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Shared { + surrogate_key: format!("ts-template {page_key}"), + }], + "should attach the URL-purge and all-purge keys independently of template eligibility (preload={preload})" + ); + } + } + + #[tokio::test] + async fn a_shareable_navigation_no_longer_forces_an_origin_miss() { + // The point of issue #852. A cookieless, ad-serving navigation used to set + // pass on every origin fetch, so every pageview paid a full origin round + // trip. + // + // This asserts the *intent* recorded on the outbound request, not a cache + // hit, because a hit is not observable here. Measured under Viceroy 0.17 with + // the gate enabled, the request judged shareable, and an origin responding + // `Cache-Control: public, max-age=60` with no `Set-Cookie`: two identical + // navigations still produced two origin fetches. Viceroy does not implement + // the readthrough cache, so the saving this gate exists for cannot be + // demonstrated locally in any form — only the decision that enables it. + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + crate::platform::reader_url_surrogate_key("http://ts.example.com/article") + ), + }], + "a shareable navigation must not force a MISS" + ); + } + + #[tokio::test] + async fn a_repeat_visitor_revalidating_still_gets_readthrough() { + // The conditional headers are stripped before the origin is asked, so it + // returns a full document that is shareable like any other. Judging the + // pre-strip request marked this unshareable and cost readthrough for exactly + // the repeat-visit population the change targets. + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); + queue_shareable_html(&stub); + + let mut request = navigation_request(); + request.headers_mut().insert( + header::IF_NONE_MATCH, + HeaderValue::from_static("\"cached\""), + ); + + let _ = run(&settings, &services, request).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + crate::platform::reader_url_surrogate_key("http://ts.example.com/article") + ), + }], + "a stripped conditional navigation asks the origin an unconditional \ + question, so its answer is shareable" + ); + } + + #[tokio::test] + async fn a_reader_asking_for_a_range_is_not_served_a_shared_template() { + // The other half, and the invariant that caught an over-broad first attempt: + // stripping changes what the origin is asked, not what the reader wanted. + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + // Warm a template with a plain navigation. + let _ = run(&settings, &services, navigation_request()).await; + + let mut request = navigation_request(); + request + .headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-31")); + let _ = run(&settings, &services, request).await; + + assert_eq!( + stub.recorded_cache_intents().len(), + 2, + "the range request must reach the origin rather than be answered from the \ + warm shared template" + ); + } + + #[tokio::test] + async fn unshareable_requests_still_bypass() { + // Each of these is a distinct reason the origin response cannot be held in a + // shared cache, and each must reach the same decision on its own. + for (label, request) in [ + ("cookie", navigation_request_with_cookie("ts-ec=abc")), + ("authorization", { + let mut request = navigation_request(); + request.headers_mut().insert( + header::AUTHORIZATION, + HeaderValue::from_static("Basic dXNlcjpwYXNz"), + ); + request + }), + ("non-GET", { + let mut request = navigation_request(); + *request.method_mut() = Method::POST; + request + }), + ("conditional", { + let mut request = navigation_request(); + request + .headers_mut() + .insert(header::IF_MATCH, HeaderValue::from_static("\"tag\"")); + request + }), + ] { + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, request).await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "a {label}-bearing request must bypass the shared cache" + ); + } + } + + #[tokio::test] + async fn a_request_that_skips_the_ad_stack_is_still_judged_on_shareability() { + // The tightening half, and the term that left the condition. A prefetch runs + // no auction, so it used to skip the bypass; whether an auction runs says + // nothing about whether the origin's response may be shared. + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::new(RecordingTelemetrySink::default()), + ); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, { + let mut request = prefetch_navigation_request(); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("ts-ec=abc")); + request + }) + .await; + + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "an ad-stack opt-out that is also unshareable must still bypass" + ); + } + + #[tokio::test] + async fn navigation_records_whether_the_origin_response_was_shareable() { + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = Arc::new(settings_with_mode("esi")); + queue_shareable_html(&stub); + + let _ = run( + &settings, + &services, + navigation_request_with_cookie("ts-ec=abc"), + ) + .await; + + assert_eq!( + last_summary_row(&sink) + .expect("should emit a summary row") + .origin_cache_shareable, + Some(0), + "a cookie-bearing request must record as not shareable" + ); + } + + #[tokio::test] + async fn cookieless_navigation_records_the_origin_response_as_shareable() { + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = Arc::new(settings_with_mode("esi")); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + + assert_eq!( + last_summary_row(&sink) + .expect("should emit a summary row") + .origin_cache_shareable, + Some(1), + "a cookieless GET navigation is the population the gate is meant to admit" + ); + } + + #[tokio::test] + async fn non_document_requests_do_not_count_toward_readthrough_gate_reach() { + for fetch_metadata in [true, false] { + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = Arc::new(settings_with_mode("esi")); + queue_shareable_html(&stub); + let mut request = navigation_request(); + if fetch_metadata { + request + .headers_mut() + .insert("sec-fetch-dest", HeaderValue::from_static("empty")); + } else { + request.headers_mut().remove("sec-fetch-dest"); + request + .headers_mut() + .insert(header::ACCEPT, HeaderValue::from_static("*/*")); + } + + let _ = run(&settings, &services, request).await; + + assert_eq!( + last_summary_row(&sink) + .expect("should emit a summary row") + .origin_cache_shareable, + Some(0), + "should exclude non-document requests from the gate's potential reach" + ); + } + } + + #[tokio::test] + async fn a_second_request_is_served_from_the_cache_without_touching_the_origin() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + + // Only one origin response is queued. If the second request reached the + // origin it would find the queue empty, so this fixture is itself part of + // the assertion. + queue_shareable_html(&stub); + + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("miss-stored"), + "a successful cold fill must be distinguishable from a bypass" + ); + assert!( + cold.headers() + .get(header::HeaderName::from_static("x-ts-c2-cache")) + .is_none(), + "the retired template cache diagnostic must be absent from cold responses" + ); + let first = body_of(cold).await; + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the cold request must fetch the origin" + ); + + let warm = run(&settings, &services, navigation_request()).await; + assert_eq!( + warm.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("hit") + ); + assert!( + warm.headers() + .get(header::HeaderName::from_static("x-ts-c2-cache")) + .is_none(), + "the retired template cache diagnostic must be absent from warm responses" + ); + let second = body_of(warm).await; + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the warm request must not fetch the origin — that saving is the point" + ); + assert_eq!( + second, first, + "the cached template must be byte-identical to what was stored" + ); + } + + #[tokio::test] + async fn a_pass_through_authorization_never_reaches_the_shared_template() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + let mut request = navigation_request(); + request.headers_mut().insert( + header::AUTHORIZATION, + HeaderValue::from_static("Basic dXNlcjpwYXNz"), + ); + + let response = run(&settings, &services, request).await; + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-request"), + "a credential TS did not terminate is bound for the origin, so its response must never be shared" + ); + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + 0, + "an unterminated authorization must bypass before the lookup, not after it" + ); + } + + #[tokio::test] + async fn repeated_authorization_never_reaches_the_shared_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); @@ -10339,7 +11295,7 @@ mod tests { stub.set_streaming_responses_supported(true); stub.set_pending_streaming_responses_supported(true); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); let lookups = Arc::new(AtomicUsize::new(0)); let http_calls_at_lookup = Arc::new(AtomicUsize::new(0)); @@ -10431,9 +11387,15 @@ mod tests { "should preserve streaming on the cold origin fetch" ); assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![true], - "should bypass platform caching for the cold origin fetch" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + reader_url_surrogate_key("http://ts.example.com/article") + ), + }], + "a shareable cold fetch must stop forcing a MISS — the origin round \ + trip issue #852 exists to take off the hot path" ); } @@ -11011,7 +11973,7 @@ mod tests { #[test] fn parser_validation_does_not_change_the_cached_schema() { - assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 4); + assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 5); assert_eq!(AD_ASSEMBLY_SEAM, ""); assert!(!contains_publisher_esi_directive( AD_ASSEMBLY_SEAM.as_bytes() @@ -11530,7 +12492,7 @@ mod tests { #[tokio::test] async fn the_cached_template_holds_the_marker_and_never_the_bids() { // Store the reader-neutral template before assembling the final per-user - // response, which must never enter the forbidden C3 cache. If + // response, which must never enter the forbidden assembled-response cache. If // these were swapped, the cache would hold one visitor's bids and serve them // to the next — and every test above would still pass, because the served // page would look correct. @@ -11559,6 +12521,89 @@ mod tests { ); } + #[tokio::test] + async fn nextjs_stored_templates_are_identical_across_request_shapes() { + let mut settings = settings_with_mode("esi"); + settings.publisher.origin_url = "https://origin.example.com".to_owned(); + settings + .integrations + .insert_config( + "nextjs", + &serde_json::json!({ + "enabled": true, + "rewrite_attributes": ["href", "link", "url"], + }), + ) + .expect("should enable Next.js processing"); + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + assert!( + !registry.html_stream_processor_factories().is_empty(), + "should exercise the Next.js stream processor" + ); + let settings = Arc::new(settings); + let html = br#"
"#; + + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut templates = Vec::new(); + for request in [navigation_request(), prefetch_navigation_request()] { + // Independent fills force distinct per-request RSC namespaces. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + html.to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + + let response = run_via(&settings, &services, request, finalizer).await; + let _ = body_of(response).await; + + let entries = cache.entries.lock().expect("should lock stored templates"); + assert_eq!( + entries.len(), + 1, + "should store a Next.js template on every fill" + ); + let template = entries + .values() + .next() + .expect("should find stored template") + .body + .clone(); + let text = core::str::from_utf8(&template).expect("should store UTF-8 HTML"); + assert!( + text.contains("ts.example.com/page"), + "should rewrite the RSC URL" + ); + assert!( + text.contains(AD_ASSEMBLY_SEAM), + "should store before per-reader assembly" + ); + for marker in [ + "__ts_rsc_", + ".adSlots", + ".bids=", + "gpt-diagnostics", + "gpt_history", + ] { + assert!( + !text.contains(marker), + "should exclude request-scoped marker {marker}" + ); + } + templates.push(template); + } + assert_eq!( + templates[0], templates[1], + "should store identical bytes across request shapes and RSC namespaces" + ); + } + } + #[tokio::test] async fn a_transform_that_overruns_its_buffer_stores_nothing() { // The 16 MB cap in production, shrunk here. A partial template in template cache is the @@ -11943,7 +12988,7 @@ mod tests { // The converse, and the failure mode a fingerprint fix can introduce: // over-invalidating is as total as under-invalidating. A fingerprint that // moves between two equal configurations is a cache that never hits, which - // the spike would report as "no measurable benefit" rather than as a bug. + // this would read as "no measurable benefit" rather than as a bug. // // The two `Settings` are parsed independently, so their `[integrations]` // maps iterate in different orders — which is what exercises the sort. @@ -11986,6 +13031,124 @@ mod tests { .expect("should build cookie-bearing request") } + #[tokio::test] + async fn readthrough_never_collapses_template_cookie_variants() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .origin_readthrough_enabled = Some(true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for arm in ["absent", "A", "B", "empty"] { + stub.push_response_with_headers( + 200, + format!("arm-{arm}").into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + } + for (index, (cookie, arm)) in [ + (None, "absent"), + (Some("ab_bucket=A"), "A"), + (Some("ab_bucket=B"), "B"), + (Some("ab_bucket="), "empty"), + (Some("ab_bucket=A"), "A"), + (Some("ab_bucket=B"), "B"), + ] + .iter() + .enumerate() + { + let fields: Vec<&[u8]> = cookie.iter().map(|value| value.as_bytes()).collect(); + let response = run(&settings, &services, cookie_policy_request(&fields)).await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + if index < 4 { "miss-stored" } else { "hit" }, + "should still cache separate templates" + ); + let body = String::from_utf8(body_of(response).await).expect("should decode HTML"); + assert!( + body.contains(&format!("arm-{arm}")), + "should preserve each cookie variant" + ); + } + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass; 4], + "should never cache origin bytes without the template cookie dimensions, even when the key cookie is absent" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 4, + "should store each template variant" + ); + } + + #[tokio::test] + async fn readthrough_respects_named_session_bypass_without_disabling_anonymous_sharing() { + let mut settings = cookie_policy_settings(None, Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .origin_readthrough_enabled = Some(true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + let _ = run(&settings, &services, cookie_policy_request(&[])).await; + let response = run(&settings, &services, cookie_policy_request(&[b"session="])).await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should not read an anonymous template for a session" + ); + assert_eq!( + stub.recorded_cache_intents(), + vec![ + PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + reader_url_surrogate_key("http://ts.example.com/article") + ) + }, + PlatformCacheIntent::Bypass, + ], + "should share anonymous bytes while refusing even an empty named session" + ); + } + + #[tokio::test] + async fn readthrough_preload_bypasses_configured_cookie_dimensions() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let config = Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.origin_readthrough_enabled = Some(true); + config.assembly_mode = Some(AssemblyMode::Inline); + let stub = Arc::new(StubHttpClient::new()); + stub.set_pending_streaming_responses_supported(true); + let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + queue_shareable_html(&stub); + run_through_ec_preload( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await; + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "should apply the cookie-dimension exclusion on the pending origin path too" + ); + } + fn cookie_policy_settings( key_names: Option<&[&str]>, bypass_names: Option<&[&str]>, @@ -13002,7 +14165,7 @@ mod tests { #[tokio::test] async fn a_declared_cookie_independent_origin_lets_repeat_visitors_share() { - // The opt-in. Without it the spike can only ever measure first-ever page + // The opt-in. Without it the cache can only ever serve first-ever page // views, which is not the population the issue cares about. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); @@ -14772,6 +15935,7 @@ mod tests { template_cache_key_cookies: None, template_cache_bypass_cookies: None, origin_is_cookie_independent: None, + origin_readthrough_enabled: None, section_segment: None, slot: vec![slot()], }); @@ -15571,8 +16735,8 @@ mod tests { // Assert assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![true], + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], "eligible publisher navigation should bypass the platform cache" ); let recorded_requests = stub.recorded_request_headers(); @@ -15671,9 +16835,9 @@ mod tests { // Assert assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![false], - "publisher navigation without matched slots should use the default cache mode" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform caching for non-ad requests while readthrough is disabled", ); let recorded_requests = stub.recorded_request_headers(); let outbound_headers = recorded_requests @@ -15776,9 +16940,9 @@ mod tests { // Assert assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![false], - "disabled server-side ad templates should not bypass the origin cache" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform caching with disabled ad templates and readthrough" ); assert_eq!( response_head @@ -16310,9 +17474,9 @@ mod tests { ); } assert_eq!( - stub.recorded_cache_bypass_flags(), - vec![false], - "noneligible publisher navigation should use the default cache mode" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform revalidation for non-ad requests with readthrough disabled" ); let recorded_requests = stub.recorded_request_headers(); let outbound_headers = recorded_requests @@ -20651,6 +21815,7 @@ mod tests { template_cache_key_cookies: None, template_cache_bypass_cookies: None, origin_is_cookie_independent: None, + origin_readthrough_enabled: None, section_segment: None, slot: Vec::new(), } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 8674429ea..46ff4bf5b 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -68,7 +68,8 @@ pub fn apply_inactive_ad_stack_browser_cache_policy(response: &mut Response) { /// /// Call this after every configurable response mutation. It deliberately overwrites /// `Cache-Control` and strips validators, expiry metadata, and runtime edge-cache -/// directives so a later integration cannot turn an assembled document into C3. +/// directives so a later integration cannot turn an assembled document into something a +/// shared cache would store. pub fn enforce_private_no_store(response: &mut Response) { CacheControlPolicy::NoStorePrivate .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 78e4006ff..cf21ed646 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -3217,6 +3217,7 @@ impl Settings { "/_ts/admin/ec", "/_ts/admin/ec/{id}", "/_ts/admin/eids", + "/_ts/admin/cache/purge", ]; /// Probes that establish handler coverage for the dynamic @@ -7171,6 +7172,7 @@ source_domain = "partner.example.com" "/_ts/admin/ec", "/_ts/admin/ec/{id}", "/_ts/admin/eids", + "/_ts/admin/cache/purge", ], "should report every admin endpoint as uncovered" ); @@ -7210,6 +7212,7 @@ source_domain = "partner.example.com" "/_ts/admin/ec", "/_ts/admin/ec/{id}", "/_ts/admin/eids", + "/_ts/admin/cache/purge", ], "should detect the admin endpoints not covered by the narrow handler" ); @@ -7221,7 +7224,7 @@ source_domain = "partner.example.com" r#"path = "^/_ts/admin" username = "admin" password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" username = "admin" password = "strong-test-password" @@ -7246,7 +7249,7 @@ source_domain = "partner.example.com" r#"path = "^/_ts/admin" username = "admin" password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" username = "admin" password = "strong-test-password" @@ -7329,7 +7332,7 @@ source_domain = "partner.example.com" r#"path = "^/_ts/admin" username = "admin" password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" username = "admin" password = "strong-test-password" @@ -7355,7 +7358,7 @@ source_domain = "partner.example.com" r#"path = "^/_ts/admin" username = "admin" password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" username = "admin" password = "strong-test-password" @@ -7380,7 +7383,7 @@ source_domain = "partner.example.com" r#"path = "^/_ts/admin" username = "admin" password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" + r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" username = "admin" password = "strong-test-password" diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index c466e2036..2428d7799 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -923,6 +923,72 @@ async fn legacy_admin_aliases_are_denied_locally_not_proxied() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn admin_cache_purge_not_implemented_parity() { + // The template cache is Fastly-backed, so the other three adapters answer 501 rather + // than letting the path fall through to the publisher origin and 404 — a CMS purge + // webhook needs to tell "not supported here" from "no such endpoint". + let body = r#"{"scope":"all"}"#; + + let (axum_status, _) = axum_authorized_json("POST", "/_ts/admin/cache/purge", body).await; + let (cf_status, _) = cf_authorized_json("POST", "/_ts/admin/cache/purge", body).await; + let (spin_status, _) = spin_authorized_json("POST", "/_ts/admin/cache/purge", body).await; + + assert_eq!(axum_status, 501, "Axum must answer cache purge with 501"); + assert_eq!( + cf_status, 501, + "Cloudflare must answer cache purge with 501" + ); + assert_eq!(spin_status, 501, "Spin must answer cache purge with 501"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn admin_cache_purge_unauthenticated_parity() { + // Guards the test above from passing for the wrong reason. Without credentials the + // path must 401, which proves the 501s were reached through auth rather than being + // the 401s of a probe that never arrived at a handler. + let body = r#"{"scope":"all"}"#; + + let (axum_status, _) = axum_post_headers("/_ts/admin/cache/purge", body).await; + let (cf_status, _) = cf_post_headers("/_ts/admin/cache/purge", body).await; + let (spin_status, _) = spin_post_headers("/_ts/admin/cache/purge", body).await; + + for (adapter, status) in [ + ("Axum", axum_status), + ("Cloudflare", cf_status), + ("Spin", spin_status), + ] { + assert_eq!( + status, 401, + "{adapter} must require auth on the cache purge path" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn admin_cache_purge_rejects_credential_forwarding_methods() { + // The guard the Fastly route exists for, asserted cross-adapter: a method the route + // does not claim falls through to the publisher with the Authorization header still + // attached. Every method must be answered locally, never forwarded. + for method in ["GET", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] { + let (axum_status, _) = axum_authorized_json(method, "/_ts/admin/cache/purge", "").await; + let (cf_status, _) = cf_authorized_json(method, "/_ts/admin/cache/purge", "").await; + let (spin_status, _) = spin_authorized_json(method, "/_ts/admin/cache/purge", "").await; + + for (adapter, status) in [ + ("Axum", axum_status), + ("Cloudflare", cf_status), + ("Spin", spin_status), + ] { + assert_eq!( + status, 501, + "{adapter} must answer {method} locally; a fallthrough would ship the \ + admin credential to the origin" + ); + } + } +} + /// A known non-regulated location permits the fixture's server-side auction. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn adapter_buffers_nextjs_auction_output() { diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 0885b6369..09a59a01f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2261,9 +2261,12 @@ TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false ### Shared template assembly (`assembly_mode = "esi"`) -This configuration is an experimental validation spike scoped to -[IABTechLab/trusted-server#1009](https://github.com/IABTechLab/trusted-server/issues/1009), -not a settled production cache interface. +`inline` remains the default. `esi` is opt-in per deployment, covered by the +`template-cache-local-test.sh` harness and by rendered-document byte-identity tests, and +originated in +[IABTechLab/trusted-server#1009](https://github.com/IABTechLab/trusted-server/issues/1009). +Enable it deliberately and verify with the harness first; the keys below are the safety +contract that makes it safe to do so. `assembly_mode` controls how initial-page slot and bid state is delivered: @@ -2464,20 +2467,37 @@ The two headers together are the reliable verification signal. Timing alone can vary with the origin, auction, compression, browser connection reuse, and local proxy buffering. +> **Upgrade note.** This release adds `/_ts/admin/cache/purge` to the admin endpoints +> startup validation covers. A configuration whose `[[handlers]]` enumerate admin paths +> individually, rather than using the `^/_ts/admin` prefix, fails to start until that path +> is covered too. The failure is at startup and explicit, not at request time. + Rollback must preserve configuration compatibility: 1. Change `assembly_mode` to `inline` and deploy/push that configuration. 2. Before rolling back to a binary that predates these fields, remove `assembly_mode`, `template_cache_vary`, `template_cache_max_age_seconds`, - `template_cache_key_cookies`, `template_cache_bypass_cookies`, and - `origin_is_cookie_independent`, then push the cleaned configuration. Older binaries + `template_cache_key_cookies`, `template_cache_bypass_cookies`, + `origin_is_cookie_independent`, and `origin_readthrough_enabled`, then push the + cleaned configuration. Older binaries use `deny_unknown_fields` and intentionally reject unknown keys, even empty lists. When rolling back only the named-cookie feature to a binary that supports ESI, remove both cookie-list fields and keep `origin_is_cookie_independent = false` or disable ESI if the origin depends on cookies. Keeping `true` after removing the lists loses variant separation and session bypass. -3. Purge the Fastly surrogate key `ts-template` using the service's normal purge - tooling, or wait for the bounded origin-derived lifetime to expire. + Remove `origin_readthrough_enabled` even when rolling it back: `false` still + serializes the field and older binaries reject it. +3. Purge the template cache with `ts cache purge --service --all`, or + `--page ` for a single reader-facing URL. Use its exact scheme, host, and + port: `http://example.com/article` and `https://example.com/article` have different + purge keys. A success acknowledges invalidation of the requested key, not that an + object existed. `--service` requires HTTPS, except for loopback development + services (`localhost`, `127.0.0.1`, or `::1`). The admin endpoint + `POST /_ts/admin/cache/purge` is the same operation for a CMS webhook. Either clears + the `ts-template` surrogate key; waiting out the bounded origin-derived lifetime also + works. With readthrough caching enabled, `--all` also purges tagged origin + documents, so the next requests refetch those documents from the origin. Check + whether the origin can absorb that load before purging during a traffic peak. Run `scripts/template-cache-local-test.sh esi` before a rollout and `scripts/template-cache-local-test.sh inline` as its control. The harness uses a temporary @@ -2489,6 +2509,165 @@ versus empty buckets, ignored compact JSON and comma-list cookies, and session bypass on warm and cold URLs. Each request checks the selected HTML, cache diagnostics, private response policy, winning-bid assembly, and origin fetch count. +### Origin readthrough caching + +`origin_readthrough_enabled` controls a **different cache** from everything above. +The template cache stores Trusted Server's own transformed HTML. Readthrough is the +platform's own cache sitting in front of the publisher origin, and it stores the +origin's bytes. + +```toml +[creative_opportunities] +# Default false. Enable only after `ts origin probe-shareability` passes on every +# axis and every verdict. +origin_readthrough_enabled = true +``` + +Left at the default, the existing caching policy is preserved: ad-serving requests +bypass the origin cache, while other publisher requests (including ordinary assets) +keep the platform's default caching behavior. Setting it to `true` applies request +shareability instead: eligible ad-serving requests can use the cache, while +ineligible non-ad requests bypass it. Eligible requests are `GET`s with a `Host`, +no disqualifying authorization or cookie, and no remaining conditional or range +semantics. + +**Document requests only.** The gate answers a question about pages — whether the +origin's HTML may be shared between readers — so it applies to document requests +(`Sec-Fetch-Dest: document` and equivalents, or a navigation when that header is +absent). Subresources keep the platform default whether the flag is on or off. +Judging them on shareability would bypass the edge cache for every cookie-bearing +or conditional asset request, which is most repeat-visitor asset traffic, and would +tag every cached asset with `ts-template`, turning the template rollback purge into +an origin-wide asset flush. + +#### This cache has far weaker guarantees than the template cache + +Read this before enabling it. The template cache refuses storage on inspection of +the origin's _response_ — `Set-Cookie`, a CSP nonce, missing positive freshness, an +uncovered `Vary`, and the rest of the list above. **Readthrough has none of those +refusals**, and cannot: the decision is made before the origin replies, and no +post-response hook is reachable on the Fastly adapter. + +What that means concretely, for each refusal the template cache performs: + +| Template-cache refusal | Covered on readthrough? | +| --------------------------- | ------------------------------------------------- | +| No positive freshness | **No** — probe verdict only | +| Origin `Set-Cookie` | **No** — probe verdict only | +| Response CSP nonce | **No** — probe verdict only | +| Origin marks it unshareable | Yes — the platform honours `private` / `no-store` | +| Non-`200` status | Yes — the platform honours status | +| Uncovered `Vary` | Yes — the platform keys on the origin's `Vary` | +| Not HTML | Not applicable; readthrough caches per origin | + +Every row marked **No** is an accepted risk carried by the operator, not by the +code. An origin that personalises HTML without saying so in its headers can +cross-serve one reader's page to another, including session fixation through a +cached `Set-Cookie`. That last case is the sharpest: readthrough admits requests +carrying _no_ cookie, which is exactly the first-time visitor an origin issues a +session cookie to. + +`origin_is_cookie_independent = true` also widens this gate: cookie-bearing +requests with unlisted cookies can become readthrough-eligible. Named bypass +cookies and malformed cookie policies still refuse admission. Configuring any +`template_cache_key_cookies` disables readthrough, even when those cookies are +absent: the platform cache does not include their variant values in its key. +The template cache continues to separate those variants. On the template cache, +an origin's `Vary: Cookie` still overrides the independence assertion. On readthrough there is no such +response-side guard. Setting both flags is the highest-risk configuration and +requires a cookie-axis probe pass specifically. + +#### You cannot verify this locally + +Viceroy does not implement the readthrough cache. Measured with the gate enabled, the +request judged shareable, and a stub origin answering `Cache-Control: public, max-age=60` +with no `Set-Cookie`, two identical navigations still produced two origin fetches. The +local harness can therefore show the _decision_ this gate makes, and never its effect. + +The first evidence either way comes from a deployed service. Treat any local timing as +saying nothing about this setting. + +#### Enablement + +1. Run `ts origin probe-shareability --url `. Publisher cookies a + real reader carries go in `TRUSTED_SERVER_PROBE_COOKIES` as one cookie header value + (`name=value; name=value`), and a bot-wall admission cookie in + `TRUSTED_SERVER_PROBE_ADMISSION_COOKIE`. Both are environment-only, never flags: + these are credentials, and an argument is visible to every process on the host + through `ps` and lands in shell history. Each `--url` must be HTTPS; plain HTTP is + accepted only for a loopback development origin. Admission-cookie runs are + diagnostic only: every request carries that cookie, so cookieless responses remain + untested and the safety gate fails. Rerun against the origin without it before + enabling caching. +2. **Every axis and every verdict must pass.** Do not enable on a partial pass. + The probe checks status and safety headers on every sampled response, including + repeats. Each axis compares what a cache would store — the body **and** the policy + headers replayed with it, such as `Content-Security-Policy` — so an origin that + serves one document under two policies fails just as a varying document does. + Crawler user agents and prefetch requests are their own axes: neither + classification blocks readthrough, so an origin that answers a bot or a prefetch + with a different document without declaring `Vary` would otherwise have that + document cross-served to a reader. Any `Age` header, including `Age: 0`, blocks the verdict because a + fresh cached response can hide origin personalization. Pass `--vary-header =` + for each representative value of an additional request header, for example + `--vary-header x-exp-variant=A --vary-header x-exp-variant=B`. Each value is compared + against the absent baseline and the other supplied values, both with and without RSC. + Bare `--vary-header ` samples `1` for compatibility; it cannot establish safety + for categorical values the origin actually uses. Supply every relevant variant. + A declared `Vary` can explain a user-agent, RSC, + or custom-header difference only when every response declares it. Cookie + differences and different decoded gzip/identity documents always fail, because + the template cache requires those representations to be identical. + Navigation samples send HTML `Accept` and navigation Fetch Metadata and must + return `text/html`. RSC uses an explicit same-origin fetch profile, permitting + HTML fallback or `text/x-component`. A separate fetch control keeps `RSC` + variation independent of `Accept` and Fetch Metadata changes. If navigation + and fetch controls differ, all changed profile headers must be declared in + `Vary`; this conservative check cannot attribute a combined-profile difference + to one header. `Vary: *`, revalidation directives, and unreadable safety headers + always fail. + The probe is the only response-safety control on the readthrough path. +3. Read the probe's stated limits. It runs from one client address, so + personalisation keyed on the reader's IP — geo, rate class — is invisible to + it, as are `Accept-Language` and client-hint variants it does not vary. +4. Set `origin_readthrough_enabled = true` and push the configuration. +5. Watch the `origin_cache_shareable` breakdown in publisher summary telemetry. + Its denominator is matching-slot candidates, including skipped auctions; it + does not measure every publisher origin fetch or a site-wide admission rate. + See the [telemetry population and query](https://github.com/IABTechLab/trusted-server/blob/main/tinybird/README.md#the-denominator-is-matching-slot-candidates-not-all-requests). + The predicate estimates eligibility in that population before or after enablement, + not actual cache hits. +6. Confirm the origin's own hit rate and page correctness before widening to more + URLs. + +#### Rollback + +1. Set `origin_readthrough_enabled = false` and push. This takes effect on the + next request with no deploy and restores the previous policy: ad-serving + requests bypass, while non-ad traffic keeps the platform default. It does not + disable origin caching globally. +2. Purge tagged objects with `ts cache purge --service --all`, + or `--page ` for one exact reader-facing URL. Both the template cache + and opted-in origin readthrough objects carry the page key and `ts-template` + purge-all key. The readthrough tags use the original reader URL, before origin + rewriting, and work in both inline and ESI assembly modes. +3. Objects stored by older versions without readthrough tags remain unreachable + through these purge keys and must expire on the origin's TTL. Changing the + origin's TTL does not shorten an already-cached object's lifetime. +4. **Use `--all` on multi-host or dual-scheme deployments.** The template cache keys + on scheme and host, so purging each spelling you serve covers it. Readthrough does + not line up the same way: reader URLs that rewrite to one origin URL — `http://` + and `https://`, or `www.` and the apex on one service — share a single stored + object, tagged with the reader URL of whichever request filled it first. A + `--page https://example.com/a` can therefore leave an `http://`-tagged object in + place, and the next template miss refetches through it and re-stores the stale page + into the freshly purged template cache. If you serve one page under more than one + reader-facing spelling, purge with `--all`. + +The Fastly SDK attaches these tags to cached objects; production hit and purge +behavior still requires validation on a deployed service, since Viceroy does not +implement readthrough caching. + ### `gam_unit_path` templating `gam_unit_path` is a template. A publisher whose ad unit varies by site section diff --git a/docs/superpowers/plans/2026-09-15-852-predicate-split-and-observability.md b/docs/superpowers/plans/2026-09-15-852-predicate-split-and-observability.md new file mode 100644 index 000000000..94659ade1 --- /dev/null +++ b/docs/superpowers/plans/2026-09-15-852-predicate-split-and-observability.md @@ -0,0 +1,861 @@ +# Issue #852 implementation plan — part 1 of 3: predicate split and observability + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Separate the "may this origin response be shared" condition from the template-specific +one, and report both caches' outcomes on the existing auction telemetry row, so the later +readthrough change is measurable from its first deploy. + +> **Status: complete, in a trimmed form this document does not describe.** +> +> This plan was written for **three** telemetry fields. One shipped: `origin_cache_shareable`. +> +> - `template_cache_state` was dropped as **structurally unreachable** — telemetry emits during +> body streaming, before the template is stored, so the terminal state does not exist yet at +> the only point that could record it. See Task 9. +> - `template_cache_bypass_reason` was dropped as **out of scope**: it diagnoses the template +> cache, which belongs to #1009, not to #852. +> +> Everything below that says "three fields", "3 nullable columns" or names either dropped field +> — including Task 0's trim instruction, which names the opposite field from the one that was +> actually cut, and the code blocks in Tasks 2 and 3 — is superseded by that trim. The task +> bodies are kept as the record of what was planned. The code is the record of what was built. + +**Architecture:** One pure refactor, one small piece of new derivation, and three new nullable +telemetry fields. The refactor extracts both predicates into pure functions and splits them, +changing no behavior. The new derivation produces a structured reason for the **request-side** +template-cache bypass, which today has none. The telemetry fields ride the existing +`AuctionObservationContext` → `AuctionEventRow` → Tinybird path. + +**Tech Stack:** Rust 2024, `wasm32-wasip1` via Viceroy, `serde_json` NDJSON, Tinybird ClickHouse +datasource. + +**Spec:** `docs/superpowers/specs/2026-09-15-852-template-and-origin-caching-design.md` — read +"Splitting the predicate" and "Observability" before starting. + +**All of #852 ships as one PR.** This document is the first of three plan parts covering that one +change set, in commit order: + +| Part | Covers | Plan | +| -------- | ------------------------------------------------------------------------------ | ------------------------------------ | +| 1 (this) | Predicate split, observability, Tinybird migration, docs caveats, CI gate list | this file | +| 2 | Origin shareability probe; purge endpoint, key plumbing and CLI | `2026-09-15-852-probe-and-purge.md` | +| 3 | Readthrough gate, `ts-origin` staging check, runbook | `2026-09-15-852-readthrough-gate.md` | + +Complete them in order. Parts 2 and 3 depend on the binding and the telemetry this part creates. + +**Behavior impact of this part alone:** no change to responses, cache decisions, or the +`x-ts-template-cache` header — nothing consumes `origin_response_is_shareable` until part 3, and +the predicate split is behavior-neutral by construction. The telemetry row shape **does** change: +three always-serialized fields, which is why the Tinybird migration must reach Tinybird before the +release deploys. + +--- + +## Two things review found that shape this plan + +**1. The bypass reason has two sources and only one exists.** `template_cache_ttl` +(`publisher.rs:6129`) is called inside `template_cache_reservation.and_then(...)` (`:4775`), and +a reservation exists only when `template_cache_key` was built — which is +`request_can_use_shared_template.then(...)` (`:4370`). So `InlineMode`, `AuthorizedRequest` and +`CookieForwarded` can never fire there; those requests never get a key. The request-side bypass +sets only `TemplateCacheResponseState::BypassRequest` (`:4386`) and free-text `log::debug!` +(`:4360-4369`). Task 7 builds the missing request-side derivation. + +**2. No test harness has both a template cache and a telemetry sink.** `services()` (`:9247`) +sets `.template_cache(...)` and no sink; `services_with_telemetry()` (`:13719`) sets the sink and +no cache. Task 5 builds the combined one. Do not attempt Tasks 6–9 before it exists. + +--- + +## File structure + +| File | Responsibility | Change | +| ----------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/publisher.rs` | Publisher path; both predicates, the observation, the cache-state local | Predicate functions near `:4325`; observation `:4461`; request-side reason `:4370-4386`; response-side reason `:4785`; state hook `:4824` | +| `crates/trusted-server-core/src/auction/telemetry.rs` | Observation context, row schema, NDJSON | 3 fields on `AuctionObservationContext` (`:99`) and `AuctionEventRow` (`:277`); wire `base()` (`:347`) | +| `tinybird/datasources/auction_events_raw.datasource` | ClickHouse columns | Add 3 nullable columns | +| `tinybird/fixtures/auction_events_raw.ndjson` | Fixture rows | Add 3 keys to all 8 rows | + +--- + +## Task 0: Confirm the approval gate, and know the trim boundary + +The spec's Open risks section flags this work specifically: + +> **Observability is the largest refactor here and is not in #852.** Turning +> `AuctionObservationContext` from an immutable snapshot into a mutable accumulator, plus a +> 35-column schema migration with quarantine risk, sits close to AGENTS.md's "no large refactors +> without approval". It needs explicit approval before PR 1. + +- [x] **Step 1: Get explicit approval before writing code.** Tasks 2 and 4 are exactly the + refactor and the migration named above. + +- [x] **Step 2: If approval is withheld, take the trim instead of abandoning the PR.** The spec's + trim is to drop `template_cache_state` — it is already on the `x-ts-template-cache` response + header — and keep `template_cache_bypass_reason` and `origin_cache_shareable`, which carry + the triage. Concretely that means: **skip Task 9 entirely**, and drop the + `template_cache_state` field from Tasks 2, 3 and 4 (struct field, `base()` wiring, + datasource column, fixture key). Everything else is unchanged. Task 9 is also the most + intricate task in the plan, so the trimmed form is substantially cheaper. + +- [x] **Step 3: Record which form you are building** in the PR description, so a reviewer does not + read a missing `template_cache_state` as an oversight. + +--- + +## Task 1: Extract the predicates as pure functions, then split + +The split must be guarded by a test that exercises **production code**. A table test that +re-types the boolean expression guards nothing — it passes even if the refactor drops a term. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs:4325-4331` +- Test: same file, `#[cfg(test)]` + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn template_eligibility_implies_origin_shareability() { + for bits in 0u8..128 { + let inputs = SharedRequestInputs { + method_is_cacheable: bits & 1 != 0, + host_present: bits & 2 != 0, + authorization_disqualifies: bits & 4 != 0, + cookie_disqualifies: bits & 8 != 0, + request_requires_origin: bits & 16 != 0, + }; + let is_esi = bits & 32 != 0; + let reader_supports_assembly = bits & 64 != 0; + + let shareable = origin_response_is_shareable(inputs); + let template = request_can_use_shared_template(inputs, is_esi, reader_supports_assembly); + + assert!( + !template || shareable, + "template eligibility must imply origin shareability, input bits {bits}" + ); + assert_eq!( + template, + shareable && is_esi && reader_supports_assembly, + "template eligibility must be the shared base plus the two template conditions, \ + input bits {bits}" + ); + } +} + +#[test] +fn every_shared_input_is_necessary_for_shareability() { + let all_good = SharedRequestInputs { + method_is_cacheable: true, + host_present: true, + authorization_disqualifies: false, + cookie_disqualifies: false, + request_requires_origin: false, + }; + assert!(origin_response_is_shareable(all_good)); + + for (label, broken) in [ + ("method", SharedRequestInputs { method_is_cacheable: false, ..all_good }), + ("host", SharedRequestInputs { host_present: false, ..all_good }), + ("authorization", SharedRequestInputs { authorization_disqualifies: true, ..all_good }), + ("cookie", SharedRequestInputs { cookie_disqualifies: true, ..all_good }), + ("requires-origin", SharedRequestInputs { request_requires_origin: true, ..all_good }), + ] { + assert!( + !origin_response_is_shareable(broken), + "dropping the {label} condition must make the request unshareable" + ); + } +} +``` + +The second test is the one that actually catches a mistyped refactor. + +- [x] **Step 2: Run to verify it fails** + +Run: `cargo test-fastly -- publisher::tests::template_eligibility_implies --nocapture` +Expected: FAIL — `SharedRequestInputs` not found. + +- [x] **Step 3: Add the type and functions** + +Above `handle_publisher_request`, add: + +```rust +/// Request-side conditions that decide whether this request's origin response may be shared +/// between readers. Necessary for both the readthrough cache and the template cache, which is +/// why it is one type rather than two parallel expressions that must be kept in step. +#[derive(Debug, Clone, Copy)] +pub(crate) struct SharedRequestInputs { + pub(crate) method_is_cacheable: bool, + pub(crate) host_present: bool, + pub(crate) authorization_disqualifies: bool, + pub(crate) cookie_disqualifies: bool, + pub(crate) request_requires_origin: bool, +} + +/// Whether this request's origin response may be shared between readers at all. +pub(crate) fn origin_response_is_shareable(inputs: SharedRequestInputs) -> bool { + inputs.method_is_cacheable + && inputs.host_present + && !inputs.authorization_disqualifies + && !inputs.cookie_disqualifies + && !inputs.request_requires_origin +} + +/// Whether this request may additionally use a shared *template*. The two extra conditions say +/// whether this pipeline can assemble one, not whether the origin's bytes may be shared. +pub(crate) fn request_can_use_shared_template( + inputs: SharedRequestInputs, + assembly_mode_is_esi: bool, + reader_supports_assembly: bool, +) -> bool { + origin_response_is_shareable(inputs) && assembly_mode_is_esi && reader_supports_assembly +} +``` + +- [x] **Step 4: Replace the inline expression** + +At `publisher.rs:4325-4331`, replace the `let request_can_use_shared_template = …` binding with: + +```rust + let shared_request_inputs = SharedRequestInputs { + method_is_cacheable, + host_present: !request_host.is_empty(), + authorization_disqualifies, + cookie_disqualifies, + request_requires_origin, + }; + let origin_response_is_shareable = origin_response_is_shareable(shared_request_inputs); + let request_can_use_shared_template = request_can_use_shared_template( + shared_request_inputs, + matches!(assembly_mode, AssemblyMode::Esi), + reader_supports_assembly, + ); +``` + +If shadowing a function name with a local trips clippy, rename the locals to +`origin_is_shareable` / `can_use_shared_template` and update their use sites. + +- [x] **Step 5: Verify** + +Run: `cargo test-fastly` +Expected: PASS, no newly failing tests. (The alias already names all four wasm packages; an +extra `-p` narrows nothing.) Any template-cache test changing outcome means the +refactor was not behavior-neutral — revert and re-derive. + +Run: `cargo clippy-fastly` +Expected: no warnings. `origin_response_is_shareable` is unused until Task 6; if clippy objects, +land Task 6 before committing rather than adding an allow. + +- [x] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Split origin shareability out of template eligibility + +The single predicate mixed two questions: whether the origin response may be +shared at all, and whether this pipeline can assemble a shared template. +Gating anything but the template cache on the combined form would couple +readthrough caching to the assembly mode for no safety reason. + +Extracted as pure functions so the invariant is testable against real code +rather than a re-typed copy of the expression. Behavior is unchanged." +``` + +--- + +## Task 2: Add the cache fields to the observation context + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/telemetry.rs` — struct `:99`, `from_parts` `:158`, `from_auction_request` `:130` + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn observation_cache_fields_default_to_absent_and_round_trip() { + let mut observation = test_observation(); + + assert_eq!(observation.origin_cache_shareable, None); + assert_eq!(observation.template_cache_state, None); + assert_eq!(observation.template_cache_bypass_reason, None); + + observation.set_origin_cache_shareable(true); + observation.set_template_cache_state("hit"); + observation.set_template_cache_bypass_reason("request carried Cookie"); + + assert_eq!(observation.origin_cache_shareable, Some(true)); + assert_eq!(observation.template_cache_state.as_deref(), Some("hit")); + assert_eq!( + observation.template_cache_bypass_reason.as_deref(), + Some("request carried Cookie") + ); +} +``` + +There is **no** `test_observation()` helper in `telemetry.rs`'s test module — add one, because +Tasks 2 and 3 both need it and they must build the context identically. Follow the existing +pattern at `telemetry.rs:1032-1052`: `EcContext::new_for_test(None, ConsentContext::default())` +then `AuctionObservationContext::from_parts(...)`. + +- [x] **Step 2: Run to verify it fails** + +Run: `cargo test-fastly -- auction::telemetry::tests::observation_cache_fields --nocapture` +Expected: FAIL — no field `origin_cache_shareable`. + +- [x] **Step 3: Add the fields and setters** + +Add to `AuctionObservationContext` after `slot_count`, before the private `started_at`: + +```rust + /// Whether the readthrough gate admitted this request. `None` on sources that do not make + /// the decision, which is not the same as `Some(false)`. + pub origin_cache_shareable: Option, + /// Terminal template-cache state, matching `x-ts-template-cache`. + pub template_cache_state: Option, + /// Why the template cache declined, when it did. + pub template_cache_bypass_reason: Option, +``` + +Initialize all three to `None` in `from_parts` only — `from_auction_request` (`:130`) has no +struct literal; it delegates to `Self::from_parts(...)` at `:146`. Add +`set_origin_cache_shareable(&mut self, bool)`, `set_template_cache_state(&mut self, &str)`, +`set_template_cache_bypass_reason(&mut self, &str)`. + +- [x] **Step 4: Run to verify it passes** + +Run: `cargo test-fastly -- auction::telemetry::tests::observation_cache_fields --nocapture` +Expected: PASS. + +- [x] **Step 5: Confirm no other construction sites break** + +Run: `cargo check-fastly` +Expected: clean. `publisher.rs:6597` and `:18127` are `from_parts` _calls_, not struct literals, +so this step is normally a no-op — it exists to catch a literal construction added since. + +- [x] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/auction/telemetry.rs +git commit -m "Carry cache outcomes on the auction observation context + +Three absent-by-default fields and their setters. Nothing writes them yet." +``` + +--- + +## Task 3: Add the columns to the telemetry row + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/telemetry.rs` — struct `:277`, `base()` `:347` + +`AuctionTerminalStatus` is declared at `telemetry.rs:49`; check the variant spelling there. +`push_summary` is at `:661`. + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn summary_row_carries_cache_outcomes_from_the_observation() { + let mut observation = test_observation(); + observation.set_origin_cache_shareable(false); + observation.set_template_cache_state("bypass-response"); + observation.set_template_cache_bypass_reason("origin response carries Set-Cookie"); + + let mut rows = Vec::new(); + push_summary( + &mut rows, + &observation, + "2026-09-15 00:00:00.000", + AuctionTerminalStatus::Completed, + None, + 12, + 1, + ); + + let row = rows.first().expect("should emit one summary row"); + assert_eq!(row.origin_cache_shareable, Some(0)); + assert_eq!(row.template_cache_state.as_deref(), Some("bypass-response")); + assert_eq!( + row.template_cache_bypass_reason.as_deref(), + Some("origin response carries Set-Cookie") + ); +} + +#[test] +fn rows_omit_cache_outcomes_when_the_observation_has_none() { + let observation = test_observation(); + let mut rows = Vec::new(); + push_summary( + &mut rows, + &observation, + "2026-09-15 00:00:00.000", + AuctionTerminalStatus::Completed, + None, + 12, + 1, + ); + + assert_eq!( + rows.first().expect("should emit one row").origin_cache_shareable, + None, + "an unmeasured source must be distinguishable from a measured miss" + ); +} +``` + +- [x] **Step 2: Run to verify it fails** + +Run: `cargo test-fastly -- auction::telemetry::tests::summary_row_carries_cache --nocapture` +Expected: FAIL — no field on `AuctionEventRow`. + +- [x] **Step 3: Add the fields and wire `base()`** + +Add to `AuctionEventRow` after `ad_id`, using `u8` not `bool` to match the existing `is_mobile` / +`gdpr_applies` ClickHouse convention: + +```rust + /// `0` or `1`; absent when this source does not make the readthrough decision. + pub origin_cache_shareable: Option, + /// Terminal template-cache state. + pub template_cache_state: Option, + /// Why the template cache declined, when it did. + pub template_cache_bypass_reason: Option, +``` + +In `base()`, after `ad_id: None,`: + +```rust + origin_cache_shareable: observation.origin_cache_shareable.map(u8::from), + template_cache_state: observation.template_cache_state.clone(), + template_cache_bypass_reason: observation.template_cache_bypass_reason.clone(), +``` + +Setting these in `base()` rather than only in `push_summary` means provider and bid rows carry +them too — three nullable columns, and no per-row joins in the dashboard. + +**Flag this in the PR description as a deliberate deviation.** The spec scopes the fields to "the +auction telemetry summary row". Widening to `base()` is defensible but multiplies the emitted +payload across every row kind, so it should be a stated choice rather than a silent one. + +- [x] **Step 4: Run to verify it passes** + +Run: `cargo test-fastly -- auction::telemetry::tests --nocapture` +Expected: PASS. + +- [x] **Step 5: Confirm the NDJSON shape** + +Run: `cargo test-fastly -- auction::telemetry::tests --nocapture 2>&1 | tail -20` + +`to_ndjson` (`:424`) uses plain `serde_json::to_string` with no `skip_serializing_if`, so the +new key is **always** on the wire including as `null`. That is what makes Task 4 mandatory +and ordered before deploy. + +- [x] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/auction/telemetry.rs +git commit -m "Emit cache outcomes on auction telemetry rows + +Fields are always serialized, including as null, so the datasource must +declare them before this ships or rows land in quarantine." +``` + +--- + +## Task 4: Migrate the Tinybird datasource + +**Files:** + +- Modify: `tinybird/datasources/auction_events_raw.datasource`, `tinybird/fixtures/auction_events_raw.ndjson` + +- [x] **Step 1: Add the columns** + +In `SCHEMA >`, after `ad_id` and **before** `event_date`: + +``` + `origin_cache_shareable` Nullable(UInt8), +``` + +`LowCardinality` matches how `terminal_status` and `terminal_reason` are declared. The two new +string columns have 9 and 16 possible values respectively (`TemplateCacheResponseState` at `:94`, +`TemplateCacheBypassReason` at `:5673`), so dictionary encoding is right for both. Do not touch `ENGINE_SORTING_KEY` or the TTL. + +- [x] **Step 2: Update every fixture row** + +```bash +python3 - <<'PY' +import json, pathlib +p = pathlib.Path("tinybird/fixtures/auction_events_raw.ndjson") +rows = [json.loads(l) for l in p.read_text().splitlines() if l.strip()] +for i, r in enumerate(rows): + # Pre-existing gap: user_agent is declared in the datasource but absent from every row. + r.setdefault("user_agent", None) + r["origin_cache_shareable"] = None + r["template_cache_state"] = None + r["template_cache_bypass_reason"] = None + if i == 0: + r["origin_cache_shareable"] = 1 + r["template_cache_state"] = "hit" +p.write_text("\n".join(json.dumps(r) for r in rows) + "\n") +PY +``` + +- [x] **Step 3: Verify the fixture matches the schema** + +```bash +python3 - <<'PY' +import json, re, pathlib +schema = pathlib.Path("tinybird/datasources/auction_events_raw.datasource").read_text() +cols = [c for c in re.findall(r"`([a-z_]+)`", schema) if c != "event_date"] +rows = [json.loads(l) for l in pathlib.Path("tinybird/fixtures/auction_events_raw.ndjson").read_text().splitlines() if l.strip()] +for i, r in enumerate(rows): + assert not set(cols) - set(r), f"row {i} missing {sorted(set(cols) - set(r))}" + assert not set(r) - set(cols), f"row {i} undeclared {sorted(set(r) - set(cols))}" +print(f"ok: {len(rows)} rows match {len(cols)} declared columns") +PY +``` + +Expected: `ok: 8 rows match 34 declared columns`. + +**The fixture has a pre-existing gap.** Before any change, the datasource declares 33 non-`event_date` +columns and each fixture row has 32 keys: `user_agent` is declared and absent from every row. The +verifier above will trip on row 0 until that is fixed. Add `"user_agent": null` to every row in the +Step 2 script (it is a legitimate nullable column), and note in the commit that it was missing +beforehand — do not let the implementer chase it as damage from this change. + +- [x] **Step 4: Cross-check the Rust struct against the columns** + +Read the `AuctionEventRow` field list (`telemetry.rs:277`) and confirm every field name appears +in the datasource column list. Do this by eye against the struct — a `grep -c "pub "` over the +file counts fields across every struct in it and is not a usable check. A mismatch is the +quarantine bug and is silent at runtime. + +- [x] **Step 5: Commit** + +```bash +git add tinybird/datasources/auction_events_raw.datasource tinybird/fixtures/auction_events_raw.ndjson +git commit -m "Declare cache outcome columns on the auction events datasource + +Must reach Tinybird before the emitting code deploys; rows with undeclared +columns are quarantined rather than rejected loudly." +``` + +--- + +## Task 5: Build a test harness with both a template cache and a telemetry sink + +Tasks 6–9 all need one. Neither existing builder provides it: `services()` (`:9247`) sets +`.template_cache(...)` and no sink; `services_with_telemetry()` (`:13719`) sets the sink and no +cache. This task is why those tasks are not blocked on scaffolding invented mid-task. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` — the `template_cache_end_to_end_tests` module (8984–12336) + +- [x] **Step 1: Add the combined builder** + +In `template_cache_end_to_end_tests`, alongside the existing `services()`: + +```rust + fn services_with_cache_and_telemetry( + http_client: Arc, + cache: Arc, + telemetry_sink: Arc, + ) -> RuntimeServices { + let telemetry_sink: Arc = telemetry_sink; + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .template_cache(cache) + .auction_telemetry_sink(telemetry_sink) + .build() + } +``` + +There are three copies of `RecordingTelemetrySink` — `:13657` (`ssat_cache_policy_tests`), +`:18073` (directly in `mod tests`), `:21402` (`navigation_publisher_domain_tests`). **Do not move +anything.** The `:18073` copy is already in the shared parent module and is reachable from +`template_cache_end_to_end_tests` through its `use super::*` (`:8989`). You need only add +`use crate::auction::telemetry::AuctionTelemetrySink;` for the `Arc` +coercion — `mod tests` uses the fully-qualified path at `:18078` and does not import the trait. + +The builder method for the sink is exactly `.auction_telemetry_sink(...)`, confirmed against +`services_with_telemetry` (`:13719`). + +- [x] **Step 2: Add a summary-row accessor** + +`RecordingTelemetrySink` has **no accessor** — it is +`#[derive(Default)] struct RecordingTelemetrySink { batches: Mutex> }` +(`:18073`) and the trait impl reads the field directly. Read the field: + +```rust + fn last_summary_row(sink: &RecordingTelemetrySink) -> Option { + sink.batches + .lock() + .expect("should lock recorded telemetry batches") + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .next_back() + .cloned() + } +``` + +`AuctionEventBatch::rows()` returns `&[AuctionEventRow]` (`telemetry.rs:401`), so the +`flat_map` typechecks and `next_back()` is available on both slice-iterator layers. + +- [x] **Step 3: Add settings that emit a summary row** + +`run()` takes `&Arc` (`:9471`), not `&Settings`. Both settings helpers below must +return `Arc` — existing tests wrap at the call site (`:9601`); returning the `Arc` from +the helper is cleaner and keeps every test body in this plan correct as written. + +A summary row is emitted only when an auction runs, so the settings need `[auction] enabled = +true` **and** matching creative-opportunity slots. `ssat_cache_policy_tests` has +`settings_with_enabled_auction_and_creative_opportunities` at `:13675`; adapt it into this module +rather than hand-rolling a second one, and have it return `Arc`. + +Add a cookie-bearing request builder alongside the existing `navigation_request()` (`:9285`): + +```rust + fn navigation_request_with_cookie(cookie: &str) -> Request { + let mut req = navigation_request(); + req.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(cookie).expect("should build a cookie header"), + ); + req + } +``` + +- [x] **Step 4: Prove the harness works before relying on it** + +```rust + #[tokio::test] + async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::new(StubHttpClient::new()), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = settings_with_auction_and_slots(); + + let _ = run(&settings, &services, navigation_request()).await; + + assert!( + last_summary_row(&sink).is_some(), + "the harness must emit a summary row, or every later assertion is vacuous" + ); + } +``` + +Run: `cargo test-fastly -- template_cache_end_to_end_tests::harness_emits_a_summary_row --nocapture` +Expected: PASS. If it fails, fix the harness here — do not carry a broken harness into Task 6, +where the failure will look like a wiring bug. + +- [x] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Add a publisher test harness with both a template cache and a telemetry sink + +Neither existing builder wires both, so cache-outcome telemetry had no way to +be asserted end to end." +``` + +--- + +## Task 6: Record whether the readthrough gate admitted the request + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs:4461-4468` + +**Know which binding you are holding.** There are two. `observation` is a plain +`AuctionObservationContext` value built at `:4461`; `auction_observation` is the +`Option` declared at `:4446`, and `observation` is moved into it at +`:4511`. So this task sets the field on the **value**, before the move, while Tasks 8 and 9 reach +the **`Option`** with `as_mut()` because they run after it. The two are not in conflict. + +That also means the spec is wrong on this point. Its Observability/Carrier section says the value +"must be stashed in a local and threaded to the construction site, not written through a setter on +a binding that does not exist yet". The binding does exist: `origin_response_is_shareable` is known +at `:4325`, construction is at `:4461`, and a setter immediately after it works. Amend the spec +rather than following it here. + +- [x] **Step 1: Write the failing test** + +```rust + #[tokio::test] + async fn navigation_records_whether_the_origin_response_was_shareable() { + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::new(StubHttpClient::new()), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = settings_with_auction_and_slots(); + + let _ = run(&settings, &services, navigation_request_with_cookie("ts-ec=abc")).await; + + assert_eq!( + last_summary_row(&sink) + .expect("should emit a summary row") + .origin_cache_shareable, + Some(0), + "a cookie-bearing request must record as not shareable" + ); + } +``` + +- [x] **Step 2: Run to verify it fails** + +Run: `cargo test-fastly -- template_cache_end_to_end_tests::navigation_records_whether --nocapture` +Expected: FAIL — `origin_cache_shareable` is `None`. + +- [x] **Step 3: Set the field** + +Change the binding at `:4461` to `let mut observation = …` and add immediately after it: + +```rust + observation.set_origin_cache_shareable(origin_response_is_shareable); +``` + +- [x] **Step 4: Run to verify it passes** + +Run: `cargo test-fastly -- template_cache_end_to_end_tests::navigation_records_whether --nocapture` +Expected: PASS. + +- [x] **Step 5: Run the full module** + +Run: `cargo test-fastly` +Expected: PASS. Run the whole module — Viceroy aborts on first panic, so a single-test run hides +later failures. + +- [x] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Record origin shareability on the auction observation + +Makes the readthrough gate's effect measurable before the gate ships." +``` + +--- + +## Task 7: Derive a structured request-side bypass reason — MOVED OUT OF SCOPE + +**Removed from this PR.** Not in issue #852; the bypass reason diagnoses the template cache, which is #1009 feature work. Moved to successor issue B, which promotes that cache out of spike status and should instrument it as part of that. + +## Task 8: Record the bypass reason from both sources — MOVED OUT OF SCOPE + +**Removed from this PR.** Not in issue #852; the bypass reason diagnoses the template cache, which is #1009 feature work. Moved to successor issue B, which promotes that cache out of spike status and should instrument it as part of that. + +## Task 9: Record the terminal template-cache state — NOT DONE, and cannot be + +**Attempted and reverted.** The store outcome cannot reach the summary row. On a cold fill +`stream_publisher_body_async` collects the auction, takes the observation and emits the batch, +and only afterwards does `store_template_if_authorized` run and the state get stamped. The store +cannot move earlier (it needs the transform) and the emit cannot move later without giving up +collecting during body streaming, which is a latency decision on the path this issue exists to +improve. + +`hit` was reachable and `miss-stored` was not, so the column would have reported hits without +misses and made hit rate compute as roughly 100%. A silently wrong metric is worse than an absent +one, so `template_cache_state` was dropped from the observation, the row, the datasource and the +fixture — the spec's own trim, reached by the code rather than by the approval gate. + +`template_cache_bypass_reason` and `origin_cache_shareable` carry the triage. The +`x-ts-template-cache` header still reports all nine states per response for debugging one request. + +--- + +## Task 11: Write the dashboard caveats + +The spec requires these be stated where a dashboard author will read them, and the plan's own +doc rule is that each item's docs land in that item's PR rather than as a lump. Neither caveat is +discoverable from the Rust doc comments. + +**Files:** + +- Modify: `docs/guide/` — wherever auction telemetry / Tinybird consumers are documented. If no + such page exists, add the caveats next to the datasource in `tinybird/` as a README rather than + inventing a new docs page. + +- [x] **Step 1: Write both gaps** + 1. **The denominator is ad-serving pageviews, not all requests.** A summary row is emitted only + when an auction runs, so a request that bypasses the template cache _because_ the ad stack did + not run — bot, prefetch, kill-switched, consent-denied — produces no row at all. + 2. **`None` is not a miss.** `AuctionObservationContext` is `Clone` and shared with the + `/auction` source, where the column is structurally `None`. A dashboard that reads + `None` as "miss" will be wrong for that whole source class. Filter on + `auction_source = 'initial_navigation'` before computing any rate. + +- [x] **Step 2: Format** + +Run: `cd docs && ./node_modules/.bin/prettier --check ` + +- [x] **Step 3: Commit** + +```bash +git add +git commit -m "Document the two caveats on cache-outcome telemetry + +The denominator is ad-serving pageviews, and a null is an unmeasured source +rather than a cache miss. Both are silent misreadings otherwise." +``` + +--- + +## Task 10: Correct the documented CI gate list — MOVED OUT OF SCOPE + +**Removed from this PR.** Not in issue #852 — a correct drive-by docs fix, but scope creep here. Worth landing as its own small PR. + +## Final verification + +- [x] **Full gate set** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +./scripts/test-cli.sh +cd docs && npm run format && cd .. +``` + +- [x] **Confirm the change is confined to the expected files** + +```bash +git diff main --stat +``` + +Compare against the branch point rather than `main` if later parts have already landed on the +branch. Expected for this part: only `publisher.rs`, `auction/telemetry.rs`, the two Tinybird +files, and the docs touched by Task 11. `adapter-fastly/src/tinybird.rs` also appears: that is a +`mod tests` row literal that must gain the new field, not the telemetry struct leaking into +adapter code. Any _other_ adapter file appearing would be the leak this check is looking for. + +This check confirms _which files changed_, nothing more. Behavior neutrality of the predicate split +rests on Task 1's `every_shared_input_is_necessary_for_shareability` test and on Task 1 Step 5 — +any template-cache test changing outcome means the refactor was not neutral. + +- [x] **Apply the Tinybird migration before deploying** + +This is a deploy-ordering constraint, not a commit-ordering one — the PR merges atomically. +Owner: whoever runs the deploy. Apply the datasource change to Tinybird first, then deploy the +code, then confirm with a staging request that a summary row carries the new column and +that `tinybird/pipes/quarantine_counts.pipe` shows no new quarantined rows. Record that +confirmation on the PR; the spec's close-out criteria require it. diff --git a/docs/superpowers/plans/2026-09-15-852-probe-and-purge.md b/docs/superpowers/plans/2026-09-15-852-probe-and-purge.md new file mode 100644 index 000000000..fd9f77eac --- /dev/null +++ b/docs/superpowers/plans/2026-09-15-852-probe-and-purge.md @@ -0,0 +1,615 @@ +# Issue #852 implementation plan — part 2 of 3: probe and purge + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give the operator the two things the readthrough gate depends on — evidence that an +origin is safe to share, and a way to purge what gets cached. + +> **Status: Sections A and B complete; Sections C and D not started.** The checkboxes below +> track this. Section A shipped one axis and one verdict more than this plan describes — the RSC +> axis, and a blocking `fronting-cache` verdict added after review found that a cache in front of +> the origin can answer every axis from one stored object and turn the whole run green. + +**Architecture:** A new `ts origin probe-shareability` command that compares origin responses +across five axes and reports five response-header verdicts, all blocking. Plus a purge surface in +two halves: a reader-facing surrogate key attached at template-cache insert, and two consumers of +it — an authenticated admin endpoint and a CLI command. + +**Tech Stack:** Rust 2024, `reqwest` with `rustls-tls` (host target), `wasm32-wasip1` via Viceroy, +Fastly `NamedRoute` routing, `fastly::http::purge`. + +**Spec:** `docs/superpowers/specs/2026-09-15-852-template-and-origin-caching-design.md` — read +"Origin shareability probe" and "Purge" before starting. + +**Part 2 of 3.** Part 1 (`2026-09-15-852-predicate-split-and-observability.md`) must be complete +first. Part 3 is the readthrough gate, which will not be safe to enable without the probe this +part builds. + +--- + +## Why the probe's verdicts are blocking + +Not a style choice. Part 3's gate is decided **before** the origin responds, and no post-response +hook is reachable on this adapter (Viceroy stubs the HTTP Cache ABI — see +`adapter-fastly/src/template_cache.rs:6-15`). So none of `template_cache_ttl`'s response-side +refusals — `Set-Cookie` (`publisher.rs:6147`), CSP nonce (`:6117`), absent freshness (`:6100`) — +can be applied to the readthrough path. This probe is the only control. Build the verdicts as +pass/fail with a non-zero exit, not as advisory output. + +--- + +## File structure + +| File | Responsibility | Change | +| --------------------------------------------------------------------- | ------------------------------ | ------------------------------------------------------------------------------------ | +| `crates/trusted-server-cli/Cargo.toml` | CLI deps | Add `reqwest` to the non-wasm block | +| `crates/trusted-server-cli/src/run.rs` | Command enum | Add `Origin` and `Cache` variants | +| `crates/trusted-server-cli/src/commands/origin/` | Probe | Create | +| `crates/trusted-server-cli/src/commands/cache/` | Purge CLI | Create | +| `crates/trusted-server-cli/tests/support/` | Fixture server | Add a portable loop-accept server | +| `crates/trusted-server-core/src/platform/template_cache.rs` | Key + trait | `request_path` field, reader-facing key, canonicalization, `purge_url_surrogate_key` | +| `crates/trusted-server-core/src/publisher.rs` | Key construction, test doubles | Populate `request_path`; implement the new trait method on two doubles | +| `crates/trusted-server-adapter-fastly/src/{template_cache.rs,app.rs}` | Purge impl + route | New trait method; `NamedRoute` entry; handler | +| `crates/trusted-server-adapter-{axum,cloudflare,spin}/src/app.rs` | 501 routes | Register the path | +| `crates/trusted-server-core/src/settings.rs` | `ADMIN_ENDPOINTS` | Add the path | +| `crates/trusted-server-integration-tests/tests/parity.rs` | Parity | Authenticated POST helpers | +| `scripts/template-cache-local-test.sh` + `.github/workflows/test.yml` | Harness | Purge leg + workflow step | + +--- + +# Section A — Origin shareability probe + +## Task A1: Add an HTTP client the CLI can use on Linux + +**Files:** `crates/trusted-server-cli/Cargo.toml` + +The CLI's existing HTTP stack (`hyper`, `rustls`, `tokio` with `net`) is under +`[target.'cfg(target_os = "macos")'.dependencies]` (`:42`). The `cfg(not(target_arch = "wasm32"))` +block has `tokio` **without** `net` and no HTTP client. CI runs the CLI suite on Linux as well as +macOS, so the probe needs a client in the portable block. + +- [x] **Step 1: Add the dependency** + +In `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]`: + +```toml +reqwest = { workspace = true } +``` + +`reqwest` is already a workspace dependency with `default-features = false` and +`features = ["json", "rustls-tls"]` (root `Cargo.toml:93`), already in `Cargo.lock`, and already +built natively by the Axum adapter and the integration-tests crate. No new TLS backend is linked. + +- [x] **Step 2: Verify both targets still build** + +Run: `cargo check-fastly` — expected clean (the CLI is not in this alias; this confirms nothing +leaked into the wasm build). + +Run: `cargo check --package trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p')` — +expected clean. + +- [x] **Step 3: Commit** + +```bash +git add crates/trusted-server-cli/Cargo.toml Cargo.lock +git commit -m "Add a portable HTTP client to the operator CLI + +The existing hyper/rustls stack is macOS-only, and the shareability probe +must run on Linux CI." +``` + +## Task A2: Add a loop-accept fixture server for CLI tests + +**Files:** `crates/trusted-server-cli/tests/support/` + +The only existing fixture server is reachable solely from `tests/proxy_e2e.rs`, which is +`#![cfg(target_os = "macos")]` for the dependency reason above. + +**It must loop-accept.** A single-accept fixture already caused a CI flake in this repo, fixed in +PR #823: a browser opens several sockets including request-less preconnects, and the one-accept +server lost the race. The probe opens N connections by design via `--repeat`. + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn fixture_server_answers_repeated_requests() { + let server = FixtureServer::start(|_req| FixtureResponse::html("")); + + for _ in 0..3 { + let body = reqwest::blocking::get(server.url("/")).expect("should fetch").text().expect("should read body"); + assert_eq!(body, "", "every request must be answered, not just the first"); + } +} +``` + +- [x] **Step 2: Run to verify it fails** + +Run: `cargo test --package trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') fixture_server_answers` +Expected: FAIL — `FixtureServer` not found. + +- [x] **Step 3: Implement** + +A `std::net::TcpListener` on port 0 in a spawned thread, looping on `accept()` until a shutdown +flag is set, answering each connection from a caller-supplied closure. Expose `url(path)` built +from `local_addr()`. Plain `std::net` and `std::thread` — no async runtime, so it works on every +target the CLI tests run on. The response builder needs to set arbitrary status, headers, and +body, since later tasks assert on `Vary`, `Set-Cookie`, `Cache-Control` and CSP. + +- [x] **Step 4: Run to verify it passes** + +Run the same command. Expected: PASS. + +- [x] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/tests/support/ +git commit -m "Add a portable loop-accept fixture server for CLI tests + +Single-accept fixtures have flaked in this repo before, and the probe opens +several connections by design." +``` + +## Task A3: Wire the `ts origin probe-shareability` command skeleton + +**Files:** `crates/trusted-server-cli/src/run.rs`, `crates/trusted-server-cli/src/commands/origin/` + +- [x] **Step 1: Add the command** + +Follow the `audit` and `dev` pattern — TS-local, not delegated to `edgezero_cli`. Add an `Origin` +variant with a `#[command(subcommand)] OriginCommand`, one variant `ProbeShareability`, and args: + +```rust +pub(crate) struct ProbeShareabilityArgs { + /// URL to probe. May be repeated. + #[arg(long, required = true)] + pub(crate) url: Vec, + /// How many times to repeat the self-identity comparison. + #[arg(long, default_value_t = 3)] + pub(crate) repeat: u32, + /// Extra cookie to send in the cookie-bearing arm, as name=value. May be repeated. + #[arg(long)] + pub(crate) cookie: Vec, + /// Emit machine-readable output. + #[arg(long)] + pub(crate) json: bool, +} +``` + +Dispatch it in `run()`'s `match` alongside the existing arms. + +- [x] **Step 2: Verify it is reachable** + +Run: `cargo run --package trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- origin probe-shareability --help` +Expected: the help text renders. + +- [x] **Step 3: Commit** + +```bash +git add crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/ +git commit -m "Add the ts origin probe-shareability command skeleton" +``` + +## Task A4: Implement the five comparison axes + +**Files:** `crates/trusted-server-cli/src/commands/origin/` + +Each axis fetches the URL twice and compares the bodies. All four are blocking. + +| Axis | Arm A | Arm B | Compare | +| ----------------- | ---------- | ------------------------------------------ | ---------------------- | +| Self-identity | bare | bare, repeated `--repeat` times | raw bytes | +| Cookie | bare | with the TS cookie set plus any `--cookie` | raw bytes | +| `Accept-Encoding` | `gzip` | `identity` | bytes **after decode** | +| `User-Agent` | desktop UA | mobile UA | raw bytes | +| RSC / router | bare | `rsc: 1` plus configured `next-router-*` | raw bytes | + +The TS cookie set for the cookie arm is `ts-ec`, the consent cookies from +`CONSENT_COOKIE_NAMES` (`core/src/cookies.rs:20`), and the tester cookie — representative of what +a real repeat visitor carries. + +**The RSC axis is specific to this change and easy to miss.** RSC fetches are not navigations — +`http_util.rs:73-82` requires `Sec-Fetch-Dest: document` — so they never set the bypass and +**already flow through the readthrough cache today**, while HTML navigations are PASS. Removing +the bypass puts both representations under one cache key for the first time. If the origin varies +on `rsc` / `next-router-*` without declaring it in `Vary`, the cache can serve a flight payload to +an HTML navigation. Recorded in the #1009 measurement findings as a risk nobody had considered. +Drive this axis from the operator's configured `template_cache_vary` list rather than a fixed set, +since the varying headers are publisher-specific. + +- [x] **Step 1: Write the failing tests** + +One test per axis against the fixture server, each asserting the axis reports a difference when +the fixture varies on that input and reports identical when it does not. For the +`Accept-Encoding` axis, the fixture must actually gzip one arm so the test proves the comparison +happens after decode, not before. + +Plus one test that a non-self-identical origin (fixture returns a counter in the body) fails the +self-identity axis — that is the most common real-world failure and must not be reported as a +cookie problem. + +- [x] **Step 2: Run to verify they fail** + +Run: `./scripts/test-cli.sh` (or the explicit host-triple command). Expected: FAIL. + +- [x] **Step 3: Implement** + +Report per axis: identical or differing, and on difference the byte offset of the first +divergence plus a short context window from each side. Keep the window small and escape it — it +is publisher HTML and may be large or binary-ish. + +- [x] **Step 4: Run to verify they pass** + +Expected: PASS. + +- [x] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/origin/ +git commit -m "Compare origin responses across the four shareability axes + +Self-identity is first because an origin that is not stable against itself +cannot be shared on any axis, and reporting that as a cookie failure would +send the operator after the wrong thing." +``` + +## Task A5: Implement the five response-header verdicts + +**Files:** `crates/trusted-server-cli/src/commands/origin/` + +| Verdict | Fails when | Why it blocks | +| ------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Positive shared freshness | no positive `Cache-Control`/`Surrogate-Control` freshness | Readthrough would store on a platform default TTL where the template cache refuses (`NoPositiveFreshness`, `publisher.rs:6100`) | +| No `Set-Cookie` | the response carries one | Cached and replayed to every later cookieless reader — cross-reader session fixation. Template cache refuses at `:6147`; readthrough cannot | +| No CSP `nonce` | CSP contains `'nonce-` | A shared nonce silently defeats the origin's own XSS defence. Template cache refuses at `:6117` | +| `Vary` coverage | a varying axis is not named in `Vary` | Readthrough keys on URL plus origin `Vary` only | + +- [x] **Step 1: Write the failing tests** + +One per verdict, fixture-driven. The `Vary`-coverage test is the interesting one: a fixture that +varies on `User-Agent` **and** declares `Vary: User-Agent` must pass, while the same fixture +without the declaration must fail. + +- [x] **Step 2–4: Run, implement, run** + +Same loop as A4. + +- [x] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/origin/ +git commit -m "Report the four blocking response-header verdicts + +These are the only control standing between the readthrough gate and +cross-serving: the gate is decided before the origin responds, so none of the +template cache's response-side refusals are reachable there." +``` + +## Task A6: Output, exit code, and stated limits + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn probe_exits_non_zero_when_any_verdict_fails() { /* fixture sets Set-Cookie */ } + +#[test] +fn probe_json_output_names_every_axis_and_verdict() { /* --json shape */ } +``` + +- [x] **Step 2: Implement** + +Human-readable by default; `--json` for CI. **Non-zero exit on any blocking failure**, so it can +gate a deploy. + +Print the limits every run, not only on failure: + +- Runs from one client IP, so origin personalization keyed on the forwarded client address (geo, + rate-class) is **undetectable** by this tool. +- The verdict covers the sampled URLs only, not the origin as a whole. + +- [x] **Step 3: Run, then commit** + +```bash +git add crates/trusted-server-cli/src/commands/origin/ +git commit -m "Gate on the probe verdict and state what the probe cannot see + +A clean result on one URL from one IP is not a statement about the origin." +``` + +--- + +# Section B — Purge key plumbing + +## Task B1: Add `request_path` to the cache key + +**Files:** `crates/trusted-server-core/src/platform/template_cache.rs`, `publisher.rs:4370` + +`TemplateCacheKey.url` is the **origin-rewritten** target URI (`publisher.rs:4372`, built at +`:4182`), not the URL an operator types. The struct has `url`, `request_host`, `request_scheme` +and **no path field**. Without one, a "reader-facing" key would have to be reconstructed from the +origin path, which is the coupling this section exists to remove. + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn reader_facing_key_ignores_origin_rewriting() { + let mut a = key(); + let mut b = key(); + a.url = "https://origin.internal.example/article".to_owned(); + b.url = "https://other-origin.example/article".to_owned(); + a.request_path = "/article".to_owned(); + b.request_path = "/article".to_owned(); + + assert_eq!( + a.reader_url_surrogate_key(), + b.reader_url_surrogate_key(), + "two origins behind one reader-facing URL must share the reader-facing purge key" + ); + assert_ne!( + a.url_surrogate_key(), + b.url_surrogate_key(), + "the origin-derived key must stay distinct; core uses it to evict one bad object" + ); +} +``` + +- [x] **Step 2–4:** run (fails), add `pub request_path: String` populated from the **pre-rewrite** + request at `publisher.rs:4370`, run again. + +Note `to_cache_key()` must include `request_path` in its canonical input, since it changes the +emitted bytes. Bump `TEMPLATE_SCHEMA_VERSION` — the version table at `template_cache.rs:30-36` +documents why, and a missed bump reads yesterday's template against today's key shape. + +- [x] **Step 5: Commit** + +## Task B2: Extract `url_surrogate_key` and define canonicalization + +**Files:** `crates/trusted-server-core/src/platform/template_cache.rs` + +`url_surrogate_key()` (`:147`) is a raw SHA-256 over exact bytes, and +`punctuation_distinct_urls_have_distinct_surrogate_keys` (`:961`) makes byte-exactness +load-bearing. Both the endpoint and the CLI must hash the same string as insert, or a purge +returns 200 and invalidates nothing — the worst failure mode on an incident path. + +- [x] **Step 1: Write the failing test** + +```rust +#[test] +fn reader_url_canonicalization_is_stable_across_operator_spellings() { + for (a, b) in [ + ("https://example.com/article", "https://example.com/article/"), + ("https://Example.COM/article", "https://example.com/article"), + ("https://example.com:443/article", "https://example.com/article"), + ("https://example.com/article?", "https://example.com/article"), + ] { + assert_eq!( + reader_url_surrogate_key(a), + reader_url_surrogate_key(b), + "{a} and {b} name the same page and must purge together" + ); + } + + assert_ne!( + reader_url_surrogate_key("https://example.com/a"), + reader_url_surrogate_key("https://example.com/b"), + ); +} +``` + +Decide and document the query-string rule explicitly: a query is **significant** (different query, +different page) but an empty `?` is not. + +- [x] **Step 2–4:** run, implement `pub fn reader_url_surrogate_key(url: &str) -> String` as a free + function with `TemplateCacheKey::reader_url_surrogate_key()` delegating to it, run again. + +Use a distinct prefix, `ts-template-readerurl-`, and assert in the existing surrogate-key test +that it never collides with `ts-template-url-`. Without the distinct prefix the two derivations +can alias when a staging edge host equals the configured origin host — over-purge rather than a +read leak, but a purge reporting success against an unrelated object. + +- [x] **Step 5:** add it to `surrogate_keys()` (`:138`) so it is attached at insert. The `Vec` + already exists; an extra entry is free. + +- [x] **Step 6: Commit** + +## Task B3: Add `purge_url_surrogate_key` to the platform trait + +**Files:** `platform/template_cache.rs:675`, and all five implementors + +`purge_url` takes `&TemplateCacheKey` (`:675`), which a handler holding only a URL cannot build — +it needs `origin_identity`, `template_fingerprint`, `vary_values` and `schema_version`. + +Implementors: `UnavailableTemplateCache` (`template_cache.rs:698`), +`adapter-fastly/src/template_cache.rs:163`, `adapter-fastly/src/app.rs:2840`, and the two test +doubles at `publisher.rs:8843` and `:9112`. The doubles are spelled +`impl crate::platform::PlatformTemplateCache`, which a naive grep misses. + +- [x] **Step 1–4:** test on the Fastly impl and the null object, add + `async fn purge_url_surrogate_key(&self, key: &str) -> Result<(), TemplateCacheError>`, + implement across all five, run. + +`UnavailableTemplateCache` must **not** silently succeed — a no-op purge reporting success is +worse than an error. Return the same unsupported signal the endpoint turns into a 501. + +- [x] **Step 5: Commit** + +--- + +# Section C — Admin purge endpoint + +## Task C1: Register the route on Fastly with its guards + +**Files:** `adapter-fastly/src/app.rs:1128`, `core/src/settings.rs:3214` + +``` +POST /_ts/admin/cache/purge +Content-Type: application/json + +{"scope": "all"} | {"scope": "url", "url": "https://example.com/page"} +``` + +Auth is inherited: `AuthMiddleware` (`app.rs:1305`) runs `enforce_basic_auth` (`core/src/auth.rs:79`) +before route matching, and fails closed when no handler regex covers the path (`auth.rs:93-100`). + +- [ ] **Step 1: Write the failing tests** + +- purge-all succeeds; purge-url succeeds +- unauthenticated request rejected +- **non-POST returns 405 and does not reach the origin** +- wrong `Content-Type` rejected +- oversized body rejected +- no legacy `/admin/cache/purge` alias resolves + +- [ ] **Step 2–4: Implement with every guard** + +- **Register the path as a string literal** in `NAMED_ROUTES`. The + `admin_endpoints_match_fastly_router` check (`settings.rs:7727`) scans literal `path:` entries + only; a named constant silently skips coverage. +- **Add the path to `Settings::ADMIN_ENDPOINTS`** (`settings.rs:3214`). Its doc comment says to. + It feeds live config validation at `:3265`, so an operator `trusted-server.toml` whose handler + regexes do not cover the new path will start failing validation — **call this out as a migration + note in the PR description.** +- **Register for all methods and 405 in-handler.** Do not rely on `primary_methods`: non-primary + methods on a named path fall through to the publisher (`app.rs:42`), and `enforce_basic_auth` + leaves the `Authorization` header in place so it "still reaches the publisher origin" + (`auth.rs:60-66`). A `GET` would authenticate, fall through, and ship the shared admin + credential upstream. Follow the `/auction` `OPTIONS` precedent (`app.rs:1209-1213`). +- **Enforce `Content-Type: application/json` exactly.** Basic-auth credentials are attached + automatically by browsers, so a cross-origin form POST with `enctype="text/plain"` sends no + preflight; method alone does not stop CSRF. +- **Cap the request body.** The `url` field is attacker-supplied and only ever hashed. +- **Audit-log the authenticated principal for `{"scope":"all"}`.** It is an unbounded cache-flush + and origin-stampede lever behind one shared static credential. There is no HTTP-handler + rate-limit primitive here (`ec/rate_limiter.rs` is partner batch/pull-sync only), so either add + a minimum interval or record the accepted risk explicitly in the PR. +- **Define the partial-failure answer.** `purge_all` returns a single `Result` from one + `purge_surrogate_key` call (`adapter-fastly/src/template_cache.rs:285`); an operator mid-incident + needs to know whether to retry. Say so in the response body. +- Response is `private, no-store`. +- Replay is not a concern — purge is idempotent. State that rather than leaving it unaddressed. + +`{"scope":"url"}` hashes the **reader-facing** key from Task B2, so the handler needs no origin +rewriting. + +- [ ] **Step 5: Commit** + +## Task C2: Register 501 on the other three adapters + +**Files:** `adapter-axum/src/app.rs:310`, `adapter-cloudflare/src/app.rs:515-545`, `adapter-spin/src/app.rs:211` and `:854` + +An unregistered path falls through to origin and 404s, which reads to a CMS webhook as "endpoint +does not exist" rather than "not supported here". + +- [ ] **Step 1–4:** test, register, run. + +Axum's `named_routes()` returns `[NamedRoute; 16]` — a fixed-size array that must become **17**. +Spin needs both the route list (`:211`) and the router (`:854`). Cloudflare uses a builder chain +and already has `StatusCode::NOT_IMPLEMENTED` at `:281`. + +- [ ] **Step 5: Commit** + +## Task C3: Add authenticated POST helpers to the parity suite + +**Files:** `crates/trusted-server-integration-tests/tests/parity.rs` + +The suite configures `path = "^/_ts/admin"` with basic auth (`:16-17`), and the existing +`axum_post`/`cf_post`/`spin_post` helpers send **no credentials** — an unauthenticated probe gets +401 and never reaches the handler, so a naive 501 test would pass for the wrong reason. + +- [ ] **Step 1–4:** add credential-carrying helpers (`spin_post_with_headers` at `:200` is a usable + template), then assert 501 on all three adapters. + +If the helper pulls a new dependency, this crate has its own lockfile with a shared-direct-dep +alignment gate — fix with a targeted `cargo update -p --precise `, never a full +update. + +- [ ] **Step 5: Commit** + +--- + +# Section D — Purge CLI command + +## Task D1: `ts cache purge` + +**Files:** `crates/trusted-server-cli/src/run.rs`, `crates/trusted-server-cli/src/commands/cache/` + +``` +ts cache purge --all +ts cache purge --url +``` + +With Task B2 landed this is a thin wrapper: hash the typed URL with the shared free function, +call the Fastly purge API. No config load, no origin logic. + +- [ ] **Step 1: Write the failing test** + +Assert the CLI computes the **same** key as core for the same logical page — the regression that +keeps the two from drifting: + +```rust +#[test] +fn cli_and_core_agree_on_the_reader_facing_purge_key() { + let url = "https://example.com/article"; + assert_eq!(cli_purge_key(url), trusted_server_core::platform::reader_url_surrogate_key(url)); +} +``` + +- [ ] **Step 2–4:** run, implement, run. + +**Token scope.** `adapter-fastly/src/management_api.rs:12` records that today's token is +write-scoped with **no purge permission**, and `edgezero_cli` exposes no credential helper — only +whole-command runners. Read `FASTLY_API_TOKEN` with a `--token` override, document the required +scope, and fail with an actionable message naming the missing scope. + +**If the scope cannot be granted, drop this commit alone.** Nothing else depends on it; the admin +endpoint stands on its own. + +- [ ] **Step 5: Commit** + +## Task D2: Extend the local harness with a purge leg + +**Files:** `scripts/template-cache-local-test.sh:17-18`, `.github/workflows/test.yml` + +Viceroy 0.17 implements `purge_surrogate_key` against the same in-process cache it serves reads +from (verified in `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md:156-158`), so +store → hit → purge → miss is end-to-end testable without a Fastly service. This is the strongest +test in the whole plan. + +- [ ] **Step 1:** the script accepts only `inline|esi` today and CI invokes those two literals, so + a new `purge` mode needs a **matching workflow step**. Add both. + +- [ ] **Step 2:** add the mode to the AGENTS.md gate list entry created in part 1, Task 10. + +- [ ] **Step 3: Commit** + +--- + +## Final verification for part 2 + +- [ ] **Full gate set** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm +cargo clippy-cli && cargo clippy-codegen +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +./scripts/test-cli.sh +./scripts/template-cache-local-test.sh purge +cd docs && npm run format && cd .. +``` + +- [ ] **Confirm the probe actually blocks** + +Run it against the fixture origin with a `Set-Cookie` response and confirm a non-zero exit. A +probe that reports a failure and exits 0 is worse than no probe, because part 3's safety argument +rests on it. + +- [ ] **Record the `ts-origin` staging verdict** + +Part 3 needs it and it cannot be tested under Viceroy. Timebox it here, while the purge trait is +already open: on a staging Fastly service, set a surrogate key on an origin fetch via +`Request::set_surrogate_key`, then purge it and confirm the object is gone. Write the result into +the PR description either way — part 3's rollback section depends on the answer. diff --git a/docs/superpowers/plans/2026-09-15-852-readthrough-gate.md b/docs/superpowers/plans/2026-09-15-852-readthrough-gate.md new file mode 100644 index 000000000..db17fc24a --- /dev/null +++ b/docs/superpowers/plans/2026-09-15-852-readthrough-gate.md @@ -0,0 +1,311 @@ +# Issue #852 implementation plan — part 3 of 3: the readthrough gate + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop forcing every ad-serving pageview to origin, and make the resulting sharing +decision deliberate for the whole request population rather than half of it. + +**Architecture:** One condition, at two call sites. Readthrough is enabled by **omitting** +`set_pass`, not by adding a TTL override. Safety rests on probe-verified operator preconditions, +because no response-side hook is reachable on this adapter. + +**Tech Stack:** Rust 2024, `wasm32-wasip1` via Viceroy, `fastly` 0.12.1 `CacheOverride`. + +**Spec:** `docs/superpowers/specs/2026-09-15-852-template-and-origin-caching-design.md` — read +"Origin readthrough" end to end before starting. Not just the gate: the mechanism section and the +response-side gap are the parts that constrain the implementation. + +**Part 3 of 3.** Parts 1 and 2 must be complete. This part is inert without part 1's +`origin_response_is_shareable` binding, and unsafe to enable without part 2's probe. + +--- + +## Read this before writing any code + +**This is the only change in #852 with new runtime blast radius.** Everything else is +instrumentation and tooling. Ask for this commit to be reviewed on its own. + +Three things are settled and must not be re-litigated mid-implementation: + +1. **Do not add a TTL override.** `Request::set_ttl` (`fastly-0.12.1/src/http/request.rs:2381`) + "overrides any previous `Request::set_pass` call and sets the `pass` behavior to `false`", and + overrides the origin's `Cache-Control` including `private` and `no-store`. Adding it would turn + the hazard this work exists to close into a first-class API. `set_ttl(0)` does not help either + — it caches with zero TTL, it does not bypass. +2. **`after_send` / `CandidateResponse` is unreachable.** Viceroy 0.17 stubs the HTTP Cache ABI + and the SDK converts that into a send error, so setting `after_send` makes every publisher + origin fetch fail under `fastly compute serve`, `cargo test-fastly`, and the parity suite. + Recorded at `adapter-fastly/src/template_cache.rs:6-15`. +3. **`set_pass` and `set_surrogate_key` are mutually exclusive, and order-dependent.** + `set_surrogate_key` (`request.rs:2462`) carries the same override note. Calling it after + `set_pass(true)` reverses the bypass. The platform layer must make that unrepresentable. + +--- + +## File structure + +| File | Responsibility | Change | +| ---------------------------------------------------------- | ---------------- | ---------------------------------------------------------- | +| `crates/trusted-server-core/src/publisher.rs` | Bypass decision | The condition at `:4415` and `:4716` | +| `crates/trusted-server-core/src/platform/http.rs` | Platform request | Replace the `bypass_cache` bool with a cache-intent enum | +| `crates/trusted-server-adapter-fastly/src/platform.rs:445` | Fastly mapping | Apply the enum; attach `ts-origin` on the shareable branch | +| `docs/guide/configuration.md` | Runbook | Enablement, rollback, preconditions | + +--- + +## Task 1: Make the pass/surrogate-key conflict unrepresentable + +**Files:** `crates/trusted-server-core/src/platform/http.rs:16-37`, `adapter-fastly/src/platform.rs:445` + +Today `PlatformHttpRequest` carries `bypass_cache: bool`. Once the shareable branch also attaches +a surrogate key, two booleans could express "bypass **and** key", which on Fastly silently means +"do not bypass". Encode the intent instead. + +- [ ] **Step 1: Write the failing test** + +```rust +#[test] +fn cache_intent_cannot_request_bypass_and_a_surrogate_key_at_once() { + let bypass = PlatformCacheIntent::Bypass; + let shared = PlatformCacheIntent::Shared { surrogate_key: "ts-origin".to_owned() }; + + assert!(bypass.surrogate_key().is_none()); + assert_eq!(shared.surrogate_key(), Some("ts-origin")); + assert!(!shared.is_bypass()); + assert!(bypass.is_bypass()); +} +``` + +- [ ] **Step 2: Run to verify it fails.** `cargo test-fastly -- platform::http::tests::cache_intent` + +- [ ] **Step 3: Implement** + +```rust +/// What the caller wants the platform's intermediary cache to do with this request. +/// +/// One enum rather than two flags because on Fastly they are not independent: +/// `set_surrogate_key` and `set_ttl` each reverse a prior `set_pass(true)` +/// (`fastly-0.12.1/src/http/request.rs:2462`, `:2381`). Two booleans could express a +/// combination that silently means the opposite of what it reads as. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlatformCacheIntent { + /// Let the platform apply its default behavior, honoring origin freshness. + Default, + /// Do not use the intermediary cache for this request. + Bypass, + /// Allow caching, tagged for purge. + Shared { surrogate_key: String }, +} +``` + +Replace `bypass_cache: bool` with `cache_intent: PlatformCacheIntent`. Keep +`with_cache_bypass()` as a builder that sets `Bypass` so existing call sites need no change, and +add `with_shared_cache(surrogate_key)`. + +In `apply_fastly_cache_bypass` (`adapter-fastly/src/platform.rs:445`), rename to +`apply_fastly_cache_intent` and match: `Bypass` → `set_pass(true)`; `Shared` → +`set_surrogate_key(...)` and **no** `set_pass`; `Default` → nothing. + +Other adapters ignore the intent as they ignore `bypass_cache` today — Spin hard-rejects only +`stream_response` (`adapter-spin/src/platform.rs:304`), so nothing there needs to change. + +- [ ] **Step 4: Run to verify it passes**, plus `cargo check-fastly && cargo check-axum && cargo check-cloudflare && cargo check-spin`. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/platform/http.rs crates/trusted-server-adapter-fastly/src/platform.rs +git commit -m "Model platform cache intent as one enum rather than two flags + +On Fastly, setting a surrogate key or a TTL reverses a prior set_pass. Two +booleans could express a combination that means the opposite of how it reads." +``` + +## Task 2: Gate the bypass on shareability, at both sites + +**Files:** `crates/trusted-server-core/src/publisher.rs:4415`, `:4716` + +**Both sites change.** They are alternative paths for the same fetch: `:4415` populates +`pending_origin` inside the EC-preload fan-out block, and `:4716` is the `else` branch of +`if let Some(pending) = pending_origin` (`:4703`). Changing only one makes readthrough eligibility +depend on whether EC preload fired — and `should_preload_ec_snapshot` (`:2955`) is +`is_navigation && is_get && has_ec_id && has_kv`, much of the population this exists for. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn ineligible_requests_bypass_the_platform_cache() { + // cookie-bearing, Authorization-bearing, non-GET, request_requires_origin + // each assert the recorded intent is Bypass +} + +#[test] +fn eligible_requests_do_not_bypass() { + // cookieless GET navigation asserts the recorded intent is not Bypass +} + +#[test] +fn preload_and_non_preload_paths_agree_on_cache_intent() { + // same inputs through both sites produce the same intent +} + +#[test] +fn ineligible_bot_traffic_now_bypasses_where_it_previously_did_not() { + // the tightening half of the change +} +``` + +`recorded_cache_bypass_flags()` (`platform/test_support.rs:450`) captures what is needed; widen it +to record the intent rather than a bool. + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + +```rust +if !origin_response_is_shareable { + platform_request = platform_request.with_cache_bypass(); +} +``` + +at both `:4415` and `:4716`. `should_run_ad_stack` is **gone** from the condition. + +- [ ] **Step 4: Run to verify they pass**, then the whole module: `cargo test-fastly`. + +- [ ] **Step 5: State the two-sided effect in the commit** + +This is **not** "strictly more conservative", and the commit message must not say so: + +- **Tightening** for cookie-bearing, `Authorization`-bearing, or otherwise ineligible bot and + prefetch traffic, which now gets `set_pass` where today it does not. +- **Widening** for cookieless traffic: eligible ad-serving requests begin reading objects that + cookieless bots and prefetchers store. Today those objects are written and read only by + non-ad-stack requests. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Gate the origin cache bypass on shareability rather than the ad stack + +should_run_ad_stack means 'this page serves ads', which is unrelated to +whether the origin response may be shared, and left every non-ad-stack +request sharing the readthrough cache with no eligibility check at all. + +Tightening for ineligible bot and prefetch traffic, which now bypasses. +Widening for cookieless traffic, which begins reading objects those requests +store. The probe's blocking verdicts are the control for the widening." +``` + +## Task 3: Attach the `ts-origin` surrogate key on the shareable branch + +**Files:** `crates/trusted-server-core/src/publisher.rs:4415`, `:4716` + +**Gated on the staging verdict from part 2's final verification.** If `set_surrogate_key` does not +tag readthrough objects on a real service, skip this task and take Task 4's fallback wording. + +The key may be applied **only** on the shareable branch. Stamping it unconditionally would +disable the bypass for every request, including disqualified ones — see the mutual-exclusion note +at the top. + +- [ ] **Step 1: Write the failing test** + +```rust +#[test] +fn shareable_requests_carry_a_purgeable_origin_key() { + // eligible request: intent is Shared with both the global and per-URL origin keys +} + +#[test] +fn ineligible_requests_carry_no_surrogate_key() { + // the mutual-exclusion invariant, asserted at the call site not just the type +} +``` + +- [ ] **Step 2–4:** run, implement `with_shared_cache(...)` on the eligible branch using + `ts-origin` plus a per-URL variant from part 2's `reader_url_surrogate_key`, run again. + +- [ ] **Step 5:** extend the purge endpoint and CLI from part 2 to purge `ts-origin` alongside + `ts-template`, so `--all` means both caches. Update their tests. + +- [ ] **Step 6: Commit** + +## Task 4: Runbook and preconditions + +**Files:** `docs/guide/configuration.md` + +- [ ] **Step 1: Write the enablement procedure** + +1. Run `ts origin probe-shareability --url `. +2. **Every axis and every verdict must pass.** Do not enable on a partial pass. The probe is the + only control — the gate is decided before the origin responds, so none of the template cache's + response-side refusals apply to this path. +3. Set `origin_readthrough_enabled = true`. (Drafted as `origin_is_cookie_independent`; that + flag only ever applies to cookie-_bearing_ requests, so it could not gate readthrough, which + admits cookieless ones. A separate flag shipped.) +4. Watch the `origin_cache_shareable` breakdown from part 1. (`template_cache_bypass_reason` was + designed alongside it and cut as out of scope for #852 — do not reach for it here.) +5. Confirm hit rate before widening to more URLs. + +- [ ] **Step 2: Write the rollback procedure, honestly** + +Two levers, in order of speed: + +1. **Config:** set `origin_readthrough_enabled = false`. Takes effect on the next request; no + deploy. This is the real rollback. +2. **Purge:** `ts cache purge --all` or the admin endpoint. + +Then state what purge covers, matching the staging verdict: + +- If Task 3 landed: both template-cache and readthrough objects. +- If it did not: template-cache objects **only**. Readthrough rollback is the config flag plus + waiting out the origin TTL. Say this plainly — do not ship a rollback step that does not affect + the cache being rolled back. + +- [ ] **Step 3: State the residual risk** + +An operator who enables this against an unverified origin can cross-serve, including session +fixation via a cached `Set-Cookie`. This is a weaker guarantee than the template cache's, and the +docs must say so rather than implying parity. + +- [ ] **Step 4: Format and commit** + +```bash +cd docs && ./node_modules/.bin/prettier --check guide/configuration.md +``` + +--- + +## Final verification for the whole PR + +- [ ] **Full gate set**, as in part 2, plus `./scripts/template-cache-local-test.sh purge`. + +- [ ] **Write the refusal-by-refusal comparison into the PR description** + +For each `template_cache_ttl` refusal, state whether the readthrough path covers it and how: + +| Refusal | Anchor | Covered on readthrough? | +| --------------------- | ------------------- | ------------------------------------------------------- | +| `NoPositiveFreshness` | `publisher.rs:6100` | By probe verdict only | +| `OriginSetCookie` | `:6147` | By probe verdict only | +| `CspNonce` | `:6117` | By probe verdict only | +| `OriginNotShareable` | `:6153` | Platform honors origin headers | +| `NonOkStatus` | `:6190` | Platform honors status | +| `NotHtml` | `:6198` | Not applicable — readthrough caches any type per origin | +| `VaryNotCovered` | `:6184` | Platform keys on origin `Vary` | + +Anything in the "probe verdict only" rows is an accepted risk, and the PR should name it as such +rather than leaving a reviewer to derive it. + +- [ ] **Confirm the gate is inert by default** + +With `origin_is_cookie_independent` unset, `cookie_disqualifies` is true for every cookie-bearing +request, so `origin_response_is_shareable` is false and the bypass behaves as it does today for +every repeat visitor. Prove it with a test, not by reasoning — it is the claim that makes this +change safe to merge ahead of any operator decision. + +- [ ] **Deploy ordering** + +The Tinybird datasource migration from part 1 must reach Tinybird **before** this code deploys, or +every emitted telemetry row is quarantined. Record the confirmation on the PR. diff --git a/docs/superpowers/specs/2026-09-15-852-template-and-origin-caching-design.md b/docs/superpowers/specs/2026-09-15-852-template-and-origin-caching-design.md new file mode 100644 index 000000000..5a8b7d27b --- /dev/null +++ b/docs/superpowers/specs/2026-09-15-852-template-and-origin-caching-design.md @@ -0,0 +1,848 @@ +# Closing out #852: Origin readthrough, purge, and cache observability + +**Date:** 2026-09-15 +**Issue:** IABTechLab/trusted-server#852 +**Branch:** `852-template-and-origin-caching` +**Status:** Draft, revision 3 (rewritten after two rounds of independent review — seven reviewers total) + +## Why this document exists + +Issue #852 was filed on 2026-07-05, before #1009 shipped. Its opening premise — "there is +no `CacheOverride`, `set_ttl`, or cache API call anywhere in the repo" — is no longer true, +and two of its four numbered items are substantially built. The issue is nonetheless not +closeable: the part that shipped does not produce a cache hit in a default production +deployment, the part that did not ship was mis-described, and the review that produced this +spec surfaced a pre-existing condition the issue never anticipated. + +## Glossary + +Three distinct caches sit on the publisher path. Earlier documents numbered them C1/C2/C3; +the 2026-08-19 terminology migration retired those labels for active prose, and this +document uses the names it chose. The distinction is load-bearing — conflating them is what +produced the original wrong conclusion in the #1009 design doc, and, as recorded below, +this spec's own first revision conflated two of them again. + +| Name | Contents | Owner | Status | +| ---------------------------- | ------------------------------------------------------------------------- | --------------------- | ------------------------------------------------------------------------------ | +| **Origin readthrough cache** | Raw origin bytes, pre-transform | The platform (Fastly) | Exists. Ad-stack requests opt out; **everything else already uses it** | +| **Template cache** | Post-`lol_html`, pre-assembly, reader-neutral HTML with a bid-shaped hole | Trusted Server | Built, Fastly-only, off by default | +| **Assembled-response cache** | Final per-reader output | Nobody | **Must never exist.** Naming it is how we keep it from being built by accident | + +If a change appears to require the assembled-response cache, that is the leak, not a design +option. + +## Current state, per issue item + +**Item 1 — cache the origin HTML template.** Mis-described, in both directions. See the two +corrections below. + +**Item 2 — cache the transformed HTML.** Built. +`crates/trusted-server-core/src/platform/template_cache.rs` stores the post-transform +template; `AD_ASSEMBLY_SEAM` (`publisher.rs:1284`) is the late-bound hole; a hit returns +before the origin fetch (the `TemplateCacheLookup::Hit` arm at `publisher.rs:4617`). +Fastly-only, stamped spike-only, off by default. + +**Item 3 — surrogate keys and purge hooks.** Half, and the half that exists covers the wrong +cache for item 1's purposes. Keys are emitted at template-cache insert +(`platform/template_cache.rs:138`) and `purge_url`/`purge_all` work inside Compute via +`fastly::http::purge::purge_surrogate_key` (`adapter-fastly/src/template_cache.rs:280`). +Nothing operator- or CMS-facing calls them, and none of it touches readthrough objects. + +**Item 4 — leave `response_privacy.rs` alone.** Held. Unchanged, and the template-cache hit +path stamps `private, no-store` itself because a hit returns before the normal stamp point. + +## Two corrections to the original analysis + +These were found by review, are verified against the code, and each invalidates a claim an +earlier revision of this document made. + +### Correction 1 — the readthrough cache is already shared, for most traffic + +The bypass is `if should_run_ad_stack` (`publisher.rs:4415`, `:4716`). That flag comes from +`should_run_server_side_ad_stack` (`publisher.rs:3077`), which requires GET, navigation, +non-prefetch, non-bot, matched slots, consent, `ad_templates_enabled`, and +`auction_enabled` — **all** of them. + +Every request failing any one of those conditions already reaches origin with no +`set_pass`, and therefore already participates in the shared readthrough cache with no +eligibility check whatsoever: bots, prefetches, consent-denied readers, any page with no +matched slot, and all traffic while either kill switch is off. + +Calibration, so this is neither ignored nor overstated: what gets stored is still governed +by the origin's own `Cache-Control`, exactly as for any CDN, and the premise of #852 is that +origins mark HTML private. So today this is most likely storing nothing. It is not a live +incident and this spec does not treat it as one. But two consequences follow: + +1. Origin readthrough is **not** "the first time publisher HTML can enter a shared cache". It makes + deliberate a sharing decision that is currently made by omission for the majority of + requests. +2. Without a fix, the change below would let an _eligible_ ad-serving request read an object stored + by an _unchecked_ bot request. Opting a second population in while leaving the first + unguarded is worse than either state alone. + +The change below therefore governs the whole population under one rule. That is a tightening +for ineligible bot and prefetch traffic and a widening for cookieless traffic; the gate section states both +sides, and the response-side gap explains why the resulting controls are preconditions rather than code. + +### Correction 2 — stripping TS-owned cookies would buy almost nothing + +An earlier revision rejected "strip TS cookies from the origin fetch" on correctness-risk +grounds. That reasoning was weak — `strip_cookies` already exists and is used on the +partner-forwarding path (`cookies.rs:89-103`), so the mechanism is precedented. + +The decisive argument is different: `cookie_disqualifies` keys on **header presence**, not +cookie identity — `req.headers().contains_key(header::COOKIE)` (`publisher.rs:4283`), feeding +`:4292`. Stripping TS-owned cookies only clears the gate for a reader carrying +_exclusively_ TS cookies. Any publisher analytics, session, or consent-vendor cookie leaves +the header present and the request disqualified. On a real publisher that is close to zero +traffic. + +So the rejected alternative trades a correctness risk for a hit-rate win that does not +exist. Same conclusion as before, on a foundation that survives someone reading `:4283`. + +## The actual blocker + +The template cache disqualifies any request carrying a `Cookie` header (`cookie_disqualifies`, +`publisher.rs:4292`; `TemplateCacheBypassReason::CookieForwarded`). Trusted Server sets its +own EC cookie, so essentially every repeat visitor is excluded. Hit rate in a default +deployment is approximately zero. + +The escape hatch is `creative_opportunities.origin_is_cookie_independent`, documented as +"unsafe unless independently verified" — with no tooling to do the verifying. An operator is +asked to assert a byte-level property of their origin on faith. + +This is the critical path. Every other item in this spec is inert until it is resolved. + +## Goals + +1. One explicit, auditable rule governing which requests may share an origin response, + applied to the whole request population rather than half of it. +2. `origin_is_cookie_independent` settable on evidence rather than faith. +3. Operator and CMS purge surfaces, with an honest statement of what each can and cannot + purge. +4. Cache outcomes measurable — for both caches, not just the template cache. +5. Documentation that matches the behavior, including a rollback procedure that works. + +## Non-goals + +- Porting the template cache backing to Cloudflare or Spin. The trait seam exists; separate + issue. +- Promoting the template cache out of spike status. Moved to successor issue B — it hides + unsettled design decisions that deserve their own review. See "Successor issues". +- The 2026-08-19 terminology cleanup. Moved to successor issue A. +- Any change to `response_privacy.rs` or the `private, max-age=0` downgrade. +- Stripping cookies from the origin fetch. Rejected in Correction 2. +- TTL and stale-while-revalidate overrides on the origin fetch, despite being explicit in #852 + item 1. They are buildable but unsafe as request-side knobs; see the mechanism section. Successor issue D. +- Fixing the pre-existing `/_ts/admin` credential forwarding. Worked around locally in the admin endpoint; + successor issue E. +- An assembled-response cache, in any form. + +--- + +## Origin readthrough + +This is the only change here with new runtime blast radius. Review it as security-sensitive code. + +### Current + +```rust +// publisher.rs:4415 and :4716 +if should_run_ad_stack { + platform_request = platform_request.with_cache_bypass(); +} +``` + +`should_run_ad_stack` means "this page serves ads", which is unrelated to whether the origin +response may be shared — and, per Correction 1, leaves every non-ad-stack request sharing by +default. + +### Splitting the predicate + +Today's `request_can_use_shared_template` (`publisher.rs:4325`) mixes two kinds of condition: + +```rust +let request_can_use_shared_template = method_is_cacheable + && matches!(assembly_mode, AssemblyMode::Esi) // template-only + && !request_host.is_empty() + && !authorization_disqualifies + && !cookie_disqualifies + && !request_requires_origin + && reader_supports_assembly; // template-only +``` + +`assembly_mode` and `reader_supports_assembly` say nothing about whether the origin's bytes +are shareable; they say whether _this_ pipeline can use a shared template. Gating readthrough +on them would mean an operator running default `inline` mode with a verified +cookie-independent origin gets no readthrough caching, ever, for no safety reason — and would +make `assembly_mode = "inline"` silently re-enable the origin bypass, which is the documented +rollback lever. + +Split them: + +```rust +// Necessary for either cache to share this request's origin response. +let origin_response_is_shareable = method_is_cacheable + && !request_host.is_empty() + && !authorization_disqualifies + && !cookie_disqualifies + && !request_requires_origin; + +// Additionally required to use a shared *template*. +let request_can_use_shared_template = origin_response_is_shareable + && matches!(assembly_mode, AssemblyMode::Esi) + && reader_supports_assembly; +``` + +This is a pure refactor with no behavior change, which is why it is the first commit rather than +part of the gate: the `origin_cache_shareable` telemetry field needs the binding to exist. + +### Gating the bypass, at both sites + +```rust +if !origin_response_is_shareable { + platform_request = platform_request.with_cache_bypass(); +} +``` + +`should_run_ad_stack` is **gone**, per Correction 1. + +**Both `:4415` and `:4716` change.** An earlier revision claimed `:4415` was dead code. That +was true of the old template-only predicate and is false of this weaker one. The two sites are +not independent: `:4415` populates `pending_origin` inside the EC-preload fan-out block, and +`:4716` is the `else` branch of `if let Some(pending) = pending_origin` (`publisher.rs:4703`). +They are alternative paths for the same fetch. Changing only one makes readthrough eligibility +depend on whether EC preload fired — and `should_preload_ec_snapshot` (`publisher.rs:2955`) is +`is_navigation && is_get && has_ec_id && has_kv`, which is much of the population this change +exists for. In default `inline` mode `template_cache_key` is always `None`, so the preload +branch is the one most eligible navigations take. + +Test the invariant directly: identical inputs must produce the same bypass flag on the preload +and non-preload paths. + +**Correcting a claim from revision 2.** That revision said dropping `should_run_ad_stack` is +"strictly more conservative". It is not, and the accurate statement is two-sided: + +- **Tightening** for cookie-bearing, `Authorization`-bearing, or otherwise ineligible bot and + prefetch traffic, which gets `set_pass` where today it does not. +- **Widening** for cookieless traffic: eligible ad-serving requests begin reading objects that + cookieless bots and prefetchers store. Today those objects are written and read only by + non-ad-stack requests. + +The widening is the reason the preconditions below are blocking rather than advisory. + +### Readthrough is enabled by omitting `set_pass`, not by a TTL override + +Revision 2 proposed adding `with_cache_ttl(Duration)` to `PlatformHttpRequest` to make the +decision "explicit rather than inherited from service defaults". **That was wrong and is +withdrawn.** The pinned `fastly` 0.12.1 crate documents the semantics +(`fastly-0.12.1/src/http/request.rs:2381-2392`, and the shared snippet +`docs/snippets/set-pass-override.md`): + +> This overrides any previous `Request::set_pass` call and sets the `pass` behavior to `false`. +> +> …calling any of those methods on Request _after_ calling `set_pass(true)` will reverse the +> effect of the `set_pass` call, and any response received when the request is sent may become +> cacheable. + +`set_ttl` overrides the origin's `Cache-Control`, `private` and `no-store` included. Adding it +would convert the hazard this section exists to close — a response the origin refused to let us +share becoming shared — from a service-config accident into a first-class TS API. It also +defeats the fail-safe this spec relies on elsewhere. `set_ttl(0)` does not help either: it +caches with zero TTL, it does not bypass. + +**The correct mechanism is the absence of a call.** Not calling `set_pass` leaves Fastly to +honor the origin's own freshness. If the origin marks HTML private, nothing is stored and +Origin readthrough is inert — which is the right failure. The only lever TS applies is `set_pass(true)` +on ineligible requests. + +Consequences, stated rather than worked around: + +- **TTL and stale-while-revalidate as overrides are out of scope**, and this is a deliberate + drop from #852 item 1 rather than an oversight. `set_stale_while_revalidate` does exist + (`request.rs:2395`), so it is buildable — but it carries the identical override semantics, + so a safe version needs a post-response decision, which means the Core Cache API. That is a + separate piece of work with its own review. Successor issue D. +- **`after_send` / `CandidateResponse` is not available as an alternative.** The snippet above + recommends it for exactly this problem, and it is unreachable here: Viceroy 0.17 stubs the + HTTP Cache ABI and the SDK converts that into a send error, so setting `after_send` makes + every publisher origin fetch fail under `fastly compute serve`, `cargo test-fastly`, and the + parity suite. Recorded at `adapter-fastly/src/template_cache.rs:6-15`; do not re-propose it. + +### The response-side gap cannot be closed request-side + +`template_cache_ttl` (`publisher.rs:6129`) refuses on response state: absent positive freshness +(`:6100`), `Set-Cookie` (`:6147`), foreign edge-cache headers (`:6153`), non-200 (`:6190`), +non-HTML (`:6198`), CSP nonce (`:6117`), uncovered `Vary` (`:6184`). + +The readthrough decision is made before the origin is contacted. Per the previous section, there is no reachable post-response hook. **So none of those refusals can be applied to the readthrough path.** An +earlier revision said "either the TTL knob is set to zero for that class or the case is +documented as accepted"; only the second branch exists. + +The sharpest case is `Set-Cookie`, and the cookie gate makes it more likely rather than less. +`cookie_disqualifies` keys on request-side header presence (`publisher.rs:4283`, `:4292`), so +`origin_response_is_shareable` is true precisely for readers carrying **no cookie at all** — +first-time visitors, which is exactly when an origin issues a session cookie. Fastly Compute's +readthrough has no VCL `hit-for-pass` boilerplate, so `Set-Cookie: sid=…` alongside +`Cache-Control: max-age=60` is a cacheable shared representation, and the stored object replays +that cookie to every subsequent cookieless reader for the TTL. That is cross-reader session +fixation, and it passes a freshness check cleanly. + +**Therefore the controls are preconditions, not code.** Readthrough may be enabled only against +an origin the probe has verified on every blocking axis — self-identity, cookie, +`User-Agent`, absence of `Set-Cookie`, absence of CSP nonce, and positive shared freshness. The +runbook states this, the config doc states this, and the probe's verdict is pass/fail rather +than informational. + +This is a weaker guarantee than the template cache's, and the spec says so plainly rather than +implying parity. An operator who enables readthrough against an unverified origin can cross-serve. +If that is judged unacceptable, the honest alternative is to drop the readthrough change and close #852 item +1 as won't-do — see Open risks. + +### Rollback + +`purge_surrogate_key` is documented as purging "a surrogate key for the current service" +(`fastly-0.12.1/src/http/purge.rs:12`), not a Core-Cache-scoped operation, and +`Request::set_surrogate_key` (`request.rs:2462`) is the request-side surrogate-key surface for +the readthrough cache. Revision 2 called this an unverified spike; **the API question is +settled** — what remains is empirical, and Viceroy cannot model it, so it needs a staging +service. + +Two constraints the implementation must respect, both from the same override snippet: + +- `set_surrogate_key` **also** cancels `set_pass`. Stamping `ts-origin` unconditionally would + disable the bypass for every request, including disqualified ones. It may be applied only on + the `origin_response_is_shareable` branch. +- `set_pass` and `set_surrogate_key` are mutually exclusive on one request, and the interaction + is order-dependent. The platform layer must make this unrepresentable rather than relying on + call ordering — one enum, not two booleans. + +If the staging check shows readthrough objects are not purgeable this way, readthrough rollback +is flag-flip plus origin TTL, and the runbook must say so. + +### Tests + +- Table test over the inputs of `origin_response_is_shareable`, asserting the recorded bypass + flag. `recorded_cache_bypass_flags()` (`platform/test_support.rs:450`) captures what is needed. +- Preload and non-preload paths produce the same bypass flag for identical inputs. +- The predicate split is behavior-neutral: same inputs, same `request_can_use_shared_template`, for + every combination. +- Regression: cookie-bearing, `Authorization`-bearing, and `request_requires_origin` requests + still bypass. +- New behavior: an ineligible bot or prefetch request now bypasses where it previously did not. +- The platform layer cannot represent `set_pass` and a surrogate key simultaneously. + +--- + +## Origin shareability probe + +``` +ts origin probe-shareability --url [--repeat N] [--cookie ]... +``` + +Renamed from "probe-cache-independence": cookies are one axis of several, and naming it for one +axis is how an operator ends up with a clean verdict on an origin that still cross-serves. + +Per the response-side gap, this probe is not a convenience. It is the only control standing +between readthrough and cross-serving, so its verdicts are blocking and its output is the artifact an operator keeps. + +### Axes, all blocking + +| Axis | Comparison | Why it blocks | +| --------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Self-identity** | Same request twice | An origin whose HTML differs per request (timestamps, CSRF nonces, A/B assignment) cannot be shared on any axis. Distinct from, and more common than, cookie personalization | +| **Cookie** | Bare vs. representative TS + publisher cookie jar | The `origin_is_cookie_independent` question | +| **`Accept-Encoding`** | `gzip` vs. `identity`, compared after decode | `STRUCTURALLY_COVERED = ["accept-encoding"]` (`platform/template_cache.rs:207`) assumes encoding variants differ only by content coding. Its own doc says operators "must leave ESI disabled if an origin changes document semantics instead" — an obligation shipped in prose with no way to check it | +| **`User-Agent`** | Desktop vs. mobile UA | An origin serving distinct mobile or prerendered HTML without `Vary: User-Agent` is cross-served, since readthrough keys on URL plus origin `Vary` only | +| **RSC** | Bare vs. an `RSC` flight-fetch header | RSC fetches already flow through the readthrough cache while HTML navigations are passed, so removing the bypass puts both representations under one cache key for the first time. An origin that varies on these without declaring it can serve a flight payload to an HTML navigation | + +### Response-header verdicts, all blocking + +- **No fronting cache.** Any `Age` header (including `Age: 0`) or a vendor hit header means a cache may have answered for the + origin, so every axis may have compared one stored object with itself and the whole run says + nothing. Judged first. Detected rather than defeated: cache-busting would change either the + cache key or the origin's own caching behaviour, and perturbing the measurement to rescue it + would make a green result mean less. +- **Positive shared freshness.** No positive `Cache-Control`/`Surrogate-Control` freshness means + readthrough must not be enabled. +- **No `Set-Cookie`.** Per the response-side gap, this is the session-fixation vector and there is no runtime guard. +- **No CSP `nonce`.** The template cache refuses these (`publisher.rs:6117`) because a shared + nonce silently defeats the origin's own XSS defence; readthrough cannot. +- **`Vary` coverage.** Report which varying axes the origin's declared `Vary` fails to cover. + +`--repeat N` runs the self-identity check N times. + +### Stated limits + +The probe must print, and the docs must repeat, what it cannot see: it runs from one client IP, +so personalization keyed on the forwarded client address (geo, rate-class) is undetectable. And +a verdict covers the sampled URLs only, not the origin as a whole. + +### Implementation notes + +`crates/trusted-server-cli/src/commands/origin/`, new `Origin` variant in `run.rs`'s `Command` +enum, following the `audit`/`dev` pattern. Human-readable output by default, `--json` for CI, +and a non-zero exit on any blocking failure so it can gate a deploy. + +**The CLI has no HTTP client on Linux.** `hyper`, `rustls`, `tokio` with `net` are under +`[target.'cfg(target_os = "macos")'.dependencies]` (`crates/trusted-server-cli/Cargo.toml:42`); +the non-wasm block has `tokio` without `net`. `chromiumoxide` drives a browser and cannot give +raw origin bytes. Add `reqwest` — already a workspace dependency with `rustls-tls` +(`Cargo.toml:93`), already built natively by the Axum adapter and the integration-tests crate — +to the `cfg(not(target_arch = "wasm32"))` block, which exists precisely to keep the +`wasm32-wasip1` default from building native networking. + +**The fixture server is its own task.** The only precedent, +`crates/trusted-server-cli/tests/support/mod.rs`, is reachable only from `tests/proxy_e2e.rs`, +which is `#![cfg(target_os = "macos")]` for that same dependency reason — and CI runs the CLI +suite on Linux too. It must loop-accept: a single-accept fixture already caused a CI flake here +(fixed in PR #823), and `--repeat N` opens N connections by design. + +--- + +## Purge + +### Admin endpoint and key plumbing + +``` +POST /_ts/admin/cache/purge +Content-Type: application/json + +{"scope": "all"} | {"scope": "url", "url": "https://example.com/page"} +``` + +Registered in the Fastly `NamedRoute` table (`adapter-fastly/src/app.rs:1128`), inheriting the +`^/_ts/admin` basic-auth middleware — which runs before route matching (`app.rs:1305` → +`core/src/auth.rs:79`) and fails closed when no handler regex covers the path +(`auth.rs:93-100`). + +**Reader-facing surrogate key.** `TemplateCacheKey.url` is the origin-rewritten target URI +(`publisher.rs:4372`, built at `:4182`), not the URL an operator types. Rather than have callers +replay origin rewriting — the seam that rots silently, since a wrong input yields a well-formed +key that purges nothing — add a **third** surrogate key derived from the reader-facing request. + +Correcting revision 2: it claimed `request_scheme` + `request_host` + path were "all already +fields" on `TemplateCacheKey`. There is **no path field** (`platform/template_cache.rs:53-59` +has `url`, `request_host`, `request_scheme`). Add `request_path`, populated pre-rewrite. Without +it the "reader-facing" key would be reconstructed from the origin path, reintroducing the +coupling it exists to remove. + +Both the endpoint and the CLI then hash the same reader-facing string, and neither needs origin +logic. + +**Canonicalization is required, not optional.** `url_surrogate_key` is a raw SHA-256 over exact +bytes (`template_cache.rs:147`) and `punctuation_distinct_urls_have_distinct_surrogate_keys` +(`:961`) makes byte-exactness load-bearing. Trailing slash, default port, host case, percent +encoding, and query string must each have a stated rule, implemented once in the extracted free +function `url_surrogate_key(url: &str) -> String` and shared by insert and purge. Every mismatch +is a silent no-op purge — a 200 response and an uninvalidated object — in the mechanism rollback relies +on. Test round-trip from an operator-typed string to the stored key in both +directions. + +Give the reader-facing key a distinct prefix (`ts-template-readerurl-`), asserted distinct from +`ts-template-url-` in the existing surrogate-key test, so the two derivations cannot alias when +a staging edge host equals the configured origin host. The failure mode would be over-purge +rather than a read leak — `to_cache_key` still includes scheme, host, and origin identity +(`template_cache.rs:95-105`) — but an aliased purge reports success against an unrelated object. + +**Trait change.** `PlatformTemplateCache::purge_url` takes `&TemplateCacheKey` +(`platform/template_cache.rs:675`), which a handler holding only a URL cannot construct. Add +`purge_url_surrogate_key(&self, key: &str)` across all five implementors: +`UnavailableTemplateCache` (`template_cache.rs:698`), `adapter-fastly/src/template_cache.rs:163`, +`adapter-fastly/src/app.rs:2840`, and the two test doubles at `publisher.rs:8843` and `:9112`. + +**Guards**, each with a specific reason: + +- Register the path as a **string literal** in `NAMED_ROUTES`. The + `admin_endpoints_match_fastly_router` check (`settings.rs:7727-7741`) scans literal `path:` + entries only; a named constant silently skips coverage. +- Add the path to `Settings::ADMIN_ENDPOINTS` (`settings.rs:3214`). That constant feeds live + config validation at `:3265`, so an operator `trusted-server.toml` whose handler regexes do + not cover the new path will start failing validation — a migration note for the release. +- **Register for all methods and return 405 in-handler.** Revision 2 asserted "`GET` rejected"; + the router does not do that. Non-primary methods on a named path fall through to the publisher + (`app.rs:42`), and `enforce_basic_auth` deliberately leaves the `Authorization` header in place + so it "still reaches the publisher origin" (`auth.rs:60-66`). A `GET` would therefore + authenticate, fall through, and ship the shared admin credential to the publisher backend. + Follow the `/auction` `OPTIONS` precedent (`app.rs:1209-1213`) and test that a non-POST does + not reach the origin. This credential-forwarding behavior is pre-existing for every `/_ts/admin` + route and deserves its own issue — successor issue E. +- No non-`/_ts` alias, ever. The legacy `/admin/keys/*` aliases bypassed the auth regex and had + to be denied locally (`app.rs:1168`). +- Reject any `Content-Type` that is not exactly `application/json`. Basic-auth credentials are + attached automatically by browsers, so a cross-origin form POST with `enctype="text/plain"` + sends no preflight; method alone does not stop CSRF, enforced content type does. +- Cap the request body. The `url` field is attacker-supplied and only ever hashed. +- `{"scope":"all"}` is privileged: audit-log the authenticated principal, and rate-limit it or + record the accepted risk explicitly. The spec calls it an unbounded cache-flush and + origin-stampede lever behind one shared static credential, and there is no HTTP-handler + rate-limit primitive in this repo (`ec/rate_limiter.rs` is partner batch/pull-sync only). +- Replay is not a concern — purge is idempotent. Say so rather than leaving it unaddressed. +- Define the partial-failure answer. `purge_all` returns a single `Result` from one + `purge_surrogate_key` call (`adapter-fastly/src/template_cache.rs:285`); an operator mid-incident + needs to know whether to retry. +- Response is `private, no-store`. +- Non-Fastly adapters return 501, not 404 or 200. + +**Four route tables, not one.** Fastly (`adapter-fastly/src/app.rs:1128`), Axum (`app.rs:310`, +the fixed `[NamedRoute; 16]` array becomes 17, plus the entry), Cloudflare (`app.rs:515-545`), +and Spin (the route list at `app.rs:211-215` and the router at `:854-861`). An unregistered path +falls through to origin and 404s, which reads to a CMS webhook as "endpoint does not exist" +rather than "not supported here". + +### Purge CLI command + +``` +ts cache purge --all +ts cache purge --url +``` + +New `Cache` variant in `run.rs`'s `Command` enum, `crates/trusted-server-cli/src/commands/cache/`. +With the reader-facing key and canonicalization landed with the endpoint, this is a thin wrapper: hash the +typed URL, call the Fastly purge API. No config load, no origin logic. + +**Open dependency, and why this ships after the endpoint.** +`adapter-fastly/src/management_api.rs:12` records that today's token is write-scoped with no +Purge permission, and `edgezero_cli` exposes no credential helper — only whole-command runners. +Read `FASTLY_API_TOKEN` with a `--token` override, document the required scope, fail with an +actionable message when it is missing. If the scope cannot be granted, the endpoint stands alone and this +PR is dropped. + +### What purge does not cover + +Template-cache objects are purgeable today. Readthrough objects are purgeable only if the staging check in rollback confirms `set_surrogate_key` behaves as documented. Until then the runbook says +readthrough rollback is flag-flip plus origin TTL. Do not ship a runbook whose rollback step +does not affect the cache being rolled back — revision 1 did, and this is the correction. + +### Tests + +- Fastly adapter: Purge-all and purge-url succeed; unauthenticated rejected; non-POST returns + 405 **and does not reach the origin**; wrong `Content-Type` rejected; oversized body rejected; + no legacy alias resolves. +- Surrogate-key round-trip: operator-typed string → stored key, both directions, across the + canonicalization cases. +- Parity suite: 501 on Cloudflare, Spin, Axum. **Needs new authenticated POST helpers** — + `parity.rs:16-17` sets `^/_ts/admin` basic auth and the existing `axum_post`/`cf_post`/ + `spin_post` helpers send no credentials, so an unauthenticated probe returns 401 and never + reaches the handler. `spin_post_with_headers` (`:200`) is a usable template. +- CLI: both flags produce the expected keys; missing token produces the actionable error. +- `scripts/template-cache-local-test.sh` gains a purge leg — store, hit, purge, confirm miss. + Viceroy 0.17 implements `purge_surrogate_key` against the same in-process cache it serves reads + from (`2026-08-08-1009-measurement-findings.md:156-158`), so this is end-to-end testable + without a Fastly service. The script accepts only `inline|esi` today (`:17-18`) and CI invokes + those literals, so a new mode needs a matching workflow step and a gate-list entry. + +--- + +## Observability + +### Current + +Two response headers, `x-ts-template-cache` and `x-ts-assembly` (`publisher.rs:122`, `:149`). +A hit rate cannot be computed from a response header without a scraping harness. There is no +access log to extend — `tinybird.access_enabled` is explicitly rejected as unwired +(`settings.rs:1945`). + +### Change + +Carry outcomes on the auction telemetry rows, wired in `AuctionEventRow::base()` +(`auction/telemetry.rs:347`) so provider and bid rows carry them too rather than the summary +alone (`AuctionEventRow` is at `:277`), which +already flows to Tinybird with `publisher_domain` and `page_path`: + +- ~~`template_cache_state`~~ — **dropped during implementation, and it cannot be added + back without restructuring when telemetry is emitted.** On a cold fill the ordering is + fixed: `stream_publisher_body_async` collects the auction, takes the observation and emits + the batch, and only then does `store_template_if_authorized` run and the state get + stamped. The store cannot move earlier (it needs the transform) and the emit cannot move + later without giving up collecting during body streaming — which is a latency decision on + the path this whole issue exists to improve. `hit` was reachable and `miss-stored` was + not, so the column would have made hit rate compute as roughly 100%. The + `x-ts-template-cache` header still carries all nine states per response. +- ~~`template_cache_bypass_reason`~~ — **moved to successor issue B during implementation.** + It diagnoses the _template_ cache's refusals, which is #1009's feature, not this issue's. + Readthrough has no refusal reasons TS controls — that is the response-side gap — so this + column says nothing about the change #852 makes. Instrumenting a spike belongs with + promoting it out of spike status. Deriving it also required new code: the variants that + matter are structurally unreachable at the existing site, per the note below. +- `origin_cache_shareable: Option` — whether `origin_response_is_shareable` was true, + i.e. whether the readthrough gate let this request use the readthrough cache + +The third field exists because revision 1 instrumented only the template cache, leaving the +one change with real blast radius shipping with zero observability — no way to tell +"the readthrough gate is working" from "it is inert". + +The bypass reason is the field that carries the triage. `TemplateCacheBypassReason` has +sixteen variants today (`publisher.rs:5673`), seventeen once the request-side derivation below +adds one: "cookie-disqualified" is an expected default, "no +positive freshness" is an origin configuration problem, "vary not covered" is a stale +`template_cache_vary` list, "malformed cache policy" is a bug. Without it a zero hit rate is +uninterpretable. + +This is the spec's own trim, reached by the code rather than by the approval gate. + +The scope line this settles: each issue instruments its own change. `origin_cache_shareable` +measures what #852 changes; the bypass reason measures what #1009 built. + +**Retained for issue B — the reason has two sources, and only one of them exists today.** `template_cache_ttl` +(`publisher.rs:6129`) returns `Result`, but it runs inside +`template_cache_reservation.and_then(...)` (`:4775`), and a reservation exists only when +`template_cache_key` was built — which is `request_can_use_shared_template.then(...)` (`:4370`). +So its first three variants, `InlineMode`, `AuthorizedRequest` and `CookieForwarded`, are +structurally unreachable from it: those requests never get a key in the first place. Only +response-derived reasons can fire there. + +The request-side bypass carries no reason value at all. It sets +`TemplateCacheResponseState::BypassRequest` (`:4386`) and writes free-text `log::debug!` lines +(`:4360-4369`), and nothing else. + +That is a problem for whoever instruments this cache, because "cookie-disqualified" — the +expected default, and the single most important thing an operator needs to see — lives on the +unreachable side. Doing it means **deriving a structured request-side reason** alongside +`template_cache_key`, reusing the existing `TemplateCacheBypassReason` variants rather than +inventing a second vocabulary. That is new code, not a wiring exercise, and **issue B must +budget it** — it is no longer in this issue's scope. + +The `#[cfg(test)]` helper named `template_cache_bypass_reason()` at `:5863` is not the hook for +either source. + +### Schema migration — this is not free + +Revision 1 claimed "no new CI gates". Wrong for this work. +`tinybird/datasources/auction_events_raw.datasource` enumerates all 35 columns explicitly, +and `to_ndjson` (`auction/telemetry.rs:422-425`) uses plain `serde_json::to_string` with no +`skip_serializing_if`, so new fields are always on the wire including as `null`. Rows with +undeclared columns go to Tinybird quarantine, which the repo already tracks +(`tinybird/pipes/quarantine_counts.pipe`). + +Required: update the datasource, update `tinybird/fixtures/auction_events_raw.ndjson`, and +sequence the Tinybird deploy **before** the code deploy. + +### Carrier + +`AuctionObservationContext` (`auction/telemetry.rs:99-125`), the request-scoped +`auction_observation` at `publisher.rs:4446`, also a params field at `:1599`. Two notes the +plan must honor: + +- The struct is currently an immutable snapshot built once from an `AuctionRequest`. These + fields make it a mutable accumulator, because the bypass reason is only known post-fetch. + Add them via a setter, not `pub` mutation. +- There are **five** write points, not one. The hit path moves the observation out and returns + at `publisher.rs:4669` before `template_cache_ttl` is reached at `:4776`; the `Hit` state is + stamped separately at `:2202` — which is unreachable from the publisher path, so the usable + hook is the `TemplateCacheLookup::Hit` arm at `:4617`; the abandon paths at `:4726` and `:4748` + also `take()` early, while `:4957` and `:4996` take _after_ the state write and do carry it. +- There are two bindings, and which one a write site holds decides the mechanism. + `observation` is a plain value built at `:4461`; `auction_observation` is the + `Option` at `:4446`, and `observation` is moved into it at `:4511`. + A write before `:4511` sets the value directly; a write after it goes through + `auction_observation.as_mut()`. An earlier revision of this section claimed the value had to be + stashed in a local because the binding did not yet exist — that is wrong, and the plan overrules + it: `origin_cache_shareable` is known at `:4325` and construction is at `:4461`, so a setter + immediately after construction works. + +### Known gaps, to be stated in the dashboard docs + +- The summary row is emitted only when an auction runs, so the denominator is "ad-serving + pageviews", not "all requests". A request that bypasses because the ad stack did not run + produces no row. +- `AuctionObservationContext` is `Clone` and shared with the `/auction` source, where these + fields are structurally `None`. A dashboard reading `None` as "miss" will be wrong for a + whole source class. + +--- + +## Configuration, documentation, tests + +### Configuration + +No new keys. Doc comments change on `origin_is_cookie_independent` — replace "Unsafe unless +independently verified" with a pointer to `ts origin probe-shareability` and a one-line +statement of what the probe does and does not prove — and on the matching Rust doc comments +on `CreativeOpportunitiesConfig`. + +### Documentation + +`docs/guide/configuration.md` gains the three-cache glossary once, with code comments +pointing here rather than re-teaching it, plus an operator runbook: run the probe → require +a green freshness verdict → set the flag on evidence → watch the bypass-reason breakdown → +confirm hit rate → purge and roll back. The rollback section must state exactly what purge +covers, per purge coverage above. + +Docs edits hit the `cd docs && npm run format` gate. Use the pinned `docs/node_modules` +prettier, not `npx` — the npx version reports false format failures. + +Each item's docs land in that item's PR, not as a lump, so no PR documents a command +that does not exist yet. + +### CI gate list + +`AGENTS.md`'s documented gate list is incomplete in more ways than this spec first counted, and +fixing one omission while leaving the rest is how it stayed wrong. Known missing today: the +`scripts/template-cache-local-test.sh` harness (`test.yml:63`, `:66`), two clippy invocations in +`format.yml:83`/`:88` (CLI and `trusted-server-openrtb-codegen`, both hardcoded to +`x86_64-unknown-linux-gnu`), the parity crate's clippy (`test.yml:200`), the openrtb-codegen test +(`test.yml:194`), the integration-tests `cargo fmt` check, `npm run lint` for JS and docs plus +the docs `npm run build` (`format.yml:116`, `:147`, `:153`), the html-processor bench smoke, the +Fastly and Spin release WASM builds, and the whole of `.github/workflows/integration-tests.yml`. + +Rather than enumerate a list that will drift again, state in AGENTS.md that the list is the +commonly-run subset and point at `.github/workflows/` as authoritative — then add the entries +above, including the new purge-mode harness step this spec introduces. + +**Parity lockfile.** The integration-tests crate has its own lockfile with a shared-direct-dep +alignment gate. If the new authenticated POST helper pulls a dependency, fix with a targeted +`cargo update -p --precise `, never a full update. + +--- + +## Sequencing + +**One PR, by decision.** An earlier revision split this into five. The work is now a single +change set, so the ordering below is commit order within one branch rather than a merge order. + +What that costs, recorded rather than glossed: the readthrough gate is the only change here with +new blast radius, and in a single PR it lands and reverts together with the telemetry that would +tell you whether to revert it. A revert takes the instrumentation with it. The mitigations are +that the gate is inert until an operator sets `origin_is_cookie_independent`, and that +`assembly_mode` stays a runtime kill switch — so the practical rollback is a config change, not a +code revert. + +Commit order, which still follows the rule that nothing changing cache behavior precedes the +tooling to observe and reverse it: + +1. **Predicate split** — pure refactor, no behavior change. First because everything else + references the binding it creates. +2. **Observability** — the `origin_cache_shareable` telemetry field and the Tinybird datasource + migration. (Two further fields were scoped out during implementation: see Observability.) The migration must reach Tinybird **before the code + deploys**, which in a single PR is a deploy-ordering constraint on the release, not on the + merge. +3. **Probe** — the `reqwest` dependency, the loop-accept fixture server, five axes and five + response-header verdicts. +4. **Purge plumbing and endpoint** — the `request_path` field, the reader-facing surrogate key + with canonicalization, the `url_surrogate_key` extraction, the trait change, four route + registrations, `ADMIN_ENDPOINTS`, parity auth helpers, the harness purge leg and its workflow + step. +5. **Purge CLI command** — thin wrapper over the key work above. Drop this commit alone if + purge-token scope cannot be granted; nothing else depends on it. +6. **Readthrough gate** — last, and reviewed as security-sensitive. Requires the `ts-origin` + staging verdict resolved and the precondition list reflected in the runbook before the PR is + marked ready. +7. **Documentation** — runbook, glossary, config comments, dashboard caveats, CI gate list. + +**The `ts-origin` staging check gates the PR, not a commit.** It cannot be verified under Viceroy +and needs a staging service. Timebox it alongside commit 4, which already touches the purge trait, +and write the verdict into the PR description. If it comes back negative, the runbook says +readthrough rollback is flag-flip plus origin TTL — the PR still ships, with an honest rollback +section. + +**Review guidance for a change set this size.** Ask for the readthrough commit to be reviewed on +its own, against the precondition list. It is one condition at two call sites, and the rest of the +diff is instrumentation and tooling that will otherwise bury it. + +## Successor issues + +Five things are deliberately not in #852. None is in the issue text, and each hides work that +deserves its own review: + +- **Issue A — finish the 2026-08-19 terminology migration.** Six active code comments still use + the retired C1/C2/C3 labels (`platform/template_cache.rs:5-11`, `publisher.rs:1727`, `:2154`, + `:5860`, `:11025`, `response_privacy.rs:71`), and four shipped #1009 design docs still read + "Approved for implementation". Editorial; the distinctions those comments draw must survive + verbatim in substance. +- **Issue B — promote the template cache out of spike status, and instrument it.** Carries + the `template_cache_bypass_reason` telemetry column moved out of #852: it diagnoses the + template cache's refusals, and the variants that matter (`InlineMode`, `AuthorizedRequest`, + `CookieForwarded`) are structurally unreachable from `template_cache_ttl`, so it needs a + request-side derivation alongside `template_cache_key`. Note a template-cache hit/miss + column is **not** available — see the Observability section for why. 30 comment sites across six + files, but not editorial: three unsettled design decisions sit underneath. The "spike-grade + choice, not a production one" `Vary`-keying caveat (`platform/template_cache.rs:230`), whose + drift guard runs only on the cold path — a hit returns before the origin fetch, so a stored + template never learns the origin added a `Vary`, and correctness is bounded by TTL rather + than by a check. The `STRUCTURALLY_COVERED` operator obligation (`:207`), which the probe's + probe now makes checkable. And `VarySpec::new`'s panic (`:240`) on a Wasm request path. Gate + on first probe results. +- **Issue C — validate readthrough and template hit rates on a production origin.** Metric: + `origin_cache_shareable` true-rate and `template_cache_state = hit` rate from the observability + fields, per `page_path`. Window: seven days after enablement. Qualifying deployment: one whose + probe returned green on every blocking axis. Triggers: a sustained hit rate below a threshold + agreed at enablement means the readthrough gate is not earning its risk and the flag goes back off; + any cross-serving report is an immediate rollback and a redesign of the gate. +- **Issue D — safe TTL and stale-while-revalidate control.** #852 item 1 asks for both. They are + buildable (`set_ttl`, `set_stale_while_revalidate`) but carry override-the-origin semantics + that make them unsafe as request-side knobs, per the mechanism above. A safe version needs a post-response + decision, which means the Core Cache API and its own review. +- **Issue E — `/_ts/admin` forwards the admin credential to the publisher origin.** Non-primary + methods on named admin paths fall through to the publisher (`app.rs:42`) and + `enforce_basic_auth` leaves the `Authorization` header in place by design (`auth.rs:60-66`). + Pre-existing for every admin route, not introduced here; the admin endpoint works around it locally + with an all-methods registration. + +Note when filing: `gh issue create --label task` fails in this repo — issue type is GraphQL-only. + +## Open risks + +**Readthrough safety rests on preconditions, not code.** As set out above, none of the template cache's +response-side refusals can be applied to the readthrough path, because the decision is made +before the origin responds and no post-response hook is reachable. The probe's blocking verdicts +are the only control. An operator who enables readthrough against an unverified origin can +cross-serve, including session fixation via a cached `Set-Cookie`. + +**Decided: the readthrough change ships.** This paragraph previously left it open. The +alternative was to close +#852 item 1 as won't-do and keep the unconditional bypass, accepting that every ad-serving +pageview pays a full origin round trip — legitimate, because the template cache already delivers +the same benefit on its hit path and enforces response-side rules readthrough cannot. Its marginal +value is confined to template-cache misses; its marginal risk is a weaker guarantee on a broader +population. + +It ships anyway, on the understanding that the probe's blocking verdicts are the control, and that +enablement stays per-operator behind `origin_is_cookie_independent` so nothing changes for anyone +who does not opt in. Review the readthrough commit against that bar specifically. + +**The gate is a no-op for cookie-varying origins.** A publisher whose HTML genuinely depends on +publisher cookies gets nothing from this work. The probe tells them quickly, which is the honest +outcome, but the addressable population is unknown until operators run it. + +**Origin readthrough's efficacy depends on origin headers we do not control.** #852's premise is that +origins mark HTML private. Where that holds, readthrough stores nothing even with the flag flipped. +That fails safe and is the correct behavior, but it must not be sold as a guaranteed latency win. + +**Template cache and streaming pull in opposite directions.** `TemplateEntry.body` is `Vec` +(`platform/template_cache.rs:687`) and inserts take an owned body, so the template-cache path +cannot stream and `MAX_PLATFORM_RESPONSE_BODY_BYTES` applies (`adapter-fastly/src/platform.rs:425`). +Relevant to issue B's promotion decision and to the streaming work; noted so the interaction is +not rediscovered later. + +**Observability is not in #852's text, and shipped in its trimmed form.** _Resolved — this is no +longer an open approval question._ As drafted it was the largest refactor here: three fields, a +mutable accumulator, and a 35-column migration with quarantine risk, close enough to AGENTS.md's +"no large refactors without approval" to need asking. What shipped is the fallback this section +already named — `origin_cache_shareable` alone, one field, one setter, one nullable column. +`template_cache_state` proved structurally unreachable and `template_cache_bypass_reason` was +scoped out to issue B as a template-cache diagnostic rather than a #852 one. + +The one field earns its place on relevance: it _is_ `origin_response_is_shareable`, the predicate +#852 introduces, not adjacent instrumentation. It also has to land ahead of the gate rather than +with it — its purpose is to answer "how much traffic would the gate admit" before anyone flips it, +and shipping it alongside the gate would destroy that baseline. Given the response-side gap leaves +an operator's config flag and a one-off probe run as the only runtime controls, going into the +first production window blind is the worse trade. + +## What closes #852 + +All five work items landed, and specifically: + +- The rollback staging verdict recorded, with the runbook matching it. +- Probe green against the harness fixture origin on all five axes and all five response-header + verdicts. +- `origin_cache_shareable` confirmed present on Tinybird rows from a staging deploy, with no + quarantine. +- The readthrough gate reviewed as its own commit against the precondition list, not as part of + the wider diff. + +Production hit-rate validation is **issue C**, not a condition of this one. Revision 1 required a +recorded measurement from a real deployment, which makes the issue un-closeable by the engineer +who builds it — it depends on an operator with a suitable origin volunteering. +Validation-by-deployment is different work and deserves its own issue and assignee. diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index 953c8a72b..bf0867bd0 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -10,17 +10,26 @@ # Usage: # ./scripts/template-cache-local-test.sh # esi mode (shared template + edge assembly) # ./scripts/template-cache-local-test.sh inline # today's shipped behaviour, as a control +# ./scripts/template-cache-local-test.sh purge # store -> hit -> purge -> miss, end to end set -euo pipefail MODE="${1:-esi}" case "$MODE" in - inline | esi) ;; + inline | esi | purge) ;; *) - echo "Unknown mode '$MODE'. Use one of: inline, esi." >&2 + echo "Unknown mode '$MODE'. Use one of: inline, esi, purge." >&2 exit 1 ;; esac + +# `purge` exercises the same shared-template configuration as `esi`, then invalidates it. +# Everything upstream of the purge assertions is identical, so the config generator and +# every esi assertion keep running unchanged. +CONFIG_MODE="$MODE" +if [ "$MODE" = "purge" ]; then + CONFIG_MODE="esi" +fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WORK="$(mktemp -d)" ORIGIN_PORT="${ORIGIN_PORT:-9099}" @@ -235,7 +244,7 @@ ORIGIN_PID=$! sleep 1 info "Generating stub config (mode: $MODE)" -python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$MODE" \ +python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$CONFIG_MODE" \ "$ORIGIN_PORT" "$BID_PORT" <<'PYEOF' import sys @@ -517,7 +526,7 @@ check_post_reaches_origin() { "$(( $(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) - before ))" "1" } -if [ "$MODE" = "inline" ]; then +if [ "$CONFIG_MODE" = "inline" ]; then check "inline fetches the origin every time" "$FETCHES" "2" check "inline writes no shared template" \ "$(grep -c 'template_cache stored' "$WORK/viceroy.log" || true)" "0" @@ -669,7 +678,7 @@ NODEEOF check_post_reaches_origin fi -if [ "$MODE" = "esi" ]; then +if [ "$CONFIG_MODE" = "esi" ]; then info "Where the marker actually lives" echo " The cached template (the shared copy — has a hole where bids go):" grep -oE "template_cache stored [0-9]+ bytes \(seam marker present: [a-z]+\)" \ @@ -759,7 +768,7 @@ COMPLETE=$(echo "$B_LINE" | sed -n 's/.*complete=\([0-9]*\)ms.*/\1/p') if ! [[ "$FIRST_BODY" =~ ^[0-9]+$ && "$COMPLETE" =~ ^[0-9]+$ ]]; then bad "socket probe did not return numeric body timings: '$B_LINE'" else - if [ "$MODE" = "inline" ]; then + if [ "$CONFIG_MODE" = "inline" ]; then check "inline delivers the article before the auction resolves" \ "$(awk -v f="$FIRST_BODY" -v c="$COMPLETE" 'BEGIN { print (f < c / 3) ? "yes" : "no" }')" \ "yes" @@ -774,7 +783,7 @@ else printf ' first body byte %sms, complete %sms\n\n' "$FIRST_BODY" "$COMPLETE" fi -if [ "$MODE" != "inline" ]; then +if [ "$CONFIG_MODE" != "inline" ]; then # Guards a regression where assembly rewrote a reader's accepted gzip origin request # to identity, making the origin send ~674KB where it would have sent ~100KB. The # cache still stores identity; that does not require changing what this reader accepts. @@ -782,8 +791,91 @@ if [ "$MODE" != "inline" ]; then "$(grep -c 'served PLAINTEXT' "$WORK/origin.log" || true)" "0" fi +if [ "$MODE" = "purge" ]; then + info "Purge invalidates the shared template" + + # The config's `password = "handler_password"` is a secret-store *reference*; the basic + # auth value is the seeded secret itself. Read it back from the generated manifest so + # this cannot drift from the seeding block above. + ADMIN_PASSWORD=$(awk -F'"' '/^key = "handler_password"/ { found = 1; next } \ + found && /^data = / { print $2; exit }' "$WORK/fastly.toml") + if [ -z "$ADMIN_PASSWORD" ]; then + bad "could not read the seeded admin password from the generated manifest" + ADMIN_PASSWORD="unreadable" + fi + + # Viceroy 0.17 implements purge_surrogate_key against the same in-process cache it + # serves reads from, so store -> hit -> purge -> miss is genuinely end to end here. No + # Fastly service is involved, which is what makes this the strongest check available + # for the purge path. + + fetch_article_state() { + local headers + headers=$(curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -D- -o /dev/null \ + -H "Host: ts.example.com" \ + -H "Accept-Encoding: gzip" \ + -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ + "http://127.0.0.1:$TS_PORT/article") + echo "$headers" > "$WORK/purge-probe.headers" + template_cache_state "$WORK/purge-probe.headers" + } + + purge() { + curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -o "$WORK/purge.out" -w '%{http_code}' \ + -X POST \ + -u "admin:$ADMIN_PASSWORD" \ + -H "Host: ts.example.com" \ + -H "Content-Type: application/json" \ + --data "$1" \ + "http://127.0.0.1:$TS_PORT/_ts/admin/cache/purge" + } + + # The suite above has already warmed the cache; confirm that before purging, or a + # "miss after purge" result would prove nothing. + check "the template is warm before the purge" "$(fetch_article_state)" "hit" + + check "purge-all is accepted" "$(purge '{"scope":"all"}')" "200" + check "purge-all reports success" \ + "$(grep -c '\"purged\":true' "$WORK/purge.out" || true)" "1" + + check "the next request misses after a purge, and refills" \ + "$(fetch_article_state)" "miss-stored" + check "the cache refills after the purge" "$(fetch_article_state)" "hit" + + info "Purge guards" + + check "an unauthenticated purge is refused" \ + "$(curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -o /dev/null -w '%{http_code}' \ + -X POST -H "Host: ts.example.com" -H "Content-Type: application/json" \ + --data '{"scope":"all"}' \ + "http://127.0.0.1:$TS_PORT/_ts/admin/cache/purge")" "401" + + # The guard the route claims every method for: an unclaimed method would fall through + # to the publisher with the Authorization header still attached. + check "a GET is answered locally, not forwarded to the origin" \ + "$(curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -o /dev/null -w '%{http_code}' \ + -u "admin:$ADMIN_PASSWORD" -H "Host: ts.example.com" \ + "http://127.0.0.1:$TS_PORT/_ts/admin/cache/purge")" "405" + + check "a form-postable content type is refused" \ + "$(curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -o /dev/null -w '%{http_code}' \ + -X POST -u "admin:$ADMIN_PASSWORD" -H "Host: ts.example.com" \ + -H "Content-Type: text/plain" --data '{"scope":"all"}' \ + "http://127.0.0.1:$TS_PORT/_ts/admin/cache/purge")" "415" + + check "scope all carrying a url is refused rather than flushing" \ + "$(purge '{"scope":"all","url":"http://ts.example.com/article"}')" "400" + + # Purging one reader-facing URL, which is the scope a CMS webhook uses. + check "the template is warm before the url purge" "$(fetch_article_state)" "hit" + check "purge-url is accepted" \ + "$(purge '{"scope":"url","url":"http://ts.example.com/article"}')" "200" + check "the next request misses after a url purge, and refills" \ + "$(fetch_article_state)" "miss-stored" +fi + info "Cookie variant isolation and session bypass (mode: $MODE)" -if python3 - "$TS_PORT" "$MODE" "$WORK/origin.log" "$REQUEST_TIMEOUT_SECONDS" <<'PYEOF' +if python3 - "$TS_PORT" "$CONFIG_MODE" "$WORK/origin.log" "$REQUEST_TIMEOUT_SECONDS" <<'PYEOF' import gzip import sys import urllib.request diff --git a/tinybird/README.md b/tinybird/README.md index ee47b14f0..c0a0e976c 100644 --- a/tinybird/README.md +++ b/tinybird/README.md @@ -25,3 +25,70 @@ Events API host configured by `tinybird.api_host`. For runtime behavior, limits, privacy properties, and the Fastly-only emission boundary, see [Auction Telemetry](../docs/guide/telemetry.md). For exact configuration fields, see [Configuration](../docs/guide/configuration.md). + +## Deploy ordering + +**Apply datasource changes to Tinybird before deploying the code that emits them.** +`AuctionEventBatch::to_ndjson` serializes with plain `serde_json` and no +`skip_serializing_if`, so every declared field is always on the wire, including as `null`. +Rows carrying a column the datasource does not declare go to quarantine rather than being +rejected loudly. Check quarantine in the Tinybird UI: `pipes/quarantine_counts.pipe` +is an unconfigured placeholder returning `NULL`, not a working monitor. Connect it to the +workspace quarantine source before relying on its counts. A code-first deploy can lose +rows silently until the schema catches up. + +Adding a field means changing three things together: the struct in +`crates/trusted-server-core/src/auction/telemetry.rs`, the `SCHEMA` block in +`datasources/auction_events_raw.datasource`, and every row in +`fixtures/auction_events_raw.ndjson`. + +## Reading `origin_cache_shareable` + +Reports whether a request's origin response **would be** eligible to share between readers. + +**It is a predicate, not an outcome.** It records whether a request _would_ be eligible, +not whether anything was cached. A row with `1` still forced an origin fetch unless the +operator had set `creative_opportunities.origin_readthrough_enabled` (default `false`), and +even then the platform stores nothing if the origin's own `Cache-Control` refuses. So it +answers "how much traffic would the gate admit", and only in combination with that setting +does it bound "how much did it admit". + +Three caveats, each of which silently produces wrong numbers if a query ignores it. + +### The denominator is matching-slot candidates, not all requests + +Publisher summary rows cover requests with matching renderable slots while creative +opportunities are enabled. They include completed auctions **and skipped candidates**: +bots, prefetches, consent-denied readers, and requests with auctions disabled can emit a +`Skipped` summary when slots match. A page with no matching slot, a non-GET request, or +traffic with creative opportunities disabled produces no publisher summary. + +The query below therefore measures shareability among matching-slot candidates, including +skipped summaries. It is neither an ad-serving-pageview rate nor a site-wide rate. The +readthrough gate also governs origin fetches outside this telemetry population; measure +those separately before estimating whole-site impact. + +### `NULL` is "not measured", not "false" + +`AuctionObservationContext` is shared with the `/auction` API source, which makes no cache +decision and leaves the column `NULL`. A query that reads `NULL` as "not shareable" will be +wrong for that whole source class. Filter on the source first: + +```sql +SELECT countIf(origin_cache_shareable = 1) / count() AS shareable_rate +FROM auction_events_raw +WHERE event_kind = 'summary' + AND auction_source = 'initial_navigation' + AND origin_cache_shareable IS NOT NULL +``` + +### There is no template-cache hit/miss column + +Deliberately, and it cannot be added without restructuring when telemetry is emitted. The +store outcome is not knowable when the row is sent: the auction is collected during body +streaming, which takes the observation and emits the batch, and the template is only stored +afterwards. `hit` was reachable and `miss-stored` was not, which would have made hit rate +compute as roughly 100%. + +For debugging one request, the `x-ts-template-cache` response header still reports all nine +states. diff --git a/tinybird/datasources/auction_events_raw.datasource b/tinybird/datasources/auction_events_raw.datasource index 97e8251bb..2227f7a7f 100644 --- a/tinybird/datasources/auction_events_raw.datasource +++ b/tinybird/datasources/auction_events_raw.datasource @@ -35,6 +35,7 @@ SCHEMA > `is_win` Nullable(UInt8), `ad_domain` Nullable(String), `ad_id` Nullable(String), + `origin_cache_shareable` Nullable(UInt8), `event_date` Date DEFAULT toDate(event_ts) ENGINE "MergeTree" diff --git a/tinybird/fixtures/auction_events_raw.ndjson b/tinybird/fixtures/auction_events_raw.ndjson index 078d0c533..ebd091083 100644 --- a/tinybird/fixtures/auction_events_raw.ndjson +++ b/tinybird/fixtures/auction_events_raw.ndjson @@ -1,8 +1,8 @@ -{"event_ts":"2026-06-23 12:00:00.000","event_kind":"summary","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":"completed","terminal_reason":null,"slot_count":2,"total_time_ms":120,"winning_bid_count":1,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:00:00.000","event_kind":"provider_call","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"success","provider_response_time_ms":80,"provider_bid_count":2,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:00:00.000","event_kind":"provider_call","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"aps","provider_role":"bidder","status":"nobid","provider_response_time_ms":95,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:00:00.000","event_kind":"bid","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":"slot-1","slot_w":300,"slot_h":250,"media_type":"banner","seat":"kargo","price_cpm":1.25,"currency":"USD","is_win":1,"ad_domain":"advertiser.example","ad_id":"ad-1"} -{"event_ts":"2026-06-23 12:01:00.000","event_kind":"summary","auction_id":"650e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/sports","country":"US","region":"CA","is_mobile":1,"is_known_browser":1,"gdpr_applies":0,"consent_present":1,"terminal_status":"abandoned","terminal_reason":"pass_through_response","slot_count":1,"total_time_ms":35,"winning_bid_count":0,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:01:00.000","event_kind":"provider_call","auction_id":"650e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/sports","country":"US","region":"CA","is_mobile":1,"is_known_browser":1,"gdpr_applies":0,"consent_present":1,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"abandoned","provider_response_time_ms":35,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:02:00.000","event_kind":"summary","auction_id":"750e8400-e29b-41d4-a716-446655440000","auction_source":"spa_navigation","publisher_domain":"test-publisher.example","page_path":"/privacy","country":"DE","region":null,"is_mobile":2,"is_known_browser":2,"gdpr_applies":1,"consent_present":1,"terminal_status":"skipped","terminal_reason":"consent_denied","slot_count":1,"total_time_ms":0,"winning_bid_count":0,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:03:00.000","event_kind":"provider_call","auction_id":"850e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"http_status_error","provider_response_time_ms":15,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} +{"event_ts": "2026-06-23 12:00:00.000", "event_kind": "summary", "auction_id": "550e8400-e29b-41d4-a716-446655440000", "auction_source": "auction_api", "publisher_domain": "test-publisher.example", "page_path": "/article/:id", "country": "US", "region": "CA", "is_mobile": 0, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 0, "terminal_status": "completed", "terminal_reason": null, "slot_count": 2, "total_time_ms": 120, "winning_bid_count": 1, "provider": null, "provider_role": null, "status": null, "provider_response_time_ms": null, "provider_bid_count": null, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": null} +{"event_ts": "2026-06-23 12:00:00.000", "event_kind": "provider_call", "auction_id": "550e8400-e29b-41d4-a716-446655440000", "auction_source": "auction_api", "publisher_domain": "test-publisher.example", "page_path": "/article/:id", "country": "US", "region": "CA", "is_mobile": 0, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 0, "terminal_status": null, "terminal_reason": null, "slot_count": null, "total_time_ms": null, "winning_bid_count": null, "provider": "prebid", "provider_role": "bidder", "status": "success", "provider_response_time_ms": 80, "provider_bid_count": 2, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": null} +{"event_ts": "2026-06-23 12:00:00.000", "event_kind": "provider_call", "auction_id": "550e8400-e29b-41d4-a716-446655440000", "auction_source": "auction_api", "publisher_domain": "test-publisher.example", "page_path": "/article/:id", "country": "US", "region": "CA", "is_mobile": 0, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 0, "terminal_status": null, "terminal_reason": null, "slot_count": null, "total_time_ms": null, "winning_bid_count": null, "provider": "aps", "provider_role": "bidder", "status": "nobid", "provider_response_time_ms": 95, "provider_bid_count": 0, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": null} +{"event_ts": "2026-06-23 12:00:00.000", "event_kind": "bid", "auction_id": "550e8400-e29b-41d4-a716-446655440000", "auction_source": "auction_api", "publisher_domain": "test-publisher.example", "page_path": "/article/:id", "country": "US", "region": "CA", "is_mobile": 0, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 0, "terminal_status": null, "terminal_reason": null, "slot_count": null, "total_time_ms": null, "winning_bid_count": null, "provider": "prebid", "provider_role": null, "status": null, "provider_response_time_ms": null, "provider_bid_count": null, "slot_id": "slot-1", "slot_w": 300, "slot_h": 250, "media_type": "banner", "seat": "kargo", "price_cpm": 1.25, "currency": "USD", "is_win": 1, "ad_domain": "advertiser.example", "ad_id": "ad-1", "user_agent": null, "origin_cache_shareable": null} +{"event_ts": "2026-06-23 12:01:00.000", "event_kind": "summary", "auction_id": "650e8400-e29b-41d4-a716-446655440000", "auction_source": "initial_navigation", "publisher_domain": "test-publisher.example", "page_path": "/sports", "country": "US", "region": "CA", "is_mobile": 1, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 1, "terminal_status": "abandoned", "terminal_reason": "pass_through_response", "slot_count": 1, "total_time_ms": 35, "winning_bid_count": 0, "provider": null, "provider_role": null, "status": null, "provider_response_time_ms": null, "provider_bid_count": null, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": 1} +{"event_ts": "2026-06-23 12:01:00.000", "event_kind": "provider_call", "auction_id": "650e8400-e29b-41d4-a716-446655440000", "auction_source": "initial_navigation", "publisher_domain": "test-publisher.example", "page_path": "/sports", "country": "US", "region": "CA", "is_mobile": 1, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 1, "terminal_status": null, "terminal_reason": null, "slot_count": null, "total_time_ms": null, "winning_bid_count": null, "provider": "prebid", "provider_role": "bidder", "status": "abandoned", "provider_response_time_ms": 35, "provider_bid_count": 0, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": 1} +{"event_ts": "2026-06-23 12:02:00.000", "event_kind": "summary", "auction_id": "750e8400-e29b-41d4-a716-446655440000", "auction_source": "spa_navigation", "publisher_domain": "test-publisher.example", "page_path": "/privacy", "country": "DE", "region": null, "is_mobile": 2, "is_known_browser": 2, "gdpr_applies": 1, "consent_present": 1, "terminal_status": "skipped", "terminal_reason": "consent_denied", "slot_count": 1, "total_time_ms": 0, "winning_bid_count": 0, "provider": null, "provider_role": null, "status": null, "provider_response_time_ms": null, "provider_bid_count": null, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": null} +{"event_ts": "2026-06-23 12:03:00.000", "event_kind": "provider_call", "auction_id": "850e8400-e29b-41d4-a716-446655440000", "auction_source": "initial_navigation", "publisher_domain": "test-publisher.example", "page_path": "/article/:id", "country": "US", "region": "CA", "is_mobile": 0, "is_known_browser": 1, "gdpr_applies": 0, "consent_present": 0, "terminal_status": null, "terminal_reason": null, "slot_count": null, "total_time_ms": null, "winning_bid_count": null, "provider": "prebid", "provider_role": "bidder", "status": "http_status_error", "provider_response_time_ms": 15, "provider_bid_count": 0, "slot_id": null, "slot_w": null, "slot_h": null, "media_type": null, "seat": null, "price_cpm": null, "currency": null, "is_win": null, "ad_domain": null, "ad_id": null, "user_agent": null, "origin_cache_shareable": 0} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 67b485016..f552c989e 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -337,11 +337,11 @@ price_granularity = "dense" # only if your SSPs need more headroom and analytics confirm the DCL slip is OK. auction_timeout_ms = 500 # -# Initial-page delivery mode (spike/experimental). `inline` is the default and -# current production behaviour; `esi` is an opt-in Fastly Core Cache experiment -# storing an inert comment in a shared, reader-neutral template cache. See -# docs/guide/configuration.md before enabling. This and the cache-safety -# keys below belong in this [creative_opportunities] table. +# Initial-page delivery mode. `inline` is the default and current production +# behaviour; `esi` is opt-in per deployment and stores an inert comment in a +# shared, reader-neutral template cache. See docs/guide/configuration.md and run +# scripts/template-cache-local-test.sh esi before enabling. This and the +# cache keys below belong in this [creative_opportunities] table. # assembly_mode = "inline" # Request headers (besides Accept-Encoding) the origin may name in Vary. Every # emitted Vary name must be covered here or template storage is refused. Never @@ -364,7 +364,9 @@ auction_timeout_ms = 500 # With either list nonempty, false bypasses unlisted cookies; true asserts those # cookies do not change origin HTML. No identity or consent cookie is exempt. # With both lists empty, false bypasses all Cookie headers, as before. -# An origin `Vary: Cookie` is always refused, regardless of these settings. +# On the template path, origin `Vary: Cookie` is always refused. Readthrough +# has no response-side guard; run `ts origin probe-shareability` before enabling. +# Configured key-cookie dimensions disable readthrough, preserving template variants. # origin_is_cookie_independent = false # Invalid names, duplicates, cross-list overlap, and TS identity cookies in the # key list reject configuration. Named policies bypass ambiguous/nonconforming @@ -374,6 +376,18 @@ auction_timeout_ms = 500 # Rollback: older binaries reject both new fields even as empty lists. Remove # both fields AND restore independence=false or disable ESI when HTML uses cookies. # +# Applies shareability checks to the platform's origin cache. Default false keeps +# the existing policy: ad-serving requests bypass; other publisher requests use +# the platform default. True admits shareable ad requests and bypasses unshareable +# non-ad requests. A DIFFERENT cache from the template cache +# above, and a weaker one: the decision is made before the origin replies, so +# none of the template cache's response-side refusals (`Set-Cookie`, CSP nonce, +# missing freshness) apply. `ts origin probe-shareability` must pass on every +# axis and verdict first. Tagged readthrough objects can be purged with +# ts cache purge (--all or the exact reader-facing --page URL). Older untagged +# objects must expire on the origin TTL; see the configuration guide's rollback note. +# origin_readthrough_enabled = false +# # `gam_unit_path` may be a template. Supported placeholders: # {network_id} -> gam_network_id # {slot_id} -> the slot's id