From 7362e251982f3548e315f23f8b4c62a46d31d13d Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 13 Nov 2026 12:46:34 +0100 Subject: [PATCH 1/2] uploader: skip redundant schema keyword scans --- tools/core/validate_payload_schema.py | 39 +++++++++++++------------ tools/tests/python/test_python_tools.py | 33 +++++++++++++++++++++ 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/tools/core/validate_payload_schema.py b/tools/core/validate_payload_schema.py index 2593ff74..2bf9bec5 100644 --- a/tools/core/validate_payload_schema.py +++ b/tools/core/validate_payload_schema.py @@ -275,25 +275,26 @@ def _validate( if not isinstance(schema, dict): return - for keyword in _UNSUPPORTED_KEYWORDS: - if keyword in schema: - if unsupported_policy == "error": - errors.append( - f"{path}: unsupported JSON Schema keyword '{keyword}' " - "is not supported by this validator" - ) - return - if keyword not in warned_unsupported: - warning = ( - f"warning: unsupported JSON Schema keyword '{keyword}' " - f"at {path} is ignored" - ) - if warning_output is None: - if enabled("WARN"): - print(warning, file=sys.stderr) - else: - warning_output(warning) - warned_unsupported.add(keyword) + if not _UNSUPPORTED_KEYWORDS.isdisjoint(schema): + for keyword in _UNSUPPORTED_KEYWORDS: + if keyword in schema: + if unsupported_policy == "error": + errors.append( + f"{path}: unsupported JSON Schema keyword '{keyword}' " + "is not supported by this validator" + ) + return + if keyword not in warned_unsupported: + warning = ( + f"warning: unsupported JSON Schema keyword '{keyword}' " + f"at {path} is ignored" + ) + if warning_output is None: + if enabled("WARN"): + print(warning, file=sys.stderr) + else: + warning_output(warning) + warned_unsupported.add(keyword) if "$ref" in schema: try: diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index f79093af..902cd5d4 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -287,6 +287,39 @@ def setUpClass(cls) -> None: "tools/core/validate_payload_schema.py", ) + def test_type_only_leaves_keep_json_types_and_diagnostics(self) -> None: + cases = [ + ("string", "ok", 3), + ("integer", 3, True), + ("number", 3.5, False), + ("boolean", True, 1), + ("null", None, "null"), + ("array", [None, {"anything": True}], {}), + ("object", {"anything": [None]}, []), + (["null", "integer"], None, True), + ] + for expected, accepted, rejected in cases: + with self.subTest(expected=expected): + schema = {"type": expected} + valid = self.mod.validate_payload(accepted, schema) + invalid = self.mod.validate_payload(rejected, schema) + self.assertTrue(valid.valid) + self.assertEqual((f"$: expected type {expected}",), invalid.errors) + self.assertEqual((), valid.warnings) + self.assertEqual(1, valid.stats["nodes"]) + self.assertEqual(1, valid.stats["type_checks"]) + self.assertEqual(valid.stats, invalid.stats) + self.assertEqual({"type": expected}, schema) + + def test_type_leaves_do_not_bypass_other_schema_constraints(self) -> None: + for constraint in ({"const": "other"}, {"enum": ["other"]}, {"not": {}}): + with self.subTest(constraint=constraint): + result = self.mod.validate_payload("value", {"type": "string", **constraint}) + self.assertFalse(result.valid) + warned = self.mod.validate_payload("value", {"type": "string", "not": {}}, unsupported_policy="warn") + self.assertTrue(warned.valid) + self.assertEqual(1, len(warned.warnings)) + def _run_main( self, schema_path: str, From 07ff02963a79485da96de842d44540fd461a7b17 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 13 Nov 2026 12:46:49 +0100 Subject: [PATCH 2/2] uploader: compact common metadata in opt-in test batches --- CHANGELOG.md | 10 ++ docs/Uploader_Reference.md | 15 +++ tools/core/uploader_py/batch_worker.py | 8 +- tools/core/uploader_py/batching.py | 46 ++++++++ .../python/test_uploader_batching_tools.py | 100 ++++++++++++++++++ 5 files changed, 178 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68491d3c..dd4de7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,16 @@ versioning. ## [Unreleased] +### Changed + +- Opt-in test batching stores identical common enrichment tags once in the + request metadata. Per-event process IDs and varying target metadata are + preserved, and the emitted body is validated before upload. Batching remains + disabled by default; real intake and representative performance validation + remain prerequisites for rollout. +- Payload schema validation avoids repeated unsupported-keyword scans when a + schema node contains none. Validation rules and diagnostics are unchanged. + ### Fixed - Go instrumentation preserves the test's `DD_SERVICE` environment. The uploader diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index 6061166b..4f053d7c 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -255,6 +255,21 @@ type and test-level scopes after they have been copied to events. The uploader does not generate a replacement process identity. Real intake acceptance and rejection behavior still need validation before rollout. +In the opt-in batch path, string tags beginning with `bazel.`, `git.`, `ci.`, +`os.`, or `runtime.` move from event metadata to `metadata["*"]` only when every +supported event has the same value. A key mentioned in another metadata scope +stays in each event, preserving the precedence above. Different target tags, +`test.*` fields, hierarchy IDs, services and per-event `runtime-id` values keep +their existing locations. Unknown versions or event shapes are left unchanged. +The emitted body is schema-validated after compaction and remains subject to +the existing uncompressed byte limit. The JSON report records its CPU and wall +time under `performance.worker_phases.metadata_compaction`. + +Compaction reduces repeated data before final validation and gzip. It does not +change the gzip level or introduce another setting; disabling batching also +disables compaction. Ordinary per-file uploads, coverage and telemetry retain +their current representations. + HTTP success acknowledges all whole sources in that request. A rejected batch with status 400, 413, or 422 gets one per-source pass to isolate the rejection. Authentication, transport, rate-limit, and server failures use the existing diff --git a/tools/core/uploader_py/batch_worker.py b/tools/core/uploader_py/batch_worker.py index 6b390160..20efdeb2 100644 --- a/tools/core/uploader_py/batch_worker.py +++ b/tools/core/uploader_py/batch_worker.py @@ -21,7 +21,7 @@ from validate_payload_schema import validate_payload as validate_schema_payload from topt_runtime.bep_execution import test_result_event_count, completed_empty_test_session -from .batching import BatchCompatibilityError, localize_runtime_ids +from .batching import BatchCompatibilityError, compact_shared_metadata, localize_runtime_ids from .file_worker import ( WorkerRuntime, WorkerTransport, _cleanup_source, _enrich_and_validate_test, _phase, _skipped, _failed, _http_failure, _prepare_test_requests, _read_json_object, @@ -324,6 +324,12 @@ def _prepare_and_send( raise ValueError("batch exceeded the uncompressed request guard") with _phase(runtime, "validation"): document = strict_json_loads(body) + with _phase(runtime, "metadata_compaction"): + if compact_shared_metadata(document): + del body + body = compact_json_bytes(document) + if len(body) > MAX_TEST_PAYLOAD_BYTES: + raise ValueError("compacted batch exceeded the uncompressed request guard") assert runtime.batch_schema is not None if not validate_schema_payload(document, runtime.batch_schema).valid: return WorkItemOutcome(tuple(_failed(source.task, "invalid_batch_representation", "outbound batch failed schema validation") for source in sources)), None diff --git a/tools/core/uploader_py/batching.py b/tools/core/uploader_py/batching.py index f19746ac..0e6dae8e 100644 --- a/tools/core/uploader_py/batching.py +++ b/tools/core/uploader_py/batching.py @@ -26,6 +26,52 @@ class BatchCompatibilityError(ValueError): _TEST_LEVEL_TYPES = frozenset(("test", "test_suite_end", "test_module_end", "test_session_end")) _FIXED_TYPES = _TEST_LEVEL_TYPES | {"span"} +_SHARED_METADATA_PREFIXES = ("bazel.", "git.", "ci.", "os.", "runtime.") + + +def compact_shared_metadata(payload: dict[str, Any]) -> int: + """Move common enrichment tags in an owned request to its global defaults. + + Only string tags present with the same value in every supported event can + move. Leave keys mentioned in another metadata scope alone: fixed-type + overrides have different precedence. Protocol fields and process IDs stay + in their existing locations. No caller-owned source file is changed. + """ + metadata, events = payload.get("metadata"), payload.get("events") + if type(payload.get("version")) is not int or payload["version"] != 1 or not isinstance(metadata, dict): + return 0 + if not isinstance(events, list) or len(events) < 2: + return 0 + if any(not isinstance(scope, dict) for scope in metadata.values()): + return 0 + global_meta = metadata.get("*") + if not isinstance(global_meta, dict): + return 0 + event_metadata = [] + for event in events: + if not isinstance(event, dict) or not isinstance(event.get("type"), str) or event["type"] not in _FIXED_TYPES: + return 0 + content = event.get("content") + if not isinstance(content, dict) or not isinstance(content.get("meta"), dict): + return 0 + event_metadata.append(content["meta"]) + overrides = {key for name, scope in metadata.items() if name != "*" for key in scope} + common = { + key: value for key, value in event_metadata[0].items() + if isinstance(key, str) and key.startswith(_SHARED_METADATA_PREFIXES) + and isinstance(value, str) and key not in overrides + } + for event_meta in event_metadata[1:]: + different = [key for key, value in common.items() if event_meta.get(key) != value] + for key in different: + del common[key] + if not common: + return 0 + global_meta.update(common) + for event_meta in event_metadata: + for key in common: + del event_meta[key] + return len(common) * len(events) def localize_runtime_ids(payload: dict[str, Any], *, max_additional_bytes: int | None = None) -> bytes: diff --git a/tools/tests/python/test_uploader_batching_tools.py b/tools/tests/python/test_uploader_batching_tools.py index e1993e04..0dadee6d 100644 --- a/tools/tests/python/test_uploader_batching_tools.py +++ b/tools/tests/python/test_uploader_batching_tools.py @@ -31,6 +31,7 @@ add_uploader_runtime_to_path() from uploader_py.batching import BatchCompatibilityError, localize_runtime_ids, schema_for_event_runtime_ids # noqa: E402 +from uploader_py import batching as batch_metadata # noqa: E402 from uploader_py.application import run_uploader # noqa: E402 from uploader_py.config import ConfigError, parse_uploader_config # noqa: E402 from uploader_py.endpoints import build_endpoints # noqa: E402 @@ -63,6 +64,105 @@ def _runtime(root, **changes): return _base_runtime(root, **values) +class SharedMetadataTests(unittest.TestCase): + def shared_payload(self): + body = payload() + body["events"] *= 2 + body["events"] = deepcopy(body["events"]) + # Build independent event metadata, including different process IDs. + body["events"][1] = deepcopy(body["events"][0]) + for index, event in enumerate(body["events"]): + event["content"]["meta"].update({ + "runtime-id": f"process-{index}", + "git.commit.sha": "a" * 40, + "ci.pipeline.id": "pipeline", + "os.platform": "linux", + "bazel.target": f"//package:test_{index}", + "test.codeowners": '["@team"]', + }) + return body + + def test_moves_only_common_enrichment_tags_and_preserves_identities(self): + body = self.shared_payload() + original = deepcopy(body) + self.assertGreater(batch_metadata.compact_shared_metadata(body), 0) + for key in ("git.commit.sha", "ci.pipeline.id", "os.platform"): + self.assertEqual(original["events"][0]["content"]["meta"][key], body["metadata"]["*"][key]) + self.assertTrue(all(key not in e["content"]["meta"] for e in body["events"])) + for before, after in zip(original["events"], body["events"]): + restored = deepcopy(after) + for key in ("git.commit.sha", "ci.pipeline.id", "os.platform"): + restored["content"]["meta"][key] = body["metadata"]["*"][key] + self.assertEqual(before, restored) + self.assertEqual("process-a", body["metadata"]["*"]["runtime-id"]) + + def test_mixed_event_types_keep_effective_defaults_and_compaction_is_idempotent(self): + body = self.shared_payload() + body["events"] = [deepcopy(body["events"][0]) for _ in range(5)] + for kind, event in zip(("test", "test_suite_end", "test_module_end", "test_session_end", "span"), body["events"]): + event["type"] = kind + body["metadata"]["*"]["git.commit.sha"] = "shadowed-default" + body["metadata"]["test_levels"] = {"ci.pipeline.id": "level-default"} + body["metadata"]["span"] = {"os.platform": "type-override"} + batch_metadata.compact_shared_metadata(body) + self.assertEqual("a" * 40, body["metadata"]["*"]["git.commit.sha"]) + for event in body["events"]: + self.assertNotIn("git.commit.sha", event["content"]["meta"]) + self.assertEqual("pipeline", event["content"]["meta"]["ci.pipeline.id"]) + self.assertEqual("linux", event["content"]["meta"]["os.platform"]) + compacted = deepcopy(body) + self.assertEqual(0, batch_metadata.compact_shared_metadata(body)) + self.assertEqual(compacted, body) + + def test_scope_overrides_missing_and_nonstring_values_are_preserved(self): + for scope in ("test", "test_levels", "opaque"): + body = self.shared_payload() + body["metadata"][scope] = {"git.commit.sha": "override"} + for event in body["events"]: + event["content"]["meta"]["ci.object"] = {"nested": "value"} + body["events"][1]["content"]["meta"].pop("ci.pipeline.id") + original = deepcopy(body) + batch_metadata.compact_shared_metadata(body) + self.assertEqual(original["metadata"][scope], body["metadata"][scope]) + for before, after in zip(original["events"], body["events"]): + for key in ("git.commit.sha", "ci.pipeline.id", "ci.object"): + self.assertEqual(before["content"]["meta"].get(key), after["content"]["meta"].get(key)) + + def test_unsupported_shapes_are_not_modified(self): + for change in ("future-version", "boolean-version", "future-event", "array-event-type", "bad-scope", "bad-meta", "single-event"): + body = self.shared_payload() + if change == "future-version": body["version"] = 2 + elif change == "boolean-version": body["version"] = True + elif change == "future-event": body["events"][1]["type"] = "future" + elif change == "array-event-type": body["events"][1]["type"] = ["test"] + elif change == "bad-scope": body["metadata"]["opaque"] = None + elif change == "bad-meta": body["events"][1]["content"]["meta"] = None + else: body["events"] = body["events"][:1] + original = deepcopy(body) + with self.subTest(change=change): + self.assertEqual(0, batch_metadata.compact_shared_metadata(body)) + self.assertEqual(original, body) + + def test_actual_batch_compacts_metadata_without_changing_retained_sources(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tasks = [] + for index in range(2): + source = root / f"{index}.json" + source.write_text(json.dumps(payload(f"process-{index}"))) + tasks.append(FileTask(str(index), source, source.name, PayloadType.TEST)) + originals = [task.source_path.read_bytes() for task in tasks] + transport = _FakeTransport() + run = run_file_workers(tasks, workers=1, runtime=_runtime(root, keep_payloads=True), + transport_factory=lambda: transport, process_file=process_file, prepare_work=prepare_test_work_items) + self.assertTrue(all(r.status is FileStatus.SUCCEEDED for r in run.results)) + self.assertEqual(1, len(transport.json_calls)) + document = json.loads(transport.json_calls[0]["body"]) + self.assertEqual("abcdef", document["metadata"]["*"]["git.commit.sha"]) + self.assertTrue(all("git.commit.sha" not in e["content"]["meta"] for e in document["events"])) + self.assertEqual(originals, [task.source_path.read_bytes() for task in tasks]) + + class BatchEnvelopeTests(unittest.TestCase): def test_localized_payload_keeps_the_canonical_global_runtime_requirement(self): schema = json.loads(resolve_runfile("tools/core/schemas/agentless-schema.json").read_text())