Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,16 @@ versioning.

## [Unreleased]

### Changed

- Opt-in test batching stores identical common enrichment tags once in the
request metadata. Per-event process IDs and varying target metadata are
preserved, and the emitted body is validated before upload. Batching remains
disabled by default; real intake and representative performance validation
remain prerequisites for rollout.
- Payload schema validation avoids repeated unsupported-keyword scans when a
schema node contains none. Validation rules and diagnostics are unchanged.

### Fixed

- Go instrumentation preserves the test's `DD_SERVICE` environment. The uploader
Expand Down
15 changes: 15 additions & 0 deletions docs/Uploader_Reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,21 @@ type and test-level scopes after they have been copied to events. The uploader
does not generate a replacement process identity.
Real intake acceptance and rejection behavior still need validation before rollout.

In the opt-in batch path, string tags beginning with `bazel.`, `git.`, `ci.`,
`os.`, or `runtime.` move from event metadata to `metadata["*"]` only when every
supported event has the same value. A key mentioned in another metadata scope
stays in each event, preserving the precedence above. Different target tags,
`test.*` fields, hierarchy IDs, services and per-event `runtime-id` values keep
their existing locations. Unknown versions or event shapes are left unchanged.
The emitted body is schema-validated after compaction and remains subject to
the existing uncompressed byte limit. The JSON report records its CPU and wall
time under `performance.worker_phases.metadata_compaction`.

Compaction reduces repeated data before final validation and gzip. It does not
change the gzip level or introduce another setting; disabling batching also
disables compaction. Ordinary per-file uploads, coverage and telemetry retain
their current representations.

HTTP success acknowledges all whole sources in that request. A rejected batch
with status 400, 413, or 422 gets one per-source pass to isolate the rejection.
Authentication, transport, rate-limit, and server failures use the existing
Expand Down
8 changes: 7 additions & 1 deletion tools/core/uploader_py/batch_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from validate_payload_schema import validate_payload as validate_schema_payload
from topt_runtime.bep_execution import test_result_event_count, completed_empty_test_session

from .batching import BatchCompatibilityError, localize_runtime_ids
from .batching import BatchCompatibilityError, compact_shared_metadata, localize_runtime_ids
from .file_worker import (
WorkerRuntime, WorkerTransport, _cleanup_source, _enrich_and_validate_test, _phase, _skipped,
_failed, _http_failure, _prepare_test_requests, _read_json_object,
Expand Down Expand Up @@ -324,6 +324,12 @@ def _prepare_and_send(
raise ValueError("batch exceeded the uncompressed request guard")
with _phase(runtime, "validation"):
document = strict_json_loads(body)
with _phase(runtime, "metadata_compaction"):
if compact_shared_metadata(document):
del body
body = compact_json_bytes(document)
if len(body) > MAX_TEST_PAYLOAD_BYTES:
raise ValueError("compacted batch exceeded the uncompressed request guard")
assert runtime.batch_schema is not None
if not validate_schema_payload(document, runtime.batch_schema).valid:
return WorkItemOutcome(tuple(_failed(source.task, "invalid_batch_representation", "outbound batch failed schema validation") for source in sources)), None
Expand Down
46 changes: 46 additions & 0 deletions tools/core/uploader_py/batching.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,52 @@ class BatchCompatibilityError(ValueError):

_TEST_LEVEL_TYPES = frozenset(("test", "test_suite_end", "test_module_end", "test_session_end"))
_FIXED_TYPES = _TEST_LEVEL_TYPES | {"span"}
_SHARED_METADATA_PREFIXES = ("bazel.", "git.", "ci.", "os.", "runtime.")


def compact_shared_metadata(payload: dict[str, Any]) -> int:
"""Move common enrichment tags in an owned request to its global defaults.

Only string tags present with the same value in every supported event can
move. Leave keys mentioned in another metadata scope alone: fixed-type
overrides have different precedence. Protocol fields and process IDs stay
in their existing locations. No caller-owned source file is changed.
"""
metadata, events = payload.get("metadata"), payload.get("events")
if type(payload.get("version")) is not int or payload["version"] != 1 or not isinstance(metadata, dict):
return 0
if not isinstance(events, list) or len(events) < 2:
return 0
if any(not isinstance(scope, dict) for scope in metadata.values()):
return 0
global_meta = metadata.get("*")
if not isinstance(global_meta, dict):
return 0
event_metadata = []
for event in events:
if not isinstance(event, dict) or not isinstance(event.get("type"), str) or event["type"] not in _FIXED_TYPES:
return 0
content = event.get("content")
if not isinstance(content, dict) or not isinstance(content.get("meta"), dict):
return 0
event_metadata.append(content["meta"])
overrides = {key for name, scope in metadata.items() if name != "*" for key in scope}
common = {
key: value for key, value in event_metadata[0].items()
if isinstance(key, str) and key.startswith(_SHARED_METADATA_PREFIXES)
and isinstance(value, str) and key not in overrides
}
for event_meta in event_metadata[1:]:
different = [key for key, value in common.items() if event_meta.get(key) != value]
for key in different:
del common[key]
if not common:
return 0
global_meta.update(common)
for event_meta in event_metadata:
for key in common:
del event_meta[key]
return len(common) * len(events)


def localize_runtime_ids(payload: dict[str, Any], *, max_additional_bytes: int | None = None) -> bytes:
Expand Down
39 changes: 20 additions & 19 deletions tools/core/validate_payload_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,25 +275,26 @@ def _validate(
if not isinstance(schema, dict):
return

for keyword in _UNSUPPORTED_KEYWORDS:
if keyword in schema:
if unsupported_policy == "error":
errors.append(
f"{path}: unsupported JSON Schema keyword '{keyword}' "
"is not supported by this validator"
)
return
if keyword not in warned_unsupported:
warning = (
f"warning: unsupported JSON Schema keyword '{keyword}' "
f"at {path} is ignored"
)
if warning_output is None:
if enabled("WARN"):
print(warning, file=sys.stderr)
else:
warning_output(warning)
warned_unsupported.add(keyword)
if not _UNSUPPORTED_KEYWORDS.isdisjoint(schema):
for keyword in _UNSUPPORTED_KEYWORDS:
if keyword in schema:
if unsupported_policy == "error":
errors.append(
f"{path}: unsupported JSON Schema keyword '{keyword}' "
"is not supported by this validator"
)
return
if keyword not in warned_unsupported:
warning = (
f"warning: unsupported JSON Schema keyword '{keyword}' "
f"at {path} is ignored"
)
if warning_output is None:
if enabled("WARN"):
print(warning, file=sys.stderr)
else:
warning_output(warning)
warned_unsupported.add(keyword)

if "$ref" in schema:
try:
Expand Down
33 changes: 33 additions & 0 deletions tools/tests/python/test_python_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,39 @@ def setUpClass(cls) -> None:
"tools/core/validate_payload_schema.py",
)

def test_type_only_leaves_keep_json_types_and_diagnostics(self) -> None:
cases = [
("string", "ok", 3),
("integer", 3, True),
("number", 3.5, False),
("boolean", True, 1),
("null", None, "null"),
("array", [None, {"anything": True}], {}),
("object", {"anything": [None]}, []),
(["null", "integer"], None, True),
]
for expected, accepted, rejected in cases:
with self.subTest(expected=expected):
schema = {"type": expected}
valid = self.mod.validate_payload(accepted, schema)
invalid = self.mod.validate_payload(rejected, schema)
self.assertTrue(valid.valid)
self.assertEqual((f"$: expected type {expected}",), invalid.errors)
self.assertEqual((), valid.warnings)
self.assertEqual(1, valid.stats["nodes"])
self.assertEqual(1, valid.stats["type_checks"])
self.assertEqual(valid.stats, invalid.stats)
self.assertEqual({"type": expected}, schema)

def test_type_leaves_do_not_bypass_other_schema_constraints(self) -> None:
for constraint in ({"const": "other"}, {"enum": ["other"]}, {"not": {}}):
with self.subTest(constraint=constraint):
result = self.mod.validate_payload("value", {"type": "string", **constraint})
self.assertFalse(result.valid)
warned = self.mod.validate_payload("value", {"type": "string", "not": {}}, unsupported_policy="warn")
self.assertTrue(warned.valid)
self.assertEqual(1, len(warned.warnings))

def _run_main(
self,
schema_path: str,
Expand Down
100 changes: 100 additions & 0 deletions tools/tests/python/test_uploader_batching_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
add_uploader_runtime_to_path()

from uploader_py.batching import BatchCompatibilityError, localize_runtime_ids, schema_for_event_runtime_ids # noqa: E402
from uploader_py import batching as batch_metadata # noqa: E402
from uploader_py.application import run_uploader # noqa: E402
from uploader_py.config import ConfigError, parse_uploader_config # noqa: E402
from uploader_py.endpoints import build_endpoints # noqa: E402
Expand Down Expand Up @@ -63,6 +64,105 @@ def _runtime(root, **changes):
return _base_runtime(root, **values)


class SharedMetadataTests(unittest.TestCase):
def shared_payload(self):
body = payload()
body["events"] *= 2
body["events"] = deepcopy(body["events"])
# Build independent event metadata, including different process IDs.
body["events"][1] = deepcopy(body["events"][0])
for index, event in enumerate(body["events"]):
event["content"]["meta"].update({
"runtime-id": f"process-{index}",
"git.commit.sha": "a" * 40,
"ci.pipeline.id": "pipeline",
"os.platform": "linux",
"bazel.target": f"//package:test_{index}",
"test.codeowners": '["@team"]',
})
return body

def test_moves_only_common_enrichment_tags_and_preserves_identities(self):
body = self.shared_payload()
original = deepcopy(body)
self.assertGreater(batch_metadata.compact_shared_metadata(body), 0)
for key in ("git.commit.sha", "ci.pipeline.id", "os.platform"):
self.assertEqual(original["events"][0]["content"]["meta"][key], body["metadata"]["*"][key])
self.assertTrue(all(key not in e["content"]["meta"] for e in body["events"]))
for before, after in zip(original["events"], body["events"]):
restored = deepcopy(after)
for key in ("git.commit.sha", "ci.pipeline.id", "os.platform"):
restored["content"]["meta"][key] = body["metadata"]["*"][key]
self.assertEqual(before, restored)
self.assertEqual("process-a", body["metadata"]["*"]["runtime-id"])

def test_mixed_event_types_keep_effective_defaults_and_compaction_is_idempotent(self):
body = self.shared_payload()
body["events"] = [deepcopy(body["events"][0]) for _ in range(5)]
for kind, event in zip(("test", "test_suite_end", "test_module_end", "test_session_end", "span"), body["events"]):
event["type"] = kind
body["metadata"]["*"]["git.commit.sha"] = "shadowed-default"
body["metadata"]["test_levels"] = {"ci.pipeline.id": "level-default"}
body["metadata"]["span"] = {"os.platform": "type-override"}
batch_metadata.compact_shared_metadata(body)
self.assertEqual("a" * 40, body["metadata"]["*"]["git.commit.sha"])
for event in body["events"]:
self.assertNotIn("git.commit.sha", event["content"]["meta"])
self.assertEqual("pipeline", event["content"]["meta"]["ci.pipeline.id"])
self.assertEqual("linux", event["content"]["meta"]["os.platform"])
compacted = deepcopy(body)
self.assertEqual(0, batch_metadata.compact_shared_metadata(body))
self.assertEqual(compacted, body)

def test_scope_overrides_missing_and_nonstring_values_are_preserved(self):
for scope in ("test", "test_levels", "opaque"):
body = self.shared_payload()
body["metadata"][scope] = {"git.commit.sha": "override"}
for event in body["events"]:
event["content"]["meta"]["ci.object"] = {"nested": "value"}
body["events"][1]["content"]["meta"].pop("ci.pipeline.id")
original = deepcopy(body)
batch_metadata.compact_shared_metadata(body)
self.assertEqual(original["metadata"][scope], body["metadata"][scope])
for before, after in zip(original["events"], body["events"]):
for key in ("git.commit.sha", "ci.pipeline.id", "ci.object"):
self.assertEqual(before["content"]["meta"].get(key), after["content"]["meta"].get(key))

def test_unsupported_shapes_are_not_modified(self):
for change in ("future-version", "boolean-version", "future-event", "array-event-type", "bad-scope", "bad-meta", "single-event"):
body = self.shared_payload()
if change == "future-version": body["version"] = 2
elif change == "boolean-version": body["version"] = True
elif change == "future-event": body["events"][1]["type"] = "future"
elif change == "array-event-type": body["events"][1]["type"] = ["test"]
elif change == "bad-scope": body["metadata"]["opaque"] = None
elif change == "bad-meta": body["events"][1]["content"]["meta"] = None
else: body["events"] = body["events"][:1]
original = deepcopy(body)
with self.subTest(change=change):
self.assertEqual(0, batch_metadata.compact_shared_metadata(body))
self.assertEqual(original, body)

def test_actual_batch_compacts_metadata_without_changing_retained_sources(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
tasks = []
for index in range(2):
source = root / f"{index}.json"
source.write_text(json.dumps(payload(f"process-{index}")))
tasks.append(FileTask(str(index), source, source.name, PayloadType.TEST))
originals = [task.source_path.read_bytes() for task in tasks]
transport = _FakeTransport()
run = run_file_workers(tasks, workers=1, runtime=_runtime(root, keep_payloads=True),
transport_factory=lambda: transport, process_file=process_file, prepare_work=prepare_test_work_items)
self.assertTrue(all(r.status is FileStatus.SUCCEEDED for r in run.results))
self.assertEqual(1, len(transport.json_calls))
document = json.loads(transport.json_calls[0]["body"])
self.assertEqual("abcdef", document["metadata"]["*"]["git.commit.sha"])
self.assertTrue(all("git.commit.sha" not in e["content"]["meta"] for e in document["events"]))
self.assertEqual(originals, [task.source_path.read_bytes() for task in tasks])


class BatchEnvelopeTests(unittest.TestCase):
def test_localized_payload_keeps_the_canonical_global_runtime_requirement(self):
schema = json.loads(resolve_runfile("tools/core/schemas/agentless-schema.json").read_text())
Expand Down
Loading