Stabilize CodSpeed microbenchmarks

.github/workflows/codspeed.yml

@@ -297,7 +297,7 @@ jobs:
               for package in meta["packages"]
               for target in package["targets"]
               if "bench" in target["kind"]
-              and "cpu_features]" in pathlib.Path(target["src_path"]).read_text()
+              and "#[vortex_bench_support::cpu_features" in pathlib.Path(target["src_path"]).read_text()
           )
           if not tagged:
               raise SystemExit("no benchmark carries `#[cpu_features]`; this job has nothing to measure")

benchmarks/bench-support/src/lib.rs

@@ -12,10 +12,10 @@ use quote::quote;
 use syn::ItemFn;
 use syn::parse_macro_input;
 
-/// Measure this benchmark on every walltime CPU-feature leg instead of in simulation.
+/// Measure this benchmark on walltime CPU-feature legs instead of in simulation.
 ///
-/// Takes no argument: the benchmark runs on all of them. Write it *above* `#[divan::bench]`,
-/// whose arguments it fills in — the name is qualified with the leg that produced it, both as
+/// Runs on every leg except those named in `exclude = ["neon", ...]`. Write it above
+/// `#[divan::bench]`, whose arguments it fills in. The name is qualified with the leg, both as
 /// a `::` component the CI filter selects on and as a suffix on the leaf name CodSpeed
 /// reports under, so the legs report one series each rather than fighting over a shared name.
 /// `ignore` takes the benchmark out of the sharded simulation job. A plain `cargo bench` runs
@@ -30,22 +30,25 @@ use syn::parse_macro_input;
 /// Spell it out in full rather than importing it: benchmark files are read a function at a
 /// time, and the path says where the behaviour comes from.
 ///
-/// This is for code that is written once and *compiled* differently per feature set — a
+/// This is for code that is written once and *compiled* differently per feature set: a
 /// shipped entry point that selects its kernel through `cfg(target_feature)`, or a scalar
 /// loop whose auto-vectorization depends on the build. A hand-written kernel for one
 /// instruction set extension is a different thing: it cannot run on the other legs, so it
 /// does not belong here. Keep those on `#[cfg(not(codspeed))]` for local A/B runs.
 #[proc_macro_attribute]
 pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream {
-    if !attr.is_empty() {
-        let attr = TokenStream2::from(attr);
-        return syn::Error::new_spanned(
-            attr,
-            "`#[cpu_features]` takes no argument; a tagged benchmark runs on every feature-set leg",
-        )
-        .to_compile_error()
-        .into();
-    }
+    let excluded = if attr.is_empty() {
+        quote!(false)
+    } else {
+        let argument = parse_macro_input!(attr as syn::MetaNameValue);
+        if !argument.path.is_ident("exclude") {
+            return syn::Error::new_spanned(argument, "expected `exclude = [\"feature\", ...]`")
+                .to_compile_error()
+                .into();
+        }
+        let features = argument.value;
+        quote!((#features).contains(&env!("VORTEX_BENCH_VARIANT")))
+    };
 
     let mut function = parse_macro_input!(item as ItemFn);
 
@@ -117,7 +120,7 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream {
                 #name,
                 env!("VORTEX_BENCH_SUFFIX"),
             ),
-            ignore = env!("VORTEX_BENCH_VARIANT") == "simulation",
+            ignore = env!("VORTEX_BENCH_VARIANT") == "simulation" || #excluded,
         )
     };
 

encodings/fastlanes/benches/bitpacking_filter.rs

@@ -69,8 +69,10 @@ fn fixture(selected_per_chunk: usize) -> (ArrayRef, Mask) {
     (packed, Mask::from_indices(LEN, indices))
 }
 
-#[vortex_bench_support::cpu_features]
-#[divan::bench(args = [64, 80])]
+// Batch short iterations to reduce variation in the fastest walltime sample.
+// The NEON cases still vary on unchanged code, so keep those measurements local.
+#[vortex_bench_support::cpu_features(exclude = ["neon"])]
+#[divan::bench(args = [64, 80], sample_size = 64)]
 fn filter(bencher: Bencher, selected_per_chunk: usize) {
     let (packed, mask) = fixture(selected_per_chunk);
 

encodings/fastlanes/benches/bitpacking_take.rs

@@ -83,8 +83,10 @@ fn threshold_fixture(selected_per_chunk: usize) -> (ArrayRef, ArrayRef) {
     (packed, indices)
 }
 
-#[vortex_bench_support::cpu_features]
-#[divan::bench(args = [64, 80])]
+// Batch short iterations to reduce variation in the fastest walltime sample.
+// The NEON cases still vary on unchanged code, so keep those measurements local.
+#[vortex_bench_support::cpu_features(exclude = ["neon"])]
+#[divan::bench(args = [64, 80], sample_size = 16)]
 fn threshold(bencher: Bencher, selected_per_chunk: usize) {
     let (packed, indices) = threshold_fixture(selected_per_chunk);
 

vortex-array/benches/binary_ops.rs

@@ -4,7 +4,7 @@
 //! Benchmarks for the binary `Operator` path, over primitive, decimal, and boolean inputs.
 //!
 //! The primitive arithmetic and comparison cases carry `#[cpu_features]`, so they are
-//! measured on every walltime CPU-feature leg rather than in simulation. Each is written
+//! measured on walltime CPU-feature legs rather than in simulation. Each is written
 //! once and compiled differently per leg: the kernel underneath is a portable lane loop, and
 //! how wide the compiler vectorizes it is decided by the build flags, not by the source.
 //!
@@ -237,7 +237,8 @@ fn mul_u32_nonnull(bencher: Bencher) {
     bench_primitive(bencher, lhs, rhs, Operator::Mul);
 }
 
-#[vortex_bench_support::cpu_features]
+// This case varies on unchanged code on the NEON leg. The cause is unknown.
+#[vortex_bench_support::cpu_features(exclude = ["neon"])]
 #[divan::bench]
 fn mul_u64_nonnull(bencher: Bencher) {
     let lhs = primitive_u64_small_nonnull(1, I64_LEN).into_array();

vortex-array/benches/chunk_array_builder.rs

@@ -63,7 +63,8 @@ fn chunked_bool_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usi
         })
 }
 
-#[divan::bench(args = BENCH_ARGS)]
+// The largest nullable case varies on unchanged code in simulation. The cause is unknown.
+#[divan::bench(args = if cfg!(codspeed) { &[(10, 100), (100, 50)] } else { BENCH_ARGS })]
 fn chunked_opt_bool_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usize)) {
     let chunk = make_opt_bool_chunks(len, chunk_count);
 

vortex-array/benches/row_fn_utf8_input.rs

@@ -6,6 +6,8 @@
     reason = "benchmark fixtures reduce values below 26 before narrowing"
 )]
 
+use std::fmt::Display;
+use std::fmt::Formatter;
 use std::sync::LazyLock;
 
 use divan::Bencher;
@@ -63,6 +65,37 @@ enum Layout {
 
 const LAYOUTS: &[Layout] = &[Layout::VarBin, Layout::VarBinView];
 
+#[derive(Clone, Copy)]
+struct Case {
+    len: usize,
+    layout: Layout,
+}
+
+impl Display for Case {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        // Preserve the report names from the former const-length and layout parameters.
+        let layout = match self.layout {
+            Layout::VarBin => "VarBin",
+            Layout::VarBinView => "VarBinView",
+        };
+
+        write!(f, "{}, {layout}", self.len)
+    }
+}
+
+fn cases() -> Vec<Case> {
+    VALUE_LENS
+        .iter()
+        .flat_map(|&len| LAYOUTS.iter().map(move |&layout| Case { len, layout }))
+        .filter(|case| {
+            // VarBin varies on unchanged code, except for short values on the NEON leg.
+            env!("VORTEX_BENCH_VARIANT") == "local"
+                || matches!(case.layout, Layout::VarBinView)
+                || (env!("VORTEX_BENCH_VARIANT") == "neon" && case.len == 8)
+        })
+        .collect()
+}
+
 #[derive(Clone)]
 struct StartsWith;
 
@@ -105,9 +138,9 @@ fn make_input(layout: Layout, len: usize) -> ArrayRef {
 }
 
 #[vortex_bench_support::cpu_features]
-#[divan::bench(args = LAYOUTS, consts = VALUE_LENS)]
-fn starts_with<const LEN: usize>(bencher: Bencher, &layout: &Layout) {
-    let args = VecExecutionArgs::new(vec![make_input(layout, LEN)], ROWS);
+#[divan::bench(args = cases())]
+fn starts_with(bencher: Bencher, case: Case) {
+    let args = VecExecutionArgs::new(vec![make_input(case.layout, case.len)], ROWS);
 
     bencher
         .counter(ItemsCount::new(ROWS))

vortex-array/benches/take_chunked.rs

@@ -147,7 +147,7 @@ impl Display for Case {
     }
 }
 
-// Chunk and index counts are sized to keep CodSpeed simulation under 1ms per case.
+// Chunk and index counts cover routing costs while keeping each iteration below 1 ms.
 fn cases() -> Vec<Case> {
     let mut cases = Vec::new();
 
@@ -347,7 +347,10 @@ fn take_indices(case: Case) -> ArrayRef {
     }
 }
 
-#[divan::bench(args = cases())]
+// Materialization can grow mimalloc's page map through system calls. Measure its full cost.
+// The x86 variants still vary between processes with larger samples. Keep those runs local.
+#[vortex_bench_support::cpu_features(exclude = ["avx2", "avx512"])]
+#[divan::bench(args = cases(), sample_size = 16)]
 fn take(bencher: Bencher, case: Case) {
     let values = chunked_values(case);
     let indices = take_indices(case);

vortex-python/benchmark/test_scalar_at.py

@@ -1,20 +1,34 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+import os
+
 import pyarrow as pa
 import pytest
 from pytest_benchmark.fixture import BenchmarkFixture
 
 import vortex as vx
 
 
+@pytest.fixture
+def scalar_vxf(request: pytest.FixtureRequest, vxf: vx.VortexFile) -> vx.VortexFile:
+    enabled = request.config.getoption("codspeed", default=False) or "CODSPEED_ENV" in os.environ
+    mode = request.config.getoption("codspeed_mode", default=None)
+    if mode is None and "CODSPEED_ENV" in os.environ:
+        mode = os.environ.get("CODSPEED_RUNNER_MODE", "simulation")
+    assert isinstance(request.node, pytest.Function)
+    if enabled and mode == "simulation" and request.node.callspec.params["vxf"] == "int":
+        pytest.skip("Reactor polling varies inside these scalar scans under simulation.")
+    return vxf
+
+
 @pytest.mark.benchmark(group="scalar_at")
-def test_scan_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
+def test_scan_scalar_at(benchmark: BenchmarkFixture, scalar_vxf: vx.VortexFile) -> None:
     indices = vx.array([50_000])
-    benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan(indices=indices)))
+    benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in scalar_vxf.scan(indices=indices)))
 
 
 @pytest.mark.benchmark(group="scalar_at")
-def test_repeated_scan_scalar_at(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
-    rscan = vxf.to_repeated_scan()
+def test_repeated_scan_scalar_at(benchmark: BenchmarkFixture, scalar_vxf: vx.VortexFile) -> None:
+    rscan = scalar_vxf.to_repeated_scan()
     benchmark(lambda: rscan.scalar_at(50_000))