From 34f039142aa708f05e1c7fa50bb66d7050a311aa Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Fri, 30 Oct 2026 18:10:14 +0100 Subject: [PATCH 1/2] ci: Measure only SDK calls in CodSpeed using instrument-hooks Run all benchmarks in one process, measuring each SDK call through a native extension built on CodSpeed's instrument-hooks. Ruby start-up, loading the SDK and building the client no longer count towards the results. --- .gitmodules | 3 ++ Makefile | 12 +++++- benchmarks/codspeed/codspeed.c | 60 ++++++++++++++++++++++++++++ benchmarks/codspeed/extconf.rb | 18 +++++++++ benchmarks/codspeed/instrument-hooks | 1 + benchmarks/run.rb | 40 ++++++++++++++----- codspeed.yml | 32 +-------------- 7 files changed, 126 insertions(+), 40 deletions(-) create mode 100644 benchmarks/codspeed/codspeed.c create mode 100644 benchmarks/codspeed/extconf.rb create mode 160000 benchmarks/codspeed/instrument-hooks diff --git a/.gitmodules b/.gitmodules index 0c86e07..72eee3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = spec/sdk-benchmarks url = https://github.com/Flagsmith/sdk-benchmarks.git branch = 0.0.2 +[submodule "benchmarks/codspeed/instrument-hooks"] + path = benchmarks/codspeed/instrument-hooks + url = https://github.com/CodSpeedHQ/instrument-hooks.git diff --git a/Makefile b/Makefile index 8689072..776d4d4 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,19 @@ SDK_BENCHMARKS := spec/sdk-benchmarks CODSPEED_VERSION := 5.6.0 CODSPEED_INSTALLER_SHA256 := bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40 +INSTRUMENT_HOOKS := benchmarks/codspeed/instrument-hooks +CODSPEED_EXT := tmp/codspeed/codspeed.$(shell ruby -e 'print RbConfig::CONFIG["DLEXT"]') + $(SDK_BENCHMARKS)/environments: git submodule update --init -- $(SDK_BENCHMARKS) +$(INSTRUMENT_HOOKS)/dist/core.c: + git submodule update --init -- $(INSTRUMENT_HOOKS) + +$(CODSPEED_EXT): benchmarks/codspeed/codspeed.c benchmarks/codspeed/extconf.rb $(INSTRUMENT_HOOKS)/dist/core.c + mkdir -p $(@D) + cd $(@D) && ruby $(CURDIR)/benchmarks/codspeed/extconf.rb && $(MAKE) + .PHONY: install-codspeed install-codspeed: $(SDK_BENCHMARKS)/environments bundle install @@ -14,5 +24,5 @@ install-codspeed: $(SDK_BENCHMARKS)/environments bash "$$installer" --quiet .PHONY: benchmark -benchmark: install-codspeed +benchmark: install-codspeed $(CODSPEED_EXT) SDK_BENCHMARKS=$(SDK_BENCHMARKS) codspeed run --mode simulation --config codspeed.yml diff --git a/benchmarks/codspeed/codspeed.c b/benchmarks/codspeed/codspeed.c new file mode 100644 index 0000000..b9000b0 --- /dev/null +++ b/benchmarks/codspeed/codspeed.c @@ -0,0 +1,60 @@ +// Ruby bindings for CodSpeed's instrument-hooks, measuring a block in-process. +#include +#include + +#include "core.h" + +static InstrumentHooks *hooks; + +// Root frame of the benchmark flame graph: everything measured runs inside it. +__attribute__((noinline)) static VALUE __codspeed_root_frame__measure(VALUE _) { + return rb_yield(Qnil); +} + +static VALUE stop_measuring(VALUE uri) { + callgrind_stop_instrumentation(); + if (instrument_hooks_stop_benchmark(hooks) != 0) + rb_warn("CodSpeed: failed to stop the benchmark"); + if (instrument_hooks_set_executed_benchmark(hooks, getpid(), StringValueCStr(uri)) != 0) + rb_warn("CodSpeed: failed to report the executed benchmark"); + return Qnil; +} + +// CodSpeed.measure(uri) { ... }: measure the block once as the benchmark at uri. +static VALUE codspeed_measure(VALUE self, VALUE uri) { + rb_need_block(); + StringValueCStr(uri); + if (instrument_hooks_start_benchmark(hooks) != 0) + rb_raise(rb_eRuntimeError, "CodSpeed: failed to start the benchmark"); + // Toggle callgrind here, rather than in start_benchmark, to measure as little harness as possible. + callgrind_start_instrumentation(); + return rb_ensure(__codspeed_root_frame__measure, Qnil, stop_measuring, uri); +} + +// CodSpeed.instrumented?: whether the process runs under the CodSpeed runner. +static VALUE codspeed_instrumented_p(VALUE self) { + return instrument_hooks_is_instrumented(hooks) ? Qtrue : Qfalse; +} + +// CodSpeed.setup(integration, version, skipped_objects): register the harness with the runner. +static VALUE codspeed_setup(VALUE self, VALUE integration, VALUE version, VALUE skipped_objects) { + if (instrument_hooks_set_integration(hooks, StringValueCStr(integration), StringValueCStr(version)) != 0) + rb_warn("CodSpeed: failed to register the integration"); + // Attribute interpreter costs to the root frame, as pytest-codspeed does for libpython. + for (long i = 0; i < RARRAY_LEN(skipped_objects); i++) { + VALUE path = rb_ary_entry(skipped_objects, i); + instrument_hooks_callgrind_add_obj_skip(StringValueCStr(path)); + } + return Qnil; +} + +void Init_codspeed(void) { + hooks = instrument_hooks_init(); + if (!hooks) rb_raise(rb_eRuntimeError, "CodSpeed: failed to initialise instrument-hooks"); + instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true); + + VALUE codspeed = rb_define_module("CodSpeed"); + rb_define_module_function(codspeed, "measure", codspeed_measure, 1); + rb_define_module_function(codspeed, "instrumented?", codspeed_instrumented_p, 0); + rb_define_module_function(codspeed, "setup", codspeed_setup, 3); +} diff --git a/benchmarks/codspeed/extconf.rb b/benchmarks/codspeed/extconf.rb new file mode 100644 index 0000000..eaefa49 --- /dev/null +++ b/benchmarks/codspeed/extconf.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Builds the CodSpeed extension from instrument-hooks' single-file distribution. +require 'mkmf' + +instrument_hooks = File.join(__dir__, 'instrument-hooks') +abort "Missing #{instrument_hooks}: run `git submodule update --init`" unless File.exist?(File.join(instrument_hooks, 'dist', 'core.c')) + +# rubocop:disable Style/GlobalVars +$INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}" +$VPATH << File.join(instrument_hooks, 'dist') +$srcs = %w[codspeed.c core.c] +# Silence warnings from the generated core.c, as instrument-hooks recommends. +$CFLAGS << ' -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter ' \ + '-Wno-unused-but-set-variable -Wno-type-limits -Wno-format -Wno-old-style-definition' +# rubocop:enable Style/GlobalVars + +create_makefile('codspeed') diff --git a/benchmarks/codspeed/instrument-hooks b/benchmarks/codspeed/instrument-hooks new file mode 160000 index 0000000..8fd5b87 --- /dev/null +++ b/benchmarks/codspeed/instrument-hooks @@ -0,0 +1 @@ +Subproject commit 8fd5b878499f819561b33219cdafaa6f8c28b4e4 diff --git a/benchmarks/run.rb b/benchmarks/run.rb index b5b3a4e..dfc969a 100644 --- a/benchmarks/run.rb +++ b/benchmarks/run.rb @@ -1,19 +1,41 @@ # frozen_string_literal: true +# Runs every SDK benchmark in a single process, measuring only the benchmarked calls. +# +# Under the CodSpeed runner, each call is measured with instrument-hooks, so Ruby start-up, +# loading the SDK and building the client stay out of the results. Elsewhere, it prints timings. require 'bundler/setup' +require 'benchmark' +require 'rbconfig' require_relative 'sdk_benchmark' -REVOLUTIONS = 10 +LOCAL_REVOLUTIONS = 100 -benchmark, environment = ARGV sdk_benchmarks_dir = ENV.fetch('SDK_BENCHMARKS', '') - -unless !sdk_benchmarks_dir.empty? && SdkBenchmark::BENCHMARKS.key?(benchmark) && SdkBenchmark::ENVIRONMENTS.include?(environment) - warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME} " \ - "<#{SdkBenchmark::BENCHMARKS.keys.join('|')}> <#{SdkBenchmark::ENVIRONMENTS.join('|')}>" +if sdk_benchmarks_dir.empty? + warn "Usage: SDK_BENCHMARKS= ruby #{$PROGRAM_NAME}" exit 2 end -sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) -REVOLUTIONS.times { sdk_benchmark.public_send(SdkBenchmark::BENCHMARKS.fetch(benchmark)) } -sdk_benchmark.close +codspeed = !ENV.fetch('CODSPEED_ENV', '').empty? +if codspeed + require_relative '../tmp/codspeed/codspeed' + abort 'CodSpeed: not running under instrumentation' unless CodSpeed.instrumented? + libruby = File.join(RbConfig::CONFIG['libdir'], RbConfig::CONFIG['LIBRUBY_SO']) + CodSpeed.setup('flagsmith-ruby-client', Flagsmith::VERSION, [RbConfig.ruby, libruby].select { File.exist?(_1) }) +end + +SdkBenchmark::ENVIRONMENTS.each do |environment| + sdk_benchmark = SdkBenchmark.new(sdk_benchmarks_dir, environment) + SdkBenchmark::BENCHMARKS.each do |name, method| + sdk_benchmark.public_send(method) # Warm up caches + if codspeed + GC.start + CodSpeed.measure("benchmarks/sdk_benchmark.rb::#{name}[#{environment}]") { sdk_benchmark.public_send(method) } + else + seconds = Benchmark.realtime { LOCAL_REVOLUTIONS.times { sdk_benchmark.public_send(method) } } + puts format('%-50s %10.3f ms', name: "#{name}[#{environment}]", ms: seconds * 1000 / LOCAL_REVOLUTIONS) + end + end + sdk_benchmark.close +end diff --git a/codspeed.yml b/codspeed.yml index 6400170..7fb6f72 100644 --- a/codspeed.yml +++ b/codspeed.yml @@ -1,33 +1,5 @@ $schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json benchmarks: - - name: getEnvironmentFlags[small] - exec: ruby benchmarks/run.rb getEnvironmentFlags small - - name: getEnvironmentFlags[medium] - exec: ruby benchmarks/run.rb getEnvironmentFlags medium - - name: getEnvironmentFlags[large] - exec: ruby benchmarks/run.rb getEnvironmentFlags large - - name: getIdentityFlagsWithIdentityOverrides[small] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides small - - name: getIdentityFlagsWithIdentityOverrides[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides medium - - name: getIdentityFlagsWithIdentityOverrides[large] - exec: ruby benchmarks/run.rb getIdentityFlagsWithIdentityOverrides large - - name: getIdentityFlagsNoTraits[small] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits small - - name: getIdentityFlagsNoTraits[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits medium - - name: getIdentityFlagsNoTraits[large] - exec: ruby benchmarks/run.rb getIdentityFlagsNoTraits large - - name: getIdentityFlagsWithTraits[small] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits small - - name: getIdentityFlagsWithTraits[medium] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits medium - - name: getIdentityFlagsWithTraits[large] - exec: ruby benchmarks/run.rb getIdentityFlagsWithTraits large - - name: getIdentitySegmentsWithTraits[small] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits small - - name: getIdentitySegmentsWithTraits[medium] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits medium - - name: getIdentitySegmentsWithTraits[large] - exec: ruby benchmarks/run.rb getIdentitySegmentsWithTraits large + - name: SDK benchmarks + entrypoint: ruby benchmarks/run.rb From d0b658f78d00426f4397dc432d0c3b9a0a3847ab Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Fri, 30 Oct 2026 18:16:52 +0100 Subject: [PATCH 2/2] ci: Compile instrument-hooks as C11 GCC 15, which ships with Ubuntu 26.04 runners, defaults to C23, where zig.h's noreturn expands to an attribute that the generated core.c places after the storage class, failing the build. Pin C11 like pytest-codspeed does. --- benchmarks/codspeed/extconf.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/benchmarks/codspeed/extconf.rb b/benchmarks/codspeed/extconf.rb index eaefa49..b0a84c5 100644 --- a/benchmarks/codspeed/extconf.rb +++ b/benchmarks/codspeed/extconf.rb @@ -10,9 +10,13 @@ $INCFLAGS << " -I#{File.join(instrument_hooks, 'includes')}" $VPATH << File.join(instrument_hooks, 'dist') $srcs = %w[codspeed.c core.c] +# Compile as C11 like pytest-codspeed: GCC 15 defaults to C23, which breaks zig.h's noreturn. +$CFLAGS << ' -std=gnu11' # Silence warnings from the generated core.c, as instrument-hooks recommends. -$CFLAGS << ' -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter ' \ - '-Wno-unused-but-set-variable -Wno-type-limits -Wno-format -Wno-old-style-definition' +$CFLAGS << %w[ + -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable + -Wno-type-limits -Wno-format -Wno-format-security -Wno-old-style-definition -Wno-unterminated-string-initialization +].join(' ').prepend(' ') # rubocop:enable Style/GlobalVars create_makefile('codspeed')