Skip to content

Commit ebda94f

Browse files
committed
chore(ci): simplify import profiler metrics and remove P90/P99 stats (#17775)
This PR improves the import profiler tool ci check (#17657) by simplifying its output metrics: - **Simplified & Reliable Stats**: Shows only `Min`, `P50` (Median), `Max`, and `StdDev`. Removed the calculation and formatting of `P90`, `P99`, and `Mean` stats as they are not reliable for the standard 10-iteration sample size. - **Enhanced Readability**: Reordered the output values from lowest to highest (`Min` -> `P50` -> `Max`). - **Sample Size Indicator**: Included the number of iterations (`[N=...]`) in the title of each metric section to make sample sizes explicit. - **Cleaned Up Dead Code**: Removed the unused `_calculate_percentiles` helper function to keep the codebase clean.
1 parent 005af88 commit ebda94f

1 file changed

Lines changed: 23 additions & 29 deletions

File tree

scripts/import_profiler/profiler.py

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -125,33 +125,34 @@ def _run_worker_and_parse(cmd):
125125
print(f"Worker stderr:\n{result.stderr}", file=sys.stderr)
126126
raise parse_err
127127

128-
def _calculate_percentiles(data_list):
129-
"""Helper method to calculate P50, P90, P99 from a list of numbers."""
130-
if len(data_list) > 1:
131-
q = statistics.quantiles(data_list, n=100)
132-
return q[49], q[89], q[98]
133-
val = data_list[0] if data_list else 0.0
134-
return val, val, val
135-
136128
def _print_outputs(target_module, iterations, loaded_modules_val, loaded_lines_val,
137-
times, p50_time, p90_time, p99_time,
138-
memories, p50_mem, p90_mem, p99_mem,
139-
rss_memories, p50_rss, p90_rss, p99_rss):
129+
times, memories, rss_memories):
140130
"""Helper method to format and print the final benchmark results."""
141-
def _format_stats(title, data, p50, p90, p99, fmt):
131+
def _format_stats(title, data, fmt):
142132
if not data:
143133
return ""
144-
std_str = f" StdDev: {statistics.stdev(data):{fmt}}\n" if len(data) > 1 else ""
145-
return f"{title}:\n P50 (Median): {p50:{fmt}}\n P90: {p90:{fmt}}\n P99: {p99:{fmt}}\n Mean: {statistics.mean(data):{fmt}}\n Min: {min(data):{fmt}}\n Max: {max(data):{fmt}}\n{std_str}"
134+
n = len(data)
135+
p50 = statistics.median(data)
136+
min_val = min(data)
137+
max_val = max(data)
138+
std_dev = statistics.stdev(data) if n > 1 else 0.0
139+
140+
stats_lines = [
141+
f"{title} [N={n}]:",
142+
f" Min: {min_val:{fmt}}",
143+
f" P50: {p50:{fmt}}",
144+
f" Max: {max_val:{fmt}}"
145+
]
146+
if n > 1:
147+
stats_lines.append(f" StdDev: {std_dev:{fmt}}")
148+
return "\n".join(stats_lines) + "\n"
146149

147150
final_output = f"""
148151
--- Results for {target_module} ({iterations} iterations) ---
149152
Code Volume (Deterministic):
150153
Loaded Modules: {loaded_modules_val}
151154
Loaded Lines: {loaded_lines_val}
152-
{_format_stats("Time (ms)", times, p50_time, p90_time, p99_time, ".2f")}
153-
{_format_stats("Tracemalloc RAM (MB)", memories, p50_mem, p90_mem, p99_mem, ".4f")}
154-
{_format_stats("Physical RSS RAM (MB)", rss_memories, p50_rss, p90_rss, p99_rss, ".4f")}"""
155+
{_format_stats("Time (ms)", times, ".2f")}{_format_stats("Tracemalloc RAM (MB)", memories, ".4f")}{_format_stats("Physical RSS RAM (MB)", rss_memories, ".4f")}"""
155156
print(final_output.strip())
156157

157158
def run_master(iterations, target_module, cpu=0, csv_path=None, clear_cache=True, fail_threshold=None, diff_baseline=None, diff_threshold=None):
@@ -224,17 +225,11 @@ def run_master(iterations, target_module, cpu=0, csv_path=None, clear_cache=True
224225
writer.writerow([idx + 1, f"{t:.2f}", f"{m:.4f}", f"{r:.4f}"])
225226
print(f"Raw metrics successfully exported to CSV: {csv_path}")
226227

227-
# Compute percentiles (P50, P90, P99)
228-
# statistics.quantiles returns 99 cut points for n=100
229-
p50_time, p90_time, p99_time = _calculate_percentiles(times)
230-
p50_mem, p90_mem, p99_mem = _calculate_percentiles(memories)
231-
p50_rss, p90_rss, p99_rss = _calculate_percentiles(rss_memories)
228+
p50_time = statistics.median(times) if times else 0.0
232229

233230
_print_outputs(
234231
target_module, iterations, loaded_modules_val, loaded_lines_val,
235-
times, p50_time, p90_time, p99_time,
236-
memories, p50_mem, p90_mem, p99_mem,
237-
rss_memories, p50_rss, p90_rss, p99_rss
232+
times, memories, rss_memories
238233
)
239234

240235
exit_code = 0
@@ -250,7 +245,7 @@ def run_master(iterations, target_module, cpu=0, csv_path=None, clear_cache=True
250245
for row in reader:
251246
baseline_times.append(float(row[1]))
252247
if baseline_times:
253-
baseline_p50, _, _ = _calculate_percentiles(baseline_times)
248+
baseline_p50 = statistics.median(baseline_times)
254249

255250
if baseline_p50 is not None:
256251
diff = p50_time - baseline_p50
@@ -293,10 +288,9 @@ def run_master(iterations, target_module, cpu=0, csv_path=None, clear_cache=True
293288

294289
if exit_code == 0:
295290
print("\nSession import_profiler was successful.")
296-
sys.exit(0)
297291
else:
298292
print("\nSession import_profiler failed.")
299-
sys.exit(1)
293+
return exit_code
300294

301295

302296
def run_trace(target_module):
@@ -461,4 +455,4 @@ def find_module_from_package(pkg):
461455
if not args.keep_pycache: clean_bytecode()
462456
run_mprofile(target_module)
463457
else:
464-
run_master(args.iterations, target_module, args.cpu, args.csv, not args.keep_pycache, args.fail_threshold, args.diff_baseline, args.diff_threshold)
458+
sys.exit(run_master(args.iterations, target_module, args.cpu, args.csv, not args.keep_pycache, args.fail_threshold, args.diff_baseline, args.diff_threshold))

0 commit comments

Comments
 (0)