chore(load): start implementation of load tests

initial test from Ed's nuc: metric pctl baseline current delta status ----------------------------------------------------------------------------------------------- folder_cascade.list_depth1 p50 0.3ms 0.3ms -6.3% ok folder_cascade.list_depth1 p95 2.3ms 0.5ms -75.7% ok folder_cascade.list_depth1 p99 4.7ms 2.5ms -48.1% ok folder_cascade.list_depth4 p50 0.4ms 0.3ms -10.0% ok folder_cascade.list_depth4 p95 0.9ms 0.6ms -31.2% ok folder_cascade.list_depth4 p99 2.4ms 1.0ms -56.7% ok folder_cascade.list_depth8 p50 0.3ms 0.3ms -8.8% ok folder_cascade.list_depth8 p95 0.6ms 0.5ms -22.2% ok folder_cascade.list_depth8 p99 1.9ms 0.5ms -71.5% ok folder_cascade.list_depth_deep p50 0.3ms 0.3ms -5.0% ok folder_cascade.list_depth_deep p95 0.6ms 0.5ms -18.6% ok folder_cascade.list_depth_deep p99 2.0ms 0.7ms -67.2% ok share_cascade_rebac.list_grants p50 0.4ms 0.3ms -27.6% ok share_cascade_rebac.list_grants p95 1.2ms 0.5ms -57.0% ok share_cascade_rebac.list_grants p99 1.7ms 1.1ms -36.7% ok share_cascade_rebac.fetch_as_grantee_depth1 p50 0.5ms 0.5ms -11.1% ok share_cascade_rebac.fetch_as_grantee_depth1 p95 1.1ms 0.7ms -41.2% ok share_cascade_rebac.fetch_as_grantee_depth1 p99 3.0ms 1.3ms -58.3% ok share_cascade_rebac.fetch_as_grantee_depth4 p50 0.5ms 0.5ms -13.7% ok share_cascade_rebac.fetch_as_grantee_depth4 p95 1.4ms 0.7ms -50.5% ok share_cascade_rebac.fetch_as_grantee_depth4 p99 2.2ms 1.1ms -49.3% ok share_cascade_rebac.fetch_as_grantee_depth8 p50 0.5ms 0.4ms -16.0% ok share_cascade_rebac.fetch_as_grantee_depth8 p95 0.9ms 0.7ms -26.1% ok share_cascade_rebac.fetch_as_grantee_depth8 p99 1.5ms 0.9ms -40.1% ok share_cascade_rebac.fetch_as_grantee_depth_deep p50 0.5ms 0.4ms -17.3% ok share_cascade_rebac.fetch_as_grantee_depth_deep p95 1.0ms 0.7ms -31.2% ok share_cascade_rebac.fetch_as_grantee_depth_deep p99 1.6ms 0.8ms -49.4% ok subject_group_nested.fetch_as_member_depth1 p50 0.5ms 0.4ms -8.4% ok subject_group_nested.fetch_as_member_depth1 p95 0.6ms 0.6ms -10.2% ok subject_group_nested.fetch_as_member_depth1 p99 1.4ms 0.6ms -55.2% ok subject_group_nested.fetch_as_member_depth4 p50 0.5ms 0.5ms -7.9% ok subject_group_nested.fetch_as_member_depth4 p95 0.6ms 0.6ms -4.0% ok subject_group_nested.fetch_as_member_depth4 p99 0.7ms 0.6ms -2.2% ok subject_group_nested.fetch_as_member_depth8 p50 0.5ms 0.4ms -8.9% ok subject_group_nested.fetch_as_member_depth8 p95 0.5ms 0.6ms +7.1% ok subject_group_nested.fetch_as_member_depth8 p99 0.6ms 0.7ms +10.3% ok subject_group_nested.fetch_as_member_depth_deep p50 0.5ms 0.4ms -7.6% ok subject_group_nested.fetch_as_member_depth_deep p95 0.6ms 0.5ms -11.9% ok subject_group_nested.fetch_as_member_depth_deep p99 0.6ms 0.7ms +8.9% ok
2026-06-14 16:42:58 +02:00
parent 1d0e4d3c42
commit 0ef4c624c5
26 changed files with 2345 additions and 15 deletions
@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+// compare.mjs — regression diff for k6 load-test runs.
+//
+// Reads a k6 --summary-export JSON and the committed baseline, diffs the
+// p50/p95/p99 of every baseline metric against the current run, prints a
+// human-readable table, and exits non-zero if any metric regresses beyond
+// its tolerance.
+//
+// Usage:
+//   node compare.mjs <summary.json> <baseline.json>
+//
+// Exit codes:
+//   0 — every metric within tolerance.
+//   1 — one or more metrics regressed, or a baseline metric is absent from
+//       the current summary (suite drift — either a scenario was deleted
+//       or a tag was renamed without updating baseline.json).
+
+import { readFileSync } from 'node:fs';
+import { argv, exit } from 'node:process';
+
+if (argv.length < 4) {
+  console.error('usage: compare.mjs <summary.json> <baseline.json>');
+  exit(2);
+}
+
+const summaryPath = argv[2];
+const baselinePath = argv[3];
+
+const summary = JSON.parse(readFileSync(summaryPath, 'utf8'));
+const baseline = JSON.parse(readFileSync(baselinePath, 'utf8'));
+
+// k6 summary metrics are keyed verbatim with the tag: e.g.
+// "http_req_duration{op:smoke.login}".
+const metricKeyFor = (op) => `http_req_duration{op:${op}}`;
+
+// Map k6 percentile field names → baseline field names.
+const PERCENTILES = [
+  { baseline: 'p50', k6: 'med' },
+  { baseline: 'p95', k6: 'p(95)' },
+  { baseline: 'p99', k6: 'p(99)' },
+];
+
+const rows = [];
+let anyRegression = false;
+let anyMissing = false;
+
+for (const [op, base] of Object.entries(baseline)) {
+  if (op.startsWith('_')) continue; // skip _comment etc.
+  const metric = summary.metrics?.[metricKeyFor(op)];
+  if (!metric) {
+    rows.push({ op, status: 'MISSING', detail: 'no data for this tag in current summary' });
+    anyMissing = true;
+    continue;
+  }
+  const tol = (base.tolerance_pct ?? 10) / 100;
+  // Noise-floor guard: at sub-millisecond percentiles, a 10% tolerance is
+  // smaller than a single context switch. Require the absolute delta to
+  // exceed `min_delta_ms` too — otherwise the swing is below the measurement
+  // floor and we don't flag it. Default 0.5ms is conservative for the load
+  // suite's typical 0.3–3ms range. Override per metric in baseline.json.
+  const minDelta = base.min_delta_ms ?? 0.5;
+
+  for (const p of PERCENTILES) {
+    const baseVal = base[p.baseline];
+    // k6 --summary-export puts percentile values directly on the metric, not
+    // inside a `.values` wrapper.
+    const curVal = metric[p.k6];
+    if (baseVal === undefined || curVal === undefined) continue;
+
+    const deltaPct = ((curVal - baseVal) / baseVal) * 100;
+    const deltaAbs = curVal - baseVal;
+    const limitPct = baseVal * (1 + tol);
+    // Regression only if BOTH the % rule AND the absolute floor are breached.
+    const regressed = curVal > limitPct && deltaAbs > minDelta;
+    if (regressed) anyRegression = true;
+    rows.push({
+      op,
+      percentile: p.baseline,
+      base: baseVal,
+      cur: curVal,
+      deltaPct,
+      regressed,
+      tol: base.tolerance_pct ?? 10,
+    });
+  }
+}
+
+// ── Render table ──────────────────────────────────────────────────────────
+
+const ms = (v) => `${v.toFixed(1)}ms`;
+const pct = (v) => `${v >= 0 ? '+' : ''}${v.toFixed(1)}%`;
+
+function pad(s, n) {
+  if (s.length >= n) return s;
+  return s + ' '.repeat(n - s.length);
+}
+
+const colOp = Math.max(20, ...rows.map((r) => r.op.length));
+const colP = 5;
+const colVal = 11;
+
+console.log();
+console.log(
+  pad('metric', colOp + 1) +
+    pad('pctl', colP + 1) +
+    pad('baseline', colVal + 1) +
+    pad('current', colVal + 1) +
+    pad('delta', 9) +
+    'status',
+);
+console.log('-'.repeat(colOp + colP + colVal * 2 + 9 + 12));
+
+for (const r of rows) {
+  if (r.status === 'MISSING') {
+    console.log(`${pad(r.op, colOp + 1)}${pad('-', colP + 1)}${pad('-', colVal + 1)}${pad('-', colVal + 1)}${pad('-', 9)}MISSING — ${r.detail}`);
+    continue;
+  }
+  const status = r.regressed ? `REGRESSION (tol ±${r.tol}%)` : 'ok';
+  console.log(
+    pad(r.op, colOp + 1) +
+      pad(r.percentile, colP + 1) +
+      pad(ms(r.base), colVal + 1) +
+      pad(ms(r.cur), colVal + 1) +
+      pad(pct(r.deltaPct), 9) +
+      status,
+  );
+}
+
+console.log();
+if (anyRegression || anyMissing) {
+  if (anyRegression) console.error('compare.mjs: one or more metrics regressed beyond tolerance.');
+  if (anyMissing) console.error('compare.mjs: one or more baseline metrics are missing from the current summary.');
+  exit(1);
+}
+console.log('compare.mjs: all metrics within tolerance.');
+exit(0);