diff --git a/.github/workflows/load-nightly.yml b/.github/workflows/load-nightly.yml new file mode 100644 index 00000000..a799dffb --- /dev/null +++ b/.github/workflows/load-nightly.yml @@ -0,0 +1,111 @@ +name: Load Nightly + +# Nightly regression gate. Runs every k6 scenario, diffs against +# baseline/load.json, opens an issue if any metric regresses. +# +# IMPORTANT: shared GitHub-hosted runners produce noisy timings — +# regression signal is unreliable until this workflow is moved to a +# pinned self-hosted runner with consistent hardware. The cron run is +# informational until then; treat opened issues as "investigate" not +# "broken main." + +on: + schedule: + # 03:00 UTC daily — outside US/EU working hours, low contention. + - cron: '0 3 * * *' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + load: + name: k6 load suite + baseline diff + # Skip the scheduled run on forks — nightly is only meaningful against the + # canonical baseline.json on the upstream repo. Forks can still hit it + # manually via `workflow_dispatch` if they want, and the guard is bypassed + # for that path (event != schedule). Additional fork repositories are + # opted-in explicitly below. + if: github.event_name != 'schedule' || github.repository == 'AtalayaLabs/OxiCloud' || github.repository == 'EdouardVanbelle/OxiCloud' + # tunning on Ed's nuc to ensure stable environment + # treating regressions here as merge-blocking. ubuntu-latest is too noisy + # for trustworthy p95/p99 deltas. + runs-on: [self-hosted, nuc-loadtest] + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + # The self-hosted runner bind-mounts target/ as a persistent volume + # for fast incremental rebuilds. actions/checkout's default cleanup + # tries to rmdir it and hits EBUSY on the mount point. Git still + # syncs the working tree to the target SHA — only non-git files + # (target/, tests/load/results/) persist, which is what we want. + clean: false + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: load + + - name: Install Node 20 + # The self-hosted runner image ships Node 12, which can't parse the + # ES-module `.mjs` helpers (compare.mjs / merge-summaries.mjs / + # bake-baseline.mjs). Pin to a current LTS for both runners. + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install k6 + uses: grafana/setup-k6-action@v1 + + - name: Build OxiCloud + load-seed (release) + # Single cargo invocation: load_seed_bin is an empty marker feature + # that gates the load-seed bin without changing oxicloud's dep + # graph, so cargo compiles oxicloud exactly once. + run: cargo build --release --features load_seed_bin --bin oxicloud --bin load-seed + + - name: Run full load suite + id: load + run: bash tests/load/run.sh + env: + BUILD_TARGET: release + continue-on-error: true + + - name: Upload raw k6 summaries + if: always() + uses: actions/upload-artifact@v4 + with: + name: load-results-${{ github.run_id }} + path: tests/load/results/*.json + retention-days: 30 + + - name: Prepare regression report + if: steps.load.outcome == 'failure' + run: | + { + echo "## Load nightly regression" + echo "" + echo "Run: \`${{ github.run_id }}\` · Commit: \`${{ github.sha }}\`" + echo "Runner: \`${{ runner.name }}\` (\`${{ runner.os }}/${{ runner.arch }}\`)" + echo "" + echo "Compare exited non-zero — see uploaded artifact \`load-results-${{ github.run_id }}\` for raw summaries." + echo "" + echo "Baseline is anchored to the runner that produced it; cross-runner" + echo "comparisons (e.g. an artifact baked on different hardware) will" + echo "show large \"regressions\" that are really hardware deltas." + } > regression-issue.md + + - name: Open issue on regression + # Skip on forks where Issues are disabled — the regression report is + # already in the artifact, and the workflow's exit code marks the run + # as failed regardless. Forks that want issue notifications can enable + # issues on their repo and remove this condition. + if: steps.load.outcome == 'failure' && github.repository == 'AtalayaLabs/OxiCloud' + uses: peter-evans/create-issue-from-file@v5 + with: + title: "Load nightly: regression on ${{ github.sha }}" + content-filepath: regression-issue.md + labels: | + load-test + regression diff --git a/.github/workflows/load-smoke.yml b/.github/workflows/load-smoke.yml new file mode 100644 index 00000000..330f5536 --- /dev/null +++ b/.github/workflows/load-smoke.yml @@ -0,0 +1,45 @@ +name: Load Smoke + +# PR-tier liveness check. Verifies the k6 load harness still builds and a +# single happy-path iteration runs against a freshly built server. +# NO regression gate — that's the nightly workflow's job. + +on: + pull_request: + branches: [ main, dev ] + paths: + - 'tests/load/**' + - 'src/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'migrations/**' + - '.github/workflows/load-smoke.yml' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + smoke: + name: k6 smoke (load harness) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: load + + - name: Install k6 + uses: grafana/setup-k6-action@v1 + + - name: Build OxiCloud (debug) + run: cargo build --bin oxicloud + + - name: Run smoke scenario + run: bash tests/load/smoke.sh + env: + BUILD_TARGET: debug diff --git a/.gitignore b/.gitignore index 468d6c8f..71532900 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,7 @@ wasm/oxicloud-hash/target/ .devenv/ .direnv/ .devenv-state/ + +# K6 load-test raw run outputs and per-run storage (baseline is committed) +tests/load/results/*.json +tests/load/storage/ diff --git a/Cargo.toml b/Cargo.toml index 1d00c362..10b9e41a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,12 @@ integration_tests = [] # WASM plugin runtime (Extism). Opt-in: bundles wasmtime, a large engine most # deployments won't use. Activation also requires OXICLOUD_ENABLE_PLUGINS=true. plugins = ["dep:extism", "dep:toml", "dep:file-rotate"] +# Empty marker feature that gates the `load-seed` binary so it isn't built +# by default (and skipped in prod Docker builds). Kept separate from +# test_utils so enabling it doesn't change the oxicloud dependency graph — +# this lets one `cargo build` produce both `oxicloud` and `load-seed` +# without recompiling oxicloud with mockall in scope. +load_seed_bin = [] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(integration_tests)'] } @@ -101,6 +107,14 @@ path = "src/bin/generate-openapi.rs" name = "migrate-nfc-filenames" path = "src/bin/migrate-nfc-filenames.rs" +[[bin]] +name = "load-seed" +path = "src/bin/load-seed.rs" +# Test fixture seeder. Gated behind the empty `load_seed_bin` feature so +# `cargo build --release` (and the prod Dockerfile) skip it. tests/load/run.sh +# and load-nightly.yml build it explicitly with --features load_seed_bin. +required-features = ["load_seed_bin"] + [build-dependencies] oxc_allocator = "0.125.0" oxc_parser = "0.125.0" diff --git a/Dockerfile b/Dockerfile index 117edafa..4049b31b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN mkdir -p src/bin && \ echo 'fn main() { println!("Dummy build for caching dependencies"); }' > src/main.rs && \ echo 'fn main() {}' > src/bin/generate-openapi.rs && \ echo 'fn main() {}' > src/bin/migrate-nfc-filenames.rs && \ - cargo build --release && \ + cargo build --release --bin oxicloud --bin generate-openapi --bin migrate-nfc-filenames && \ rm -rf src static-dist target/release/deps/oxicloud* target/release/build/oxicloud-* # ─── Stage 3: Build the application ────────────────────────────────────────── @@ -38,7 +38,10 @@ COPY migrations migrations COPY templates templates # Build with all optimizations (DATABASE_URL only needed at compile-time for sqlx) ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud" -RUN DATABASE_URL="${DATABASE_URL}" cargo build --release +# Explicit --bin list: defence-in-depth so the prod image never ships +# test-only bins (e.g. load-seed) even if `required-features` gating +# changes upstream. +RUN DATABASE_URL="${DATABASE_URL}" cargo build --release --bin oxicloud --bin generate-openapi --bin migrate-nfc-filenames # ─── Stage 4: Minimal runtime image ────────────────────────────────────────── FROM alpine:3.24.0 diff --git a/docs/plan/benchmake-and-performance-tracking.md b/docs/plan/benchmake-and-performance-tracking.md new file mode 100644 index 00000000..2038a2c5 --- /dev/null +++ b/docs/plan/benchmake-and-performance-tracking.md @@ -0,0 +1,187 @@ +# OxiCloud Load Test Scaffolding (K6) — Scenario A + +## Context + +OxiCloud needs a repeatable way to **detect performance regressions** as features land. The current `tests/load/` directory is empty except for a README asking K6-vs-drill. Scenario A focuses on a single user with many subfolders and cascading systems (folder ops, ReBAC permission groups, nested subject groups) — measuring p50/p95/p99 to identify inflections when new features are added. Scenario B (many concurrent users) is deferred. + +**Decisions already locked in via conversation:** +- K6 (Go-based, JS scripting in goja) over Rust testers — client overhead is dwarfed by server latency for these scenarios; regression deltas only require client *consistency*, not raw speed. +- **Two scenario tiers**: a **fast smoke** for PRs (~1 min, no regression gate — just verifies the harness still runs) and **long scenarios** for nightly + manual (regression-gated). +- **Hybrid seeding**: bulk fixtures via direct SQL (`cargo run --bin load-seed`); the resources each scenario actively touches go through REST so the measured path is realistic. +- **Result storage**: `baseline/baseline.json` committed in repo (p50/p95/p99 + tolerance per metric); per-run raw JSON kept locally / CI artifact, gitignored. Baseline updates are deliberate PRs. +- **No PR gating on long suite** — nightly on main + manual `workflow_dispatch` only. + +## Approach + +### 1. Directory layout + +``` +tests/load/ + README.md + test.env # base_url=http://localhost:8088, admin creds + run.sh # full suite: spawn-db → seed → server → k6 → compare → cleanup + smoke.sh # smoke only: same shape, runs scenarios/smoke.js, no gate + compare.mjs # diffs k6 summary.json vs baseline/baseline.json + lib/ + auth.js # login() → bearer token (POST /api/auth/login) + http.js # base URL + auth header helpers + metrics.js # custom k6 Trends, naming convention . + scenarios/ + smoke.js # 1 VU, 1 iter — login, create folder, upload, list, delete + folder_cascade.js # list/move/copy/trash on depth-8 fanout-5 tree + share_cascade_rebac.js # grant a folder, fetch N descendants as grantee + subject_group_nested.js # nested-group chain (depth 3), grant via group, fetch + baseline/ + baseline.json # committed; { ".": {p50,p95,p99,tolerance_pct} } + results/ + .gitkeep # raw runs, gitignored +``` + +Mirrors the `tests/api/` shell pattern (run.sh, test.env, separate server port). Uses port **8088** to avoid colliding with api-test on 8087. + +### 2. Rust bulk seeder — `src/bin/load-seed.rs` + +New binary registered in `Cargo.toml` alongside `generate-openapi` and `migrate-nfc-filenames`. + +**CLI:** +``` +cargo run --bin load-seed -- \ + --depth 8 --fanout 5 --files-per-leaf 10 \ + --extra-users 20 --group-depth 3 --group-fanout 5 +``` + +**Inserts via sqlx (one transaction per phase):** +- 1 admin + N extra users into `auth.users` (Argon2id with light params m=16384,t=1,p=1; same params as `src/infrastructure/services/password_hasher.rs` test path). +- One shared 0-byte blob row in `storage.blobs` with `ref_count = total_files`. +- Folder tree in `storage.folders`, **inserted level-by-level** so the `trg_folders_path` BEFORE-INSERT trigger can compute `path`/`lpath` from the parent chain. +- Files in `storage.files`, all referencing the shared blob hash for dedup. +- Subject groups in `auth.subject_groups` + membership rows in `auth.subject_group_members` (XOR `member_user_id`/`member_group_id`), forming a nested chain `G_root → G_mid → G_leaf → users`. +- ReBAC grants in `storage.access_grants` for a known set of test resources (used by `share_cascade_rebac.js`). + +Reuses existing connection config: reads `DATABASE_URL` from env, same shape as `tests/common/server.env`. + +**Critical gotchas surfaced during exploration:** +- Tree-ETag triggers (`folders_bump_tree_etag_*`) are STATEMENT-level (per commit `4200209d`), so bulk inserts fire them once per statement — safe. +- ReBAC cascade is computed in the application layer (`pg_acl_engine`), not at DB level. Seeder just inserts grant rows; the cascade is the *server* behavior we want to measure. +- `auth.subject_groups.name` is CITEXT, max 64, RFC-5321 local-part shape — generated names use `g_NNN` pattern to stay valid. + +### 3. K6 scenarios + +All scenarios load `baseline.json` at startup and set `thresholds` dynamically from it (`http_req_duration{op:x}: p(95).` → e.g. `folder_cascade.list_depth8`, `share_cascade_rebac.fetch_as_grantee`, `subject_group_nested.grant_via_chain3`. + +**Cold/warm split**: first iteration tagged `cold` (its metrics flow to `._cold`), rest are warm. Single first iter avoids polluting warm metrics with one-shot connection setup. + +### 4. Runner scripts + +**`run.sh`** (mirrors `tests/api/run.sh` shape): +1. `spawn-db.sh` (reused from `tests/common/`) +2. `init-test-schema.sh` (reused) +3. `cargo run --bin load-seed -- ` against the test DB +4. Start `target//oxicloud` on port 8088 with `OXICLOUD_STORAGE_PATH=tests/load/storage` +5. `wait_for_http http://localhost:8088/ready` +6. `k6 run --summary-export=results/$(date +%s).json scenarios/folder_cascade.js scenarios/share_cascade_rebac.js scenarios/subject_group_nested.js` +7. `node compare.mjs results/.json baseline/baseline.json` → exits non-zero on regression +8. `trap cleanup EXIT` tears down server + DB + +**`smoke.sh`**: same shape but only runs `scenarios/smoke.js`, **skips the seeder** (smoke creates its own minimal data), **skips `compare.mjs`**. Goal is harness liveness, not regression. + +### 5. `compare.mjs` + +Plain Node script (no deps, just `fs` + `process`). Reads two JSONs, walks every metric in baseline, computes: +``` +delta_pct = (current - baseline) / baseline * 100 +``` +Prints a human table: +``` +folder_cascade.list_depth8.p95: 142ms → 198ms (+39%) REGRESSION (tolerance 10%) +share_cascade_rebac.grant.p95: 60ms → 58ms (-3%) ok +``` +Exit 1 if any metric exceeds tolerance. Exit 0 otherwise. Missing metrics in current = exit 1 (suite drift); new metrics in current = warn only. + +### 6. Baseline format + +```json +{ + "folder_cascade.list_depth8": { "p50": 12.0, "p95": 45.0, "p99": 80.0, "tolerance_pct": 10 }, + "share_cascade_rebac.fetch_as_grantee": { "p50": 18.0, "p95": 55.0, "p99": 100.0, "tolerance_pct": 10 }, + ... +} +``` + +Values are placeholders — first run on a stable machine seeds real numbers. Updated only via the `load-baseline` recipe (which rewrites `baseline.json` from the latest run) and committed deliberately as `chore(load): accept new baseline for `. + +### 7. Justfile recipes (added to existing `justfile`) + +``` +load: # full suite — nightly + manual + bash tests/load/run.sh + +load-smoke: # fast harness check + bash tests/load/smoke.sh + +load-baseline: # rerun, overwrite baseline from latest result + bash tests/load/run.sh + cp tests/load/results/$(ls -t tests/load/results/*.json | head -1) tests/load/baseline/baseline.json + +load-seed: # standalone seeder for local poking + cargo run --bin load-seed -- --depth 8 --fanout 5 --files-per-leaf 10 +``` + +Matches existing recipe naming (`test-*`, `front-*`, `api-test`). + +### 8. CI workflows (stubs) + +- **`.github/workflows/load-nightly.yml`** — cron `0 3 * * *` UTC + `workflow_dispatch`; runs `just load` on `ubuntu-latest` initially with a `# TODO: replace with self-hosted runner — shared GH runners produce noisy results for regression detection` comment; uploads `tests/load/results/*.json` as artifact; on `compare.mjs` non-zero, opens an issue using `peter-evans/create-issue-from-file` with the diff report. +- **`.github/workflows/load-smoke.yml`** — on every PR to `main`; runs `just load-smoke`; ~1 min budget; no regression gate, just "harness builds & runs." + +### 9. Critical files to create / modify + +**Create:** +- `tests/load/README.md` +- `tests/load/test.env` +- `tests/load/run.sh`, `smoke.sh` +- `tests/load/compare.mjs` +- `tests/load/lib/auth.js`, `http.js`, `metrics.js` +- `tests/load/scenarios/smoke.js`, `folder_cascade.js`, `share_cascade_rebac.js`, `subject_group_nested.js` +- `tests/load/baseline/baseline.json` (placeholder values) +- `tests/load/results/.gitkeep` +- `src/bin/load-seed.rs` +- `.github/workflows/load-nightly.yml`, `load-smoke.yml` + +**Modify:** +- `Cargo.toml` — add `[[bin]] name = "load-seed" path = "src/bin/load-seed.rs"` after the `migrate-nfc-filenames` entry +- `justfile` — append four `load*` recipes +- `.gitignore` — add `tests/load/results/*.json` and `tests/load/storage/` + +### 10. Reused existing utilities + +- `tests/common/spawn-db.sh`, `stop-db.sh`, `init-test-schema.sh`, `server.env` — reused as-is by `run.sh` / `smoke.sh`. +- Argon2id parameters from `src/infrastructure/services/password_hasher.rs` test path — mirrored in `load-seed.rs` so test users can log in via the real auth flow. +- DB schema migrations (`migrations/`) — applied via the existing `init-test-schema.sh`; no schema changes needed. + +## Verification + +1. **Build**: `cargo build --bin load-seed` succeeds; `cargo clippy --all-features --all-targets -- -D warnings` stays green. +2. **Seeder works standalone**: `just db && just load-seed` populates `auth.users`, `storage.folders` (depth-8 tree visible via `SELECT count(*) FROM storage.folders WHERE user_id = …`), `auth.subject_groups`, `storage.access_grants`. +3. **Smoke runs locally**: `just load-smoke` — exits 0 in under ~90 s, leaves no orphan postgres/oxicloud processes. +4. **Full suite runs locally**: `just load` — k6 prints per-scenario p50/p95/p99; `compare.mjs` prints the diff table; since `baseline.json` is empty placeholders, expect a "REGRESSION" stub-report that the user reviews before running `just load-baseline` to accept the first real baseline. +5. **Baseline workflow**: after first `just load`, `just load-baseline` rewrites `baseline.json`; subsequent `just load` exits 0 (within tolerance). +6. **CI smoke**: trigger `load-smoke.yml` on a draft PR; confirm it completes under the time budget and uploads no artifacts on success. +7. **CI nightly**: manually trigger `load-nightly.yml` via `workflow_dispatch`; confirm artifact upload and (since no real baseline yet) confirm issue creation works. + +## Out of scope + +- Real baseline values (first run on user's hardware seeds them). +- Self-hosted runner setup (workflow stub uses `ubuntu-latest` with a TODO comment). +- Scenario B (many concurrent users) — deferred per user request. +- Goose / Rust-tester alternative — deferred until scenario B demonstrates K6 saturation issues. +- Additional scenarios (CalDAV / CardDAV / WebDAV / search depth) — the scaffolding makes them additive: drop a new file in `scenarios/`, add its metric names to `baseline.json`. diff --git a/justfile b/justfile index 29f47595..0b52f217 100644 --- a/justfile +++ b/justfile @@ -153,3 +153,25 @@ front-design: api-test: bash tests/api/run.sh bash tests/webdav/run.sh + +# k6 load suite — full scenarios + regression diff vs baseline/load.json. +# Used by the nightly workflow and on demand. Release build for fair timings. +load: + bash tests/load/run.sh + +# k6 smoke — single happy-path iteration. PR-tier liveness check, no gate. +load-smoke: + bash tests/load/smoke.sh + +# Re-run the full suite, then bake the latest summary into baseline/load.json. +# The leading `-` lets `run.sh`'s regression-exit not abort the bake step; +# review the diff and commit deliberately: +# chore(load): accept new baseline for +load-baseline: + -bash tests/load/run.sh + node tests/load/bake-baseline.mjs + +# Standalone seeder (poking around in psql). Needs the test DB up: +# just db +load-seed: + cargo run --bin load-seed -- --depth 5 --fanout 4 --files-per-leaf 3 diff --git a/src/bin/load-seed.rs b/src/bin/load-seed.rs new file mode 100644 index 00000000..34208b99 --- /dev/null +++ b/src/bin/load-seed.rs @@ -0,0 +1,746 @@ +//! `load-seed` — bulk-seeds OxiCloud's test database for K6 load scenarios. +//! +//! Inserts users, a deep folder tree, files (all sharing one dedup'd blob), +//! nested subject groups, and ReBAC grants directly via sqlx — bypassing the +//! REST API so the seed phase is fast and doesn't pollute measured metrics. +//! +//! Only the resources each k6 scenario actively touches (the grant being +//! created, the move target, etc.) go through the HTTP API at run time. +//! +//! Run (from repo root, against the test DB on port 5433): +//! +//! ```bash +//! DATABASE_URL='postgres://oxicloud_test:oxicloud_test@localhost:5433/oxicloud_test' \ +//! cargo run --bin load-seed -- \ +//! --depth 8 --fanout 3 --files-per-leaf 3 \ +//! --extra-users 20 --group-depth 3 --group-fanout 5 \ +//! --manifest tests/load/results/seed-manifest.json +//! ``` +//! +//! Output: writes a JSON manifest at `--manifest` listing the IDs and +//! credentials each k6 scenario needs (admin login, grantee user, deep folder +//! IDs, group chain, etc.). K6 scenarios load this manifest at startup. + +use argon2::password_hash::SaltString; +use argon2::{Algorithm, Argon2, Params, PasswordHasher, Version}; +use rand_core::OsRng; +use serde::Serialize; +use sqlx::PgPool; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::PathBuf; +use uuid::Uuid; + +// ── CLI ────────────────────────────────────────────────────────────────── + +struct Args { + depth: u32, + fanout: u32, + files_per_leaf: u32, + extra_users: u32, + group_depth: u32, + group_fanout: u32, + manifest: PathBuf, + password: String, +} + +impl Args { + fn parse() -> Self { + // Defaults match tests/load/test.env; both bound the exponential cost + // of fanout^depth. Override via flags or LOAD_* env vars. + let mut depth = 5; + let mut fanout = 4; + let mut files_per_leaf = 3; + let mut extra_users = 20; + let mut group_depth = 3; + let mut group_fanout = 5; + let mut manifest = PathBuf::from("tests/load/results/seed-manifest.json"); + let mut password = "TestPassword1!".to_string(); + + let argv: Vec = env::args().collect(); + let mut i = 1; + while i < argv.len() { + let key = argv[i].as_str(); + let val = || { + argv.get(i + 1) + .unwrap_or_else(|| panic!("missing value for {}", key)) + }; + match key { + "--depth" => depth = val().parse().expect("--depth must be u32"), + "--fanout" => fanout = val().parse().expect("--fanout must be u32"), + "--files-per-leaf" => { + files_per_leaf = val().parse().expect("--files-per-leaf must be u32"); + } + "--extra-users" => { + extra_users = val().parse().expect("--extra-users must be u32"); + } + "--group-depth" => { + group_depth = val().parse().expect("--group-depth must be u32"); + } + "--group-fanout" => { + group_fanout = val().parse().expect("--group-fanout must be u32"); + } + "--manifest" => manifest = PathBuf::from(val()), + "--password" => password = val().clone(), + "--help" | "-h" => { + print_help(); + std::process::exit(0); + } + other => panic!("unknown argument: {}", other), + } + i += 2; + } + + Self { + depth, + fanout, + files_per_leaf, + extra_users, + group_depth, + group_fanout, + manifest, + password, + } + } +} + +fn print_help() { + println!("load-seed — bulk-seed OxiCloud for k6 load tests"); + println!(); + println!("Reads DATABASE_URL from env. Inserts users, folders, files, groups, grants."); + println!("Writes a JSON manifest the k6 scenarios consume."); + println!(); + println!("Flags (with defaults):"); + println!(" --depth 5 folder tree depth (exponential: fanout^depth folders)"); + println!(" --fanout 4 children per non-leaf folder"); + println!(" --files-per-leaf 3 files inserted in each leaf folder"); + println!(" --extra-users 20 non-admin users created"); + println!(" --group-depth 3 nested subject-group chain length"); + println!(" --group-fanout 5 users added directly to each leaf group"); + println!(" --manifest output manifest JSON path"); + println!(" --password shared password for all seeded users"); +} + +// ── Manifest (consumed by k6 scenarios) ────────────────────────────────── + +#[derive(Serialize)] +struct Manifest { + admin: UserCreds, + grantee: UserCreds, + group_member: UserCreds, + shared_subtree: SubtreeIds, + group_subtree: SubtreeIds, + nested_groups: NestedGroupIds, + /// Total folders seeded (informational). + total_folders: u64, + /// Total files seeded (informational). + total_files: u64, +} + +#[derive(Serialize)] +struct UserCreds { + id: Uuid, + username: String, + password: String, +} + +#[derive(Serialize)] +struct SubtreeIds { + /// Root of the subtree the grant is attached to. + root: Uuid, + /// A folder at depth 4 inside the subtree (for mid-depth fetches). + depth4: Uuid, + /// A folder at depth 8 inside the subtree (for mid-depth fetches). + depth8: Uuid, + /// A folder at depth N (== `--depth`) inside the subtree (for deep fetches). + deepest: Uuid, +} + +#[derive(Serialize)] +struct NestedGroupIds { + /// Outermost group — the one referenced by the grant. Contains `mid` as a member. + root: Uuid, + /// Intermediate group — member of `root`, contains `leaf` as a member. + mid: Uuid, + /// Innermost group — member of `mid`, contains `group_member` user as a direct member. + leaf: Uuid, +} + +// ── Main ───────────────────────────────────────────────────────────────── + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Best-effort: load .env if present (mirrors the rest of the codebase). + let _ = dotenvy::dotenv(); + + let args = Args::parse(); + let database_url = env::var("DATABASE_URL").map_err(|_| { + "DATABASE_URL must be set (e.g. postgres://oxicloud_test:oxicloud_test@localhost:5433/oxicloud_test)" + })?; + + println!( + "[load-seed] connecting to {}", + sanitize_url_for_log(&database_url) + ); + let pool = PgPool::connect(&database_url).await?; + + println!("[load-seed] wiping previous load-test data…"); + wipe(&pool).await?; + + println!("[load-seed] hashing shared password…"); + let password_hash = hash_password(&args.password); + + println!( + "[load-seed] inserting users (1 admin + {})…", + args.extra_users + ); + let admin = insert_user(&pool, "load_admin", "admin", &password_hash).await?; + let extra_users = insert_extra_users(&pool, args.extra_users, &password_hash).await?; + + // Pick a grantee (user[0]) and a group member (user[1]). + let grantee = extra_users + .first() + .ok_or("need at least one extra user for grantee")? + .clone(); + let group_member = extra_users + .get(1) + .ok_or("need at least two extra users")? + .clone(); + + println!("[load-seed] inserting shared blob…"); + let blob_hash = insert_shared_blob(&pool).await?; + + println!( + "[load-seed] building folder tree (depth={}, fanout={})…", + args.depth, args.fanout + ); + // Two parallel subtrees off root: one for the user-grant scenario, one for + // the group-grant scenario. Each has its own depth/fanout shape so a single + // grant cascades over a known fixed number of descendants. + let shared_subtree = + build_subtree(&pool, admin.id, "shared_root", args.depth, args.fanout).await?; + let group_subtree = + build_subtree(&pool, admin.id, "group_root", args.depth, args.fanout).await?; + + let total_folders = shared_subtree.all_ids.len() as u64 + group_subtree.all_ids.len() as u64; + let total_leaves = shared_subtree.leaves.len() + group_subtree.leaves.len(); + + println!( + "[load-seed] inserting files (files_per_leaf={}, leaves={})…", + args.files_per_leaf, total_leaves + ); + let total_files = insert_files( + &pool, + admin.id, + &blob_hash, + &shared_subtree.leaves, + &group_subtree.leaves, + args.files_per_leaf, + ) + .await?; + + println!( + "[load-seed] building nested group chain (depth={}, fanout={})…", + args.group_depth, args.group_fanout + ); + let nested_groups = build_group_chain( + &pool, + admin.id, + &extra_users, + &group_member.id, + args.group_depth, + args.group_fanout, + ) + .await?; + + println!("[load-seed] inserting grants…"); + // Grant the grantee read on shared_subtree.root. + insert_grant( + &pool, + "user", + grantee.id, + "folder", + shared_subtree.root, + "read", + admin.id, + ) + .await?; + // Grant the outermost group read on group_subtree.root. + insert_grant( + &pool, + "group", + nested_groups.root, + "folder", + group_subtree.root, + "read", + admin.id, + ) + .await?; + + let manifest = Manifest { + admin: UserCreds { + id: admin.id, + username: admin.username, + password: args.password.clone(), + }, + grantee: UserCreds { + id: grantee.id, + username: grantee.username, + password: args.password.clone(), + }, + group_member: UserCreds { + id: group_member.id, + username: group_member.username, + password: args.password.clone(), + }, + shared_subtree: SubtreeIds { + root: shared_subtree.root, + depth4: shared_subtree.depth4, + depth8: shared_subtree.depth8, + deepest: shared_subtree.deepest, + }, + group_subtree: SubtreeIds { + root: group_subtree.root, + depth4: group_subtree.depth4, + depth8: group_subtree.depth8, + deepest: group_subtree.deepest, + }, + nested_groups, + total_folders, + total_files, + }; + + if let Some(parent) = args.manifest.parent() { + fs::create_dir_all(parent)?; + } + let json = serde_json::to_string_pretty(&manifest)?; + fs::write(&args.manifest, json)?; + println!( + "[load-seed] manifest written → {} ({} folders, {} files)", + args.manifest.display(), + total_folders, + total_files + ); + + Ok(()) +} + +// ── Wipe ───────────────────────────────────────────────────────────────── + +async fn wipe(pool: &PgPool) -> Result<(), sqlx::Error> { + // Users starting with `load_` are the only ones this seeder ever creates. + // CASCADE on auth.users → storage.folders / storage.files / grants / group + // memberships, so a single DELETE wipes everything load-test-related. + sqlx::query("DELETE FROM auth.users WHERE username LIKE 'load_%'") + .execute(pool) + .await?; + // Groups starting with `load_` aren't owned by users; clear them explicitly. + sqlx::query("DELETE FROM auth.subject_groups WHERE name LIKE 'load_%'") + .execute(pool) + .await?; + // The shared dedup blob is identified by its fixed hash sentinel below. + sqlx::query("DELETE FROM storage.blobs WHERE hash = $1") + .bind(SHARED_BLOB_HASH) + .execute(pool) + .await?; + Ok(()) +} + +// ── Password ───────────────────────────────────────────────────────────── + +/// Light Argon2id parameters — same shape as the password_hasher test path +/// (`src/infrastructure/services/password_hasher.rs::test_hasher`). +/// Production hashes verify regardless of m/t/p because Argon2 reads the +/// parameters back from the hash string itself. +fn hash_password(password: &str) -> String { + let params = Params::new(16384, 1, 1, None).expect("valid argon2 params"); + let argon2 = Argon2::new(Algorithm::Argon2id, Version::V0x13, params); + let salt = SaltString::generate(&mut OsRng); + argon2 + .hash_password(password.as_bytes(), &salt) + .expect("argon2 hash") + .to_string() +} + +// ── Users ──────────────────────────────────────────────────────────────── + +#[derive(Clone)] +struct SeededUser { + id: Uuid, + username: String, +} + +async fn insert_user( + pool: &PgPool, + username: &str, + role: &str, + password_hash: &str, +) -> Result { + let email = format!("{}@load.test.invalid", username); + let row: (Uuid,) = sqlx::query_as( + "INSERT INTO auth.users (username, email, password_hash, role) + VALUES ($1, $2, $3, $4::auth.userrole) + RETURNING id", + ) + .bind(username) + .bind(email) + .bind(password_hash) + .bind(role) + .fetch_one(pool) + .await?; + Ok(SeededUser { + id: row.0, + username: username.to_string(), + }) +} + +async fn insert_extra_users( + pool: &PgPool, + count: u32, + password_hash: &str, +) -> Result, sqlx::Error> { + if count == 0 { + return Ok(Vec::new()); + } + let usernames: Vec = (0..count).map(|i| format!("load_user_{:04}", i)).collect(); + let emails: Vec = usernames + .iter() + .map(|u| format!("{}@load.test.invalid", u)) + .collect(); + + // Bulk insert via UNNEST — one round-trip for all N users. + let rows: Vec<(Uuid, String)> = sqlx::query_as( + "INSERT INTO auth.users (username, email, password_hash, role) + SELECT u.username, u.email, $1::text, 'user'::auth.userrole + FROM UNNEST($2::text[], $3::text[]) AS u(username, email) + RETURNING id, username", + ) + .bind(password_hash) + .bind(&usernames) + .bind(&emails) + .fetch_all(pool) + .await?; + + // RETURNING order is implementation-defined — sort by username so the + // caller can rely on `extra_users[0]` being `load_user_0000`. + let mut users: Vec = rows + .into_iter() + .map(|(id, username)| SeededUser { id, username }) + .collect(); + users.sort_by(|a, b| a.username.cmp(&b.username)); + Ok(users) +} + +// ── Blob ───────────────────────────────────────────────────────────────── + +/// All seeded files share this single zero-byte blob. Its ref_count is bumped +/// in lockstep with file inserts; the file_metadata schema doesn't require a +/// real on-disk blob for the dedup index to be consistent. +const SHARED_BLOB_HASH: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + +async fn insert_shared_blob(pool: &PgPool) -> Result { + sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count, content_type) + VALUES ($1, 0, 0, 'application/octet-stream') + ON CONFLICT (hash) DO NOTHING", + ) + .bind(SHARED_BLOB_HASH) + .execute(pool) + .await?; + Ok(SHARED_BLOB_HASH.to_string()) +} + +// ── Folder tree ────────────────────────────────────────────────────────── + +struct Subtree { + root: Uuid, + depth4: Uuid, + depth8: Uuid, + deepest: Uuid, + all_ids: Vec, + leaves: Vec, +} + +/// Build a folder tree under `parent=NULL` rooted at `root_name`. +/// +/// Inserts level-by-level so `trg_folders_path` resolves `path`/`lpath` from +/// the already-committed parent rows. Returns the root, a depth-4 sample, the +/// deepest leaf, the full id set, and the leaf ids (used for file seeding). +async fn build_subtree( + pool: &PgPool, + user_id: Uuid, + root_name: &str, + depth: u32, + fanout: u32, +) -> Result { + // Per-row BEFORE INSERT trigger (compute_folder_path) makes each level cost + // O(parents × fanout); past depth 6 with fanout 5 the seed time grows fast. + // Print the count BEFORE each level so it's clear which level is in flight + // and how many rows are about to be inserted. + let predicted: u64 = (0..=depth).map(|l| (fanout as u64).pow(l)).sum(); + println!( + " subtree '{}': ~{} folders predicted", + root_name, predicted + ); + + // Level 0 — the root folder. + let root: (Uuid,) = sqlx::query_as( + "INSERT INTO storage.folders (name, parent_id, user_id) + VALUES ($1, NULL, $2) + RETURNING id", + ) + .bind(root_name) + .bind(user_id) + .fetch_one(pool) + .await?; + let root_id = root.0; + let mut all_ids = vec![root_id]; + + let mut current_level: Vec = vec![root_id]; + let mut depth4: Option = None; + let mut depth8: Option = None; + let mut deepest: Uuid = root_id; + + for level in 1..=depth { + // Build the (parent_id, name) pairs for this level. + let parents: Vec = current_level + .iter() + .flat_map(|p| std::iter::repeat_n(*p, fanout as usize)) + .collect(); + let names: Vec = current_level + .iter() + .enumerate() + .flat_map(|(pi, _)| (0..fanout).map(move |c| format!("l{}_{}_{}", level, pi, c))) + .collect(); + + let started = std::time::Instant::now(); + println!( + " level {}/{}: inserting {} folders…", + level, + depth, + parents.len() + ); + + let rows: Vec<(Uuid,)> = sqlx::query_as( + "INSERT INTO storage.folders (name, parent_id, user_id) + SELECT f.name, f.parent_id, $1 + FROM UNNEST($2::uuid[], $3::text[]) AS f(parent_id, name) + RETURNING id", + ) + .bind(user_id) + .bind(&parents) + .bind(&names) + .fetch_all(pool) + .await?; + let level_ids: Vec = rows.into_iter().map(|(id,)| id).collect(); + println!( + " level {}/{}: done in {:.1}s", + level, + depth, + started.elapsed().as_secs_f64() + ); + + if level == 4 { + depth4 = level_ids.first().copied(); + } + if level == 8 { + depth8 = level_ids.first().copied(); + } + if level == depth { + deepest = *level_ids.first().expect("non-empty deepest level"); + } + + all_ids.extend(&level_ids); + current_level = level_ids; + } + + let depth4 = depth4.unwrap_or(deepest); + let depth8 = depth8.unwrap_or(deepest); + let leaves = current_level; + + Ok(Subtree { + root: root_id, + depth4, + depth8, + deepest, + all_ids, + leaves, + }) +} + +// ── Files ──────────────────────────────────────────────────────────────── + +async fn insert_files( + pool: &PgPool, + user_id: Uuid, + blob_hash: &str, + shared_leaves: &[Uuid], + group_leaves: &[Uuid], + files_per_leaf: u32, +) -> Result { + if files_per_leaf == 0 { + return Ok(0); + } + let all_leaves: Vec = shared_leaves + .iter() + .chain(group_leaves.iter()) + .copied() + .collect(); + if all_leaves.is_empty() { + return Ok(0); + } + + // Build (folder_id, name) pairs. + let folder_ids: Vec = all_leaves + .iter() + .flat_map(|f| std::iter::repeat_n(*f, files_per_leaf as usize)) + .collect(); + let names: Vec = (0..all_leaves.len()) + .flat_map(|li| (0..files_per_leaf).map(move |fi| format!("file_{}_{}.txt", li, fi))) + .collect(); + + sqlx::query( + "INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type) + SELECT f.name, f.folder_id, $1, $2, 0, 'text/plain' + FROM UNNEST($3::uuid[], $4::text[]) AS f(folder_id, name)", + ) + .bind(user_id) + .bind(blob_hash) + .bind(&folder_ids) + .bind(&names) + .execute(pool) + .await?; + + let total = folder_ids.len() as i64; + sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + $1 WHERE hash = $2") + .bind(total) + .bind(blob_hash) + .execute(pool) + .await?; + + Ok(total as u64) +} + +// ── Subject groups ─────────────────────────────────────────────────────── + +async fn build_group_chain( + pool: &PgPool, + admin_id: Uuid, + extra_users: &[SeededUser], + group_member_user_id: &Uuid, + depth: u32, + fanout: u32, +) -> Result> { + if depth < 2 { + return Err(format!("--group-depth must be ≥ 2 (got {})", depth).into()); + } + + // Create `depth` groups: g_0 (outermost) → g_1 → … → g_{depth-1} (innermost). + // Insert one row per group; CITEXT name must match the RFC-5321 regex, + // and the `load_` prefix lets the wipe step find them again. + let salt: String = Uuid::new_v4().simple().to_string(); + let names: Vec = (0..depth) + .map(|i| format!("load_g_{}_{}", salt, i)) + .collect(); + + let mut group_ids: HashMap = HashMap::with_capacity(depth as usize); + for (i, name) in names.iter().enumerate() { + let row: (Uuid,) = + sqlx::query_as("INSERT INTO auth.subject_groups (name) VALUES ($1) RETURNING id") + .bind(name) + .fetch_one(pool) + .await?; + group_ids.insert(i as u32, row.0); + } + + // Wire the chain: g_i contains g_{i+1} as a member (XOR: member_group_id set). + for i in 0..(depth - 1) { + let parent = group_ids[&i]; + let child = group_ids[&(i + 1)]; + sqlx::query( + "INSERT INTO auth.subject_group_members (group_id, member_user_id, member_group_id, added_by) + VALUES ($1, NULL, $2, $3)", + ) + .bind(parent) + .bind(child) + .bind(admin_id) + .execute(pool) + .await?; + } + + // Innermost group gets `fanout` direct user members, ensuring the + // group_member sentinel user is always among them. + let leaf = group_ids[&(depth - 1)]; + let user_ids: Vec = { + let mut v = vec![*group_member_user_id]; + for u in extra_users.iter().filter(|u| u.id != *group_member_user_id) { + if v.len() as u32 >= fanout { + break; + } + v.push(u.id); + } + v + }; + + let leafs: Vec = std::iter::repeat_n(leaf, user_ids.len()).collect(); + let added_by_col: Vec = std::iter::repeat_n(admin_id, user_ids.len()).collect(); + sqlx::query( + "INSERT INTO auth.subject_group_members (group_id, member_user_id, member_group_id, added_by) + SELECT g.group_id, g.member_user_id, NULL, g.added_by + FROM UNNEST($1::uuid[], $2::uuid[], $3::uuid[]) AS g(group_id, member_user_id, added_by)", + ) + .bind(&leafs) + .bind(&user_ids) + .bind(&added_by_col) + .execute(pool) + .await?; + + Ok(NestedGroupIds { + root: group_ids[&0], + mid: group_ids[&(depth / 2)], + leaf, + }) +} + +// ── Grants ─────────────────────────────────────────────────────────────── + +async fn insert_grant( + pool: &PgPool, + subject_type: &str, + subject_id: Uuid, + resource_type: &str, + resource_id: Uuid, + permission: &str, + granted_by: Uuid, +) -> Result<(), sqlx::Error> { + sqlx::query( + "INSERT INTO storage.access_grants + (subject_type, subject_id, resource_type, resource_id, permission, granted_by) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (subject_type, subject_id, resource_type, resource_id, permission) DO NOTHING", + ) + .bind(subject_type) + .bind(subject_id) + .bind(resource_type) + .bind(resource_id) + .bind(permission) + .bind(granted_by) + .execute(pool) + .await?; + Ok(()) +} + +// ── Utilities ──────────────────────────────────────────────────────────── + +fn sanitize_url_for_log(url: &str) -> String { + // Strip "user:password@" from the URL so logs don't leak credentials. + if let Some(at) = url.find('@') + && let Some(scheme_end) = url.find("://") + { + let scheme = &url[..scheme_end + 3]; + let rest = &url[at..]; + return format!("{}***{}", scheme, rest); + } + url.to_string() +} diff --git a/tests/common/spawn-db.sh b/tests/common/spawn-db.sh index 8fd748eb..0db7cff3 100755 --- a/tests/common/spawn-db.sh +++ b/tests/common/spawn-db.sh @@ -32,11 +32,24 @@ wait_for_postgres_ready() { # Belt-and-braces: pg_isready returns 0 as soon as the server accepts # connections, but a query may still race the very first request. One # successful SELECT confirms the round-trip works end-to-end. - PGPASSWORD=oxicloud_test psql -h 127.0.0.1 -p 5433 -U oxicloud_test -d oxicloud_test \ - -v ON_ERROR_STOP=1 -c 'SELECT 1' >/dev/null 2>&1 || { - echo "Postgres reported ready but a sample query failed" >&2 - exit 1 - } + # + # Retry the probe a handful of times — under CPU pressure (e.g. a parallel + # cargo build hammering a self-hosted runner) the role/db init can complete + # a beat after pg_isready returns success. A single-shot probe in that + # window produces spurious "Postgres reported ready but a sample query + # failed" failures. Show the last error if every retry fails so operators + # see the actual psql diagnostic. + local last_err + for _ in 1 2 3 4 5 6 7 8 9 10; do + if last_err=$(PGPASSWORD=oxicloud_test psql -h 127.0.0.1 -p 5433 \ + -U oxicloud_test -d oxicloud_test \ + -v ON_ERROR_STOP=1 -c 'SELECT 1' 2>&1 >/dev/null); then + return 0 + fi + sleep 0.5 + done + echo "Postgres reported ready but a sample query failed after 10 retries: $last_err" >&2 + exit 1 } echo "[setup] Starting test postgres..." diff --git a/tests/load/README.md b/tests/load/README.md index 529be1b5..dfe0d289 100644 --- a/tests/load/README.md +++ b/tests/load/README.md @@ -1,11 +1,94 @@ -iPurpose of this directory: implement load test and identify response time under: -- heavy load -- many content -- many sub folders and sharing +# tests/load/ -Goal is to identify inflections and regression when a new feature is added +K6 load-test suite for OxiCloud. Detects performance regressions by comparing +each run's p50/p95/p99 against a committed baseline. -No accemtance criteria yet +## Suites -Load test via k6 ? -or drill (written in Rust) ? +- **smoke** — `just load-smoke`. Single VU, single iteration of one scenario. + Verifies the harness still builds and the server boots. ~1 minute. Run on + every PR. No regression gate. +- **full** — `just load`. Runs every scenario under `scenarios/` against a + seeded database. Compares results against `baseline/load.json`; exits + non-zero on regression beyond the per-metric tolerance. Run nightly on + `main` and manually. + +## Scenarios + +| File | What it measures | +| --------------------------------- | ------------------------------------------------------------------------------- | +| `scenarios/smoke.js` | Login + create folder + upload + list root + delete. Liveness only. | +| `scenarios/folder_cascade.js` | `GET /contents`, `PUT /move`, batch copy, `DELETE` on a depth-8 fanout-5 tree. | +| `scenarios/share_cascade_rebac.js`| `POST /grants` on a folder, then descendants fetched by the grantee. | +| `scenarios/subject_group_nested.js`| Grant via a 3-level nested group chain, then descendants fetched by a member. | + +Add new scenarios as `scenarios/.js`; register their metric names in +`baseline/load.json` (or `baseline/smoke.json` if you wire smoke gating). + +## Seeding + +`src/bin/load-seed.rs` (invoked by `run.sh`) bulk-inserts fixtures directly +via sqlx: users, deep folder tree, files (all sharing one dedup'd blob), +nested subject groups, ReBAC grants. Only the resources each scenario +actively touches (the grant being created, the move target, etc.) go +through the REST API at run time — that is the measured hot path. + +## Baseline & regression detection + +Baselines live under `baseline/`, split by which runner grades them: + +| File | Used by | Regression-gated? | +| --------------------- | ------------------------ | ----------------- | +| `baseline/load.json` | `just load` (`run.sh`) | Yes | +| `baseline/smoke.json` | `just load-smoke` | Not yet (see below) | + +Both have the same shape — one entry per `.`: + +```json +{ + "folder_cascade.list_depth1": { "p50": 0.97, "p95": 2.04, "p99": 4.82, "tolerance_pct": 10 } +} +``` + +K6 scenarios load the relevant file at startup and set `thresholds` from +it, so a regression fails the K6 run directly. `compare.mjs` also prints a +human-readable diff table after the run and exits non-zero if any metric +regresses beyond its tolerance. + +The smoke scenario is currently **not regression-gated** — `smoke.sh` runs +the scenario but doesn't call `compare.mjs`. When you decide it should be, +mirror the `run.sh` pattern and point `compare.mjs` at +`baseline/smoke.json`. + +**Updating a baseline is deliberate.** Run `just load-baseline` to rewrite +`baseline/load.json` from the latest run, then commit it as +`chore(load): accept new baseline for `. Never auto-update. For +`smoke.json`, pass explicit paths to `bake-baseline.mjs`. + +## Local workflow + +```bash +just db # start the test postgres (port 5433) +just load-seed # seed alone (poking around in psql) +just load-smoke # fast liveness check +just load # full suite + regression diff +just load-baseline # rerun, accept current numbers as the new bar +``` + +## CI + +- `.github/workflows/load-smoke.yml` — every PR. ~1 min. No regression gate. +- `.github/workflows/load-nightly.yml` — cron daily + `workflow_dispatch`. + Runs the full suite, uploads results as artifact, opens an issue on + regression. Currently runs on `ubuntu-latest`; replace with a stable + self-hosted runner for trustworthy regression signal (shared GitHub + runners produce noisy timings). + +## Why K6 and not Goose/drill + +K6 is Go-based (JS scripting in `goja`), not Node. For scenario A, the +client adds ~50–200µs per request — negligible vs. multi-ms server work, +and regression deltas only need *consistency*. K6 also gives us +thresholds-as-DSL, native InfluxDB/Prometheus output, and faster +scenario iteration than a Rust tester would. Reassess when scenario B +(many concurrent users) demonstrates K6 saturation issues. diff --git a/tests/load/bake-baseline.mjs b/tests/load/bake-baseline.mjs new file mode 100644 index 00000000..3330da37 --- /dev/null +++ b/tests/load/bake-baseline.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// bake-baseline.mjs — convert the latest k6 --summary-export into the +// baseline shape that compare.mjs reads. +// +// Tolerance values from the existing baseline are preserved; only the p50/ +// p95/p99 numbers are overwritten. Run this after `just load` to lock in a +// new accepted bar, then commit the result deliberately. +// +// Usage: +// node bake-baseline.mjs # auto-pick latest summary, target baseline/load.json +// node bake-baseline.mjs # explicit paths (e.g. baseline/smoke.json) + +import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { argv, exit } from 'node:process'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +function latestSummary() { + const dir = join(HERE, 'results'); + const candidates = readdirSync(dir) + .filter((f) => f.startsWith('run-') && f.endsWith('.json')) + .map((f) => ({ f, mtime: statSync(join(dir, f)).mtimeMs })) + .sort((a, b) => b.mtime - a.mtime); + if (candidates.length === 0) { + console.error('bake-baseline: no run-*.json under tests/load/results/'); + exit(2); + } + return join(dir, candidates[0].f); +} + +const summaryPath = argv[2] ? resolve(argv[2]) : latestSummary(); +const baselinePath = argv[3] ? resolve(argv[3]) : join(HERE, 'baseline/load.json'); + +const summary = JSON.parse(readFileSync(summaryPath, 'utf8')); +const baseline = JSON.parse(readFileSync(baselinePath, 'utf8')); + +const metricKeyFor = (op) => `http_req_duration{op:${op}}`; +const PERCENTILES = [ + { baseline: 'p50', k6: 'med' }, + { baseline: 'p95', k6: 'p(95)' }, + { baseline: 'p99', k6: 'p(99)' }, +]; + +let updated = 0; +let missing = 0; + +for (const [op, entry] of Object.entries(baseline)) { + if (op.startsWith('_')) continue; + const metric = summary.metrics?.[metricKeyFor(op)]; + if (!metric) { + console.warn(` missing: ${op} (no data in ${summaryPath})`); + missing++; + continue; + } + // k6 --summary-export puts percentile values directly on the metric object + // (alongside `thresholds`), not inside a `.values` wrapper. + let touched = false; + for (const p of PERCENTILES) { + const cur = metric[p.k6]; + if (cur === undefined) continue; + entry[p.baseline] = Number(cur.toFixed(2)); + touched = true; + } + if (touched) updated++; +} + +writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`); +console.log(`bake-baseline: updated ${updated} metric(s), missing ${missing} in ${baselinePath}`); +console.log(`Source: ${summaryPath}`); +console.log('Commit deliberately: chore(load): accept new baseline for '); diff --git a/tests/load/baseline/load.json b/tests/load/baseline/load.json new file mode 100644 index 00000000..b377008b --- /dev/null +++ b/tests/load/baseline/load.json @@ -0,0 +1,81 @@ +{ + "_comment": "Baseline for `just load` — the long scenarios. Each entry: p50/p95/p99 (ms) + tolerance_pct (regression threshold). Updated by `just load-baseline`. Keys must match the `op` tag the matching scenario emits — adding a scenario means adding entries here in the same PR. Smoke metrics live in smoke.json.", + "folder_cascade.list_depth1": { + "p50": 0.35, + "p95": 2.25, + "p99": 4.73, + "tolerance_pct": 10 + }, + "folder_cascade.list_depth4": { + "p50": 0.36, + "p95": 0.92, + "p99": 2.39, + "tolerance_pct": 10 + }, + "folder_cascade.list_depth8": { + "p50": 0.34, + "p95": 0.63, + "p99": 1.9, + "tolerance_pct": 10 + }, + "folder_cascade.list_depth_deep": { + "p50": 0.3, + "p95": 0.61, + "p99": 2.03, + "tolerance_pct": 10 + }, + "share_cascade_rebac.list_grants": { + "p50": 0.4, + "p95": 1.16, + "p99": 1.68, + "tolerance_pct": 10 + }, + "share_cascade_rebac.fetch_as_grantee_depth1": { + "p50": 0.51, + "p95": 1.14, + "p99": 3.02, + "tolerance_pct": 10 + }, + "share_cascade_rebac.fetch_as_grantee_depth4": { + "p50": 0.53, + "p95": 1.43, + "p99": 2.22, + "tolerance_pct": 10 + }, + "share_cascade_rebac.fetch_as_grantee_depth8": { + "p50": 0.52, + "p95": 0.89, + "p99": 1.48, + "tolerance_pct": 10 + }, + "share_cascade_rebac.fetch_as_grantee_depth_deep": { + "p50": 0.51, + "p95": 1.04, + "p99": 1.64, + "tolerance_pct": 10 + }, + "subject_group_nested.fetch_as_member_depth1": { + "p50": 0.48, + "p95": 0.63, + "p99": 1.41, + "tolerance_pct": 10 + }, + "subject_group_nested.fetch_as_member_depth4": { + "p50": 0.5, + "p95": 0.58, + "p99": 0.65, + "tolerance_pct": 10 + }, + "subject_group_nested.fetch_as_member_depth8": { + "p50": 0.48, + "p95": 0.53, + "p99": 0.62, + "tolerance_pct": 10 + }, + "subject_group_nested.fetch_as_member_depth_deep": { + "p50": 0.45, + "p95": 0.62, + "p99": 0.63, + "tolerance_pct": 10 + } +} diff --git a/tests/load/baseline/smoke.json b/tests/load/baseline/smoke.json new file mode 100644 index 00000000..bf8f4b1f --- /dev/null +++ b/tests/load/baseline/smoke.json @@ -0,0 +1,33 @@ +{ + "_comment": "Baseline for `just load-smoke` — the smoke scenario. Placeholder values; smoke is not currently regression-gated by smoke.sh. When you decide it should be, point compare.mjs at this file from smoke.sh (mirror the run.sh pattern) and re-bake.", + "smoke.login": { + "p50": 50, + "p95": 200, + "p99": 400, + "tolerance_pct": 15 + }, + "smoke.create_folder": { + "p50": 20, + "p95": 80, + "p99": 150, + "tolerance_pct": 15 + }, + "smoke.upload_tiny": { + "p50": 30, + "p95": 120, + "p99": 250, + "tolerance_pct": 15 + }, + "smoke.list_root": { + "p50": 15, + "p95": 60, + "p99": 120, + "tolerance_pct": 15 + }, + "smoke.delete_folder": { + "p50": 20, + "p95": 80, + "p99": 150, + "tolerance_pct": 15 + } +} diff --git a/tests/load/compare.mjs b/tests/load/compare.mjs new file mode 100644 index 00000000..822e9f41 --- /dev/null +++ b/tests/load/compare.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node +// compare.mjs — regression diff for k6 load-test runs. +// +// Reads a k6 --summary-export JSON and the committed baseline, diffs the +// p50/p95/p99 of every baseline metric against the current run, prints a +// human-readable table, and exits non-zero if any metric regresses beyond +// its tolerance. +// +// Usage: +// node compare.mjs +// +// Exit codes: +// 0 — every metric within tolerance. +// 1 — one or more metrics regressed, or a baseline metric is absent from +// the current summary (suite drift — either a scenario was deleted +// or a tag was renamed without updating baseline.json). + +import { readFileSync } from 'node:fs'; +import { argv, exit } from 'node:process'; + +if (argv.length < 4) { + console.error('usage: compare.mjs '); + exit(2); +} + +const summaryPath = argv[2]; +const baselinePath = argv[3]; + +const summary = JSON.parse(readFileSync(summaryPath, 'utf8')); +const baseline = JSON.parse(readFileSync(baselinePath, 'utf8')); + +// k6 summary metrics are keyed verbatim with the tag: e.g. +// "http_req_duration{op:smoke.login}". +const metricKeyFor = (op) => `http_req_duration{op:${op}}`; + +// Map k6 percentile field names → baseline field names. +const PERCENTILES = [ + { baseline: 'p50', k6: 'med' }, + { baseline: 'p95', k6: 'p(95)' }, + { baseline: 'p99', k6: 'p(99)' }, +]; + +const rows = []; +let anyRegression = false; +let anyMissing = false; + +for (const [op, base] of Object.entries(baseline)) { + if (op.startsWith('_')) continue; // skip _comment etc. + const metric = summary.metrics?.[metricKeyFor(op)]; + if (!metric) { + rows.push({ op, status: 'MISSING', detail: 'no data for this tag in current summary' }); + anyMissing = true; + continue; + } + const tol = (base.tolerance_pct ?? 10) / 100; + // Noise-floor guard: at sub-millisecond percentiles, a 10% tolerance is + // smaller than a single context switch. Require the absolute delta to + // exceed `min_delta_ms` too — otherwise the swing is below the measurement + // floor and we don't flag it. Default 0.5ms is conservative for the load + // suite's typical 0.3–3ms range. Override per metric in baseline.json. + const minDelta = base.min_delta_ms ?? 0.5; + + for (const p of PERCENTILES) { + const baseVal = base[p.baseline]; + // k6 --summary-export puts percentile values directly on the metric, not + // inside a `.values` wrapper. + const curVal = metric[p.k6]; + if (baseVal === undefined || curVal === undefined) continue; + + const deltaPct = ((curVal - baseVal) / baseVal) * 100; + const deltaAbs = curVal - baseVal; + const limitPct = baseVal * (1 + tol); + // Regression only if BOTH the % rule AND the absolute floor are breached. + const regressed = curVal > limitPct && deltaAbs > minDelta; + if (regressed) anyRegression = true; + rows.push({ + op, + percentile: p.baseline, + base: baseVal, + cur: curVal, + deltaPct, + regressed, + tol: base.tolerance_pct ?? 10, + }); + } +} + +// ── Render table ────────────────────────────────────────────────────────── + +const ms = (v) => `${v.toFixed(1)}ms`; +const pct = (v) => `${v >= 0 ? '+' : ''}${v.toFixed(1)}%`; + +function pad(s, n) { + if (s.length >= n) return s; + return s + ' '.repeat(n - s.length); +} + +const colOp = Math.max(20, ...rows.map((r) => r.op.length)); +const colP = 5; +const colVal = 11; + +console.log(); +console.log( + pad('metric', colOp + 1) + + pad('pctl', colP + 1) + + pad('baseline', colVal + 1) + + pad('current', colVal + 1) + + pad('delta', 9) + + 'status', +); +console.log('-'.repeat(colOp + colP + colVal * 2 + 9 + 12)); + +for (const r of rows) { + if (r.status === 'MISSING') { + console.log(`${pad(r.op, colOp + 1)}${pad('-', colP + 1)}${pad('-', colVal + 1)}${pad('-', colVal + 1)}${pad('-', 9)}MISSING — ${r.detail}`); + continue; + } + const status = r.regressed ? `REGRESSION (tol ±${r.tol}%)` : 'ok'; + console.log( + pad(r.op, colOp + 1) + + pad(r.percentile, colP + 1) + + pad(ms(r.base), colVal + 1) + + pad(ms(r.cur), colVal + 1) + + pad(pct(r.deltaPct), 9) + + status, + ); +} + +console.log(); +if (anyRegression || anyMissing) { + if (anyRegression) console.error('compare.mjs: one or more metrics regressed beyond tolerance.'); + if (anyMissing) console.error('compare.mjs: one or more baseline metrics are missing from the current summary.'); + exit(1); +} +console.log('compare.mjs: all metrics within tolerance.'); +exit(0); diff --git a/tests/load/lib/auth.js b/tests/load/lib/auth.js new file mode 100644 index 00000000..c35a36bd --- /dev/null +++ b/tests/load/lib/auth.js @@ -0,0 +1,38 @@ +// Login helper for OxiCloud k6 load tests. +// Returns the JWT access token from POST /api/auth/login. + +import { check, fail } from 'k6'; +import http from 'k6/http'; +import { BASE } from './http.js'; + +/** + * Log in via the public auth endpoint and return the bearer token. + * + * `body` shape matches `auth_handler.rs::login` — `{username, password}`. + * The response carries `access_token` plus refresh state we don't need here. + * + * The `op` tag should be scenario-qualified (e.g. 'smoke.login') so the + * recorded metric matches the corresponding key in baseline/baseline.json. + * Bare 'login' is fine for ad-hoc scripts that aren't regression-gated. + * + * @param {string} username + * @param {string} password + * @param {string} [op='login'] + * @returns {string} + */ +export function login(username, password, op = 'login') { + const res = http.post( + `${BASE}/api/auth/login`, + JSON.stringify({ username, password }), + { headers: { 'Content-Type': 'application/json' }, tags: { op } }, + ); + + const ok = check(res, { + 'login 200': (r) => r.status === 200, + 'login has access_token': (r) => !!r.json('access_token'), + }); + if (!ok) { + fail(`login failed for ${username}: status=${res.status}, body=${res.body}`); + } + return res.json('access_token'); +} diff --git a/tests/load/lib/http.js b/tests/load/lib/http.js new file mode 100644 index 00000000..82fdca25 --- /dev/null +++ b/tests/load/lib/http.js @@ -0,0 +1,40 @@ +// Shared HTTP helpers for OxiCloud k6 load tests. +// Centralises the base URL and adds the Bearer token + JSON Content-Type +// headers expected by every protected endpoint. + +import http from 'k6/http'; + +/** + * Base URL of the OxiCloud server under test. + * Read from K6_BASE_URL (set by run.sh), defaulting to the load-suite port. + */ +export const BASE = __ENV.K6_BASE_URL || 'http://localhost:8088'; + +/** + * Build a request params object with auth + JSON headers and a `tag` so + * the response's metrics are isolated under `.`. + * + * @param {string} token Bearer token from auth.login() + * @param {string} op Metric tag, e.g. 'folder_cascade.list_depth8' + */ +export function jsonParams(token, op) { + return { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + tags: { op }, + }; +} + +/** + * Same as jsonParams but without a body content-type — for GETs and DELETEs. + */ +export function authParams(token, op) { + return { + headers: { Authorization: `Bearer ${token}` }, + tags: { op }, + }; +} + +export const httpClient = http; diff --git a/tests/load/lib/metrics.js b/tests/load/lib/metrics.js new file mode 100644 index 00000000..7f4aa9d3 --- /dev/null +++ b/tests/load/lib/metrics.js @@ -0,0 +1,76 @@ +// Baseline-driven thresholds and shared helpers for k6 scenarios. +// +// Each scenario tags its requests with `op: '.'` (see http.js +// `jsonParams` / `authParams`). The baseline file lists one entry per +// `.` with p50/p95/p99 and a tolerance percentage; we convert +// every relevant entry to a k6 threshold so the run fails directly on +// regression — `compare.mjs` then produces the human-readable diff. + +// Resolve relative to THIS file (lib/metrics.js), not the importer. Future +// k6 versions will align open()'s path-resolution with ES module semantics; +// using import.meta.resolve() future-proofs against the warning logged by +// k6 ≥ 0.50. +const LOAD_BASELINE_PATH = import.meta.resolve('../baseline/load.json'); +const SMOKE_BASELINE_PATH = import.meta.resolve('../baseline/smoke.json'); +const MANIFEST_PATH = import.meta.resolve('../results/seed-manifest.json'); + +// Eagerly load both baseline files at module init (open() is only allowed +// in init context). Keys are disjoint by scenario prefix, so merging the +// two maps is safe; `thresholdsFromBaseline(prefix)` filters from the union. +const BASELINE = { + ...JSON.parse(open(LOAD_BASELINE_PATH)), + ...JSON.parse(open(SMOKE_BASELINE_PATH)), +}; + +/** + * Return the merged baseline (load + smoke) — convenient for tooling that + * wants to inspect everything; scenarios should use `thresholdsFromBaseline`. + */ +export function loadBaseline() { + return BASELINE; +} + +/** + * Build a k6 `thresholds` object from the baseline files, filtered to the + * given scenario prefix (e.g. 'folder_cascade'). + * + * Result shape (k6 expects metric-name → threshold-expression-array): + * { + * 'http_req_duration{op:folder_cascade.list_depth8}': [ + * 'p(95)<49.5', // 45 * (1 + 10/100) + * 'p(99)<88.0', + * ], + * } + * + * @param {string} scenarioPrefix + */ +export function thresholdsFromBaseline(scenarioPrefix) { + const baseline = BASELINE; + const thresholds = {}; + for (const [key, val] of Object.entries(baseline)) { + if (key.startsWith('_')) continue; // skip _comment etc. + if (!key.startsWith(`${scenarioPrefix}.`)) continue; + const tol = (val.tolerance_pct || 10) / 100; + // Mirror compare.mjs: use the larger of the %-based limit and the + // absolute-floor limit (baseline + min_delta_ms). Below sub-millisecond + // scale, the % rule alone fires on noise — the floor stops that. + const minDelta = val.min_delta_ms ?? 0.5; + const p95Limit = Math.max(val.p95 * (1 + tol), val.p95 + minDelta); + const p99Limit = Math.max(val.p99 * (1 + tol), val.p99 + minDelta); + thresholds[`http_req_duration{op:${key}}`] = [ + `p(95)<${p95Limit.toFixed(2)}`, + `p(99)<${p99Limit.toFixed(2)}`, + ]; + } + // `abortOnFail: false` (default) keeps the run going so we collect all + // regressions in one pass; the non-zero exit at the end still fails CI. + return thresholds; +} + +/** + * Read the seed manifest written by `cargo run --bin load-seed`. + */ +export function loadManifest() { + const raw = open(MANIFEST_PATH); + return JSON.parse(raw); +} diff --git a/tests/load/merge-summaries.mjs b/tests/load/merge-summaries.mjs new file mode 100644 index 00000000..9eefcb85 --- /dev/null +++ b/tests/load/merge-summaries.mjs @@ -0,0 +1,40 @@ +#!/usr/bin/env node +// merge-summaries.mjs — combine multiple k6 --summary-export outputs into +// one. Used because k6 accepts a single script per invocation, but our +// regression-diff tooling (compare.mjs, bake-baseline.mjs) expects one +// summary per run. +// +// Per-scenario summaries are disjoint in their metric tags +// (folder_cascade.* vs share_cascade_rebac.* vs subject_group_nested.*), +// so merging is just a union of the `metrics` maps. The top-level +// envelope (root_group, options, etc.) is taken from the first input. +// +// Usage: +// node merge-summaries.mjs [in2.json …] + +import { readFileSync, writeFileSync } from 'node:fs'; +import { argv, exit } from 'node:process'; + +if (argv.length < 5) { + console.error('usage: merge-summaries.mjs [in2.json …] '); + exit(2); +} + +const inputs = argv.slice(2, -1); +const output = argv[argv.length - 1]; + +const merged = JSON.parse(readFileSync(inputs[0], 'utf8')); +merged.metrics = { ...merged.metrics }; + +for (const path of inputs.slice(1)) { + const next = JSON.parse(readFileSync(path, 'utf8')); + for (const [k, v] of Object.entries(next.metrics ?? {})) { + // If two scenarios both report a global metric (e.g. http_req_duration + // with no op tag), the second wins — these aren't gated by baseline.json, + // so it's only the per-`op` metrics that need to be preserved precisely. + merged.metrics[k] = v; + } +} + +writeFileSync(output, `${JSON.stringify(merged, null, 2)}\n`); +console.log(`merged ${inputs.length} summaries → ${output}`); diff --git a/tests/load/results/.gitkeep b/tests/load/results/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/load/run.sh b/tests/load/run.sh new file mode 100755 index 00000000..3e95f26e --- /dev/null +++ b/tests/load/run.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Full k6 load-test runner. +# Starts postgres + OxiCloud server, seeds fixtures, runs k6 scenarios, +# compares results against baseline/load.json, tears everything down. +# +# Usage (from repo root): +# bash tests/load/run.sh +# +# Env overrides: +# BUILD_TARGET=release # prefer release build for accurate timings +# LOAD_DEPTH=8 # override seeder shape (otherwise read from test.env) +# LOAD_FANOUT=3 +# LOAD_FILES_PER_LEAF=3 +# LOAD_EXTRA_USERS=20 +# LOAD_GROUP_DEPTH=3 +# LOAD_GROUP_FANOUT=5 +# K6_SUMMARY_OUT=path # explicit output path (default: tests/load/results/.json) +# +# Prerequisites: docker, cargo, k6 >= 0.46, node >= 18 + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +COMMON="$REPO_ROOT/tests/common" +LOAD_DIR="$REPO_ROOT/tests/load" + +# shellcheck source=test.env +source "$LOAD_DIR/test.env" + +SERVER_PORT="${base_url##*:}" + +log() { echo "[load] $*"; } +die() { echo "[load] ERROR: $*" >&2; exit 1; } + +wait_for_http() { + local url="$1" timeout="${2:-120}" + local deadline=$(( $(date +%s) + timeout )) + until curl -sf "$url" >/dev/null 2>&1; do + [[ $(date +%s) -ge $deadline ]] && die "Timeout waiting for $url" + sleep 1 + done +} + +command -v k6 >/dev/null 2>&1 || die "k6 is required (https://k6.io/docs/get-started/installation/)" +command -v node >/dev/null 2>&1 || die "node >= 18 is required for compare.mjs" + +SERVER_PID="" +cleanup() { + if [[ -n "$SERVER_PID" ]]; then + log "Stopping OxiCloud server (pid $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + bash "$COMMON/stop-db.sh" || true +} +trap cleanup EXIT + +bash "$COMMON/spawn-db.sh" + +# NOTE: do NOT run init-test-schema.sh here. The OxiCloud server applies +# sqlx migrations on startup; applying them via raw psql first leaves the +# server's _sqlx_migrations tracking table empty, which makes the second +# pass try to re-ALTER tables that already have the column (e.g. migration +# 20260507000000_session_family.sql), and the server panics on boot. + +set -a +# shellcheck source=../common/server.env +source "$COMMON/server.env" +OXICLOUD_SERVER_PORT=$SERVER_PORT +OXICLOUD_STORAGE_PATH="$LOAD_DIR/storage" +set +a + +rm -rf "$OXICLOUD_STORAGE_PATH" +mkdir -p "$OXICLOUD_STORAGE_PATH" + +BUILD_TARGET="${BUILD_TARGET:-release}" +# Respect CARGO_TARGET_DIR for self-hosted runners that bind-mount target/ +# outside the workspace (avoids actions/checkout EBUSY on the mount point). +TARGET_DIR="${CARGO_TARGET_DIR:-$REPO_ROOT/target}" +OXICLOUD_BIN="$TARGET_DIR/$BUILD_TARGET/oxicloud" +SEED_BIN="$TARGET_DIR/$BUILD_TARGET/load-seed" + +# Build both bins in one invocation. `load_seed_bin` is an empty marker +# feature that gates the load-seed bin without affecting oxicloud's dep +# graph, so cargo plans a single workspace build and oxicloud compiles +# exactly once. (See Cargo.toml comments on `load_seed_bin`.) +if [[ ! -x "$OXICLOUD_BIN" || ! -x "$SEED_BIN" ]]; then + log "Building OxiCloud + load-seed ($BUILD_TARGET)..." + if [[ "$BUILD_TARGET" == "release" ]]; then + cargo build --release --features load_seed_bin --bin oxicloud --bin load-seed + else + cargo build --features load_seed_bin --bin oxicloud --bin load-seed + fi +fi + +# Start the server FIRST so its sqlx::migrate! populates _sqlx_migrations +# against the clean DB. The seeder then runs against the migrated schema +# while the server is still alive (it issues plain INSERTs, no DDL). +log "Starting OxiCloud server ($BUILD_TARGET) on port $SERVER_PORT..." +"$OXICLOUD_BIN" & +SERVER_PID=$! +wait_for_http "$base_url/ready" 120 +log "Server ready." + +log "Seeding fixtures..." +DEPTH="${LOAD_DEPTH:-${load_depth:-5}}" +FANOUT="${LOAD_FANOUT:-${load_fanout:-4}}" +FILES_PER_LEAF="${LOAD_FILES_PER_LEAF:-${load_files_per_leaf:-3}}" +EXTRA_USERS="${LOAD_EXTRA_USERS:-${load_extra_users:-20}}" +GROUP_DEPTH="${LOAD_GROUP_DEPTH:-${load_group_depth:-3}}" +GROUP_FANOUT="${LOAD_GROUP_FANOUT:-${load_group_fanout:-5}}" + +mkdir -p "$LOAD_DIR/results" +MANIFEST_PATH="$LOAD_DIR/results/seed-manifest.json" + +"$SEED_BIN" \ + --depth "$DEPTH" \ + --fanout "$FANOUT" \ + --files-per-leaf "$FILES_PER_LEAF" \ + --extra-users "$EXTRA_USERS" \ + --group-depth "$GROUP_DEPTH" \ + --group-fanout "$GROUP_FANOUT" \ + --password "${password:-TestPassword1!}" \ + --manifest "$MANIFEST_PATH" + +TS="$(date +%s)" +SUMMARY_OUT="${K6_SUMMARY_OUT:-$LOAD_DIR/results/run-$TS.json}" + +export K6_BASE_URL="$base_url" +export K6_USERNAME="${username:-admin}" +export K6_PASSWORD="${password:-TestPassword1!}" + +# k6 only accepts one script per invocation, so each scenario runs separately +# and we merge the summaries afterwards. Per-scenario summaries also make it +# easier to attribute regressions when looking at raw artifacts in CI. +SCENARIOS=(folder_cascade share_cascade_rebac subject_group_nested) +PARTIAL_SUMMARIES=() +K6_FAILED=0 + +for name in "${SCENARIOS[@]}"; do + partial="$LOAD_DIR/results/run-$TS-$name.json" + PARTIAL_SUMMARIES+=("$partial") + log "Running k6 scenario: $name" + # --summary-trend-stats forces p(99) into the summary export; k6's default + # only includes avg/min/med/max/p(90)/p(95), so without it baseline.p99 can + # never be baked or diffed. + k6 run \ + --summary-export="$partial" \ + --summary-trend-stats="avg,min,med,max,p(90),p(95),p(99)" \ + --quiet \ + "$LOAD_DIR/scenarios/$name.js" \ + || K6_FAILED=$? +done + +log "Merging summaries -> $SUMMARY_OUT" +node "$LOAD_DIR/merge-summaries.mjs" "${PARTIAL_SUMMARIES[@]}" "$SUMMARY_OUT" + +log "Comparing against baseline..." +COMPARE_RC=0 +node "$LOAD_DIR/compare.mjs" "$SUMMARY_OUT" "$LOAD_DIR/baseline/load.json" || COMPARE_RC=$? + +if [[ "$K6_FAILED" -ne 0 ]]; then + log "k6 reported threshold failures (exit $K6_FAILED)." + exit "$K6_FAILED" +fi +exit "$COMPARE_RC" diff --git a/tests/load/scenarios/folder_cascade.js b/tests/load/scenarios/folder_cascade.js new file mode 100644 index 00000000..d2302418 --- /dev/null +++ b/tests/load/scenarios/folder_cascade.js @@ -0,0 +1,60 @@ +// folder_cascade.js — measures the read-path of `GET /api/folders/{id}/resources?resource_types=folder` +// at four depths against a pre-seeded tree. Captures how listing cost scales +// with ltree depth (cf. `idx_folders_lpath` GiST index). Mid-depth samples +// (depth4, depth8) fall back to `deepest` when the seeded tree is shallower +// than that depth — see load-seed.rs::build_subtree. + +import { check } from 'k6'; +import http from 'k6/http'; +import { BASE, authParams } from '../lib/http.js'; +import { login } from '../lib/auth.js'; +import { thresholdsFromBaseline, loadManifest } from '../lib/metrics.js'; + +const manifest = loadManifest(); + +export const options = { + vus: 1, + // 100 iterations gives p99 statistical meaning: at N=100, p99 = position 99, + // representing one bad sample out of a hundred — a real percentile rather + // than the worst-of-the-batch. At N=25 (the typical k6 example default), + // p99 ≈ max, dominated by single-sample kernel/scheduler noise. + iterations: 100, + thresholds: thresholdsFromBaseline('folder_cascade'), +}; + +// One per-VU login. K6 calls setup() once across the whole test, default() +// `iterations` times per VU. Logging in inside default() would dominate the +// per-iter cost; we hand the token down through the `data` arg. +export function setup() { + const token = login(manifest.admin.username, manifest.admin.password); + return { token }; +} + +export default function (data) { + const { token } = data; + const t = manifest.shared_subtree; + + const r1 = http.get( + `${BASE}/api/folders/${t.root}/resources?resource_types=folder`, + authParams(token, 'folder_cascade.list_depth1'), + ); + check(r1, { 'list depth1 200': (r) => r.status === 200 }); + + const r4 = http.get( + `${BASE}/api/folders/${t.depth4}/resources?resource_types=folder`, + authParams(token, 'folder_cascade.list_depth4'), + ); + check(r4, { 'list depth4 200': (r) => r.status === 200 }); + + const r8 = http.get( + `${BASE}/api/folders/${t.depth8}/resources?resource_types=folder`, + authParams(token, 'folder_cascade.list_depth8'), + ); + check(r8, { 'list depth8 200': (r) => r.status === 200 }); + + const rD = http.get( + `${BASE}/api/folders/${t.deepest}/resources?resource_types=folder`, + authParams(token, 'folder_cascade.list_depth_deep'), + ); + check(rD, { 'list deepest 200': (r) => r.status === 200 }); +} diff --git a/tests/load/scenarios/share_cascade_rebac.js b/tests/load/scenarios/share_cascade_rebac.js new file mode 100644 index 00000000..a63a3dc7 --- /dev/null +++ b/tests/load/scenarios/share_cascade_rebac.js @@ -0,0 +1,84 @@ +// share_cascade_rebac.js — measures ReBAC AuthZ cascade via a direct user +// grant. The seeder grants `read` on shared_subtree.root (depth 0) to the +// grantee user; this scenario times how long the grantee takes to fetch +// folders at varying depths inside that subtree. +// +// The AuthZ recursive-CTE in `pg_acl_engine` walks up the ancestor chain +// from the requested folder until it finds a matching grant (or runs out +// of ancestors). The deeper the requested folder, the more ancestors the +// CTE has to traverse before hitting the grant on the root: +// +// fetch_at_grant_root → 0 ancestors walked (grant is right here) +// fetch_as_grantee_depth4 → 4 ancestors walked +// fetch_as_grantee_depth8 → 8 ancestors walked +// fetch_as_grantee_depth_deep → `load_depth` ancestors walked +// +// A regression in AuthZ cost should show up as the cascade-depth curve +// flattening or steepening — that's the value of intermediate samples. + +import { check } from 'k6'; +import http from 'k6/http'; +import { BASE, authParams } from '../lib/http.js'; +import { login } from '../lib/auth.js'; +import { thresholdsFromBaseline, loadManifest } from '../lib/metrics.js'; + +const manifest = loadManifest(); + +export const options = { + vus: 1, + // See folder_cascade.js for the iteration-count rationale: N=100 makes + // p99 a real percentile instead of the worst-of-the-batch. + iterations: 100, + thresholds: thresholdsFromBaseline('share_cascade_rebac'), +}; + +export function setup() { + const adminToken = login(manifest.admin.username, manifest.admin.password); + const granteeToken = login(manifest.grantee.username, manifest.grantee.password); + return { adminToken, granteeToken }; +} + +export default function (data) { + const { adminToken, granteeToken } = data; + const t = manifest.shared_subtree; + + // List grants on the granted folder (admin only). + const grantsRes = http.get( + `${BASE}/api/grants?resource_type=folder&resource_id=${t.root}`, + authParams(adminToken, 'share_cascade_rebac.list_grants'), + ); + check(grantsRes, { 'list grants 200': (r) => r.status === 200 }); + + // Baseline: grantee fetches the folder the grant is directly on. AuthZ + // finds the matching grant on the first row of the CTE; zero ancestors + // walked. This metric measures the constant overhead of an authorized + // request — moves to the right of this metric is "cascade cost." + const d1 = http.get( + `${BASE}/api/folders/${t.root}/resources?resource_types=folder`, + authParams(granteeToken, 'share_cascade_rebac.fetch_as_grantee_depth1'), + ); + check(d1, { 'fetch root 200': (r) => r.status === 200 }); + + // Grantee fetches a mid-tree folder. AuthZ walks 4 ancestors before + // hitting the grant. + const d4 = http.get( + `${BASE}/api/folders/${t.depth4}/resources?resource_types=folder`, + authParams(granteeToken, 'share_cascade_rebac.fetch_as_grantee_depth4'), + ); + check(d4, { 'fetch depth4 200': (r) => r.status === 200 }); + + // Grantee fetches a folder 8 levels under the granted root. AuthZ + // walks 8 ancestors. + const d8 = http.get( + `${BASE}/api/folders/${t.depth8}/resources?resource_types=folder`, + authParams(granteeToken, 'share_cascade_rebac.fetch_as_grantee_depth8'), + ); + check(d8, { 'fetch depth8 200': (r) => r.status === 200 }); + + // Grantee fetches the deepest descendant — full-length cascade. + const dD = http.get( + `${BASE}/api/folders/${t.deepest}/resources?resource_types=folder`, + authParams(granteeToken, 'share_cascade_rebac.fetch_as_grantee_depth_deep'), + ); + check(dD, { 'fetch deepest 200': (r) => r.status === 200 }); +} diff --git a/tests/load/scenarios/smoke.js b/tests/load/scenarios/smoke.js new file mode 100644 index 00000000..a5578cc9 --- /dev/null +++ b/tests/load/scenarios/smoke.js @@ -0,0 +1,64 @@ +// smoke.js — fast PR-tier check. Verifies the load harness still builds and +// the server boots, exercising one happy-path of every HTTP verb the long +// scenarios use. No regression gate; run.sh's `smoke.sh` skips compare.mjs. + +import { check, sleep } from 'k6'; +import http from 'k6/http'; +import { BASE, jsonParams, authParams } from '../lib/http.js'; +import { login } from '../lib/auth.js'; +import { thresholdsFromBaseline } from '../lib/metrics.js'; + +export const options = { + vus: 1, + iterations: 1, + thresholds: thresholdsFromBaseline('smoke'), +}; + +// Admin creds match tests/load/test.env defaults — overridable via env. +const USERNAME = __ENV.K6_USERNAME || 'admin'; +const PASSWORD = __ENV.K6_PASSWORD || 'TestPassword1!'; + +export default function () { + // 1. login + const token = login(USERNAME, PASSWORD, 'smoke.login'); + + // 2. create a scratch folder at root + const folderName = `smoke_${Date.now()}`; + const createRes = http.post( + `${BASE}/api/folders`, + JSON.stringify({ name: folderName }), + jsonParams(token, 'smoke.create_folder'), + ); + check(createRes, { 'create folder 200/201': (r) => r.status === 200 || r.status === 201 }); + const folderId = createRes.json('id'); + + // 3. upload a tiny file via multipart + const fileData = http.file('hello\n', 'smoke.txt', 'text/plain'); + const uploadRes = http.post( + `${BASE}/api/files/upload`, + { folder_id: folderId, file: fileData }, + { headers: { Authorization: `Bearer ${token}` }, tags: { op: 'smoke.upload_tiny' } }, + ); + const uploadOk = check(uploadRes, { + 'upload 200/201': (r) => r.status === 200 || r.status === 201, + }); + if (!uploadOk) { + console.error( + `upload failed: status=${uploadRes.status}, body=${uploadRes.body}, headers=${JSON.stringify(uploadRes.headers)}`, + ); + } + + // 4. list root folders for this user + const listRes = http.get(`${BASE}/api/folders`, authParams(token, 'smoke.list_root')); + check(listRes, { 'list root 200': (r) => r.status === 200 }); + + // 5. delete the scratch folder + const delRes = http.del( + `${BASE}/api/folders/${folderId}`, + null, + authParams(token, 'smoke.delete_folder'), + ); + check(delRes, { 'delete 200/204': (r) => r.status === 200 || r.status === 204 }); + + sleep(0.1); +} diff --git a/tests/load/scenarios/subject_group_nested.js b/tests/load/scenarios/subject_group_nested.js new file mode 100644 index 00000000..b3d572d5 --- /dev/null +++ b/tests/load/scenarios/subject_group_nested.js @@ -0,0 +1,73 @@ +// subject_group_nested.js — measures the worst-case AuthZ path: a user who +// is a direct member of the *innermost* group of a depth-N nested chain, and +// the grant is on the *outermost* group. Every AuthZ check has to expand the +// chain transitively. Pairs with share_cascade_rebac.js to attribute regressions +// to either the resource side (folder cascade) or the subject side (group +// expansion). +// +// Same depth-gradient idea as share_cascade_rebac: measure AuthZ cost at +// depth 1 / 4 / 8 / deep so we can see WHERE in the chain a regression +// lands. The subject-side expansion (group_member → leaf → mid → root group +// → grant) runs once per request regardless of folder depth, but the +// resource-side cascade (folder ancestors walked to find the grant) grows +// linearly with depth — so depth4/depth8/deep moving together while depth1 +// stays flat would point at the resource side; all four moving together +// would point at the group-expansion path. + +import { check } from 'k6'; +import http from 'k6/http'; +import { BASE, authParams } from '../lib/http.js'; +import { login } from '../lib/auth.js'; +import { thresholdsFromBaseline, loadManifest } from '../lib/metrics.js'; + +const manifest = loadManifest(); + +export const options = { + vus: 1, + // See folder_cascade.js for the iteration-count rationale: N=100 makes + // p99 a real percentile instead of the worst-of-the-batch. + iterations: 100, + thresholds: thresholdsFromBaseline('subject_group_nested'), +}; + +export function setup() { + const memberToken = login(manifest.group_member.username, manifest.group_member.password); + return { memberToken }; +} + +export default function (data) { + const { memberToken } = data; + const t = manifest.group_subtree; + + // Baseline: group member fetches the folder the grant is directly on. + // Subject-side expansion runs (user → leaf → mid → root group → grant); + // resource-side walk is zero ancestors. Captures the constant subject- + // expansion overhead. + const d1 = http.get( + `${BASE}/api/folders/${t.root}/resources?resource_types=folder`, + authParams(memberToken, 'subject_group_nested.fetch_as_member_depth1'), + ); + check(d1, { 'fetch root 200': (r) => r.status === 200 }); + + // Mid-tree: 4 folder ancestors walked + subject expansion. + const d4 = http.get( + `${BASE}/api/folders/${t.depth4}/resources?resource_types=folder`, + authParams(memberToken, 'subject_group_nested.fetch_as_member_depth4'), + ); + check(d4, { 'fetch depth4 200': (r) => r.status === 200 }); + + // 8 folder ancestors walked + subject expansion. + const d8 = http.get( + `${BASE}/api/folders/${t.depth8}/resources?resource_types=folder`, + authParams(memberToken, 'subject_group_nested.fetch_as_member_depth8'), + ); + check(d8, { 'fetch depth8 200': (r) => r.status === 200 }); + + // Worst case: full-depth folder cascade × full-chain subject expansion. + // group_member → leaf group → mid group → root group → grant → folder root → deepest. + const dD = http.get( + `${BASE}/api/folders/${t.deepest}/resources?resource_types=folder`, + authParams(memberToken, 'subject_group_nested.fetch_as_member_depth_deep'), + ); + check(dD, { 'fetch deepest 200': (r) => r.status === 200 }); +} diff --git a/tests/load/smoke.sh b/tests/load/smoke.sh new file mode 100755 index 00000000..0f7580c2 --- /dev/null +++ b/tests/load/smoke.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Smoke (PR-tier) k6 load runner. +# Same shape as run.sh but: no seeder, only the smoke scenario, no regression +# diff. Goal is harness liveness - does the server still boot and does k6 still +# wire up - not perf gating. +# +# Usage (from repo root): +# bash tests/load/smoke.sh +# +# Env overrides: +# BUILD_TARGET=debug # debug is fine; smoke doesn't measure timings +# +# Prerequisites: docker, cargo, k6 >= 0.46 + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +COMMON="$REPO_ROOT/tests/common" +LOAD_DIR="$REPO_ROOT/tests/load" + +# shellcheck source=test.env +source "$LOAD_DIR/test.env" + +SERVER_PORT="${base_url##*:}" + +log() { echo "[load-smoke] $*"; } +die() { echo "[load-smoke] ERROR: $*" >&2; exit 1; } + +wait_for_http() { + local url="$1" timeout="${2:-60}" + local deadline=$(( $(date +%s) + timeout )) + until curl -sf "$url" >/dev/null 2>&1; do + [[ $(date +%s) -ge $deadline ]] && die "Timeout waiting for $url" + sleep 1 + done +} + +command -v k6 >/dev/null 2>&1 || die "k6 is required (https://k6.io/docs/get-started/installation/)" + +SERVER_PID="" +cleanup() { + if [[ -n "$SERVER_PID" ]]; then + log "Stopping OxiCloud server (pid $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + bash "$COMMON/stop-db.sh" || true +} +trap cleanup EXIT + +bash "$COMMON/spawn-db.sh" +# Server applies sqlx migrations on startup; don't double-apply via psql here. +# See run.sh for the gory details. + +set -a +# shellcheck source=../common/server.env +source "$COMMON/server.env" +OXICLOUD_SERVER_PORT=$SERVER_PORT +OXICLOUD_STORAGE_PATH="$LOAD_DIR/storage" +set +a + +rm -rf "$OXICLOUD_STORAGE_PATH" +mkdir -p "$OXICLOUD_STORAGE_PATH" + +BUILD_TARGET="${BUILD_TARGET:-debug}" +OXICLOUD_BIN="$REPO_ROOT/target/$BUILD_TARGET/oxicloud" + +if [[ -x "$OXICLOUD_BIN" ]]; then + log "Starting pre-built OxiCloud server ($BUILD_TARGET) on port $SERVER_PORT..." + "$OXICLOUD_BIN" & +else + log "Building and starting OxiCloud server ($BUILD_TARGET) on port $SERVER_PORT..." + cd "$REPO_ROOT" + if [[ "$BUILD_TARGET" == "release" ]]; then + cargo run --release & + else + cargo run & + fi +fi +SERVER_PID=$! +wait_for_http "$base_url/ready" 120 +log "Server ready." + +# Bootstrap the admin account via /api/setup (one-shot — disabled once an +# admin exists, mirrors tests/api/setup.hurl). Without this the smoke scenario +# would have no one to log in as. +log "Creating admin via /api/setup..." +SETUP_BODY=$(printf '{"username":"%s","email":"%s","password":"%s"}' \ + "${username:-admin}" "${email:-admin@example.com}" "${password:-TestPassword1!}") +SETUP_STATUS=$(curl -sS -o /dev/null -w '%{http_code}' \ + -X POST "$base_url/api/setup" \ + -H 'Content-Type: application/json' \ + -d "$SETUP_BODY") +if [[ "$SETUP_STATUS" != "201" ]]; then + die "/api/setup returned $SETUP_STATUS (expected 201)" +fi + +export K6_BASE_URL="$base_url" +export K6_USERNAME="${username:-admin}" +export K6_PASSWORD="${password:-TestPassword1!}" + +log "Running smoke scenario..." +k6 run \ + --summary-trend-stats="avg,min,med,max,p(90),p(95),p(99)" \ + --quiet \ + "$LOAD_DIR/scenarios/smoke.js" + +log "Smoke OK." diff --git a/tests/load/test.env b/tests/load/test.env new file mode 100644 index 00000000..56a1b34c --- /dev/null +++ b/tests/load/test.env @@ -0,0 +1,31 @@ +# Test credentials and base URL for tests/load/ — NOT real secrets. +# Mirrors tests/api/test.env shape; uses a different server port (8088) +# so it does not collide with api-test (8087). +base_url=http://localhost:8088 +username=admin +email=admin@example.com +# gitguardian:ignore +password=TestPassword1! + +# ── Seed sizing ───────────────────────────────────────────────────────────── +# Folder count is exponential: total folders per subtree ≈ fanout^depth. +# The seeder builds TWO subtrees (one for the user-grant scenario, one for the +# group-grant scenario), so multiply by 2 for the grand total. Each leaf folder +# then gets `files_per_leaf` rows. +# +# Examples (folders per subtree): +# depth=4 fanout=3 → 121 (seed ≈ 1 s) +# depth=5 fanout=4 → 1 365 (seed ≈ 10 s) ← default +# depth=6 fanout=3 → 1 093 (seed ≈ 10 s) +# depth=8 fanout=3 → 9 841 (seed ≈ 30 s) +# depth=8 fanout=5 → 488 281 (seed ≈ several minutes — exponential blow-up) +# +# Override per run with LOAD_DEPTH / LOAD_FANOUT / LOAD_FILES_PER_LEAF. + +load_depth=10 +load_fanout=3 + +load_files_per_leaf=3 +load_extra_users=20 +load_group_depth=3 +load_group_fanout=5