Optimize storage, GC, and upload hot paths
This commit is contained in:
DioCrafts
Generated
+1979
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,53 @@
|
||||
[package]
|
||||
name = "oxicloud-perf-audit"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "gc_manifest_batch"
|
||||
path = "gc_manifest_batch.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "verify_integrity_phase1"
|
||||
path = "verify_integrity_phase1.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "verify_integrity_borrowed"
|
||||
path = "verify_integrity_borrowed.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "verify_integrity_streaming"
|
||||
path = "verify_integrity_streaming.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "migration_workset"
|
||||
path = "migration_workset.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "cached_range_ab"
|
||||
path = "cached_range_ab.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "admin_user_listing_e2e"
|
||||
path = "admin_user_listing_e2e.rs"
|
||||
|
||||
[dependencies]
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
foldhash = "0.2"
|
||||
futures = "0.3.32"
|
||||
moka = { version = "0.12.15", features = ["future"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
sqlx = { version = "0.8.6", default-features = false, features = ["chrono", "json", "postgres", "runtime-tokio", "uuid"] }
|
||||
tokio = { version = "1.52.3", features = ["fs", "macros", "rt-multi-thread", "sync", "time"] }
|
||||
uuid = { version = "1", features = ["serde", "v4"] }
|
||||
|
||||
# This audit utility is intentionally independent of the repository package:
|
||||
# compiling it must not build or link the OxiCloud server just to issue SQL.
|
||||
[workspace]
|
||||
|
||||
[profile.release]
|
||||
codegen-units = 1
|
||||
lto = "thin"
|
||||
opt-level = 3
|
||||
@@ -0,0 +1,456 @@
|
||||
# OxiCloud performance audit
|
||||
|
||||
This directory is the reproducible evidence log for the 2026-07-21--22
|
||||
performance audit. It deliberately lives outside every `benches/` directory; the audit did
|
||||
not use or inspect those directories. Production changes were accepted only
|
||||
after an A/B gate preserved observable behaviour. Rejected candidates remain
|
||||
here as evidence, but their production changes were rolled back.
|
||||
|
||||
Result labels have six meanings:
|
||||
|
||||
- `accepted`: the measured candidate passed correctness and resource gates.
|
||||
- `rejected`: the candidate regressed a gate or weakened semantics; production
|
||||
is unchanged.
|
||||
- `pending_gate`: evidence is incomplete or a measured resource regression has
|
||||
not been explicitly authorized; this is not an acceptance decision.
|
||||
- `pending_representative_gate`: the benchmark population may not represent the
|
||||
production cost distribution closely enough to authorize a tradeoff.
|
||||
- `pending_explicit_user_tradeoff`: the representative gate is complete, but a
|
||||
measured resource regression still needs explicit authorization or rollback.
|
||||
- `accepted_by_explicit_user_tradeoff`: not Pareto-superior, but the user
|
||||
explicitly chose the documented resource tradeoff after seeing both costs.
|
||||
|
||||
Heap/RSS figures from in-process Node runs are indicative. Where memory decided
|
||||
the result, a fresh-process gate was used. SQL harnesses use temporary tables or
|
||||
disposable databases. Raw samples and environment metadata are retained under
|
||||
`results/`.
|
||||
|
||||
## Decision summary
|
||||
|
||||
| Area | Decision | Key measured evidence |
|
||||
| --- | --- | --- |
|
||||
| Admin user summary projection | Accepted | Minimal full component path 1.296 -> 0.966 ms, JSON -34.35%, RSS -112 KiB; heavy JSON -99.946%, RSS -132.20 MiB |
|
||||
| Admin newest-first index | Explicit tradeoff accepted | Unique timestamps: first page 57.217 -> 0.215 ms (266.13x), offset 50k 126.377 -> 5.219 ms (24.21x); 11,255,808 B index; +0.680 us/insert |
|
||||
| Admin compound newest-first index | Rejected | Unique first page 0.186 -> 0.198 ms; index bytes +80.13%-327.76%; burst inserts +0.445 us/user versus the narrow index |
|
||||
| Admin `COUNT(*) OVER()` fusion | Rejected | 5.604 -> 42.287 ms (7.55x slower) |
|
||||
| Folder-upload progress accumulator | Accepted | 1-file repeat 0.797 -> 0.289 us (2.76x); 100-file/5k-update case 6,256.84 -> 13.33 us (469.48x) |
|
||||
| Delta-worker upload queue cursor | Explicit tradeoff | 1.18x-246.78x faster; median max RSS +112 to +480 KiB; producer-ahead retained RSS +1,008 KiB |
|
||||
| Frontend whole-file dedup above 10k | Rejected | All-miss 11,384.656 -> 12,545.523 ms with more heap/RSS; production reverted |
|
||||
| Local blob sync preparation | Accepted | Empty call 25.607 -> 22.946 ns (1.116x; 10/11 process wins); path grouping 1.22x-2.39x; directory preparation 1.23x-212.32x |
|
||||
| Cached bounded-range length | Accepted | Removed exactly one surplus byte/read; 10k A/B p50 unchanged at 8.459 us, p95 28.292 -> 26.917 us (-4.86%) |
|
||||
| Manifest-GC hybrid aggregation | Explicit tradeoff accepted | 500 manifests 60.03x with +720 KiB RSS; 1,000 manifests 51.16x with +1,008 KiB RSS; N=0/1 keeps the serial path |
|
||||
| Integrity verification sorted windows | Explicit tradeoff accepted | Real-FS full method 1.086x-5.766x and remote full method 5.691x-39.056x faster; fresh-process RSS +112 KiB phase 1 and +80 KiB full method |
|
||||
| Migration work-set paging | Rejected | 65,536-row pages cut RSS 82.74% but were 2.52x slower; 262,144-row pages cut RSS 42.64% but were 2.99x slower |
|
||||
| Indexed migration verification window | Rejected | Query 267.13x faster, but a 1% contiguous failure range was detected about 1% vs 63.4% for 100 independent samples |
|
||||
| Loose-chunk DB prefilter | Rejected | Added work on the normal negotiated-miss path and removed backend-missing self-healing semantics |
|
||||
| Identical-overwrite refcount CTE | Rejected | Mixed legacy/CDC ownership is ambiguous; candidate could undercount live data or leak the shadowed representation |
|
||||
|
||||
The original tied-timestamp table also evaluated a compound `(created_at, id)`
|
||||
index, but posting-list compression invalidated its resource comparison. The
|
||||
isolated representative A/B/C rerun below supersedes that exploratory result:
|
||||
the compound candidate failed the no-regression gate and is absent from
|
||||
production.
|
||||
|
||||
## Frontend upload algorithms
|
||||
|
||||
`frontend-upload-algorithms.mjs` isolates three algorithms: queue drain,
|
||||
aggregate progress, and the proposed >10k whole-file dedup batching. It
|
||||
alternates A/B order, accumulates tiny cases above timer resolution, forces GC
|
||||
when available, and validates checksums/progress/protocol counts.
|
||||
|
||||
Run the general harness from the repository root:
|
||||
|
||||
node --expose-gc tools/perf-audit/frontend-upload-algorithms.mjs \
|
||||
--warmup 3 \
|
||||
--samples 15 \
|
||||
--queue-counts 64,256,1024,10000,50000 \
|
||||
--progress-cases 1:100,10:500,100:5000,1000:10000,10000:5000 \
|
||||
--hash-counts 1000,10000,10001,25000 \
|
||||
--output /tmp/oxicloud-frontend-upload.json
|
||||
|
||||
The accepted progress implementation maintains the aggregate sum with
|
||||
`new_fraction - old_fraction`; restart-to-zero and finalization are covered by
|
||||
the frontend unit test. The preliminary common run improved every median but
|
||||
had a noisy one-file p95 regression (2.136 -> 11.240 ms per 1,000-run block,
|
||||
equivalent to a 2.136 -> 11.240 us/run block average, not a per-event p95), so
|
||||
it is evidence-only. The
|
||||
subsequent 41-sample focused gate improved both the normalized one-file median
|
||||
(0.797 -> 0.289 us/run) and block p95 (14.158 -> 6.338 ms per 1,000 runs,
|
||||
equivalent to a 14.158 -> 6.338 us/run block average) with identical output.
|
||||
`progress-common-node26-macos-arm64.json` and
|
||||
`progress-one-file-repeat-node26-macos-arm64.json` retain both sets of samples.
|
||||
|
||||
### Delta-worker queue memory gate
|
||||
|
||||
The in-process queue microbenchmark favored the cursor but was biased because
|
||||
`Array.shift()` ran long enough for V8 to collect while the cursor finished
|
||||
before the next GC. `queue-memory-gate.mjs` therefore runs every sample in a
|
||||
fresh process, keeps the permanent ordered chunk table alive, and measures
|
||||
wall time, max RSS, post-GC retained RSS, and heap for prefilled,
|
||||
producer-ahead, and balanced shapes.
|
||||
|
||||
node --expose-gc tools/perf-audit/queue-memory-gate.mjs \
|
||||
--count 100000 \
|
||||
--samples 5 \
|
||||
--output tools/perf-audit/results/queue-memory-process-node26-macos-arm64.json
|
||||
|
||||
Production uses `cursor-clear-4096`. Against `shift()`, medians were:
|
||||
|
||||
| Shape | Wall speedup | Median max RSS delta | Retained RSS delta | Retained heap delta |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Prefilled | 246.783x | +448 KiB | +448 KiB | -3,080 B |
|
||||
| Producer ahead | 37.137x | +480 KiB | +1,008 KiB | -3,576 B |
|
||||
| Balanced | 1.184x | +112 KiB | 0 | +3,168 B |
|
||||
|
||||
This, the representative admin index, the manifest-GC hybrid, and the sorted
|
||||
integrity windows are the audit's explicitly accepted non-Pareto changes. The
|
||||
queue result JSON marks it
|
||||
`accepted_by_explicit_user_tradeoff` and retains the rejected thresholds,
|
||||
`splice`, `slice`, no-clear, and array-reset variants.
|
||||
|
||||
### Rejected whole-file dedup batching
|
||||
|
||||
The microbenchmark correctly showed that one >10k request is rejected while
|
||||
bounded requests recover owned hashes. That is functional evidence, not an
|
||||
acceptance result: the rejected control returns no hashes and does less work.
|
||||
The decisive loopback workflow includes every dedup, by-hash, and content
|
||||
request at production upload concurrency:
|
||||
|
||||
node --expose-gc tools/perf-audit/frontend-dedup-workflow.mjs \
|
||||
--samples 3 \
|
||||
--bytes-per-file 4096 \
|
||||
--output tools/perf-audit/results/frontend-dedup-workflow-node26-macos-arm64.json
|
||||
|
||||
At 10,001 all-miss files, batching was 10.2% slower and increased median peak
|
||||
heap/RSS. At 50% hits it saved 50.005% of content bytes and was 1.051x faster,
|
||||
but roughly doubled peak heap and added about 27.9 MiB RSS. Backend SQL for the
|
||||
two accepted ownership queries is not modeled, making the candidate optimistic.
|
||||
The feature was rejected and reverted. The four `dedup-*` JSON files remain
|
||||
labelled evidence-only so their invalid-control speedups cannot be mistaken for
|
||||
production acceptance.
|
||||
|
||||
## Admin user listing
|
||||
|
||||
The accepted compact response fetches only fields rendered by the table;
|
||||
full-detail API clients keep the previous shape unless `summary=true` is sent.
|
||||
It avoids detoasting/transporting avatars and preferences. The service-layer
|
||||
system-admin gate is authoritative, and deterministic pagination uses
|
||||
`ORDER BY created_at DESC, id DESC`.
|
||||
|
||||
Run the projection, count-fusion rejection, representative three-way index
|
||||
gate, and full component-path gate:
|
||||
|
||||
psql "$DATABASE_URL" -f tools/perf-audit/admin_user_projection.sql
|
||||
psql "$DATABASE_URL" -f tools/perf-audit/admin_user_count.sql
|
||||
psql "$DATABASE_URL" -f \
|
||||
tools/perf-audit/admin_user_order_index_representative.sql
|
||||
cargo build --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin admin_user_listing_e2e
|
||||
DATABASE_URL="$DATABASE_URL" \
|
||||
tools/perf-audit/target/release/admin_user_listing_e2e timing minimal 31
|
||||
DATABASE_URL="$DATABASE_URL" \
|
||||
tools/perf-audit/target/release/admin_user_listing_e2e timing heavy 11
|
||||
/usr/bin/time -l env DATABASE_URL="$DATABASE_URL" \
|
||||
tools/perf-audit/target/release/admin_user_listing_e2e memory-historical minimal
|
||||
/usr/bin/time -l env DATABASE_URL="$DATABASE_URL" \
|
||||
tools/perf-audit/target/release/admin_user_listing_e2e memory-candidate minimal
|
||||
/usr/bin/time -l env DATABASE_URL="$DATABASE_URL" \
|
||||
tools/perf-audit/target/release/admin_user_listing_e2e memory-historical heavy
|
||||
/usr/bin/time -l env DATABASE_URL="$DATABASE_URL" \
|
||||
tools/perf-audit/target/release/admin_user_listing_e2e memory-candidate heavy
|
||||
|
||||
Repeat each `memory-*` command in three fresh processes; the result file retains
|
||||
all twelve max-RSS/elapsed samples rather than only the medians.
|
||||
|
||||
The projection fixture has 100 users with a 512 KiB avatar and 8 KiB preference
|
||||
bag each. Its 117.93x timing is the `psql` query/row-transfer/client-decode
|
||||
gate, not an end-to-end HTTP claim; it excludes Serde and the service-layer
|
||||
authorization check. The first index fixture has 500,000 users with 100-way
|
||||
timestamp ties and checks exact order equivalence, first/deep pages, index
|
||||
bytes, and 10,000-row insert cost. It is not an acceptance result: the
|
||||
ties deliberately stress incremental sorting, but they also let PostgreSQL
|
||||
compress the one-column B-tree into posting lists. Its 3.45 MB size and +0.596
|
||||
us per inserted user understated a normal mostly-unique registration workload.
|
||||
|
||||
The decisive component harness includes the full historical SQL/DTO/Serde path
|
||||
and the candidate's hot Moka flags lookup, system-admin policy check, compact
|
||||
SQL/DTO, count query, and Serde. It excludes common router/JWT/socket work and
|
||||
deliberately omits the old handler's intermediate `serde_json::Value`
|
||||
materialization, making the historical baseline optimistic. It is therefore a
|
||||
conservative component-path gate rather than a whole HTTP-stack claim. On the
|
||||
minimal profile, 31 interleaved samples improved median latency 1.296 -> 0.966
|
||||
ms (1.341x), JSON fell 43,759 -> 28,726 bytes (-34.35%), and three fresh
|
||||
processes saved 114,688 bytes (112 KiB) median max RSS. With 512 KiB avatars and
|
||||
8 KiB preferences, median latency improved 1,141.422 -> 0.966 ms, JSON fell
|
||||
53,306,743 -> 28,726 bytes (-99.946%), and median max RSS fell by 138,625,024
|
||||
bytes (132.20 MiB). Exact rendered fields, ordering, and counts matched.
|
||||
|
||||
The follow-up ran three independent rollback-only transactions, for 15 A/B
|
||||
samples per shape, with the UUID primary-key index present on both sides. At
|
||||
500,000 unique timestamps the index was 11,255,808 bytes (3.263x the prior
|
||||
disclosure), first-page/deep-page reads improved 266.13x/24.21x, and 10,000-row
|
||||
insert medians imply +0.680 us per user. Ten-user bursts used 4,751,360 bytes,
|
||||
improved reads 321.57x/18.29x, and added +0.582 us per user. Every initial/final
|
||||
order and row-count check passed in all three transactions. Because the
|
||||
representative unique-key disk cost is materially larger than the original
|
||||
disclosure, the first authorization was invalidated. After seeing the corrected
|
||||
11,255,808-byte/+0.680-us unique cost and the 4,751,360-byte/+0.582-us burst10
|
||||
cost, the user explicitly reauthorized the timestamp-only index. It is therefore
|
||||
`accepted_by_explicit_user_tradeoff`.
|
||||
|
||||
The later isolated A/B/C gate retained 15 samples per shape. Versus that
|
||||
accepted narrow index, the compound index regressed the common unique-timestamp
|
||||
first page 0.186 -> 0.198 ms (+6.45%), enlarged the unique index 11,255,808 ->
|
||||
20,275,200 bytes (+80.13%), and enlarged the ten-user-burst index 4,751,360 ->
|
||||
20,324,352 bytes (+327.76%). It did accelerate deep pages 1.63x-2.30x, but burst
|
||||
insert medians regressed 44.475 -> 48.921 ms per 10,000 rows (+0.445 us/user).
|
||||
It therefore failed the no-regression gate and was rejected. Raw A/B and A/B/C
|
||||
samples are in `admin-user-index-representative-postgres18-macos-arm64.json`;
|
||||
component-path samples are in
|
||||
`admin-user-listing-e2e-postgres18-macos-arm64.json`. Production keeps one
|
||||
narrow online `CREATE INDEX CONCURRENTLY` statement.
|
||||
|
||||
## Local blob durability preparation
|
||||
|
||||
`local_sync_grouping.rs` A/Bs only the CPU/allocation preparation around the
|
||||
unchanged fsync work: moving owned `PathBuf`s into task groups instead of
|
||||
cloning, and a fixed exact-case prefix bitmap instead of sort/dedup of parent
|
||||
paths. It checks ordered file-path equivalence plus case-sensitive `af`/`aF`
|
||||
directory equivalence before timing.
|
||||
|
||||
rustc --edition 2024 -O tools/perf-audit/local_sync_grouping.rs -o /tmp/local-sync-grouping
|
||||
/tmp/local-sync-grouping
|
||||
|
||||
The zero-path gate caught avoidable candidate setup and led to a production
|
||||
fast return. In 11 fresh processes, each running 31 alternating samples of
|
||||
100,000 repetitions, it won 10/11 times; the median of process medians improved
|
||||
25.607 -> 22.946 ns (1.116x). All measured non-empty sizes from 1 to 100,000
|
||||
paths also improved. The bitmap is fixed at 22x22 slots so uppercase and
|
||||
lowercase directory names remain distinct on case-sensitive filesystems. Raw
|
||||
process medians are in `backend-audit-macos-arm64.json`.
|
||||
|
||||
## Cached blob bounded ranges
|
||||
|
||||
`CachedBlobBackend` was the only backend treating `end` as inclusive even
|
||||
though the port, Local, S3, Azure, encrypted, CDC, RAM-cache, and HTTP adapter
|
||||
paths all use `[start, end)`. It consequently read one surplus byte on every
|
||||
bounded cold-after-fill or hot-cache range. The production fix changes only the
|
||||
two cached-file limits to `end.saturating_sub(start)` and adds a cold/hot
|
||||
regression test, including the empty `[3,3)` range.
|
||||
|
||||
Run the focused semantic test and standalone hot-file A/B:
|
||||
|
||||
cargo test --lib \
|
||||
cached_blob_backend::tests::range_end_is_exclusive_on_cold_and_hot_cache_reads
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin cached_range_ab
|
||||
|
||||
For 10,000 interleaved `[1,3)` reads, the historical path returned 30,000
|
||||
bytes versus the correct 20,000 (-33.333% for this two-byte fixture). Median
|
||||
latency stayed 8.459 us; p95 improved from 28.292 to 26.917 us (-4.86%). The
|
||||
cold fill remained exactly one origin GET/six bytes. The fixed Local/cached
|
||||
length vectors both equal `[1,2,0,4]`; the historical cached vector was
|
||||
`[2,3,1,4]`. Exact evidence is in
|
||||
`cached_range_exclusive_2026-07-22.json`.
|
||||
|
||||
## Manifest garbage collection
|
||||
|
||||
`gc_manifest_batch.rs` compares the historical serial update per deleted
|
||||
manifest with several measured candidates. Production keeps the accepted
|
||||
hybrid: the dominant empty sweep uses the original simple `DELETE RETURNING`,
|
||||
one returned manifest uses the original serial update, and batches of two or
|
||||
more aggregate exact distinct-per-manifest decrements in an owned `HashMap`
|
||||
before one `UPDATE FROM unnest`.
|
||||
|
||||
The crossover was positive at two manifests. At 500, statements fell 502 -> 3
|
||||
and median latency 1,456.096 -> 24.255 ms (60.03x); at 1,000, 1,003 -> 5 and
|
||||
3,163.190 -> 61.829 ms (51.16x). Five fresh processes measured no RSS change at
|
||||
two, +720 KiB at 500, and +1,008 KiB at 1,000. The user explicitly accepted
|
||||
that bounded memory tradeoff, so the result is
|
||||
`accepted_by_explicit_user_tradeoff`.
|
||||
|
||||
The atomic all-in-one CTE was rejected because an all-live sweep regressed
|
||||
15.59%-44.84%. Borrowed SQLx binds saved memory but regressed large-batch
|
||||
latency; sorted/RLE scratch was not Pareto either. All variants validate live
|
||||
controls, shared and repeated chunks, exact refcounts, underflow,
|
||||
`orphaned_at`, and exact statement counts.
|
||||
|
||||
Reproduce the threshold, large-batch, and fresh-process resource gates:
|
||||
|
||||
OXICLOUD_POSTGRES_HOST=192.168.107.2 \
|
||||
GC_SCENARIOS='0:500,1:499,2:498,4:496,8:492,32:468' \
|
||||
GC_HYBRID_THRESHOLDS='2,4,8,32,500' GC_WARMUPS=2 GC_SAMPLES=9 \
|
||||
bash tools/perf-audit/run_gc_manifest_batch.sh
|
||||
|
||||
OXICLOUD_POSTGRES_HOST=192.168.107.2 \
|
||||
GC_SCENARIOS='500:10,1000:10' GC_HYBRID_THRESHOLDS=2 \
|
||||
GC_WARMUPS=1 GC_SAMPLES=5 \
|
||||
bash tools/perf-audit/run_gc_manifest_batch.sh
|
||||
|
||||
OXICLOUD_POSTGRES_HOST=192.168.107.2 GC_RSS_RUNS=5 \
|
||||
bash tools/perf-audit/run_gc_manifest_bind_rss.sh
|
||||
|
||||
The scripts create randomly named disposable databases and drop them on
|
||||
success, failure, or interruption. Exact samples and rejected candidates are
|
||||
in `gc_manifest_batch_2026-07-21.json`.
|
||||
|
||||
## Integrity verification
|
||||
|
||||
`verify_integrity_borrowed.rs` compares the historical serial backend-size
|
||||
probe per manifest occurrence with owned, borrowed-hash-map, and sorted
|
||||
borrowed-key candidates. The accepted implementation keeps the exact serial
|
||||
path through four valid occurrences. Above that gate it processes bounded
|
||||
256-occurrence windows, sorts and deduplicates borrowed `&str` keys, probes at
|
||||
concurrency 8 with `FuturesUnordered`, and replays issue generation in original
|
||||
manifest/occurrence order. Malformed manifests retain their historical
|
||||
no-probe behaviour.
|
||||
|
||||
OXICLOUD_AUDIT_CONCURRENCY=8 \
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin verify_integrity_borrowed -- --real-fs
|
||||
|
||||
OXICLOUD_AUDIT_CONCURRENCY=8 \
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin verify_integrity_borrowed -- --remote-only
|
||||
|
||||
The first unbounded table doubled max RSS and was rejected. A concurrent path
|
||||
for two/four immediate probes was 62x-67x slower and was also rejected. The
|
||||
intermediate owned-key window at concurrency 16 added 176 KiB RSS and was
|
||||
superseded. The final sorted/borrowed concurrency-8 scheduler was tested with
|
||||
the same boxed-future shape used by production. Across 31-sample real-filesystem
|
||||
gates it improved the full method 1.086x for unique hashes, 1.095x for a mixed
|
||||
existing/missing set, and 5.766x for shared hashes. The remote full-method gates
|
||||
improved 5.691x for unique and 39.056x for shared hashes. Backend calls never
|
||||
increased, issue order was exact, malformed manifests performed zero probes,
|
||||
and the `1x2`, `2x1`, and `1x4` cases execute the same serial code.
|
||||
|
||||
Eleven fresh-process runs over 250,000 unique occurrences measured the accepted
|
||||
candidate at +112 KiB (+0.4284%) RSS for phase 1 and +80 KiB (+0.3053%) for the
|
||||
full method. After disclosure of a measured peak cost up to 112 KiB, the user
|
||||
explicitly reauthorized retaining the candidate in exchange for the measured
|
||||
speedup. Build once, then reproduce the RSS modes separately so the compiler is
|
||||
not part of the measurement. Run each timed command in 11 fresh processes and
|
||||
compare medians:
|
||||
|
||||
cargo build --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin verify_integrity_borrowed
|
||||
OXICLOUD_AUDIT_CONCURRENCY=8 /usr/bin/time -l \
|
||||
tools/perf-audit/target/release/verify_integrity_borrowed \
|
||||
--memory historical phase
|
||||
OXICLOUD_AUDIT_CONCURRENCY=8 /usr/bin/time -l \
|
||||
tools/perf-audit/target/release/verify_integrity_borrowed \
|
||||
--memory sorted phase
|
||||
OXICLOUD_AUDIT_CONCURRENCY=8 /usr/bin/time -l \
|
||||
tools/perf-audit/target/release/verify_integrity_borrowed \
|
||||
--memory historical full
|
||||
OXICLOUD_AUDIT_CONCURRENCY=8 /usr/bin/time -l \
|
||||
tools/perf-audit/target/release/verify_integrity_borrowed \
|
||||
--memory sorted full
|
||||
|
||||
`verify_integrity_streaming.rs` additionally tested direct SQLx streaming and a
|
||||
bounded producer/channel with 16 prefetched manifest rows against a disposable
|
||||
PostgreSQL database. The producer/channel candidate cut RSS 74.23% and made
|
||||
phase 1 1.722x faster, but its same-round full-method median regressed 4.17%, so
|
||||
it was rejected and no streaming code entered production. Reproduce both SQLx
|
||||
experiments with:
|
||||
|
||||
bash tools/perf-audit/run_verify_integrity_streaming.sh
|
||||
bash tools/perf-audit/run_verify_integrity_prefetch.sh
|
||||
|
||||
The accepted measurements and raw gates are in
|
||||
`verify_integrity_sorted_c8_2026-07-22.json`; the rejected SQLx result is in
|
||||
`verify_integrity_streaming_2026-07-22.json`. The earlier owned-window evidence
|
||||
is retained in `verify_integrity_phase1_2026-07-21.json` as a rejected,
|
||||
superseded candidate.
|
||||
|
||||
## Rejected migration work-set paging
|
||||
|
||||
`migration_workset.rs` compares the current one-million-row ordered work-set
|
||||
materialization with bounded keyset pages. Every mode ran in a fresh client
|
||||
process and had to return exactly 1,000,000 rows in the same order/checksum.
|
||||
|
||||
Seed and run against a disposable PostgreSQL database:
|
||||
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin migration_workset -- seed 1000000
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin migration_workset -- current
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin migration_workset -- paged 65536
|
||||
cargo run --release --manifest-path tools/perf-audit/Cargo.toml \
|
||||
--bin migration_workset -- paged 262144
|
||||
|
||||
The 65,536-row page reduced median process RSS from 106,053,632 to 18,300,928
|
||||
bytes (-82.74%) but increased median query/consume time from 644.398 to
|
||||
1,622.317 ms (+151.76%, 2.52x). The 262,144-row page used 60,833,792 bytes
|
||||
(-42.64%) and took 1,928.731 ms (+199.31%, 2.99x). Both candidates therefore
|
||||
failed the no-latency-regression gate and production remains unchanged. The
|
||||
container bridge was noisy; only complete three-way rounds were retained, and
|
||||
every transport failure is listed in
|
||||
`migration-workset-postgres18-macos-arm64.json`.
|
||||
|
||||
## Rejected migration verification sampler
|
||||
|
||||
`migration_verify_sampling.sql` measures replacing `ORDER BY random()` with a
|
||||
random pivot followed by one contiguous indexed hash window:
|
||||
|
||||
psql "$DATABASE_URL" -f tools/perf-audit/migration_verify_sampling.sql
|
||||
|
||||
The query improved from 100.442 to 0.376 ms on one million rows (267.13x), but
|
||||
the samples are correlated. For a 1% contiguous/prefix failure range, one
|
||||
100-row successor window detects the failure about 1% of the time; 100
|
||||
independent samples detect it with probability `1 - 0.99^100 = 63.4%`. The
|
||||
semantic regression rejected the candidate and production was reverted. See
|
||||
`migration-verify-sampling-postgres18-macos-arm64.json`.
|
||||
|
||||
## Rejected storage candidates
|
||||
|
||||
`rejected_storage_candidates_2026-07-21.json` records two fully rolled-back
|
||||
experiments. Their Rust files are archived diagnostic source snapshots rather
|
||||
than registered binaries in the standalone perf Cargo package.
|
||||
|
||||
### Loose-chunk prefilter
|
||||
|
||||
`rejected_delta_loose_hit_probe.rs` counted physical object-store PUTs/bytes for
|
||||
400 x 256 KiB frames. The browser protocol already negotiates missing hashes,
|
||||
so all-miss is the normal receive path. A DB prefilter would add queries and up
|
||||
to 8 MiB request buffering there. More importantly, a metadata row does not
|
||||
prove the backend object exists: skipping PUT based only on PostgreSQL would
|
||||
remove the current self-healing overwrite for missing objects. No candidate
|
||||
showed a Pareto win across miss latency, RAM, remote bytes, and repair semantics.
|
||||
|
||||
### Identical-overwrite refcount CTE
|
||||
|
||||
`rejected_refcount_overwrite_probe.rs` exercised the public write port on
|
||||
legacy, CDC-manifest, different-hash, delete/GC, missing-file, SQL-error, and
|
||||
lifecycle-hook fixtures. The proposed CTE fixed unambiguous same-representation
|
||||
cases, but `storage.files` stores only a hash. When legacy `storage.blobs` and a
|
||||
new `storage.chunk_manifests` row coexist under that hash, the swap cannot know
|
||||
which representation owns the displaced reference. It can decrement live CDC
|
||||
state or preserve a shadowed legacy reference/bytes. Timing samples also had
|
||||
enough container jitter that no non-regression claim was possible. The CTE was
|
||||
rejected and fully reverted.
|
||||
|
||||
The result also exposes a pre-existing baseline issue: repeated identical
|
||||
legacy overwrites increased refcount from 1 to 1,001 in the 1,000-iteration
|
||||
fixture. It remains unfixed because the attempted shortcut could turn a leak
|
||||
into undercount/data loss. A future fix needs explicit representation ownership
|
||||
or normalization before another benchmarked candidate is safe.
|
||||
|
||||
## Video thumbnail diagnostic utility
|
||||
|
||||
`video-thumbnail-server.mjs` is a browser-side real-media gate for a possible
|
||||
thumbnail fallback change. It can serve failed thumbnail responses, a
|
||||
range-capable WebM original, and thumbnail PUT sinks. No production decision in
|
||||
this audit depends on it.
|
||||
|
||||
Generate a deterministic fixture:
|
||||
|
||||
ffmpeg -y -hide_banner -loglevel error -f lavfi \
|
||||
-i testsrc2=size=640x360:rate=30 -t 8 -c:v libvpx-vp9 \
|
||||
-b:v 2M -deadline realtime -cpu-used 8 -an \
|
||||
/tmp/oxicloud-thumbnail-perf.webm
|
||||
|
||||
Any future candidate using this gate must run in fresh browser contexts,
|
||||
alternate A/B order, and reject a supposedly no-download path if it emits any
|
||||
original-video GET or thumbnail PUT.
|
||||
@@ -0,0 +1,121 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
\pset format unaligned
|
||||
\pset tuples_only on
|
||||
|
||||
-- Compare the endpoint's narrow page + independent count with a tempting
|
||||
-- COUNT(*) OVER() fusion. The transaction/temp table leave no persistent
|
||||
-- database state. This benchmark exists to reject the fusion if the window
|
||||
-- forces PostgreSQL to materialise too much of a large directory.
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE perf_admin_count (
|
||||
id uuid NOT NULL,
|
||||
username text,
|
||||
email text NOT NULL,
|
||||
role_text text NOT NULL,
|
||||
storage_quota_bytes bigint NOT NULL,
|
||||
storage_used_bytes bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
last_login_at timestamptz,
|
||||
active boolean NOT NULL,
|
||||
oidc_provider text,
|
||||
is_external boolean NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO perf_admin_count
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'user-' || n,
|
||||
'user-' || n || '@example.invalid',
|
||||
CASE WHEN n % 20 = 0 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
n::bigint * 1048576,
|
||||
clock_timestamp() - make_interval(secs => n),
|
||||
clock_timestamp() - make_interval(mins => n),
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
n % 7 = 0
|
||||
FROM generate_series(1, 100000) AS n;
|
||||
|
||||
CREATE INDEX perf_admin_count_created_idx
|
||||
ON perf_admin_count (created_at DESC);
|
||||
ANALYZE perf_admin_count;
|
||||
|
||||
\o /dev/null
|
||||
\timing on
|
||||
|
||||
-- A sample consists of these two statements; add their reported times.
|
||||
\echo current_warmup_page
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_count
|
||||
ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
\echo current_warmup_count
|
||||
SELECT COUNT(*) FROM perf_admin_count;
|
||||
|
||||
\echo candidate_warmup
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external,
|
||||
COUNT(*) OVER () AS total
|
||||
FROM perf_admin_count
|
||||
ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo current_1_page
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_1_count
|
||||
SELECT COUNT(*) FROM perf_admin_count;
|
||||
\echo candidate_1
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external, COUNT(*) OVER () AS total
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
|
||||
\echo candidate_2
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external, COUNT(*) OVER () AS total
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_2_page
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_2_count
|
||||
SELECT COUNT(*) FROM perf_admin_count;
|
||||
|
||||
\echo current_3_page
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_3_count
|
||||
SELECT COUNT(*) FROM perf_admin_count;
|
||||
\echo candidate_3
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external, COUNT(*) OVER () AS total
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
|
||||
\echo candidate_4
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external, COUNT(*) OVER () AS total
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_4_page
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_4_count
|
||||
SELECT COUNT(*) FROM perf_admin_count;
|
||||
|
||||
\echo current_5_page
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
\echo current_5_count
|
||||
SELECT COUNT(*) FROM perf_admin_count;
|
||||
\echo candidate_5
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external, COUNT(*) OVER () AS total
|
||||
FROM perf_admin_count ORDER BY created_at DESC LIMIT 100;
|
||||
|
||||
\timing off
|
||||
\o
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,533 @@
|
||||
//! Component-faithful A/B for `GET /api/admin/users`.
|
||||
//!
|
||||
//! Historical path: full-row SQL -> full DTO -> count SQL -> direct Serde JSON.
|
||||
//! Candidate path: hot Moka `get_user_flags` equivalent -> policy check ->
|
||||
//! summary SQL -> summary DTO -> count SQL -> Serde JSON.
|
||||
//!
|
||||
//! The harness uses the exact production column sets and response fields but
|
||||
//! deliberately stays independent of the OxiCloud crate. That keeps it small
|
||||
//! enough for fresh-process max-RSS gates while disclosing that router/JWT and
|
||||
//! socket-level HTTP framing are common work and are not modeled. It also omits
|
||||
//! the old handler's intermediate `serde_json::Value` materialization, making
|
||||
//! the historical side optimistic and the accepted speedup conservative.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use moka::future::Cache;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use sqlx::{PgPool, Row};
|
||||
use std::env;
|
||||
use std::hint::black_box;
|
||||
use std::time::{Duration, Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
const USERS: i64 = 100;
|
||||
const LIMIT: i64 = 100;
|
||||
const OFFSET: i64 = 0;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Profile {
|
||||
Minimal,
|
||||
Heavy,
|
||||
}
|
||||
|
||||
impl Profile {
|
||||
fn parse(value: &str) -> Self {
|
||||
match value {
|
||||
"minimal" => Self::Minimal,
|
||||
"heavy" => Self::Heavy,
|
||||
_ => panic!("profile must be minimal or heavy"),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Minimal => "minimal",
|
||||
Self::Heavy => "heavy",
|
||||
}
|
||||
}
|
||||
|
||||
fn is_heavy(self) -> bool {
|
||||
matches!(self, Self::Heavy)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct UserFlags {
|
||||
admin: bool,
|
||||
is_external: bool,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct FullUserDto {
|
||||
id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
username: Option<String>,
|
||||
email: String,
|
||||
role: String,
|
||||
storage_quota_bytes: i64,
|
||||
storage_used_bytes: i64,
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
last_login_at: Option<DateTime<Utc>>,
|
||||
active: bool,
|
||||
auth_provider: String,
|
||||
image: Option<String>,
|
||||
can_edit_image: bool,
|
||||
is_external: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
given_name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
family_name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
email_verified_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
preferred_locale: Option<String>,
|
||||
notify_on_share: bool,
|
||||
ui_preferences: Value,
|
||||
}
|
||||
|
||||
impl FullUserDto {
|
||||
fn summary(&self) -> SummaryUserDto {
|
||||
SummaryUserDto {
|
||||
id: self.id.clone(),
|
||||
username: self.username.clone(),
|
||||
email: self.email.clone(),
|
||||
role: self.role.clone(),
|
||||
storage_quota_bytes: self.storage_quota_bytes,
|
||||
storage_used_bytes: self.storage_used_bytes,
|
||||
last_login_at: self.last_login_at,
|
||||
active: self.active,
|
||||
auth_provider: self.auth_provider.clone(),
|
||||
is_external: self.is_external,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Serialize)]
|
||||
struct SummaryUserDto {
|
||||
id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
username: Option<String>,
|
||||
email: String,
|
||||
role: String,
|
||||
storage_quota_bytes: i64,
|
||||
storage_used_bytes: i64,
|
||||
last_login_at: Option<DateTime<Utc>>,
|
||||
active: bool,
|
||||
auth_provider: String,
|
||||
is_external: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Page<T> {
|
||||
users: Vec<T>,
|
||||
total: i64,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct TimingReport {
|
||||
profile: &'static str,
|
||||
users: i64,
|
||||
warmups: usize,
|
||||
samples: usize,
|
||||
order: &'static str,
|
||||
historical_full_samples_ms: Vec<f64>,
|
||||
candidate_summary_hot_authz_samples_ms: Vec<f64>,
|
||||
historical_full_median_ms: f64,
|
||||
candidate_summary_hot_authz_median_ms: f64,
|
||||
speedup: f64,
|
||||
historical_json_bytes: usize,
|
||||
candidate_json_bytes: usize,
|
||||
byte_reduction_percent: f64,
|
||||
summary_projection_equal: bool,
|
||||
total_equal: bool,
|
||||
}
|
||||
|
||||
async fn setup(pool: &PgPool, profile: Profile) {
|
||||
sqlx::query(
|
||||
"CREATE TEMP TABLE perf_admin_endpoint_users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text,
|
||||
email text NOT NULL,
|
||||
password_hash text,
|
||||
role text NOT NULL,
|
||||
storage_quota_bytes bigint NOT NULL,
|
||||
storage_used_bytes bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
last_login_at timestamptz,
|
||||
active boolean NOT NULL,
|
||||
oidc_provider text,
|
||||
oidc_subject text,
|
||||
image text,
|
||||
is_external boolean NOT NULL,
|
||||
given_name text,
|
||||
family_name text,
|
||||
email_verified_at timestamptz,
|
||||
preferred_locale text,
|
||||
notify_on_share boolean NOT NULL,
|
||||
ui_preferences jsonb NOT NULL
|
||||
)",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("create fixture table");
|
||||
|
||||
sqlx::query(
|
||||
"WITH payload AS (
|
||||
SELECT string_agg(md5(i::text || ':admin-e2e'), '') AS random_hex
|
||||
FROM generate_series(1, 16384) AS i
|
||||
)
|
||||
INSERT INTO perf_admin_endpoint_users
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'perf-user-' || n,
|
||||
'perf-user-' || n || '@example.invalid',
|
||||
'$argon2id$v=19$m=19456,t=2,p=1$benchmark-only',
|
||||
CASE WHEN n = 1 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
n::bigint * 1048576,
|
||||
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second',
|
||||
timestamptz '2026-01-02 00:00:00+00' + n * interval '1 second',
|
||||
timestamptz '2026-01-03 00:00:00+00' + n * interval '1 second',
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
CASE WHEN n % 3 = 0 THEN 'subject-' || n END,
|
||||
CASE WHEN $1 THEN 'data:image/webp;base64,' || payload.random_hex END,
|
||||
false,
|
||||
CASE WHEN $1 THEN 'Given' || n END,
|
||||
CASE WHEN $1 THEN 'Family' || n END,
|
||||
CASE WHEN $1 THEN timestamptz '2026-01-04 00:00:00+00' END,
|
||||
CASE WHEN $1 THEN 'es' END,
|
||||
true,
|
||||
CASE WHEN $1
|
||||
THEN jsonb_build_object('perf_blob', left(payload.random_hex, 8192))
|
||||
ELSE '{}'::jsonb
|
||||
END
|
||||
FROM generate_series(1, $2::bigint) AS n
|
||||
CROSS JOIN payload",
|
||||
)
|
||||
.bind(profile.is_heavy())
|
||||
.bind(USERS)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed fixture users");
|
||||
|
||||
sqlx::query(
|
||||
"CREATE INDEX perf_admin_endpoint_created_at_idx
|
||||
ON perf_admin_endpoint_users (created_at DESC)",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("create listing index");
|
||||
sqlx::query("ANALYZE perf_admin_endpoint_users")
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("analyze fixture");
|
||||
}
|
||||
|
||||
async fn load_full(pool: &PgPool) -> Vec<FullUserDto> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT
|
||||
id, username, email, password_hash, role AS role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image,
|
||||
is_external, given_name, family_name, email_verified_at,
|
||||
preferred_locale, notify_on_share, ui_preferences
|
||||
FROM perf_admin_endpoint_users
|
||||
WHERE ($3 OR is_external = FALSE)
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT $1 OFFSET $2",
|
||||
)
|
||||
.bind(LIMIT)
|
||||
.bind(OFFSET)
|
||||
.bind(true)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.expect("fetch full users");
|
||||
|
||||
rows.into_iter()
|
||||
.map(|row| {
|
||||
// Decode the two fetched-but-not-serialized fields as production's
|
||||
// full User construction does; omitting them would flatter history.
|
||||
let _password_hash: Option<String> = row.get("password_hash");
|
||||
let _oidc_subject: Option<String> = row.get("oidc_subject");
|
||||
let oidc_provider: Option<String> = row.get("oidc_provider");
|
||||
let can_edit_image = oidc_provider.is_none();
|
||||
FullUserDto {
|
||||
id: row.get::<Uuid, _>("id").to_string(),
|
||||
username: row.get("username"),
|
||||
email: row.get("email"),
|
||||
role: row.get("role_text"),
|
||||
storage_quota_bytes: row.get("storage_quota_bytes"),
|
||||
storage_used_bytes: row.get("storage_used_bytes"),
|
||||
created_at: row.get("created_at"),
|
||||
updated_at: row.get("updated_at"),
|
||||
last_login_at: row.get("last_login_at"),
|
||||
active: row.get("active"),
|
||||
auth_provider: oidc_provider.unwrap_or_else(|| "local".to_owned()),
|
||||
image: row.get("image"),
|
||||
can_edit_image,
|
||||
is_external: row.get("is_external"),
|
||||
given_name: row.get("given_name"),
|
||||
family_name: row.get("family_name"),
|
||||
email_verified_at: row.get("email_verified_at"),
|
||||
preferred_locale: row.get("preferred_locale"),
|
||||
notify_on_share: row.get("notify_on_share"),
|
||||
ui_preferences: row.get("ui_preferences"),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn load_summary(pool: &PgPool) -> Vec<SummaryUserDto> {
|
||||
sqlx::query(
|
||||
"SELECT
|
||||
id, username, email, role AS role_text,
|
||||
storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_endpoint_users
|
||||
WHERE ($3 OR is_external = FALSE)
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT $1 OFFSET $2",
|
||||
)
|
||||
.bind(LIMIT)
|
||||
.bind(OFFSET)
|
||||
.bind(true)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.expect("fetch summary users")
|
||||
.into_iter()
|
||||
.map(|row| SummaryUserDto {
|
||||
id: row.get::<Uuid, _>("id").to_string(),
|
||||
username: row.get("username"),
|
||||
email: row.get("email"),
|
||||
role: row.get("role_text"),
|
||||
storage_quota_bytes: row.get("storage_quota_bytes"),
|
||||
storage_used_bytes: row.get("storage_used_bytes"),
|
||||
last_login_at: row.get("last_login_at"),
|
||||
active: row.get("active"),
|
||||
auth_provider: row
|
||||
.get::<Option<String>, _>("oidc_provider")
|
||||
.unwrap_or_else(|| "local".to_owned()),
|
||||
is_external: row.get("is_external"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn count_users(pool: &PgPool) -> i64 {
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM perf_admin_endpoint_users")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("count fixture users")
|
||||
}
|
||||
|
||||
async fn historical_response(pool: &PgPool) -> Vec<u8> {
|
||||
let users = load_full(pool).await;
|
||||
let total = count_users(pool).await;
|
||||
serde_json::to_vec(&Page {
|
||||
users,
|
||||
total,
|
||||
limit: LIMIT,
|
||||
offset: OFFSET,
|
||||
})
|
||||
.expect("serialize full response")
|
||||
}
|
||||
|
||||
async fn candidate_response(
|
||||
pool: &PgPool,
|
||||
flags_cache: &Cache<Uuid, UserFlags>,
|
||||
admin_id: Uuid,
|
||||
) -> Vec<u8> {
|
||||
let flags = flags_cache
|
||||
.try_get_with(admin_id, async {
|
||||
Err::<UserFlags, &'static str>("unexpected miss in hot-cache gate")
|
||||
})
|
||||
.await
|
||||
.expect("hot flags cache");
|
||||
assert!(flags.admin && !flags.is_external && flags.active);
|
||||
|
||||
let users = load_summary(pool).await;
|
||||
let total = count_users(pool).await;
|
||||
serde_json::to_vec(&Page {
|
||||
users,
|
||||
total,
|
||||
limit: LIMIT,
|
||||
offset: OFFSET,
|
||||
})
|
||||
.expect("serialize summary response")
|
||||
}
|
||||
|
||||
async fn correctness(pool: &PgPool) {
|
||||
let full = load_full(pool).await;
|
||||
let summary = load_summary(pool).await;
|
||||
let projected: Vec<SummaryUserDto> = full.iter().map(FullUserDto::summary).collect();
|
||||
assert_eq!(
|
||||
projected, summary,
|
||||
"summary projection changed table fields/order"
|
||||
);
|
||||
assert_eq!(count_users(pool).await, USERS);
|
||||
}
|
||||
|
||||
fn elapsed_ms(start: Instant) -> f64 {
|
||||
start.elapsed().as_secs_f64() * 1_000.0
|
||||
}
|
||||
|
||||
fn median(values: &[f64]) -> f64 {
|
||||
let mut sorted = values.to_vec();
|
||||
sorted.sort_by(f64::total_cmp);
|
||||
sorted[sorted.len() / 2]
|
||||
}
|
||||
|
||||
async fn run_timing(
|
||||
pool: &PgPool,
|
||||
profile: Profile,
|
||||
cache: &Cache<Uuid, UserFlags>,
|
||||
admin_id: Uuid,
|
||||
samples: usize,
|
||||
) {
|
||||
correctness(pool).await;
|
||||
|
||||
let warmups = 3;
|
||||
for warmup in 0..warmups {
|
||||
if warmup % 2 == 0 {
|
||||
black_box(historical_response(pool).await);
|
||||
black_box(candidate_response(pool, cache, admin_id).await);
|
||||
} else {
|
||||
black_box(candidate_response(pool, cache, admin_id).await);
|
||||
black_box(historical_response(pool).await);
|
||||
}
|
||||
}
|
||||
|
||||
let mut historical = Vec::with_capacity(samples);
|
||||
let mut candidate = Vec::with_capacity(samples);
|
||||
let mut historical_bytes = 0;
|
||||
let mut candidate_bytes = 0;
|
||||
for sample in 0..samples {
|
||||
if sample % 2 == 0 {
|
||||
let start = Instant::now();
|
||||
let body = historical_response(pool).await;
|
||||
historical.push(elapsed_ms(start));
|
||||
historical_bytes = body.len();
|
||||
black_box(body);
|
||||
|
||||
let start = Instant::now();
|
||||
let body = candidate_response(pool, cache, admin_id).await;
|
||||
candidate.push(elapsed_ms(start));
|
||||
candidate_bytes = body.len();
|
||||
black_box(body);
|
||||
} else {
|
||||
let start = Instant::now();
|
||||
let body = candidate_response(pool, cache, admin_id).await;
|
||||
candidate.push(elapsed_ms(start));
|
||||
candidate_bytes = body.len();
|
||||
black_box(body);
|
||||
|
||||
let start = Instant::now();
|
||||
let body = historical_response(pool).await;
|
||||
historical.push(elapsed_ms(start));
|
||||
historical_bytes = body.len();
|
||||
black_box(body);
|
||||
}
|
||||
}
|
||||
|
||||
let historical_median = median(&historical);
|
||||
let candidate_median = median(&candidate);
|
||||
let report = TimingReport {
|
||||
profile: profile.as_str(),
|
||||
users: USERS,
|
||||
warmups,
|
||||
samples,
|
||||
order: "interleaved and alternated",
|
||||
historical_full_samples_ms: historical,
|
||||
candidate_summary_hot_authz_samples_ms: candidate,
|
||||
historical_full_median_ms: historical_median,
|
||||
candidate_summary_hot_authz_median_ms: candidate_median,
|
||||
speedup: historical_median / candidate_median,
|
||||
historical_json_bytes: historical_bytes,
|
||||
candidate_json_bytes: candidate_bytes,
|
||||
byte_reduction_percent: (1.0 - candidate_bytes as f64 / historical_bytes as f64) * 100.0,
|
||||
summary_projection_equal: true,
|
||||
total_equal: true,
|
||||
};
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&report).expect("serialize timing report")
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_memory(
|
||||
pool: &PgPool,
|
||||
profile: Profile,
|
||||
mode: &str,
|
||||
cache: &Cache<Uuid, UserFlags>,
|
||||
admin_id: Uuid,
|
||||
) {
|
||||
let start = Instant::now();
|
||||
let body = match mode {
|
||||
"historical" => historical_response(pool).await,
|
||||
"candidate" => candidate_response(pool, cache, admin_id).await,
|
||||
_ => panic!("memory mode must be historical or candidate"),
|
||||
};
|
||||
let elapsed = elapsed_ms(start);
|
||||
black_box(&body);
|
||||
println!(
|
||||
"mode={mode} profile={} users={USERS} elapsed_ms={elapsed:.6} json_bytes={}",
|
||||
profile.as_str(),
|
||||
body.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn main() {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let command = args.get(1).map(String::as_str).unwrap_or("timing");
|
||||
let profile = Profile::parse(args.get(2).map(String::as_str).unwrap_or("minimal"));
|
||||
let samples = args
|
||||
.get(3)
|
||||
.map(|value| value.parse::<usize>().expect("samples must be an integer"))
|
||||
.unwrap_or(21);
|
||||
|
||||
let database_url = env::var("DATABASE_URL").expect("DATABASE_URL is required");
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.acquire_timeout(Duration::from_secs(10))
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect benchmark database");
|
||||
setup(&pool, profile).await;
|
||||
|
||||
let admin_id: Uuid =
|
||||
sqlx::query_scalar("SELECT id FROM perf_admin_endpoint_users WHERE role = 'admin' LIMIT 1")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("fixture admin id");
|
||||
let cache = Cache::builder()
|
||||
.max_capacity(10_000)
|
||||
.time_to_live(Duration::from_secs(30))
|
||||
.build();
|
||||
cache
|
||||
.insert(
|
||||
admin_id,
|
||||
UserFlags {
|
||||
admin: true,
|
||||
is_external: false,
|
||||
active: true,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
match command {
|
||||
"timing" => run_timing(&pool, profile, &cache, admin_id, samples).await,
|
||||
"memory-historical" => run_memory(&pool, profile, "historical", &cache, admin_id).await,
|
||||
"memory-candidate" => run_memory(&pool, profile, "candidate", &cache, admin_id).await,
|
||||
_ => panic!(
|
||||
"usage: admin_user_listing_e2e [timing|memory-historical|memory-candidate] [minimal|heavy] [samples]"
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
\pset format unaligned
|
||||
\pset tuples_only on
|
||||
|
||||
-- SUPERSEDED EXPLORATORY FIXTURE: every generated row shares one timestamp,
|
||||
-- so PostgreSQL posting-list compression understates representative btree
|
||||
-- storage cost. Keep this file only as raw historical evidence; do not use it
|
||||
-- to accept or reject an index. Use admin_user_order_index_representative.sql.
|
||||
|
||||
-- A/B the missing ORDER BY index used by GET /api/admin/users. Two temporary
|
||||
-- tables keep both variants resident and let samples alternate without DDL
|
||||
-- contaminating timings. No persistent state survives the transaction.
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE perf_users_no_index (
|
||||
id uuid NOT NULL,
|
||||
username text,
|
||||
email text NOT NULL,
|
||||
role_text text NOT NULL,
|
||||
storage_quota_bytes bigint NOT NULL,
|
||||
storage_used_bytes bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
last_login_at timestamptz,
|
||||
active boolean NOT NULL,
|
||||
oidc_provider text,
|
||||
is_external boolean NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO perf_users_no_index
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'user-' || n,
|
||||
'user-' || n || '@example.invalid',
|
||||
CASE WHEN n % 20 = 0 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
n::bigint * 1048576,
|
||||
-- One hundred accounts intentionally share each timestamp. The real
|
||||
-- default is statement-stable CURRENT_TIMESTAMP, so bulk/JIT creation can
|
||||
-- produce ties; `id` must make page boundaries deterministic.
|
||||
timestamptz '2026-01-01 00:00:00+00' + make_interval(secs => n / 100),
|
||||
timestamptz '2026-01-01 00:00:00+00' + make_interval(secs => n / 2),
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
n % 7 = 0
|
||||
FROM generate_series(1, 500000) AS n;
|
||||
|
||||
CREATE TEMP TABLE perf_users_indexed
|
||||
(LIKE perf_users_no_index INCLUDING ALL);
|
||||
INSERT INTO perf_users_indexed SELECT * FROM perf_users_no_index;
|
||||
CREATE INDEX perf_users_indexed_created_at_id
|
||||
ON perf_users_indexed (created_at DESC, id DESC);
|
||||
|
||||
ANALYZE perf_users_no_index;
|
||||
ANALYZE perf_users_indexed;
|
||||
|
||||
SELECT 'created_at_id_index_bytes|' || pg_relation_size('perf_users_indexed_created_at_id');
|
||||
SELECT 'tied_timestamp_groups|' || COUNT(*)
|
||||
FROM (
|
||||
SELECT created_at FROM perf_users_no_index GROUP BY created_at HAVING COUNT(*) > 1
|
||||
) tied;
|
||||
SELECT 'stable_order_match|' || (
|
||||
ARRAY(
|
||||
SELECT id FROM perf_users_no_index
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900
|
||||
) = ARRAY(
|
||||
SELECT id FROM perf_users_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900
|
||||
)
|
||||
);
|
||||
SELECT 'adjacent_page_overlap|' || COUNT(*)
|
||||
FROM (
|
||||
SELECT id FROM perf_users_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0
|
||||
) first_page
|
||||
JOIN (
|
||||
SELECT id FROM perf_users_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 100
|
||||
) second_page USING (id);
|
||||
|
||||
\o /dev/null
|
||||
\timing on
|
||||
|
||||
-- Warm both table variants and both page depths.
|
||||
\echo no_index_warmup_first
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_warmup_first
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo no_index_warmup_deep
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo indexed_warmup_deep
|
||||
SELECT id, username, email, role_text, storage_quota_bytes, storage_used_bytes,
|
||||
last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo no_index_first_1
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_first_1
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_deep_1
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo no_index_deep_1
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo indexed_first_2
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo no_index_first_2
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo no_index_deep_2
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo indexed_deep_2
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo no_index_first_3
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_first_3
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_deep_3
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo no_index_deep_3
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo indexed_first_4
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo no_index_first_4
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo no_index_deep_4
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo indexed_deep_4
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo no_index_first_5
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_first_5
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 0;
|
||||
\echo indexed_deep_5
|
||||
SELECT * FROM perf_users_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo no_index_deep_5
|
||||
SELECT * FROM perf_users_no_index ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
-- Write-cost gate: an ordering index is not free. Copy the same 10k-row shape
|
||||
-- into each variant and report the insertion tax alongside the read win.
|
||||
\echo no_index_insert_1
|
||||
INSERT INTO perf_users_no_index SELECT * FROM perf_users_no_index LIMIT 10000;
|
||||
\echo indexed_insert_1
|
||||
INSERT INTO perf_users_indexed SELECT * FROM perf_users_indexed LIMIT 10000;
|
||||
\echo indexed_insert_2
|
||||
INSERT INTO perf_users_indexed SELECT * FROM perf_users_indexed LIMIT 10000;
|
||||
\echo no_index_insert_2
|
||||
INSERT INTO perf_users_no_index SELECT * FROM perf_users_no_index LIMIT 10000;
|
||||
\echo no_index_insert_3
|
||||
INSERT INTO perf_users_no_index SELECT * FROM perf_users_no_index LIMIT 10000;
|
||||
\echo indexed_insert_3
|
||||
INSERT INTO perf_users_indexed SELECT * FROM perf_users_indexed LIMIT 10000;
|
||||
\echo indexed_insert_4
|
||||
INSERT INTO perf_users_indexed SELECT * FROM perf_users_indexed LIMIT 10000;
|
||||
\echo no_index_insert_4
|
||||
INSERT INTO perf_users_no_index SELECT * FROM perf_users_no_index LIMIT 10000;
|
||||
\echo no_index_insert_5
|
||||
INSERT INTO perf_users_no_index SELECT * FROM perf_users_no_index LIMIT 10000;
|
||||
\echo indexed_insert_5
|
||||
INSERT INTO perf_users_indexed SELECT * FROM perf_users_indexed LIMIT 10000;
|
||||
|
||||
\timing off
|
||||
\o
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,184 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
\pset format unaligned
|
||||
\pset tuples_only on
|
||||
|
||||
-- SUPERSEDED EXPLORATORY FIXTURE: the tied-timestamp distribution is useful as
|
||||
-- a stress shape, but PostgreSQL posting-list compression makes its index-size
|
||||
-- result non-representative. Use admin_user_order_index_representative.sql for
|
||||
-- decisions; this file is retained only as historical evidence.
|
||||
|
||||
-- Three-way Pareto gate for the stable admin pagination order:
|
||||
-- A. no index;
|
||||
-- B. created_at only (smaller btree + incremental sort inside ties);
|
||||
-- C. created_at,id (fully ordered scan).
|
||||
-- One hundred rows share each timestamp to reproduce CURRENT_TIMESTAMP ties.
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE perf_users_base (
|
||||
id uuid NOT NULL,
|
||||
username text,
|
||||
email text NOT NULL,
|
||||
role_text text NOT NULL,
|
||||
storage_quota_bytes bigint NOT NULL,
|
||||
storage_used_bytes bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
last_login_at timestamptz,
|
||||
active boolean NOT NULL,
|
||||
oidc_provider text,
|
||||
is_external boolean NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO perf_users_base
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'user-' || n,
|
||||
'user-' || n || '@example.invalid',
|
||||
CASE WHEN n % 20 = 0 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
n::bigint * 1048576,
|
||||
timestamptz '2026-01-01 00:00:00+00' + make_interval(secs => n / 100),
|
||||
timestamptz '2026-01-01 00:00:00+00' + make_interval(secs => n / 2),
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
n % 7 = 0
|
||||
FROM generate_series(1, 500000) AS n;
|
||||
|
||||
CREATE TEMP TABLE perf_users_timestamp (LIKE perf_users_base INCLUDING ALL);
|
||||
CREATE TEMP TABLE perf_users_compound (LIKE perf_users_base INCLUDING ALL);
|
||||
INSERT INTO perf_users_timestamp SELECT * FROM perf_users_base;
|
||||
INSERT INTO perf_users_compound SELECT * FROM perf_users_base;
|
||||
CREATE INDEX perf_users_timestamp_idx ON perf_users_timestamp (created_at DESC);
|
||||
CREATE INDEX perf_users_compound_idx ON perf_users_compound (created_at DESC, id DESC);
|
||||
ANALYZE perf_users_base;
|
||||
ANALYZE perf_users_timestamp;
|
||||
ANALYZE perf_users_compound;
|
||||
|
||||
SELECT 'timestamp_index_bytes|' || pg_relation_size('perf_users_timestamp_idx');
|
||||
SELECT 'compound_index_bytes|' || pg_relation_size('perf_users_compound_idx');
|
||||
SELECT 'tied_timestamp_groups|' || COUNT(*)
|
||||
FROM (SELECT created_at FROM perf_users_base GROUP BY created_at HAVING COUNT(*) > 1) tied;
|
||||
SELECT 'all_orders_match|' || (
|
||||
ARRAY(SELECT id FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
AND ARRAY(SELECT id FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
);
|
||||
|
||||
\o /dev/null
|
||||
\timing on
|
||||
|
||||
\echo warmup_no_index_first
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo warmup_timestamp_first
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo warmup_compound_first
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo warmup_no_index_deep
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo warmup_timestamp_deep
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo warmup_compound_deep
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
-- Rotate execution order between samples.
|
||||
\echo no_index_first_1
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo timestamp_first_1
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo compound_first_1
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo timestamp_deep_1
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo compound_deep_1
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo no_index_deep_1
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo compound_first_2
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo no_index_first_2
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo timestamp_first_2
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo no_index_deep_2
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo timestamp_deep_2
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo compound_deep_2
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo timestamp_first_3
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo compound_first_3
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo no_index_first_3
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo compound_deep_3
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo no_index_deep_3
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo timestamp_deep_3
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo no_index_first_4
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo compound_first_4
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo timestamp_first_4
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo timestamp_deep_4
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo no_index_deep_4
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo compound_deep_4
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo compound_first_5
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo timestamp_first_5
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo no_index_first_5
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo no_index_deep_5
|
||||
SELECT * FROM perf_users_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo compound_deep_5
|
||||
SELECT * FROM perf_users_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo timestamp_deep_5
|
||||
SELECT * FROM perf_users_timestamp ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
-- Quantify both index write taxes over identical 10k-row inserts.
|
||||
\echo no_index_insert_1
|
||||
INSERT INTO perf_users_base SELECT * FROM perf_users_base LIMIT 10000;
|
||||
\echo timestamp_insert_1
|
||||
INSERT INTO perf_users_timestamp SELECT * FROM perf_users_timestamp LIMIT 10000;
|
||||
\echo compound_insert_1
|
||||
INSERT INTO perf_users_compound SELECT * FROM perf_users_compound LIMIT 10000;
|
||||
\echo compound_insert_2
|
||||
INSERT INTO perf_users_compound SELECT * FROM perf_users_compound LIMIT 10000;
|
||||
\echo no_index_insert_2
|
||||
INSERT INTO perf_users_base SELECT * FROM perf_users_base LIMIT 10000;
|
||||
\echo timestamp_insert_2
|
||||
INSERT INTO perf_users_timestamp SELECT * FROM perf_users_timestamp LIMIT 10000;
|
||||
\echo timestamp_insert_3
|
||||
INSERT INTO perf_users_timestamp SELECT * FROM perf_users_timestamp LIMIT 10000;
|
||||
\echo compound_insert_3
|
||||
INSERT INTO perf_users_compound SELECT * FROM perf_users_compound LIMIT 10000;
|
||||
\echo no_index_insert_3
|
||||
INSERT INTO perf_users_base SELECT * FROM perf_users_base LIMIT 10000;
|
||||
\echo no_index_insert_4
|
||||
INSERT INTO perf_users_base SELECT * FROM perf_users_base LIMIT 10000;
|
||||
\echo timestamp_insert_4
|
||||
INSERT INTO perf_users_timestamp SELECT * FROM perf_users_timestamp LIMIT 10000;
|
||||
\echo compound_insert_4
|
||||
INSERT INTO perf_users_compound SELECT * FROM perf_users_compound LIMIT 10000;
|
||||
\echo compound_insert_5
|
||||
INSERT INTO perf_users_compound SELECT * FROM perf_users_compound LIMIT 10000;
|
||||
\echo timestamp_insert_5
|
||||
INSERT INTO perf_users_timestamp SELECT * FROM perf_users_timestamp LIMIT 10000;
|
||||
\echo no_index_insert_5
|
||||
INSERT INTO perf_users_base SELECT * FROM perf_users_base LIMIT 10000;
|
||||
|
||||
\timing off
|
||||
\o
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,519 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
\pset format unaligned
|
||||
\pset tuples_only on
|
||||
|
||||
-- Representative A/B/C gate for admin-list indexes:
|
||||
-- A. no ordering index;
|
||||
-- B. created_at DESC (the accepted narrow production variant);
|
||||
-- C. created_at DESC, id DESC (the rejected compound candidate).
|
||||
--
|
||||
-- The original Pareto fixture intentionally put 100 users under every
|
||||
-- timestamp to stress the incremental id sort. PostgreSQL can compress those
|
||||
-- duplicate B-tree keys into posting lists, however, so that fixture may
|
||||
-- materially understate index bytes and insert cost for normal registrations.
|
||||
-- This gate keeps the same primary-key index on every A/B/C table and covers:
|
||||
-- * unique timestamps (one normal registration per transaction), and
|
||||
-- * ten-user bursts (small provisioning/import transactions).
|
||||
-- New insert batches use new timestamps instead of duplicating old keys.
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE perf_unique_base (
|
||||
id uuid PRIMARY KEY,
|
||||
username text,
|
||||
email text NOT NULL,
|
||||
role_text text NOT NULL,
|
||||
storage_quota_bytes bigint NOT NULL,
|
||||
storage_used_bytes bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
last_login_at timestamptz,
|
||||
active boolean NOT NULL,
|
||||
oidc_provider text,
|
||||
is_external boolean NOT NULL
|
||||
);
|
||||
CREATE TEMP TABLE perf_unique_indexed (LIKE perf_unique_base INCLUDING ALL);
|
||||
CREATE TEMP TABLE perf_unique_compound (LIKE perf_unique_base INCLUDING ALL);
|
||||
|
||||
INSERT INTO perf_unique_base
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'unique-user-' || n,
|
||||
'unique-user-' || n || '@example.invalid',
|
||||
CASE WHEN n % 20 = 0 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
n::bigint * 1048576,
|
||||
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 microsecond',
|
||||
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second',
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
n % 7 = 0
|
||||
FROM generate_series(1, 500000) AS n;
|
||||
INSERT INTO perf_unique_indexed SELECT * FROM perf_unique_base;
|
||||
INSERT INTO perf_unique_compound SELECT * FROM perf_unique_base;
|
||||
CREATE INDEX perf_unique_created_at_idx
|
||||
ON perf_unique_indexed (created_at DESC);
|
||||
CREATE INDEX perf_unique_created_at_id_idx
|
||||
ON perf_unique_compound (created_at DESC, id DESC);
|
||||
|
||||
CREATE TEMP TABLE perf_burst_base (LIKE perf_unique_base INCLUDING ALL);
|
||||
CREATE TEMP TABLE perf_burst_indexed (LIKE perf_unique_base INCLUDING ALL);
|
||||
CREATE TEMP TABLE perf_burst_compound (LIKE perf_unique_base INCLUDING ALL);
|
||||
INSERT INTO perf_burst_base
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'burst-user-' || n,
|
||||
'burst-user-' || n || '@example.invalid',
|
||||
CASE WHEN n % 20 = 0 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
n::bigint * 1048576,
|
||||
timestamptz '2026-01-01 00:00:00+00'
|
||||
+ ((n - 1) / 10) * interval '1 millisecond',
|
||||
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second',
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
n % 7 = 0
|
||||
FROM generate_series(1, 500000) AS n;
|
||||
INSERT INTO perf_burst_indexed SELECT * FROM perf_burst_base;
|
||||
INSERT INTO perf_burst_compound SELECT * FROM perf_burst_base;
|
||||
CREATE INDEX perf_burst_created_at_idx
|
||||
ON perf_burst_indexed (created_at DESC);
|
||||
CREATE INDEX perf_burst_created_at_id_idx
|
||||
ON perf_burst_compound (created_at DESC, id DESC);
|
||||
|
||||
-- Pre-build deterministic new-row batches. All A/B/C tables receive identical
|
||||
-- values; the indexed sample column keeps batch-selection work bounded/common.
|
||||
CREATE TEMP TABLE perf_unique_insert_rows AS
|
||||
SELECT
|
||||
sample,
|
||||
md5('perf-unique-' || sample || '-' || n)::uuid AS id,
|
||||
'new-unique-' || sample || '-' || n AS username,
|
||||
'new-unique-' || sample || '-' || n || '@example.invalid' AS email,
|
||||
'user'::text AS role_text,
|
||||
10737418240::bigint AS storage_quota_bytes,
|
||||
n::bigint * 1048576 AS storage_used_bytes,
|
||||
timestamptz '2027-01-01 00:00:00+00'
|
||||
+ ((sample - 1) * 10000 + n) * interval '1 microsecond' AS created_at,
|
||||
NULL::timestamptz AS last_login_at,
|
||||
true AS active,
|
||||
NULL::text AS oidc_provider,
|
||||
false AS is_external
|
||||
FROM generate_series(1, 5) AS sample
|
||||
CROSS JOIN generate_series(1, 10000) AS n;
|
||||
CREATE INDEX perf_unique_insert_sample_idx ON perf_unique_insert_rows (sample);
|
||||
|
||||
CREATE TEMP TABLE perf_burst_insert_rows AS
|
||||
SELECT
|
||||
sample,
|
||||
md5('perf-burst-' || sample || '-' || n)::uuid AS id,
|
||||
'new-burst-' || sample || '-' || n AS username,
|
||||
'new-burst-' || sample || '-' || n || '@example.invalid' AS email,
|
||||
'user'::text AS role_text,
|
||||
10737418240::bigint AS storage_quota_bytes,
|
||||
n::bigint * 1048576 AS storage_used_bytes,
|
||||
timestamptz '2027-01-01 00:00:00+00'
|
||||
+ (((sample - 1) * 10000 + n - 1) / 10) * interval '1 millisecond'
|
||||
AS created_at,
|
||||
NULL::timestamptz AS last_login_at,
|
||||
true AS active,
|
||||
NULL::text AS oidc_provider,
|
||||
false AS is_external
|
||||
FROM generate_series(1, 5) AS sample
|
||||
CROSS JOIN generate_series(1, 10000) AS n;
|
||||
CREATE INDEX perf_burst_insert_sample_idx ON perf_burst_insert_rows (sample);
|
||||
|
||||
ANALYZE perf_unique_base;
|
||||
ANALYZE perf_unique_indexed;
|
||||
ANALYZE perf_unique_compound;
|
||||
ANALYZE perf_burst_base;
|
||||
ANALYZE perf_burst_indexed;
|
||||
ANALYZE perf_burst_compound;
|
||||
ANALYZE perf_unique_insert_rows;
|
||||
ANALYZE perf_burst_insert_rows;
|
||||
|
||||
SELECT 'unique_index_bytes|' || pg_relation_size('perf_unique_created_at_idx');
|
||||
SELECT 'unique_compound_index_bytes|'
|
||||
|| pg_relation_size('perf_unique_created_at_id_idx');
|
||||
SELECT 'burst10_index_bytes|' || pg_relation_size('perf_burst_created_at_idx');
|
||||
SELECT 'burst10_compound_index_bytes|'
|
||||
|| pg_relation_size('perf_burst_created_at_id_idx');
|
||||
SELECT 'unique_distinct_timestamps|' || COUNT(DISTINCT created_at)
|
||||
FROM perf_unique_base;
|
||||
SELECT 'burst10_distinct_timestamps|' || COUNT(DISTINCT created_at)
|
||||
FROM perf_burst_base;
|
||||
SELECT 'unique_order_match|' || (
|
||||
ARRAY(SELECT id FROM perf_unique_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_unique_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
AND ARRAY(SELECT id FROM perf_unique_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_unique_compound
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
);
|
||||
SELECT 'burst10_order_match|' || (
|
||||
ARRAY(SELECT id FROM perf_burst_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_burst_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
AND ARRAY(SELECT id FROM perf_burst_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_burst_compound
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
);
|
||||
|
||||
\o /dev/null
|
||||
\timing on
|
||||
|
||||
-- Warmups.
|
||||
\echo unique_no_index_first_warmup
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_first_warmup
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_compound_first_warmup
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_no_index_deep_warmup
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_index_deep_warmup
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_compound_deep_warmup
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_no_index_first_warmup
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_first_warmup
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_compound_first_warmup
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_no_index_deep_warmup
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_index_deep_warmup
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_deep_warmup
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
-- Five samples per read shape, with A/B/C order rotated.
|
||||
\echo unique_no_index_first_1
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_first_1
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_compound_first_1
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_deep_1
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_compound_deep_1
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_no_index_deep_1
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_no_index_first_1
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_first_1
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_compound_first_1
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_deep_1
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_deep_1
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_no_index_deep_1
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo unique_compound_first_2
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_first_2
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_no_index_first_2
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_no_index_deep_2
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_index_deep_2
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_compound_deep_2
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_first_2
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_first_2
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_no_index_first_2
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_no_index_deep_2
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_index_deep_2
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_deep_2
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo unique_compound_first_3
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_no_index_first_3
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_first_3
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_no_index_deep_3
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_index_deep_3
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_compound_deep_3
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_first_3
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_no_index_first_3
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_first_3
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_no_index_deep_3
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_index_deep_3
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_deep_3
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo unique_index_first_4
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_compound_first_4
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_no_index_first_4
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_deep_4
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_compound_deep_4
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_no_index_deep_4
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_index_first_4
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_compound_first_4
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_no_index_first_4
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_deep_4
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_deep_4
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_no_index_deep_4
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
\echo unique_no_index_first_5
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_compound_first_5
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_first_5
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo unique_index_deep_5
|
||||
SELECT * FROM perf_unique_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_compound_deep_5
|
||||
SELECT * FROM perf_unique_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo unique_no_index_deep_5
|
||||
SELECT * FROM perf_unique_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_no_index_first_5
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_compound_first_5
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_first_5
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100;
|
||||
\echo burst10_index_deep_5
|
||||
SELECT * FROM perf_burst_indexed ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_compound_deep_5
|
||||
SELECT * FROM perf_burst_compound ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
\echo burst10_no_index_deep_5
|
||||
SELECT * FROM perf_burst_base ORDER BY created_at DESC, id DESC LIMIT 100 OFFSET 50000;
|
||||
|
||||
-- Five 10k-row inserts per distribution. Rotate A/B/C order.
|
||||
\echo unique_no_index_insert_1
|
||||
INSERT INTO perf_unique_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 1;
|
||||
\echo unique_index_insert_1
|
||||
INSERT INTO perf_unique_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 1;
|
||||
\echo unique_compound_insert_1
|
||||
INSERT INTO perf_unique_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 1;
|
||||
\echo burst10_no_index_insert_1
|
||||
INSERT INTO perf_burst_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 1;
|
||||
\echo burst10_index_insert_1
|
||||
INSERT INTO perf_burst_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 1;
|
||||
\echo burst10_compound_insert_1
|
||||
INSERT INTO perf_burst_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 1;
|
||||
|
||||
\echo unique_compound_insert_2
|
||||
INSERT INTO perf_unique_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 2;
|
||||
\echo unique_index_insert_2
|
||||
INSERT INTO perf_unique_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 2;
|
||||
\echo unique_no_index_insert_2
|
||||
INSERT INTO perf_unique_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 2;
|
||||
\echo burst10_compound_insert_2
|
||||
INSERT INTO perf_burst_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 2;
|
||||
\echo burst10_index_insert_2
|
||||
INSERT INTO perf_burst_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 2;
|
||||
\echo burst10_no_index_insert_2
|
||||
INSERT INTO perf_burst_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 2;
|
||||
|
||||
\echo unique_compound_insert_3
|
||||
INSERT INTO perf_unique_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 3;
|
||||
\echo unique_no_index_insert_3
|
||||
INSERT INTO perf_unique_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 3;
|
||||
\echo unique_index_insert_3
|
||||
INSERT INTO perf_unique_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 3;
|
||||
\echo burst10_compound_insert_3
|
||||
INSERT INTO perf_burst_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 3;
|
||||
\echo burst10_no_index_insert_3
|
||||
INSERT INTO perf_burst_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 3;
|
||||
\echo burst10_index_insert_3
|
||||
INSERT INTO perf_burst_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 3;
|
||||
|
||||
\echo unique_index_insert_4
|
||||
INSERT INTO perf_unique_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 4;
|
||||
\echo unique_compound_insert_4
|
||||
INSERT INTO perf_unique_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 4;
|
||||
\echo unique_no_index_insert_4
|
||||
INSERT INTO perf_unique_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 4;
|
||||
\echo burst10_index_insert_4
|
||||
INSERT INTO perf_burst_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 4;
|
||||
\echo burst10_compound_insert_4
|
||||
INSERT INTO perf_burst_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 4;
|
||||
\echo burst10_no_index_insert_4
|
||||
INSERT INTO perf_burst_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 4;
|
||||
|
||||
\echo unique_no_index_insert_5
|
||||
INSERT INTO perf_unique_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 5;
|
||||
\echo unique_compound_insert_5
|
||||
INSERT INTO perf_unique_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 5;
|
||||
\echo unique_index_insert_5
|
||||
INSERT INTO perf_unique_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_unique_insert_rows WHERE sample = 5;
|
||||
\echo burst10_no_index_insert_5
|
||||
INSERT INTO perf_burst_base SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 5;
|
||||
\echo burst10_compound_insert_5
|
||||
INSERT INTO perf_burst_compound SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 5;
|
||||
\echo burst10_index_insert_5
|
||||
INSERT INTO perf_burst_indexed SELECT id, username, email, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, last_login_at,
|
||||
active, oidc_provider, is_external
|
||||
FROM perf_burst_insert_rows WHERE sample = 5;
|
||||
|
||||
\timing off
|
||||
\o
|
||||
|
||||
SELECT 'unique_final_count_match|' || (
|
||||
(SELECT COUNT(*) FROM perf_unique_base)
|
||||
= (SELECT COUNT(*) FROM perf_unique_indexed)
|
||||
AND (SELECT COUNT(*) FROM perf_unique_base)
|
||||
= (SELECT COUNT(*) FROM perf_unique_compound)
|
||||
);
|
||||
SELECT 'burst10_final_count_match|' || (
|
||||
(SELECT COUNT(*) FROM perf_burst_base)
|
||||
= (SELECT COUNT(*) FROM perf_burst_indexed)
|
||||
AND (SELECT COUNT(*) FROM perf_burst_base)
|
||||
= (SELECT COUNT(*) FROM perf_burst_compound)
|
||||
);
|
||||
SELECT 'unique_final_order_match|' || (
|
||||
ARRAY(SELECT id FROM perf_unique_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_unique_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
AND ARRAY(SELECT id FROM perf_unique_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_unique_compound
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
);
|
||||
SELECT 'burst10_final_order_match|' || (
|
||||
ARRAY(SELECT id FROM perf_burst_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_burst_indexed
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
AND ARRAY(SELECT id FROM perf_burst_base
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
= ARRAY(SELECT id FROM perf_burst_compound
|
||||
ORDER BY created_at DESC, id DESC LIMIT 200 OFFSET 49900)
|
||||
);
|
||||
SELECT 'unique_index_bytes_after_50k_inserts|'
|
||||
|| pg_relation_size('perf_unique_created_at_idx');
|
||||
SELECT 'unique_compound_index_bytes_after_50k_inserts|'
|
||||
|| pg_relation_size('perf_unique_created_at_id_idx');
|
||||
SELECT 'burst10_index_bytes_after_50k_inserts|'
|
||||
|| pg_relation_size('perf_burst_created_at_idx');
|
||||
SELECT 'burst10_compound_index_bytes_after_50k_inserts|'
|
||||
|| pg_relation_size('perf_burst_created_at_id_idx');
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,189 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
\pset format unaligned
|
||||
\pset tuples_only on
|
||||
|
||||
-- Isolated reproduction of auth.users' payload shape. The transaction and
|
||||
-- temporary table guarantee that the developer database is unchanged.
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE perf_admin_users (
|
||||
id uuid NOT NULL,
|
||||
username text,
|
||||
email text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
role_text text NOT NULL,
|
||||
storage_quota_bytes bigint NOT NULL,
|
||||
storage_used_bytes bigint NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
last_login_at timestamptz,
|
||||
active boolean NOT NULL,
|
||||
oidc_provider text,
|
||||
oidc_subject text,
|
||||
image text,
|
||||
is_external boolean NOT NULL,
|
||||
given_name text,
|
||||
family_name text,
|
||||
email_verified_at timestamptz,
|
||||
preferred_locale text,
|
||||
notify_on_share boolean NOT NULL,
|
||||
ui_preferences jsonb NOT NULL
|
||||
);
|
||||
|
||||
-- One incompressible-ish 512 KiB base64/data-URI-shaped avatar and an 8 KiB
|
||||
-- JSON preference bag per row. This is the documented maximum avatar size and
|
||||
-- intentionally models the expensive end of the admin endpoint.
|
||||
WITH payload AS (
|
||||
SELECT string_agg(md5(i::text || ':oxicloud-perf'), '') AS random_hex
|
||||
FROM generate_series(1, 16384) AS i
|
||||
)
|
||||
INSERT INTO perf_admin_users
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
'perf-user-' || n,
|
||||
'perf-user-' || n || '@example.invalid',
|
||||
'$argon2id$v=19$m=19456,t=2,p=1$benchmark-only',
|
||||
CASE WHEN n % 20 = 0 THEN 'admin' ELSE 'user' END,
|
||||
10737418240,
|
||||
(n * 1048576)::bigint,
|
||||
clock_timestamp() - make_interval(secs => n),
|
||||
clock_timestamp(),
|
||||
clock_timestamp() - make_interval(mins => n),
|
||||
true,
|
||||
CASE WHEN n % 3 = 0 THEN 'keycloak' END,
|
||||
CASE WHEN n % 3 = 0 THEN 'subject-' || n END,
|
||||
'data:image/webp;base64,' || payload.random_hex,
|
||||
false,
|
||||
'Given' || n,
|
||||
'Family' || n,
|
||||
clock_timestamp(),
|
||||
'es',
|
||||
true,
|
||||
jsonb_build_object('perf_blob', left(payload.random_hex, 8192))
|
||||
FROM generate_series(1, 100) AS n
|
||||
CROSS JOIN payload;
|
||||
|
||||
ANALYZE perf_admin_users;
|
||||
|
||||
-- Bytes serialized by the current endpoint versus the proposed summary DTO.
|
||||
-- These include exactly the JSON fields each HTTP response shape emits.
|
||||
SELECT 'current_json_bytes|' || sum(octet_length(jsonb_build_object(
|
||||
'id', id::text,
|
||||
'username', username,
|
||||
'email', email,
|
||||
'role', role_text,
|
||||
'storage_quota_bytes', storage_quota_bytes,
|
||||
'storage_used_bytes', storage_used_bytes,
|
||||
'created_at', created_at,
|
||||
'updated_at', updated_at,
|
||||
'last_login_at', last_login_at,
|
||||
'active', active,
|
||||
'auth_provider', coalesce(oidc_provider, 'local'),
|
||||
'image', image,
|
||||
'can_edit_image', oidc_provider IS NULL,
|
||||
'is_external', is_external,
|
||||
'given_name', given_name,
|
||||
'family_name', family_name,
|
||||
'email_verified_at', email_verified_at,
|
||||
'preferred_locale', preferred_locale,
|
||||
'notify_on_share', notify_on_share,
|
||||
'ui_preferences', ui_preferences
|
||||
)::text))
|
||||
FROM perf_admin_users;
|
||||
|
||||
SELECT 'summary_json_bytes|' || sum(octet_length(jsonb_build_object(
|
||||
'id', id::text,
|
||||
'username', username,
|
||||
'email', email,
|
||||
'role', role_text,
|
||||
'storage_quota_bytes', storage_quota_bytes,
|
||||
'storage_used_bytes', storage_used_bytes,
|
||||
'last_login_at', last_login_at,
|
||||
'active', active,
|
||||
'auth_provider', coalesce(oidc_provider, 'local'),
|
||||
'is_external', is_external
|
||||
)::text))
|
||||
FROM perf_admin_users;
|
||||
|
||||
-- psql's timer covers server execution, transfer and client decoding. Query
|
||||
-- output goes to /dev/null so terminal rendering does not dominate the result.
|
||||
\o /dev/null
|
||||
\timing on
|
||||
|
||||
\echo current_warmup
|
||||
SELECT id, username, email, password_hash, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image, is_external,
|
||||
given_name, family_name, email_verified_at, preferred_locale,
|
||||
notify_on_share, ui_preferences
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo summary_warmup
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo current_sample_1
|
||||
SELECT id, username, email, password_hash, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image, is_external,
|
||||
given_name, family_name, email_verified_at, preferred_locale,
|
||||
notify_on_share, ui_preferences
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
\echo summary_sample_1
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo current_sample_2
|
||||
SELECT id, username, email, password_hash, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image, is_external,
|
||||
given_name, family_name, email_verified_at, preferred_locale,
|
||||
notify_on_share, ui_preferences
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
\echo summary_sample_2
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo current_sample_3
|
||||
SELECT id, username, email, password_hash, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image, is_external,
|
||||
given_name, family_name, email_verified_at, preferred_locale,
|
||||
notify_on_share, ui_preferences
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
\echo summary_sample_3
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo current_sample_4
|
||||
SELECT id, username, email, password_hash, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image, is_external,
|
||||
given_name, family_name, email_verified_at, preferred_locale,
|
||||
notify_on_share, ui_preferences
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
\echo summary_sample_4
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\echo current_sample_5
|
||||
SELECT id, username, email, password_hash, role_text,
|
||||
storage_quota_bytes, storage_used_bytes, created_at, updated_at,
|
||||
last_login_at, active, oidc_provider, oidc_subject, image, is_external,
|
||||
given_name, family_name, email_verified_at, preferred_locale,
|
||||
notify_on_share, ui_preferences
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
\echo summary_sample_5
|
||||
SELECT id, username, email, role_text, storage_quota_bytes,
|
||||
storage_used_bytes, last_login_at, active, oidc_provider, is_external
|
||||
FROM perf_admin_users ORDER BY created_at DESC LIMIT 100 OFFSET 0;
|
||||
|
||||
\timing off
|
||||
\o
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,191 @@
|
||||
//! Reproducible local-file A/B for the cached blob range length calculation.
|
||||
//!
|
||||
//! This exercises the hot-cache filesystem shape (open, seek, limit, read)
|
||||
//! while changing only the historical inclusive-end arithmetic versus the
|
||||
//! `BlobStorageBackend` contract's exclusive end. It intentionally lives
|
||||
//! outside production tests and outside `benches/`.
|
||||
|
||||
use std::env;
|
||||
use std::error::Error;
|
||||
use std::fs::{self, File};
|
||||
use std::hint::black_box;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
const DATA: &[u8] = b"abcdef";
|
||||
const START: u64 = 1;
|
||||
const END_EXCLUSIVE: u64 = 3;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Algorithm {
|
||||
HistoricalInclusive,
|
||||
CorrectedExclusive,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Samples {
|
||||
elapsed_ns: Vec<u128>,
|
||||
bytes: u64,
|
||||
checksum: u64,
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
dir: PathBuf,
|
||||
blob: PathBuf,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn create() -> Result<Self, Box<dyn Error>> {
|
||||
let nonce = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
|
||||
let dir = env::temp_dir().join(format!(
|
||||
"oxicloud-cached-range-ab-{}-{nonce}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir(&dir)?;
|
||||
let blob = dir.join("fixture.blob");
|
||||
fs::write(&blob, DATA)?;
|
||||
Ok(Self { dir, blob })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.dir);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_count(name: &str, default: usize) -> Result<usize, Box<dyn Error>> {
|
||||
match env::var(name) {
|
||||
Ok(raw) => {
|
||||
let value = raw.parse::<usize>()?;
|
||||
if value == 0 {
|
||||
return Err(format!("{name} must be greater than zero").into());
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
Err(env::VarError::NotPresent) => Ok(default),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_range(path: &Path, algorithm: Algorithm) -> Result<Vec<u8>, Box<dyn Error>> {
|
||||
let mut file = File::open(path)?;
|
||||
file.seek(SeekFrom::Start(START))?;
|
||||
let take_len = match algorithm {
|
||||
Algorithm::HistoricalInclusive => END_EXCLUSIVE - START + 1,
|
||||
Algorithm::CorrectedExclusive => END_EXCLUSIVE.saturating_sub(START),
|
||||
};
|
||||
let mut output = Vec::with_capacity(take_len as usize);
|
||||
file.take(take_len).read_to_end(&mut output)?;
|
||||
black_box(&output);
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn record(path: &Path, algorithm: Algorithm, samples: &mut Samples) -> Result<(), Box<dyn Error>> {
|
||||
let started = Instant::now();
|
||||
let output = read_range(path, algorithm)?;
|
||||
samples.elapsed_ns.push(started.elapsed().as_nanos());
|
||||
samples.bytes += output.len() as u64;
|
||||
samples.checksum = samples
|
||||
.checksum
|
||||
.wrapping_add(output.iter().map(|byte| u64::from(*byte)).sum::<u64>());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn percentile_us(samples: &mut [u128], percentile: usize) -> f64 {
|
||||
samples.sort_unstable();
|
||||
let rank = ((samples.len() - 1) * percentile) / 100;
|
||||
samples[rank] as f64 / 1_000.0
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let iterations = parse_count("CACHED_RANGE_ITERATIONS", 10_000)?;
|
||||
let warmups = parse_count("CACHED_RANGE_WARMUPS", 1_000)?;
|
||||
let fixture = Fixture::create()?;
|
||||
|
||||
let historical = read_range(&fixture.blob, Algorithm::HistoricalInclusive)?;
|
||||
let corrected = read_range(&fixture.blob, Algorithm::CorrectedExclusive)?;
|
||||
if historical != b"bcd" || corrected != b"bc" {
|
||||
return Err("fixture did not expose the historical extra byte".into());
|
||||
}
|
||||
|
||||
for iteration in 0..warmups {
|
||||
let order = if iteration % 2 == 0 {
|
||||
[
|
||||
Algorithm::HistoricalInclusive,
|
||||
Algorithm::CorrectedExclusive,
|
||||
]
|
||||
} else {
|
||||
[
|
||||
Algorithm::CorrectedExclusive,
|
||||
Algorithm::HistoricalInclusive,
|
||||
]
|
||||
};
|
||||
for algorithm in order {
|
||||
black_box(read_range(&fixture.blob, algorithm)?);
|
||||
}
|
||||
}
|
||||
|
||||
let mut historical = Samples::default();
|
||||
let mut corrected = Samples::default();
|
||||
for iteration in 0..iterations {
|
||||
if iteration % 2 == 0 {
|
||||
record(
|
||||
&fixture.blob,
|
||||
Algorithm::HistoricalInclusive,
|
||||
&mut historical,
|
||||
)?;
|
||||
record(&fixture.blob, Algorithm::CorrectedExclusive, &mut corrected)?;
|
||||
} else {
|
||||
record(&fixture.blob, Algorithm::CorrectedExclusive, &mut corrected)?;
|
||||
record(
|
||||
&fixture.blob,
|
||||
Algorithm::HistoricalInclusive,
|
||||
&mut historical,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
let historical_p50 = percentile_us(&mut historical.elapsed_ns, 50);
|
||||
let historical_p95 = percentile_us(&mut historical.elapsed_ns, 95);
|
||||
let corrected_p50 = percentile_us(&mut corrected.elapsed_ns, 50);
|
||||
let corrected_p95 = percentile_us(&mut corrected.elapsed_ns, 95);
|
||||
let p50_delta = (corrected_p50 / historical_p50 - 1.0) * 100.0;
|
||||
let p95_delta = (corrected_p95 / historical_p95 - 1.0) * 100.0;
|
||||
|
||||
println!(
|
||||
concat!(
|
||||
"{{\n",
|
||||
" \"benchmark\": \"cached_range_exclusive_ab\",\n",
|
||||
" \"environment\": {{ \"os\": \"{}\", \"arch\": \"{}\" }},\n",
|
||||
" \"range\": {{ \"start\": {}, \"end_exclusive\": {} }},\n",
|
||||
" \"warmups_per_variant\": {},\n",
|
||||
" \"iterations_per_variant\": {},\n",
|
||||
" \"historical_inclusive\": {{ \"bytes\": {}, \"bytes_per_read\": {}, \"p50_us\": {:.3}, \"p95_us\": {:.3}, \"checksum\": {} }},\n",
|
||||
" \"corrected_exclusive\": {{ \"bytes\": {}, \"bytes_per_read\": {}, \"p50_us\": {:.3}, \"p95_us\": {:.3}, \"checksum\": {} }},\n",
|
||||
" \"delta_percent\": {{ \"bytes\": -33.333, \"p50_latency\": {:.3}, \"p95_latency\": {:.3} }}\n",
|
||||
"}}"
|
||||
),
|
||||
env::consts::OS,
|
||||
env::consts::ARCH,
|
||||
START,
|
||||
END_EXCLUSIVE,
|
||||
warmups,
|
||||
iterations,
|
||||
historical.bytes,
|
||||
historical.bytes / iterations as u64,
|
||||
historical_p50,
|
||||
historical_p95,
|
||||
historical.checksum,
|
||||
corrected.bytes,
|
||||
corrected.bytes / iterations as u64,
|
||||
corrected_p50,
|
||||
corrected_p95,
|
||||
corrected.checksum,
|
||||
p50_delta,
|
||||
p95_delta,
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// End-to-end loopback gate for the >10k whole-file dedup batching change.
|
||||
// Unlike the probe-only microbenchmark, this executes every subsequent
|
||||
// by-hash or content-upload request with the production upload concurrency.
|
||||
|
||||
import { createServer } from 'node:http';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import process from 'node:process';
|
||||
|
||||
const MAX_HASHES = 10_000;
|
||||
const HASH_COUNT = 10_001;
|
||||
const BATCH_CONCURRENCY = 4;
|
||||
const UPLOAD_CONCURRENCY = 2;
|
||||
|
||||
const args = new Map();
|
||||
for (let index = 2; index < process.argv.length; index += 2) {
|
||||
args.set(process.argv[index], process.argv[index + 1]);
|
||||
}
|
||||
const samples = Number(args.get('--samples') ?? 3);
|
||||
const bytesPerFile = Number(args.get('--bytes-per-file') ?? 4096);
|
||||
const output = args.get('--output');
|
||||
if (!Number.isInteger(samples) || samples < 1) throw new Error('samples must be >= 1');
|
||||
if (!Number.isInteger(bytesPerFile) || bytesPerFile < 1) {
|
||||
throw new Error('bytes-per-file must be >= 1');
|
||||
}
|
||||
|
||||
const hashes = Array.from({ length: HASH_COUNT }, (_, index) =>
|
||||
index.toString(16).padStart(64, '0'),
|
||||
);
|
||||
const content = Buffer.alloc(bytesPerFile, 0x5a);
|
||||
|
||||
function emptyStats(hitPercent) {
|
||||
return {
|
||||
hitPercent,
|
||||
dedupAccepted: 0,
|
||||
dedupRejected: 0,
|
||||
dedupRequestBytes: 0,
|
||||
dedupResponseBytes: 0,
|
||||
uploadRequests: 0,
|
||||
uploadContentBytes: 0,
|
||||
byHashRequests: 0,
|
||||
byHashRequestBytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let stats = emptyStats(0);
|
||||
function owned(hash) {
|
||||
return stats.hitPercent === 50 && (Number.parseInt(hash.at(-1), 16) & 1) === 0;
|
||||
}
|
||||
|
||||
function send(response, status, body) {
|
||||
response.writeHead(status, { 'content-type': 'application/json' });
|
||||
response.end(body);
|
||||
}
|
||||
|
||||
const server = createServer(async (request, response) => {
|
||||
const chunks = [];
|
||||
for await (const chunk of request) chunks.push(chunk);
|
||||
const body = Buffer.concat(chunks);
|
||||
|
||||
if (request.url === '/api/dedup/check-batch') {
|
||||
stats.dedupRequestBytes += body.byteLength;
|
||||
const parsed = JSON.parse(body.toString('utf8'));
|
||||
const requestHashes = Array.isArray(parsed.hashes) ? parsed.hashes : [];
|
||||
if (requestHashes.length > MAX_HASHES) {
|
||||
stats.dedupRejected++;
|
||||
const responseBody = JSON.stringify({ error: 'Too many hashes' });
|
||||
stats.dedupResponseBytes += Buffer.byteLength(responseBody);
|
||||
send(response, 400, responseBody);
|
||||
return;
|
||||
}
|
||||
stats.dedupAccepted++;
|
||||
const responseBody = JSON.stringify({ owned: requestHashes.filter(owned) });
|
||||
stats.dedupResponseBytes += Buffer.byteLength(responseBody);
|
||||
send(response, 200, responseBody);
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.url === '/api/files/by-hash') {
|
||||
stats.byHashRequests++;
|
||||
stats.byHashRequestBytes += body.byteLength;
|
||||
send(response, 201, '{"ok":true}');
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.url === '/api/files/upload') {
|
||||
stats.uploadRequests++;
|
||||
stats.uploadContentBytes += body.byteLength;
|
||||
send(response, 201, '{"ok":true}');
|
||||
return;
|
||||
}
|
||||
|
||||
send(response, 404, '{}');
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') throw new Error('server address unavailable');
|
||||
const baseUrl = `http://127.0.0.1:${address.port}`;
|
||||
|
||||
async function post(path, body, contentType) {
|
||||
const response = await fetch(baseUrl + path, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': contentType },
|
||||
body,
|
||||
});
|
||||
const text = await response.text();
|
||||
return { ok: response.ok, text };
|
||||
}
|
||||
|
||||
async function requestOwned(requestHashes) {
|
||||
const response = await post(
|
||||
'/api/dedup/check-batch',
|
||||
JSON.stringify({ hashes: requestHashes }),
|
||||
'application/json',
|
||||
);
|
||||
if (!response.ok) return null;
|
||||
const decoded = JSON.parse(response.text);
|
||||
return Array.isArray(decoded.owned) ? decoded.owned : null;
|
||||
}
|
||||
|
||||
async function currentProbe() {
|
||||
return new Set((await requestOwned(hashes)) ?? []);
|
||||
}
|
||||
|
||||
async function candidateProbe() {
|
||||
const ownedHashes = new Set();
|
||||
const waveSize = MAX_HASHES * BATCH_CONCURRENCY;
|
||||
for (let waveStart = 0; waveStart < hashes.length; waveStart += waveSize) {
|
||||
const requests = [];
|
||||
const waveEnd = Math.min(hashes.length, waveStart + waveSize);
|
||||
for (let start = waveStart; start < waveEnd; start += MAX_HASHES) {
|
||||
requests.push(requestOwned(hashes.slice(start, Math.min(start + MAX_HASHES, waveEnd))));
|
||||
}
|
||||
const responses = await Promise.all(requests);
|
||||
if (responses.some((batch) => batch === null)) return new Set();
|
||||
for (const batch of responses) for (const hash of batch) ownedHashes.add(hash);
|
||||
}
|
||||
return ownedHashes;
|
||||
}
|
||||
|
||||
async function mapIndexes(limit, operation) {
|
||||
let next = 0;
|
||||
await Promise.all(
|
||||
Array.from({ length: limit }, async () => {
|
||||
while (next < hashes.length) {
|
||||
const index = next++;
|
||||
await operation(index);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function runWorkflow(hitPercent, probe) {
|
||||
stats = emptyStats(hitPercent);
|
||||
if (globalThis.gc) globalThis.gc();
|
||||
const before = process.memoryUsage();
|
||||
let peakHeap = before.heapUsed;
|
||||
let peakRss = before.rss;
|
||||
const sampler = setInterval(() => {
|
||||
const memory = process.memoryUsage();
|
||||
peakHeap = Math.max(peakHeap, memory.heapUsed);
|
||||
peakRss = Math.max(peakRss, memory.rss);
|
||||
}, 1);
|
||||
|
||||
const start = performance.now();
|
||||
const ownedHashes = await probe();
|
||||
await mapIndexes(UPLOAD_CONCURRENCY, async (index) => {
|
||||
const hash = hashes[index];
|
||||
if (ownedHashes.has(hash)) {
|
||||
const response = await post(
|
||||
'/api/files/by-hash',
|
||||
JSON.stringify({ folder_id: 'folder', name: `file-${index}`, hash }),
|
||||
'application/json',
|
||||
);
|
||||
if (!response.ok) throw new Error('by-hash request failed');
|
||||
} else {
|
||||
const response = await post('/api/files/upload', content, 'application/octet-stream');
|
||||
if (!response.ok) throw new Error('content upload failed');
|
||||
}
|
||||
});
|
||||
const wallMs = performance.now() - start;
|
||||
clearInterval(sampler);
|
||||
const after = process.memoryUsage();
|
||||
peakHeap = Math.max(peakHeap, after.heapUsed);
|
||||
peakRss = Math.max(peakRss, after.rss);
|
||||
|
||||
return {
|
||||
wallMs,
|
||||
ownedCount: ownedHashes.size,
|
||||
peakHeapDeltaBytes: Math.max(0, peakHeap - before.heapUsed),
|
||||
peakRssDeltaBytes: Math.max(0, peakRss - before.rss),
|
||||
...stats,
|
||||
};
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
return sorted[Math.floor(sorted.length / 2)];
|
||||
}
|
||||
|
||||
function summarize(runs) {
|
||||
return {
|
||||
wallSamplesMs: runs.map((run) => Number(run.wallMs.toFixed(3))),
|
||||
wallMedianMs: Number(median(runs.map((run) => run.wallMs)).toFixed(3)),
|
||||
peakHeapDeltaBytesMedian: median(runs.map((run) => run.peakHeapDeltaBytes)),
|
||||
peakRssDeltaBytesMedian: median(runs.map((run) => run.peakRssDeltaBytes)),
|
||||
protocol: Object.fromEntries(
|
||||
Object.entries(runs[0]).filter(([key]) => !key.includes('Delta') && key !== 'wallMs'),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const cases = [];
|
||||
try {
|
||||
// Warm undici's connection pool and JIT without exercising the measured
|
||||
// >10k workflow.
|
||||
await post('/api/files/upload', content, 'application/octet-stream');
|
||||
await post(
|
||||
'/api/files/by-hash',
|
||||
JSON.stringify({ folder_id: 'folder', name: 'warm', hash: hashes[0] }),
|
||||
'application/json',
|
||||
);
|
||||
|
||||
for (const hitPercent of [0, 50]) {
|
||||
const currentRuns = [];
|
||||
const candidateRuns = [];
|
||||
for (let sample = 0; sample < samples; sample++) {
|
||||
if (sample % 2 === 0) {
|
||||
currentRuns.push(await runWorkflow(hitPercent, currentProbe));
|
||||
candidateRuns.push(await runWorkflow(hitPercent, candidateProbe));
|
||||
} else {
|
||||
candidateRuns.push(await runWorkflow(hitPercent, candidateProbe));
|
||||
currentRuns.push(await runWorkflow(hitPercent, currentProbe));
|
||||
}
|
||||
}
|
||||
|
||||
const expectedOwned = hitPercent === 50 ? Math.ceil(HASH_COUNT / 2) : 0;
|
||||
for (const run of candidateRuns) {
|
||||
if (run.ownedCount !== expectedOwned || run.dedupRejected !== 0) {
|
||||
throw new Error(`candidate correctness failure at ${hitPercent}% hits`);
|
||||
}
|
||||
}
|
||||
for (const run of currentRuns) {
|
||||
if (run.ownedCount !== 0 || run.dedupRejected !== 1) {
|
||||
throw new Error(`control did not reproduce >10k rejection at ${hitPercent}% hits`);
|
||||
}
|
||||
}
|
||||
|
||||
const current = summarize(currentRuns);
|
||||
const candidate = summarize(candidateRuns);
|
||||
cases.push({
|
||||
hitPercent,
|
||||
current,
|
||||
candidate,
|
||||
wallSpeedup: Number((current.wallMedianMs / candidate.wallMedianMs).toFixed(3)),
|
||||
uploadByteReductionPercent: Number(
|
||||
(
|
||||
100 *
|
||||
(1 -
|
||||
candidate.protocol.uploadContentBytes / current.protocol.uploadContentBytes)
|
||||
).toFixed(3),
|
||||
),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await new Promise((resolve, reject) =>
|
||||
server.close((error) => (error ? reject(error) : resolve())),
|
||||
);
|
||||
}
|
||||
|
||||
const result = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
environment: { node: process.version, platform: process.platform, arch: process.arch },
|
||||
fixture: { hashes: HASH_COUNT, bytesPerFile, uploadConcurrency: UPLOAD_CONCURRENCY },
|
||||
note: 'Loopback mock includes every dedup, by-hash and content request. Hashing is excluded. Backend SQL is unmodeled: current rejects before ownership lookup while candidate would execute two accepted queries, so candidate wall time is optimistic.',
|
||||
cases,
|
||||
};
|
||||
const rendered = JSON.stringify(result, null, 2) + '\n';
|
||||
if (output) writeFileSync(output, rendered);
|
||||
process.stdout.write(rendered);
|
||||
@@ -0,0 +1,798 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createServer } from "node:http";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { parseArgs } from "node:util";
|
||||
import os from "node:os";
|
||||
|
||||
const MAX_DEDUP_HASHES = 10_000;
|
||||
const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024;
|
||||
const CURSOR_COMPACT_AT = 4_096;
|
||||
|
||||
let blackhole = 0;
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
suite: { type: "string", default: "all" },
|
||||
warmup: { type: "string", default: "3" },
|
||||
samples: { type: "string", default: "15" },
|
||||
"queue-counts": { type: "string", default: "64,256,1024,10000,50000" },
|
||||
"progress-cases": {
|
||||
type: "string",
|
||||
default: "1:100,10:500,100:5000,1000:10000,10000:5000",
|
||||
},
|
||||
"hash-counts": { type: "string", default: "1000,10000,10001,25000" },
|
||||
"dedup-batch-size": { type: "string", default: "10000" },
|
||||
"dedup-concurrency": { type: "string", default: "4" },
|
||||
"server-latency-ms": { type: "string", default: "0" },
|
||||
"modeled-file-bytes": { type: "string", default: "65536" },
|
||||
output: { type: "string" },
|
||||
},
|
||||
strict: true,
|
||||
allowPositionals: false,
|
||||
});
|
||||
|
||||
function positiveInteger(name, raw, allowZero = false) {
|
||||
const value = Number(raw);
|
||||
const lowerBound = allowZero ? 0 : 1;
|
||||
if (!Number.isInteger(value) || value < lowerBound) {
|
||||
throw new Error(
|
||||
name + " must be an integer >= " + lowerBound + "; received " + raw,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function numberList(name, raw) {
|
||||
const parsed = raw
|
||||
.split(",")
|
||||
.filter(Boolean)
|
||||
.map((part) => positiveInteger(name, part));
|
||||
if (parsed.length === 0) throw new Error(name + " must not be empty");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function progressCases(raw) {
|
||||
const parsed = raw
|
||||
.split(",")
|
||||
.filter(Boolean)
|
||||
.map((entry) => {
|
||||
const parts = entry.split(":");
|
||||
if (parts.length !== 2)
|
||||
throw new Error("Invalid progress case: " + entry);
|
||||
return {
|
||||
items: positiveInteger("progress items", parts[0]),
|
||||
updates: positiveInteger("progress updates", parts[1]),
|
||||
};
|
||||
});
|
||||
if (parsed.length === 0) throw new Error("progress-cases must not be empty");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const config = {
|
||||
suite: values.suite,
|
||||
warmup: positiveInteger("warmup", values.warmup, true),
|
||||
samples: positiveInteger("samples", values.samples),
|
||||
queueCounts: numberList("queue-counts", values["queue-counts"]),
|
||||
progressCases: progressCases(values["progress-cases"]),
|
||||
hashCounts: numberList("hash-counts", values["hash-counts"]),
|
||||
dedupBatchSize: positiveInteger(
|
||||
"dedup-batch-size",
|
||||
values["dedup-batch-size"],
|
||||
),
|
||||
dedupConcurrency: positiveInteger(
|
||||
"dedup-concurrency",
|
||||
values["dedup-concurrency"],
|
||||
),
|
||||
serverLatencyMs: positiveInteger(
|
||||
"server-latency-ms",
|
||||
values["server-latency-ms"],
|
||||
true,
|
||||
),
|
||||
modeledFileBytes: positiveInteger(
|
||||
"modeled-file-bytes",
|
||||
values["modeled-file-bytes"],
|
||||
),
|
||||
};
|
||||
|
||||
if (!["all", "queue", "progress", "dedup"].includes(config.suite)) {
|
||||
throw new Error("suite must be all, queue, progress, or dedup");
|
||||
}
|
||||
if (config.dedupBatchSize > MAX_DEDUP_HASHES) {
|
||||
throw new Error(
|
||||
"dedup-batch-size must be <= the server limit of " + MAX_DEDUP_HASHES,
|
||||
);
|
||||
}
|
||||
|
||||
function median(sorted) {
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 === 0
|
||||
? (sorted[middle - 1] + sorted[middle]) / 2
|
||||
: sorted[middle];
|
||||
}
|
||||
|
||||
function summarize(samples) {
|
||||
const times = samples.map((sample) => sample.ms).sort((a, b) => a - b);
|
||||
const heaps = samples
|
||||
.map((sample) => sample.heapDeltaBytes)
|
||||
.sort((a, b) => a - b);
|
||||
const rss = samples
|
||||
.map((sample) => sample.rssDeltaBytes)
|
||||
.sort((a, b) => a - b);
|
||||
const p95Index = Math.max(0, Math.ceil(times.length * 0.95) - 1);
|
||||
return {
|
||||
sampleCount: samples.length,
|
||||
medianMs: median(times),
|
||||
p95Ms: times[p95Index],
|
||||
minMs: times[0],
|
||||
maxMs: times[times.length - 1],
|
||||
medianHeapDeltaBytes: median(heaps),
|
||||
medianRssDeltaBytes: median(rss),
|
||||
};
|
||||
}
|
||||
|
||||
function consume(result) {
|
||||
const token = Number(
|
||||
result.checksum ??
|
||||
result.ownedCount ??
|
||||
result.chunkCount ??
|
||||
result.lastPercent ??
|
||||
0,
|
||||
);
|
||||
blackhole = (blackhole ^ (token >>> 0)) >>> 0;
|
||||
}
|
||||
|
||||
async function measureOne(fn) {
|
||||
if (global.gc) global.gc();
|
||||
const before = process.memoryUsage();
|
||||
const started = performance.now();
|
||||
const result = await fn();
|
||||
const ms = performance.now() - started;
|
||||
const after = process.memoryUsage();
|
||||
consume(result);
|
||||
return {
|
||||
sample: {
|
||||
ms,
|
||||
heapDeltaBytes: after.heapUsed - before.heapUsed,
|
||||
rssDeltaBytes: after.rss - before.rss,
|
||||
},
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
async function benchmarkPair(currentFn, candidateFn, verify) {
|
||||
const checkedCurrent = await currentFn();
|
||||
const checkedCandidate = await candidateFn();
|
||||
verify(checkedCurrent, checkedCandidate);
|
||||
|
||||
for (let i = 0; i < config.warmup; i++) {
|
||||
if (i % 2 === 0) {
|
||||
consume(await currentFn());
|
||||
consume(await candidateFn());
|
||||
} else {
|
||||
consume(await candidateFn());
|
||||
consume(await currentFn());
|
||||
}
|
||||
}
|
||||
|
||||
const currentSamples = [];
|
||||
const candidateSamples = [];
|
||||
let currentResult = checkedCurrent;
|
||||
let candidateResult = checkedCandidate;
|
||||
for (let i = 0; i < config.samples; i++) {
|
||||
const order =
|
||||
i % 2 === 0
|
||||
? [
|
||||
["current", currentFn],
|
||||
["candidate", candidateFn],
|
||||
]
|
||||
: [
|
||||
["candidate", candidateFn],
|
||||
["current", currentFn],
|
||||
];
|
||||
for (const [kind, fn] of order) {
|
||||
const measured = await measureOne(fn);
|
||||
if (kind === "current") {
|
||||
currentSamples.push(measured.sample);
|
||||
currentResult = measured.result;
|
||||
} else {
|
||||
candidateSamples.push(measured.sample);
|
||||
candidateResult = measured.result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const current = summarize(currentSamples);
|
||||
const candidate = summarize(candidateSamples);
|
||||
return {
|
||||
current,
|
||||
candidate,
|
||||
speedup: current.medianMs / candidate.medianMs,
|
||||
representative: {
|
||||
current: currentResult,
|
||||
candidate: candidateResult,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeChunks(count) {
|
||||
const chunks = new Array(count);
|
||||
let offset = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const size =
|
||||
(1 + ((Math.imul(i + 1, 2_654_435_761) >>> 28) & 7)) * 32 * 1024;
|
||||
chunks[i] = { h: "chunk-" + i, s: size, offset };
|
||||
offset += size;
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function foldBatch(batch, checksum) {
|
||||
let next = checksum;
|
||||
for (const chunk of batch) {
|
||||
next = Math.imul(next ^ chunk.s ^ (chunk.offset >>> 0), 16_777_619) >>> 0;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function drainWithShift(source) {
|
||||
const uploadQueue = source.slice();
|
||||
let checksum = 2_166_136_261;
|
||||
let chunkCount = 0;
|
||||
let totalBytes = 0;
|
||||
let batchCount = 0;
|
||||
|
||||
while (uploadQueue.length > 0) {
|
||||
const batch = [];
|
||||
let bytes = 0;
|
||||
while (uploadQueue.length > 0 && bytes < UPLOAD_BATCH_BYTES) {
|
||||
const chunk = uploadQueue.shift();
|
||||
batch.push(chunk);
|
||||
bytes += chunk.s;
|
||||
}
|
||||
checksum = foldBatch(batch, checksum);
|
||||
chunkCount += batch.length;
|
||||
totalBytes += bytes;
|
||||
batchCount++;
|
||||
}
|
||||
|
||||
return { checksum, chunkCount, totalBytes, batchCount };
|
||||
}
|
||||
|
||||
function drainWithCursor(source) {
|
||||
const uploadQueue = source.slice();
|
||||
let head = 0;
|
||||
let checksum = 2_166_136_261;
|
||||
let chunkCount = 0;
|
||||
let totalBytes = 0;
|
||||
let batchCount = 0;
|
||||
let compactions = 0;
|
||||
|
||||
while (head < uploadQueue.length) {
|
||||
const batch = [];
|
||||
let bytes = 0;
|
||||
while (head < uploadQueue.length && bytes < UPLOAD_BATCH_BYTES) {
|
||||
const chunk = uploadQueue[head];
|
||||
uploadQueue[head] = undefined;
|
||||
head++;
|
||||
batch.push(chunk);
|
||||
bytes += chunk.s;
|
||||
}
|
||||
checksum = foldBatch(batch, checksum);
|
||||
chunkCount += batch.length;
|
||||
totalBytes += bytes;
|
||||
batchCount++;
|
||||
|
||||
if (head === uploadQueue.length) {
|
||||
uploadQueue.length = 0;
|
||||
head = 0;
|
||||
} else if (head >= CURSOR_COMPACT_AT && head * 2 >= uploadQueue.length) {
|
||||
uploadQueue.copyWithin(0, head);
|
||||
uploadQueue.length -= head;
|
||||
head = 0;
|
||||
compactions++;
|
||||
}
|
||||
}
|
||||
|
||||
return { checksum, chunkCount, totalBytes, batchCount, compactions };
|
||||
}
|
||||
|
||||
function verifyQueue(current, candidate) {
|
||||
for (const key of ["checksum", "chunkCount", "totalBytes", "batchCount"]) {
|
||||
if (current[key] !== candidate[key]) {
|
||||
throw new Error(
|
||||
"Queue candidate changed " +
|
||||
key +
|
||||
": " +
|
||||
current[key] +
|
||||
" vs " +
|
||||
candidate[key],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function makeProgressEvents(items, updateCount) {
|
||||
const indices = new Uint32Array(updateCount);
|
||||
const values = new Float64Array(updateCount);
|
||||
let state = 0x9e3779b9;
|
||||
for (let i = 0; i < updateCount; i++) {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
state >>>= 0;
|
||||
indices[i] = state % items;
|
||||
values[i] = (state & 1023) / 1024;
|
||||
}
|
||||
return { indices, values };
|
||||
}
|
||||
|
||||
function recordProgress(checksum, sum, total) {
|
||||
const percent = Math.round((sum / total) * 100);
|
||||
const done = Math.round(sum);
|
||||
return {
|
||||
checksum: (checksum + Math.imul(percent + 1, done + 1)) >>> 0,
|
||||
percent,
|
||||
};
|
||||
}
|
||||
|
||||
function progressWithFullScan(items, events) {
|
||||
const fractions = new Array(items).fill(0);
|
||||
let checksum = 0;
|
||||
let lastPercent = 0;
|
||||
let finalSum = 0;
|
||||
for (let update = 0; update < events.indices.length; update++) {
|
||||
fractions[events.indices[update]] = Math.min(1, events.values[update]);
|
||||
let sum = 0;
|
||||
for (const fraction of fractions) sum += fraction;
|
||||
const recorded = recordProgress(checksum, sum, items);
|
||||
checksum = recorded.checksum;
|
||||
lastPercent = recorded.percent;
|
||||
finalSum = sum;
|
||||
}
|
||||
return { checksum, lastPercent, finalSum };
|
||||
}
|
||||
|
||||
function progressWithAccumulator(items, events) {
|
||||
const fractions = new Array(items).fill(0);
|
||||
let sum = 0;
|
||||
let checksum = 0;
|
||||
let lastPercent = 0;
|
||||
for (let update = 0; update < events.indices.length; update++) {
|
||||
const index = events.indices[update];
|
||||
const next = Math.min(1, events.values[update]);
|
||||
sum += next - fractions[index];
|
||||
fractions[index] = next;
|
||||
const recorded = recordProgress(checksum, sum, items);
|
||||
checksum = recorded.checksum;
|
||||
lastPercent = recorded.percent;
|
||||
}
|
||||
return { checksum, lastPercent, finalSum: sum };
|
||||
}
|
||||
|
||||
function verifyProgress(current, candidate) {
|
||||
if (
|
||||
current.checksum !== candidate.checksum ||
|
||||
current.lastPercent !== candidate.lastPercent
|
||||
) {
|
||||
throw new Error("Progress candidate changed user-visible progress values");
|
||||
}
|
||||
if (Math.abs(current.finalSum - candidate.finalSum) > Number.EPSILON * 8) {
|
||||
throw new Error("Progress candidate changed final sum");
|
||||
}
|
||||
}
|
||||
|
||||
function makeHashes(count) {
|
||||
const hashes = new Array(count);
|
||||
for (let i = 0; i < count; i++) hashes[i] = i.toString(16).padStart(64, "0");
|
||||
return hashes;
|
||||
}
|
||||
|
||||
function isOwnedHash(hash) {
|
||||
return (Number.parseInt(hash.at(-1), 16) & 1) === 0;
|
||||
}
|
||||
|
||||
function emptyServerStats() {
|
||||
return {
|
||||
requests: 0,
|
||||
acceptedRequests: 0,
|
||||
rejectedRequests: 0,
|
||||
requestBytes: 0,
|
||||
responseBytes: 0,
|
||||
maxBatchHashes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
async function startDedupServer() {
|
||||
let activeStats = emptyServerStats();
|
||||
const server = createServer(async (request, response) => {
|
||||
const parts = [];
|
||||
for await (const part of request) parts.push(part);
|
||||
const body = Buffer.concat(parts);
|
||||
const parsed = JSON.parse(body.toString("utf8"));
|
||||
const hashes = Array.isArray(parsed.hashes) ? parsed.hashes : [];
|
||||
|
||||
activeStats.requests++;
|
||||
activeStats.requestBytes += body.byteLength;
|
||||
activeStats.maxBatchHashes = Math.max(
|
||||
activeStats.maxBatchHashes,
|
||||
hashes.length,
|
||||
);
|
||||
|
||||
if (config.serverLatencyMs > 0) {
|
||||
await new Promise((resolveDelay) =>
|
||||
setTimeout(resolveDelay, config.serverLatencyMs),
|
||||
);
|
||||
}
|
||||
|
||||
let status;
|
||||
let responseBody;
|
||||
if (hashes.length > MAX_DEDUP_HASHES) {
|
||||
status = 400;
|
||||
activeStats.rejectedRequests++;
|
||||
responseBody = JSON.stringify({ error: "Too many hashes" });
|
||||
} else {
|
||||
status = 200;
|
||||
activeStats.acceptedRequests++;
|
||||
responseBody = JSON.stringify({ owned: hashes.filter(isOwnedHash) });
|
||||
}
|
||||
activeStats.responseBytes += Buffer.byteLength(responseBody);
|
||||
response.writeHead(status, { "content-type": "application/json" });
|
||||
response.end(responseBody);
|
||||
});
|
||||
|
||||
await new Promise((resolveListen, rejectListen) => {
|
||||
server.once("error", rejectListen);
|
||||
server.listen(0, "127.0.0.1", resolveListen);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string")
|
||||
throw new Error("Could not determine mock server address");
|
||||
|
||||
return {
|
||||
url: "http://127.0.0.1:" + address.port + "/api/dedup/check-batch",
|
||||
resetStats() {
|
||||
activeStats = emptyServerStats();
|
||||
return activeStats;
|
||||
},
|
||||
async close() {
|
||||
await new Promise((resolveClose, rejectClose) => {
|
||||
server.close((error) => (error ? rejectClose(error) : resolveClose()));
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function postHashes(url, hashes) {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ hashes }),
|
||||
});
|
||||
if (!response.ok) return new Set();
|
||||
const data = await response.json().catch(() => null);
|
||||
return new Set(data?.owned ?? []);
|
||||
}
|
||||
|
||||
async function dedupCurrent(url, hashes) {
|
||||
return postHashes(url, hashes);
|
||||
}
|
||||
|
||||
async function dedupBatched(url, hashes) {
|
||||
// Production keeps the former one-request path exact for the overwhelmingly
|
||||
// common valid case: no slice, batching array, or Promise pool below the cap.
|
||||
if (hashes.length <= MAX_DEDUP_HASHES) return postHashes(url, hashes);
|
||||
|
||||
const batches = [];
|
||||
for (let start = 0; start < hashes.length; start += config.dedupBatchSize) {
|
||||
batches.push(hashes.slice(start, start + config.dedupBatchSize));
|
||||
}
|
||||
|
||||
const owned = new Set();
|
||||
let next = 0;
|
||||
const worker = async () => {
|
||||
while (next < batches.length) {
|
||||
const index = next++;
|
||||
const batchOwned = await postHashes(url, batches[index]);
|
||||
for (const hash of batchOwned) owned.add(hash);
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from(
|
||||
{ length: Math.min(config.dedupConcurrency, batches.length) },
|
||||
worker,
|
||||
),
|
||||
);
|
||||
return owned;
|
||||
}
|
||||
|
||||
function dedupRun(server, hashes, implementation) {
|
||||
return async () => {
|
||||
const stats = server.resetStats();
|
||||
const owned = await implementation(server.url, hashes);
|
||||
let checksum = 0;
|
||||
for (const hash of owned)
|
||||
checksum = (checksum + Number.parseInt(hash.slice(-8), 16)) >>> 0;
|
||||
return {
|
||||
checksum,
|
||||
ownedCount: owned.size,
|
||||
contentBytesAvoided: owned.size * config.modeledFileBytes,
|
||||
...stats,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function verifyDedup(current, candidate, hashCount) {
|
||||
const expectedOwned = Math.ceil(hashCount / 2);
|
||||
if (hashCount <= MAX_DEDUP_HASHES) {
|
||||
if (
|
||||
current.ownedCount !== expectedOwned ||
|
||||
candidate.ownedCount !== expectedOwned ||
|
||||
current.checksum !== candidate.checksum
|
||||
) {
|
||||
throw new Error("Dedup fast path changed the ownership result");
|
||||
}
|
||||
if (
|
||||
current.requests !== 1 ||
|
||||
candidate.requests !== 1 ||
|
||||
current.rejectedRequests !== 0 ||
|
||||
candidate.rejectedRequests !== 0
|
||||
) {
|
||||
throw new Error("Dedup fast path must remain one accepted request");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (current.ownedCount !== 0 || current.rejectedRequests !== 1) {
|
||||
throw new Error(
|
||||
"Current >10k control did not reproduce the expected rejection",
|
||||
);
|
||||
}
|
||||
if (candidate.ownedCount !== expectedOwned) {
|
||||
throw new Error(
|
||||
"Batched candidate found " +
|
||||
candidate.ownedCount +
|
||||
" owned hashes; expected " +
|
||||
expectedOwned,
|
||||
);
|
||||
}
|
||||
if (
|
||||
candidate.rejectedRequests !== 0 ||
|
||||
candidate.maxBatchHashes > MAX_DEDUP_HASHES
|
||||
) {
|
||||
throw new Error("Batched candidate exceeded the server request limit");
|
||||
}
|
||||
}
|
||||
|
||||
function repeatQueueDrain(fn, repetitions) {
|
||||
let checksum = 0;
|
||||
let chunkCount = 0;
|
||||
let totalBytes = 0;
|
||||
let batchCount = 0;
|
||||
for (let iteration = 0; iteration < repetitions; iteration++) {
|
||||
const result = fn();
|
||||
checksum = (checksum + result.checksum) >>> 0;
|
||||
chunkCount += result.chunkCount;
|
||||
totalBytes += result.totalBytes;
|
||||
batchCount += result.batchCount;
|
||||
}
|
||||
return { checksum, chunkCount, totalBytes, batchCount };
|
||||
}
|
||||
|
||||
function repeatProgress(fn, repetitions) {
|
||||
let checksum = 0;
|
||||
let lastPercent = 0;
|
||||
let finalSum = 0;
|
||||
for (let iteration = 0; iteration < repetitions; iteration++) {
|
||||
const result = fn();
|
||||
checksum = (checksum + result.checksum) >>> 0;
|
||||
lastPercent = result.lastPercent;
|
||||
finalSum += result.finalSum;
|
||||
}
|
||||
return { checksum, lastPercent, finalSum };
|
||||
}
|
||||
|
||||
function formatMs(value) {
|
||||
if (value >= 100) return value.toFixed(1);
|
||||
if (value >= 10) return value.toFixed(2);
|
||||
return value.toFixed(3);
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
const absolute = Math.abs(value);
|
||||
const sign = value < 0 ? "-" : "";
|
||||
if (absolute >= 1024 * 1024 * 1024)
|
||||
return sign + (absolute / (1024 * 1024 * 1024)).toFixed(2) + " GiB";
|
||||
if (absolute >= 1024 * 1024)
|
||||
return sign + (absolute / (1024 * 1024)).toFixed(2) + " MiB";
|
||||
if (absolute >= 1024) return sign + (absolute / 1024).toFixed(2) + " KiB";
|
||||
return sign + absolute.toFixed(0) + " B";
|
||||
}
|
||||
|
||||
function printPair(label, result) {
|
||||
console.log(label);
|
||||
console.log(
|
||||
" current median " +
|
||||
formatMs(result.current.medianMs) +
|
||||
" ms; p95 " +
|
||||
formatMs(result.current.p95Ms) +
|
||||
" ms; heap delta " +
|
||||
formatBytes(result.current.medianHeapDeltaBytes),
|
||||
);
|
||||
console.log(
|
||||
" candidate median " +
|
||||
formatMs(result.candidate.medianMs) +
|
||||
" ms; p95 " +
|
||||
formatMs(result.candidate.p95Ms) +
|
||||
" ms; heap delta " +
|
||||
formatBytes(result.candidate.medianHeapDeltaBytes),
|
||||
);
|
||||
console.log(" median speedup " + result.speedup.toFixed(2) + "x");
|
||||
}
|
||||
|
||||
async function runQueueSuite(output) {
|
||||
output.queue = [];
|
||||
for (const count of config.queueCounts) {
|
||||
const source = makeChunks(count);
|
||||
const repetitions = count <= 1024 ? Math.ceil(200_000 / count) : 1;
|
||||
const result = await benchmarkPair(
|
||||
() => repeatQueueDrain(() => drainWithShift(source), repetitions),
|
||||
() => repeatQueueDrain(() => drainWithCursor(source), repetitions),
|
||||
verifyQueue,
|
||||
);
|
||||
output.queue.push({
|
||||
chunkCount: count,
|
||||
repetitions,
|
||||
normalizedMedianUsPerDrain: {
|
||||
current: (result.current.medianMs * 1000) / repetitions,
|
||||
candidate: (result.candidate.medianMs * 1000) / repetitions,
|
||||
},
|
||||
...result,
|
||||
});
|
||||
printPair(
|
||||
"A queue drain, " +
|
||||
count.toLocaleString("en-US") +
|
||||
" chunks x " +
|
||||
repetitions.toLocaleString("en-US"),
|
||||
result,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function runProgressSuite(output) {
|
||||
output.progress = [];
|
||||
for (const scenario of config.progressCases) {
|
||||
const events = makeProgressEvents(scenario.items, scenario.updates);
|
||||
const repetitions =
|
||||
scenario.items <= 100 ? Math.ceil(100_000 / scenario.updates) : 1;
|
||||
const result = await benchmarkPair(
|
||||
() =>
|
||||
repeatProgress(
|
||||
() => progressWithFullScan(scenario.items, events),
|
||||
repetitions,
|
||||
),
|
||||
() =>
|
||||
repeatProgress(
|
||||
() => progressWithAccumulator(scenario.items, events),
|
||||
repetitions,
|
||||
),
|
||||
verifyProgress,
|
||||
);
|
||||
output.progress.push({
|
||||
...scenario,
|
||||
repetitions,
|
||||
normalizedMedianUsPerRun: {
|
||||
current: (result.current.medianMs * 1000) / repetitions,
|
||||
candidate: (result.candidate.medianMs * 1000) / repetitions,
|
||||
},
|
||||
...result,
|
||||
});
|
||||
printPair(
|
||||
"B aggregate progress, " +
|
||||
scenario.items.toLocaleString("en-US") +
|
||||
" files x " +
|
||||
scenario.updates.toLocaleString("en-US") +
|
||||
" updates",
|
||||
result,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function runDedupSuite(output) {
|
||||
output.dedup = [];
|
||||
const server = await startDedupServer();
|
||||
try {
|
||||
for (const hashCount of config.hashCounts) {
|
||||
const hashes = makeHashes(hashCount);
|
||||
const result = await benchmarkPair(
|
||||
dedupRun(server, hashes, dedupCurrent),
|
||||
dedupRun(server, hashes, dedupBatched),
|
||||
(current, candidate) => verifyDedup(current, candidate, hashCount),
|
||||
);
|
||||
output.dedup.push({ hashCount, ...result });
|
||||
printPair(
|
||||
"C dedup HTTP probe, " + hashCount.toLocaleString("en-US") + " hashes",
|
||||
result,
|
||||
);
|
||||
if (hashCount <= MAX_DEDUP_HASHES) {
|
||||
console.log(
|
||||
" both paths used one accepted request and found " +
|
||||
result.representative.current.ownedCount +
|
||||
" owned hashes",
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
" current rejected " +
|
||||
result.representative.current.rejectedRequests +
|
||||
" request and found " +
|
||||
result.representative.current.ownedCount +
|
||||
" owned hashes",
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
" candidate used " +
|
||||
result.representative.candidate.requests +
|
||||
" accepted batches, found " +
|
||||
result.representative.candidate.ownedCount +
|
||||
", and avoided " +
|
||||
formatBytes(result.representative.candidate.contentBytesAvoided) +
|
||||
" of modeled content upload",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
}
|
||||
|
||||
const output = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
environment: {
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
release: os.release(),
|
||||
arch: process.arch,
|
||||
cpu: os.cpus()[0]?.model ?? "unknown",
|
||||
logicalCpus: os.cpus().length,
|
||||
gcExposed: Boolean(global.gc),
|
||||
},
|
||||
config,
|
||||
notes: {
|
||||
heap: "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
dedup:
|
||||
"The current >10k path is faster only because it is rejected and returns no owned hashes.",
|
||||
},
|
||||
suites: {},
|
||||
};
|
||||
|
||||
if (!global.gc) {
|
||||
console.warn("Warning: run with --expose-gc for less noisy heap deltas.");
|
||||
}
|
||||
console.log(
|
||||
"Node " +
|
||||
process.version +
|
||||
"; warmup " +
|
||||
config.warmup +
|
||||
"; samples " +
|
||||
config.samples +
|
||||
"; GC exposed " +
|
||||
Boolean(global.gc),
|
||||
);
|
||||
|
||||
if (config.suite === "all" || config.suite === "queue")
|
||||
await runQueueSuite(output.suites);
|
||||
if (config.suite === "all" || config.suite === "progress")
|
||||
await runProgressSuite(output.suites);
|
||||
if (config.suite === "all" || config.suite === "dedup")
|
||||
await runDedupSuite(output.suites);
|
||||
|
||||
output.blackhole = blackhole;
|
||||
|
||||
if (values.output) {
|
||||
const destination = resolve(values.output);
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
await writeFile(destination, JSON.stringify(output, null, 2) + "\n");
|
||||
console.log("Wrote JSON result to " + destination);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,272 @@
|
||||
//! Standalone A/B for allocation work in LocalBlobBackend::sync_blobs.
|
||||
//!
|
||||
//! Compile directly with rustc so this audit is independent of Cargo's
|
||||
//! benchmark targets:
|
||||
//! rustc --edition 2024 -O tools/perf-audit/local_sync_grouping.rs -o /tmp/local-sync-grouping
|
||||
|
||||
use std::hint::black_box;
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const CONCURRENCY: usize = 16;
|
||||
|
||||
type EmptyPrepFuture = Pin<Box<dyn Future<Output = usize>>>;
|
||||
|
||||
// Historical empty-call shape: build the paths Vec, then discover emptiness
|
||||
// inside the boxed future.
|
||||
#[inline(never)]
|
||||
fn current_empty_prep(root: &Path, hashes: &[String]) -> EmptyPrepFuture {
|
||||
let paths: Vec<PathBuf> = hashes
|
||||
.iter()
|
||||
.map(|hash| root.join(&hash[..2]).join(format!("{hash}.blob")))
|
||||
.collect();
|
||||
Box::pin(async move {
|
||||
if paths.is_empty() {
|
||||
0
|
||||
} else {
|
||||
paths.len()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Accepted candidate shape: an empty durability sweep has no observable work,
|
||||
// so return a capture-free ready future before allocating/preparing anything.
|
||||
#[inline(never)]
|
||||
fn fast_empty_prep(_root: &Path, hashes: &[String]) -> EmptyPrepFuture {
|
||||
if hashes.is_empty() {
|
||||
return Box::pin(async { 0 });
|
||||
}
|
||||
let paths: Vec<PathBuf> = hashes.iter().map(PathBuf::from).collect();
|
||||
Box::pin(async move { paths.len() })
|
||||
}
|
||||
|
||||
fn paths(count: usize) -> Vec<PathBuf> {
|
||||
(0..count)
|
||||
.map(|i| {
|
||||
let prefix = format!("{:02x}", i & 255);
|
||||
let hash = format!("{prefix}{:062x}", i);
|
||||
Path::new("/tmp/oxicloud/.blobs")
|
||||
.join(prefix)
|
||||
.join(format!("{hash}.blob"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// Exact grouping shape currently used before spawning the blocking tasks.
|
||||
fn current_groups(paths: Vec<PathBuf>) -> Vec<Vec<PathBuf>> {
|
||||
let group_size = paths.len().div_ceil(CONCURRENCY);
|
||||
paths.chunks(group_size).map(<[PathBuf]>::to_vec).collect()
|
||||
}
|
||||
|
||||
// Candidate: the caller already owns the Vec, so move each PathBuf into its
|
||||
// task group instead of cloning every path and keeping the original alive.
|
||||
fn moved_groups(paths: Vec<PathBuf>) -> Vec<Vec<PathBuf>> {
|
||||
let group_size = paths.len().div_ceil(CONCURRENCY);
|
||||
let mut source = paths.into_iter();
|
||||
let mut groups = Vec::with_capacity(CONCURRENCY.min(source.len()));
|
||||
loop {
|
||||
let group: Vec<PathBuf> = source.by_ref().take(group_size).collect();
|
||||
if group.is_empty() {
|
||||
break;
|
||||
}
|
||||
groups.push(group);
|
||||
}
|
||||
groups
|
||||
}
|
||||
|
||||
// Exact current distinct-parent preparation.
|
||||
fn current_dirs(paths: &[PathBuf]) -> Vec<PathBuf> {
|
||||
let mut dirs: Vec<PathBuf> = paths
|
||||
.iter()
|
||||
.filter_map(|path| path.parent().map(Path::to_path_buf))
|
||||
.collect();
|
||||
dirs.sort_unstable();
|
||||
dirs.dedup();
|
||||
dirs
|
||||
}
|
||||
|
||||
fn hex_prefix_symbol(byte: u8) -> Option<usize> {
|
||||
match byte {
|
||||
b'0'..=b'9' => Some((byte - b'0') as usize),
|
||||
b'a'..=b'f' => Some((byte - b'a' + 10) as usize),
|
||||
b'A'..=b'F' => Some((byte - b'A' + 16) as usize),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
// Candidate: exact-case hex prefixes fit in a tiny fixed bitmap. Case must
|
||||
// not be folded because `af/` and `AF/` differ on a case-sensitive filesystem.
|
||||
fn prefix_dirs(root: &Path, hashes: &[String], paths: &[PathBuf]) -> Vec<PathBuf> {
|
||||
if hashes.len() == 1 {
|
||||
let mut dirs = Vec::with_capacity(1);
|
||||
if let Some(parent) = paths[0].parent() {
|
||||
dirs.push(parent.to_owned());
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
let mut seen = [false; 22 * 22];
|
||||
let mut dirs = Vec::with_capacity(256.min(hashes.len()));
|
||||
for hash in hashes {
|
||||
let bytes = hash.as_bytes();
|
||||
let slot = hex_prefix_symbol(bytes[0])
|
||||
.zip(hex_prefix_symbol(bytes[1]))
|
||||
.map(|(high, low)| high * 22 + low);
|
||||
if slot.is_none_or(|slot| !std::mem::replace(&mut seen[slot], true)) {
|
||||
dirs.push(root.join(&hash[..2]));
|
||||
}
|
||||
}
|
||||
dirs
|
||||
}
|
||||
|
||||
fn median(mut values: Vec<Duration>) -> Duration {
|
||||
values.sort_unstable();
|
||||
values[values.len() / 2]
|
||||
}
|
||||
|
||||
fn measure_pair<T>(
|
||||
samples: usize,
|
||||
mut current: impl FnMut() -> T,
|
||||
mut candidate: impl FnMut() -> T,
|
||||
) -> (Duration, Duration) {
|
||||
for _ in 0..3 {
|
||||
black_box(current());
|
||||
black_box(candidate());
|
||||
}
|
||||
let mut current_times = Vec::with_capacity(samples);
|
||||
let mut candidate_times = Vec::with_capacity(samples);
|
||||
for sample in 0..samples {
|
||||
let run = |operation: &mut dyn FnMut() -> T, times: &mut Vec<Duration>| {
|
||||
let start = Instant::now();
|
||||
black_box(operation());
|
||||
times.push(start.elapsed());
|
||||
};
|
||||
// Alternate order so allocator/cache/thermal drift cannot consistently
|
||||
// favour either implementation.
|
||||
if sample % 2 == 0 {
|
||||
run(&mut current, &mut current_times);
|
||||
run(&mut candidate, &mut candidate_times);
|
||||
} else {
|
||||
run(&mut candidate, &mut candidate_times);
|
||||
run(&mut current, &mut current_times);
|
||||
}
|
||||
}
|
||||
(median(current_times), median(candidate_times))
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let empty: Vec<String> = Vec::new();
|
||||
let empty_repetitions = 100_000;
|
||||
let (current_empty, fast_empty) = measure_pair(
|
||||
31,
|
||||
|| {
|
||||
for _ in 0..empty_repetitions {
|
||||
let _ = black_box(current_empty_prep(
|
||||
Path::new("/tmp/oxicloud/.blobs"),
|
||||
&empty,
|
||||
));
|
||||
}
|
||||
},
|
||||
|| {
|
||||
for _ in 0..empty_repetitions {
|
||||
let _ = black_box(fast_empty_prep(Path::new("/tmp/oxicloud/.blobs"), &empty));
|
||||
}
|
||||
},
|
||||
);
|
||||
println!(
|
||||
"empty,current_ns,fast_return_ns,speedup\n0,{:.3},{:.3},{:.2}",
|
||||
current_empty.as_secs_f64() * 1e9 / empty_repetitions as f64,
|
||||
fast_empty.as_secs_f64() * 1e9 / empty_repetitions as f64,
|
||||
current_empty.as_secs_f64() / fast_empty.as_secs_f64(),
|
||||
);
|
||||
if std::env::args().any(|argument| argument == "--empty") {
|
||||
return;
|
||||
}
|
||||
|
||||
let mixed_case = vec![format!("af{}", "0".repeat(62)), format!("aF{}", "0".repeat(62))];
|
||||
let mixed_paths: Vec<PathBuf> = mixed_case
|
||||
.iter()
|
||||
.map(|hash| Path::new("/tmp/oxicloud/.blobs").join(&hash[..2]).join(hash))
|
||||
.collect();
|
||||
let mut current_mixed = current_dirs(&mixed_paths);
|
||||
let mut candidate_mixed = prefix_dirs(
|
||||
Path::new("/tmp/oxicloud/.blobs"),
|
||||
&mixed_case,
|
||||
&mixed_paths,
|
||||
);
|
||||
current_mixed.sort_unstable();
|
||||
candidate_mixed.sort_unstable();
|
||||
assert_eq!(current_mixed, candidate_mixed);
|
||||
|
||||
println!("count,current_group_us,moved_group_us,group_speedup,current_dirs_us,prefix_dirs_us,dirs_speedup");
|
||||
for count in [1, 8, 32, 128, 400, 1_600, 10_000, 100_000] {
|
||||
let source = paths(count);
|
||||
let hashes: Vec<String> = (0..count)
|
||||
.map(|i| format!("{:02x}{:062x}", i & 255, i))
|
||||
.collect();
|
||||
|
||||
let current_check = current_groups(source.clone());
|
||||
let moved_check = moved_groups(source.clone());
|
||||
assert_eq!(
|
||||
current_check.iter().flatten().collect::<Vec<_>>(),
|
||||
moved_check.iter().flatten().collect::<Vec<_>>()
|
||||
);
|
||||
let current_dir_check = current_dirs(&source);
|
||||
let mut candidate_dir_check =
|
||||
prefix_dirs(Path::new("/tmp/oxicloud/.blobs"), &hashes, &source);
|
||||
candidate_dir_check.sort_unstable();
|
||||
assert_eq!(current_dir_check, candidate_dir_check);
|
||||
|
||||
// Accumulate tiny cases inside each timed sample so sub-microsecond
|
||||
// operations are not decided by one timer tick. Report normalized
|
||||
// per-operation medians below.
|
||||
let repetitions = match count {
|
||||
1 => 10_000,
|
||||
8 => 1_000,
|
||||
32 => 250,
|
||||
_ => 1,
|
||||
};
|
||||
let (current_group, moved_group) = measure_pair(
|
||||
31,
|
||||
|| {
|
||||
for _ in 0..repetitions {
|
||||
black_box(current_groups(source.clone()));
|
||||
}
|
||||
},
|
||||
|| {
|
||||
for _ in 0..repetitions {
|
||||
black_box(moved_groups(source.clone()));
|
||||
}
|
||||
},
|
||||
);
|
||||
let (current_dir, candidate_dir) = measure_pair(
|
||||
31,
|
||||
|| {
|
||||
for _ in 0..repetitions {
|
||||
black_box(current_dirs(&source));
|
||||
}
|
||||
},
|
||||
|| {
|
||||
for _ in 0..repetitions {
|
||||
black_box(prefix_dirs(
|
||||
Path::new("/tmp/oxicloud/.blobs"),
|
||||
&hashes,
|
||||
&source,
|
||||
));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
let divisor = repetitions as f64;
|
||||
let current_group_us = current_group.as_secs_f64() * 1e6 / divisor;
|
||||
let moved_group_us = moved_group.as_secs_f64() * 1e6 / divisor;
|
||||
let current_dir_us = current_dir.as_secs_f64() * 1e6 / divisor;
|
||||
let candidate_dir_us = candidate_dir.as_secs_f64() * 1e6 / divisor;
|
||||
println!(
|
||||
"{count},{current_group_us:.3},{moved_group_us:.3},{:.2},{current_dir_us:.3},{candidate_dir_us:.3},{:.2}",
|
||||
current_group_us / moved_group_us,
|
||||
current_dir_us / candidate_dir_us,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
\pset format unaligned
|
||||
\pset tuples_only on
|
||||
|
||||
-- A/B the post-migration integrity sampler. `ORDER BY random()` assigns and
|
||||
-- sorts a random float for every blob. BLAKE3/SHA-style hex hashes are already
|
||||
-- uniformly distributed, so a cryptographically random pivot plus an indexed
|
||||
-- ordered window yields a rotating sample in O(log N + sample) work.
|
||||
BEGIN;
|
||||
CREATE TEMP TABLE perf_verify_blobs (
|
||||
hash varchar(64) PRIMARY KEY,
|
||||
size bigint NOT NULL
|
||||
);
|
||||
INSERT INTO perf_verify_blobs
|
||||
SELECT md5(n::text) || md5((n + 1000003)::text), 262144
|
||||
FROM generate_series(1, 1000000) n;
|
||||
ANALYZE perf_verify_blobs;
|
||||
|
||||
-- Exact sample-size/equivalence gates for a middle pivot and wraparound pivot.
|
||||
SELECT 'middle_count|' || COUNT(*) FROM (
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= '8000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100
|
||||
) sample;
|
||||
WITH tail AS (
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= 'fffff000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100
|
||||
), wrapped AS (
|
||||
SELECT * FROM tail
|
||||
UNION ALL
|
||||
(SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash < 'fffff000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash
|
||||
LIMIT (100 - (SELECT COUNT(*) FROM tail)))
|
||||
)
|
||||
SELECT 'wrap_count|' || COUNT(*) FROM wrapped;
|
||||
|
||||
\o /dev/null
|
||||
\timing on
|
||||
\echo random_warmup
|
||||
SELECT hash, size FROM perf_verify_blobs ORDER BY random() LIMIT 100;
|
||||
\echo indexed_warmup
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= '8000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100;
|
||||
|
||||
\echo random_1
|
||||
SELECT hash, size FROM perf_verify_blobs ORDER BY random() LIMIT 100;
|
||||
\echo indexed_1
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= '8000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100;
|
||||
\echo indexed_2
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= '4000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100;
|
||||
\echo random_2
|
||||
SELECT hash, size FROM perf_verify_blobs ORDER BY random() LIMIT 100;
|
||||
\echo random_3
|
||||
SELECT hash, size FROM perf_verify_blobs ORDER BY random() LIMIT 100;
|
||||
\echo indexed_3
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= 'c000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100;
|
||||
\echo indexed_4
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= '2000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100;
|
||||
\echo random_4
|
||||
SELECT hash, size FROM perf_verify_blobs ORDER BY random() LIMIT 100;
|
||||
\echo random_5
|
||||
SELECT hash, size FROM perf_verify_blobs ORDER BY random() LIMIT 100;
|
||||
\echo indexed_5
|
||||
SELECT hash, size FROM perf_verify_blobs
|
||||
WHERE hash >= 'e000000000000000000000000000000000000000000000000000000000000000'
|
||||
ORDER BY hash LIMIT 100;
|
||||
\timing off
|
||||
\o
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,170 @@
|
||||
//! Real-PostgreSQL gate for the blob-migration work set.
|
||||
//!
|
||||
//! `current` reproduces `run_migration` collecting every `(hash, size)` row
|
||||
//! before starting backend work. `paged` uses indexed keyset pages and drops
|
||||
//! each page after consuming it. The checksum/count gate proves that both
|
||||
//! consume the exact same ordered rows.
|
||||
|
||||
use futures::TryStreamExt;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::hint::black_box;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
async fn connect_with_retry(url: &str) -> sqlx::PgPool {
|
||||
let mut last_error = None;
|
||||
for _ in 0..12 {
|
||||
match PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.acquire_timeout(Duration::from_secs(5))
|
||||
.connect(url)
|
||||
.await
|
||||
{
|
||||
Ok(pool) => return pool,
|
||||
Err(error) => {
|
||||
last_error = Some(error);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!(
|
||||
"connect PostgreSQL after retries: {}",
|
||||
last_error.expect("at least one connection attempt")
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed(pool: &sqlx::PgPool, rows: i64) {
|
||||
sqlx::query("CREATE SCHEMA IF NOT EXISTS storage")
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("create storage schema");
|
||||
sqlx::query(
|
||||
"CREATE TABLE IF NOT EXISTS storage.blobs (
|
||||
hash text PRIMARY KEY,
|
||||
size bigint NOT NULL
|
||||
)",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("create blob table");
|
||||
sqlx::query("TRUNCATE storage.blobs")
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("truncate blob table");
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.blobs(hash, size)
|
||||
SELECT lpad(to_hex(n), 64, '0'), 1024 + (n % 1048576)
|
||||
FROM generate_series(1, $1) AS n",
|
||||
)
|
||||
.bind(rows)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed blob table");
|
||||
sqlx::query("ANALYZE storage.blobs")
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("analyze blob table");
|
||||
}
|
||||
|
||||
fn consume(checksum: &mut u64, hash: &str, size: i64) {
|
||||
let first = hash.as_bytes().first().copied().unwrap_or_default() as u64;
|
||||
let last = hash.as_bytes().last().copied().unwrap_or_default() as u64;
|
||||
*checksum = checksum
|
||||
.wrapping_mul(0x100_0000_01b3)
|
||||
.wrapping_add(first)
|
||||
.wrapping_add(last << 8)
|
||||
.wrapping_add(size as u64);
|
||||
black_box(checksum);
|
||||
}
|
||||
|
||||
async fn current(pool: &sqlx::PgPool) -> (usize, u64, usize) {
|
||||
let work: Vec<(String, i64)> =
|
||||
sqlx::query_as("SELECT hash, size FROM storage.blobs ORDER BY hash")
|
||||
.fetch(pool)
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("fetch current work set");
|
||||
let peak_rows = work.len();
|
||||
let mut checksum = 0_u64;
|
||||
for (hash, size) in &work {
|
||||
consume(&mut checksum, hash, *size);
|
||||
}
|
||||
(work.len(), checksum, peak_rows)
|
||||
}
|
||||
|
||||
async fn streamed(pool: &sqlx::PgPool) -> (usize, u64, usize) {
|
||||
let mut rows =
|
||||
sqlx::query_as::<_, (String, i64)>("SELECT hash, size FROM storage.blobs ORDER BY hash")
|
||||
.fetch(pool);
|
||||
let mut count = 0usize;
|
||||
let mut checksum = 0_u64;
|
||||
while let Some(row) = rows.try_next().await.expect("stream work row") {
|
||||
consume(&mut checksum, &row.0, row.1);
|
||||
count += 1;
|
||||
}
|
||||
(count, checksum, 1)
|
||||
}
|
||||
|
||||
async fn paged(pool: &sqlx::PgPool, page_size: i64) -> (usize, u64, usize) {
|
||||
let mut after = String::new();
|
||||
let mut count = 0usize;
|
||||
let mut checksum = 0_u64;
|
||||
let mut peak_rows = 0usize;
|
||||
loop {
|
||||
let page: Vec<(String, i64)> = sqlx::query_as(
|
||||
"SELECT hash, size FROM storage.blobs
|
||||
WHERE hash > $1 ORDER BY hash LIMIT $2",
|
||||
)
|
||||
.bind(&after)
|
||||
.bind(page_size)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.expect("fetch keyset page");
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
peak_rows = peak_rows.max(page.len());
|
||||
after.clone_from(&page.last().expect("non-empty page").0);
|
||||
for (hash, size) in &page {
|
||||
consume(&mut checksum, hash, *size);
|
||||
}
|
||||
count += page.len();
|
||||
}
|
||||
(count, checksum, peak_rows)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let url = std::env::var("DATABASE_URL").expect("DATABASE_URL is required");
|
||||
let mode = std::env::args().nth(1).unwrap_or_else(|| "current".into());
|
||||
let value = std::env::args()
|
||||
.nth(2)
|
||||
.and_then(|v| v.parse::<i64>().ok())
|
||||
.unwrap_or(1_000_000);
|
||||
// The local Docker bridge occasionally drops a new host-side connection.
|
||||
// Connection retries happen before the timed region and apply identically
|
||||
// to every mode, so transport setup cannot skew an algorithm sample.
|
||||
let pool = connect_with_retry(&url).await;
|
||||
|
||||
if mode == "seed" {
|
||||
seed(&pool, value).await;
|
||||
println!("seeded_rows={value}");
|
||||
return;
|
||||
}
|
||||
|
||||
// Warm the PostgreSQL/index pages without retaining Rust rows.
|
||||
let _: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("warm count");
|
||||
let started = Instant::now();
|
||||
let (rows, checksum, peak_rows) = match mode.as_str() {
|
||||
"current" => current(&pool).await,
|
||||
"stream" => streamed(&pool).await,
|
||||
"paged" => paged(&pool, value).await,
|
||||
other => panic!("unknown mode: {other}"),
|
||||
};
|
||||
println!(
|
||||
"mode={mode} value={value} rows={rows} checksum={checksum} peak_rows={peak_rows} elapsed_ms={:.3}",
|
||||
started.elapsed().as_secs_f64() * 1_000.0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Process-isolated memory gate for the delta worker's upload queue.
|
||||
//
|
||||
// The earlier in-process heap delta was biased: Array.shift() runs long enough
|
||||
// for V8 to collect garbage during the measurement, while the cursor finishes
|
||||
// before the next GC. This harness gives every sample a fresh Node process and
|
||||
// compares max RSS plus post-GC retained RSS/heap. It models the worker's
|
||||
// permanent ordered `chunks` array as well as the second uploadQueue reference.
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import process from 'node:process';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024;
|
||||
const MODES = [
|
||||
{ name: 'current-shift', cursor: false, clear: false, threshold: 0, compact: 'none' },
|
||||
{
|
||||
name: 'cursor-clear-1024',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 1024,
|
||||
compact: 'copy',
|
||||
},
|
||||
{
|
||||
name: 'cursor-clear-4096',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 4096,
|
||||
compact: 'copy',
|
||||
},
|
||||
{
|
||||
name: 'cursor-clear-16384',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 16384,
|
||||
compact: 'copy',
|
||||
},
|
||||
{
|
||||
name: 'cursor-no-clear-4096',
|
||||
cursor: true,
|
||||
clear: false,
|
||||
threshold: 4096,
|
||||
compact: 'copy',
|
||||
},
|
||||
{
|
||||
name: 'cursor-splice-4096',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 4096,
|
||||
compact: 'splice',
|
||||
},
|
||||
{
|
||||
name: 'cursor-splice-16384',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 16384,
|
||||
compact: 'splice',
|
||||
},
|
||||
{
|
||||
name: 'cursor-slice-4096',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 4096,
|
||||
compact: 'slice',
|
||||
},
|
||||
{
|
||||
name: 'cursor-reset-4096',
|
||||
cursor: true,
|
||||
clear: true,
|
||||
threshold: 4096,
|
||||
compact: 'copy',
|
||||
resetWhenEmpty: true,
|
||||
},
|
||||
];
|
||||
const SHAPES = ['prefilled', 'streaming-ahead', 'streaming-balanced'];
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = new Map();
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
out.set(argv[index], argv[index + 1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function positiveInteger(name, raw) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 1) {
|
||||
throw new Error(`${name} must be an integer >= 1; received ${raw}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function chunkAt(index) {
|
||||
const size = (1 + ((Math.imul(index + 1, 2_654_435_761) >>> 28) & 7)) * 32 * 1024;
|
||||
return { h: `chunk-${index}`, s: size, offset: index * 32 * 1024 };
|
||||
}
|
||||
|
||||
function fold(checksum, chunk) {
|
||||
return Math.imul(checksum ^ chunk.s ^ (chunk.offset >>> 0), 16_777_619) >>> 0;
|
||||
}
|
||||
|
||||
function runChild(mode, shape, count) {
|
||||
if (typeof globalThis.gc !== 'function') {
|
||||
throw new Error('child must run with --expose-gc');
|
||||
}
|
||||
|
||||
const chunks = Array.from({ length: count }, (_, index) => chunkAt(index));
|
||||
let queue = shape === 'prefilled' ? chunks.slice() : [];
|
||||
let head = 0;
|
||||
let checksum = 2_166_136_261;
|
||||
let consumed = 0;
|
||||
let batches = 0;
|
||||
let compactions = 0;
|
||||
|
||||
const available = () => (mode.cursor ? queue.length - head : queue.length);
|
||||
const drainBatch = () => {
|
||||
if (available() === 0) return false;
|
||||
let bytes = 0;
|
||||
while (available() > 0 && bytes < UPLOAD_BATCH_BYTES) {
|
||||
let chunk;
|
||||
if (mode.cursor) {
|
||||
chunk = queue[head];
|
||||
if (mode.clear) queue[head] = undefined;
|
||||
head++;
|
||||
} else {
|
||||
chunk = queue.shift();
|
||||
}
|
||||
checksum = fold(checksum, chunk);
|
||||
bytes += chunk.s;
|
||||
consumed++;
|
||||
}
|
||||
|
||||
if (mode.cursor) {
|
||||
if (head === queue.length) {
|
||||
if (mode.resetWhenEmpty) queue = [];
|
||||
else queue.length = 0;
|
||||
head = 0;
|
||||
} else if (head >= mode.threshold && head * 2 >= queue.length) {
|
||||
if (mode.compact === 'splice') {
|
||||
queue.splice(0, head);
|
||||
} else if (mode.compact === 'slice') {
|
||||
queue = queue.slice(head);
|
||||
} else {
|
||||
queue.copyWithin(0, head);
|
||||
queue.length -= head;
|
||||
}
|
||||
head = 0;
|
||||
compactions++;
|
||||
}
|
||||
}
|
||||
batches++;
|
||||
return true;
|
||||
};
|
||||
|
||||
globalThis.gc();
|
||||
const baseline = process.memoryUsage();
|
||||
const started = performance.now();
|
||||
|
||||
if (shape === 'prefilled') {
|
||||
while (drainBatch()) {}
|
||||
} else {
|
||||
const drainsPerProduce = shape === 'streaming-ahead' ? 1 : 5;
|
||||
const produceBatch = 256;
|
||||
for (let start = 0; start < chunks.length; start += produceBatch) {
|
||||
queue.push(...chunks.slice(start, Math.min(start + produceBatch, chunks.length)));
|
||||
for (let drain = 0; drain < drainsPerProduce; drain++) {
|
||||
if (!drainBatch()) break;
|
||||
}
|
||||
}
|
||||
while (drainBatch()) {}
|
||||
}
|
||||
|
||||
const wallMs = performance.now() - started;
|
||||
const maxRssBytes = process.resourceUsage().maxRSS * 1024;
|
||||
globalThis.gc();
|
||||
const after = process.memoryUsage();
|
||||
|
||||
// Keep the production-equivalent ordered chunk table live through the final
|
||||
// measurement. The queue must be logically empty in every implementation.
|
||||
checksum ^= chunks.length;
|
||||
if (consumed !== count || available() !== 0) {
|
||||
throw new Error(`queue invariant failed: consumed=${consumed}, available=${available()}`);
|
||||
}
|
||||
|
||||
return {
|
||||
mode: mode.name,
|
||||
shape,
|
||||
count,
|
||||
wallMs,
|
||||
checksum: checksum >>> 0,
|
||||
consumed,
|
||||
batches,
|
||||
compactions,
|
||||
baselineRssBytes: baseline.rss,
|
||||
baselineHeapBytes: baseline.heapUsed,
|
||||
maxRssBytes,
|
||||
peakRssDeltaBytes: Math.max(0, maxRssBytes - baseline.rss),
|
||||
retainedRssDeltaBytes: after.rss - baseline.rss,
|
||||
retainedHeapDeltaBytes: after.heapUsed - baseline.heapUsed,
|
||||
};
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
return sorted[Math.floor(sorted.length / 2)];
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.has('--child')) {
|
||||
const modeName = args.get('--mode');
|
||||
const mode = MODES.find((candidate) => candidate.name === modeName);
|
||||
if (!mode) throw new Error(`unknown mode ${modeName}`);
|
||||
const shape = args.get('--shape');
|
||||
if (!SHAPES.includes(shape)) throw new Error(`unknown shape ${shape}`);
|
||||
const count = positiveInteger('count', args.get('--count'));
|
||||
process.stdout.write(`${JSON.stringify(runChild(mode, shape, count))}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const count = positiveInteger('count', args.get('--count') ?? '100000');
|
||||
const samples = positiveInteger('samples', args.get('--samples') ?? '5');
|
||||
const output = args.get('--output');
|
||||
const rows = [];
|
||||
|
||||
for (let sample = 0; sample < samples; sample++) {
|
||||
const modes = sample % 2 === 0 ? MODES : [...MODES].reverse();
|
||||
const shapes = sample % 2 === 0 ? SHAPES : [...SHAPES].reverse();
|
||||
for (const shape of shapes) {
|
||||
for (const mode of modes) {
|
||||
const child = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
'--expose-gc',
|
||||
new URL(import.meta.url).pathname,
|
||||
'--child',
|
||||
'1',
|
||||
'--mode',
|
||||
mode.name,
|
||||
'--shape',
|
||||
shape,
|
||||
'--count',
|
||||
String(count),
|
||||
],
|
||||
{ encoding: 'utf8', maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
if (child.status !== 0) {
|
||||
throw new Error(`child failed (${mode.name}/${shape}): ${child.stderr || child.stdout}`);
|
||||
}
|
||||
rows.push(JSON.parse(child.stdout.trim()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const shape of SHAPES) {
|
||||
const reference = rows.find((row) => row.shape === shape && row.mode === MODES[0].name);
|
||||
for (const row of rows.filter((candidate) => candidate.shape === shape)) {
|
||||
if (
|
||||
row.checksum !== reference.checksum ||
|
||||
row.consumed !== reference.consumed ||
|
||||
row.batches !== reference.batches
|
||||
) {
|
||||
throw new Error(`semantic mismatch for ${shape}/${row.mode}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const results = SHAPES.flatMap((shape) =>
|
||||
MODES.map((mode) => {
|
||||
const samplesForMode = rows.filter((row) => row.shape === shape && row.mode === mode.name);
|
||||
return {
|
||||
shape,
|
||||
mode: mode.name,
|
||||
wallMedianMs: Number(median(samplesForMode.map((row) => row.wallMs)).toFixed(3)),
|
||||
maxRssBytesMedian: median(samplesForMode.map((row) => row.maxRssBytes)),
|
||||
peakRssDeltaBytesMedian: median(samplesForMode.map((row) => row.peakRssDeltaBytes)),
|
||||
retainedRssDeltaBytesMedian: median(samplesForMode.map((row) => row.retainedRssDeltaBytes)),
|
||||
retainedHeapDeltaBytesMedian: median(samplesForMode.map((row) => row.retainedHeapDeltaBytes)),
|
||||
maxRssSamplesBytes: samplesForMode.map((row) => row.maxRssBytes),
|
||||
peakRssDeltaSamplesBytes: samplesForMode.map((row) => row.peakRssDeltaBytes),
|
||||
compactions: samplesForMode[0].compactions,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
const rendered = `${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
environment: { node: process.version, platform: process.platform, arch: process.arch },
|
||||
fixture: { chunks: count, samples, uploadBatchBytes: UPLOAD_BATCH_BYTES },
|
||||
note: 'Each row is a fresh process; maxRSS is process.resourceUsage().maxRSS. The ordered chunks table remains live through final GC.',
|
||||
results,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`;
|
||||
if (output) writeFileSync(output, rendered);
|
||||
process.stdout.write(rendered);
|
||||
@@ -0,0 +1,242 @@
|
||||
//! Rejected diagnostic harness for the delta loose-chunk write path.
|
||||
//!
|
||||
//! It models an idempotent remote object store whose PUT overwrites the same
|
||||
//! key (the behaviour of the current S3/Azure adapters) and counts physical
|
||||
//! PUT calls/bytes. The measured prefilter candidate was rolled back because
|
||||
//! it regressed the normal negotiated-miss path and weakened self-healing.
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::stream;
|
||||
use oxicloud::application::ports::blob_storage_ports::{
|
||||
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
||||
};
|
||||
use oxicloud::domain::errors::DomainError;
|
||||
use oxicloud::infrastructure::services::dedup_service::DedupService;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
use uuid::Uuid;
|
||||
|
||||
type BoxFut<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
#[derive(Default)]
|
||||
struct CountingRemote {
|
||||
enable_prefilter: bool,
|
||||
objects: Mutex<HashMap<String, Bytes>>,
|
||||
puts: AtomicU64,
|
||||
put_bytes: AtomicU64,
|
||||
exists_calls: AtomicU64,
|
||||
sync_calls: AtomicU64,
|
||||
sync_hashes: AtomicU64,
|
||||
}
|
||||
|
||||
impl CountingRemote {
|
||||
fn new(enable_prefilter: bool) -> Self {
|
||||
Self {
|
||||
enable_prefilter,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn reset(&self) {
|
||||
self.puts.store(0, Ordering::Relaxed);
|
||||
self.put_bytes.store(0, Ordering::Relaxed);
|
||||
self.exists_calls.store(0, Ordering::Relaxed);
|
||||
self.sync_calls.store(0, Ordering::Relaxed);
|
||||
self.sync_hashes.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStorageBackend for CountingRemote {
|
||||
fn initialize(&self) -> BoxFut<'_, Result<(), DomainError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn put_blob(&self, _hash: &str, _source_path: &Path) -> BoxFut<'_, Result<u64, DomainError>> {
|
||||
Box::pin(async {
|
||||
Err(DomainError::internal_error(
|
||||
"probe",
|
||||
"put_blob is outside this probe",
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn put_blob_from_bytes(&self, hash: &str, data: Bytes) -> BoxFut<'_, Result<u64, DomainError>> {
|
||||
self.puts.fetch_add(1, Ordering::Relaxed);
|
||||
self.put_bytes
|
||||
.fetch_add(data.len() as u64, Ordering::Relaxed);
|
||||
self.objects
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(hash.to_string(), data.clone());
|
||||
Box::pin(async move { Ok(data.len() as u64) })
|
||||
}
|
||||
|
||||
fn put_blob_from_bytes_unsynced(
|
||||
&self,
|
||||
hash: &str,
|
||||
data: Bytes,
|
||||
) -> BoxFut<'_, Result<u64, DomainError>> {
|
||||
self.put_blob_from_bytes(hash, data)
|
||||
}
|
||||
|
||||
fn sync_blobs(&self, hashes: &[String]) -> BoxFut<'_, Result<(), DomainError>> {
|
||||
self.sync_calls.fetch_add(1, Ordering::Relaxed);
|
||||
self.sync_hashes
|
||||
.fetch_add(hashes.len() as u64, Ordering::Relaxed);
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>> {
|
||||
let data = self.objects.lock().unwrap().get(hash).cloned();
|
||||
Box::pin(async move {
|
||||
let data = data.ok_or_else(|| DomainError::not_found("probe blob", "missing"))?;
|
||||
Ok(Box::pin(stream::once(async move { Ok(data) })) as BlobStream)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_blob_range_stream(
|
||||
&self,
|
||||
hash: &str,
|
||||
_start: u64,
|
||||
_end: Option<u64>,
|
||||
) -> BoxFut<'_, Result<BlobStream, DomainError>> {
|
||||
self.get_blob_stream(hash)
|
||||
}
|
||||
|
||||
fn delete_blob(&self, hash: &str) -> BoxFut<'_, Result<(), DomainError>> {
|
||||
self.objects.lock().unwrap().remove(hash);
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn blob_exists(&self, hash: &str) -> BoxFut<'_, Result<bool, DomainError>> {
|
||||
self.exists_calls.fetch_add(1, Ordering::Relaxed);
|
||||
let present = self.objects.lock().unwrap().contains_key(hash);
|
||||
Box::pin(async move { Ok(present) })
|
||||
}
|
||||
|
||||
fn blob_size(&self, hash: &str) -> BoxFut<'_, Result<u64, DomainError>> {
|
||||
let size = self
|
||||
.objects
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(hash)
|
||||
.map_or(0, |data| data.len() as u64);
|
||||
Box::pin(async move { Ok(size) })
|
||||
}
|
||||
|
||||
fn health_check(&self) -> BoxFut<'_, Result<StorageHealthStatus, DomainError>> {
|
||||
Box::pin(async {
|
||||
Ok(StorageHealthStatus {
|
||||
connected: true,
|
||||
backend_type: "counting-remote".into(),
|
||||
message: "probe".into(),
|
||||
available_bytes: None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn backend_type(&self) -> &'static str {
|
||||
if self.enable_prefilter {
|
||||
"counting-remote"
|
||||
} else {
|
||||
// Selects the production raw-local fast path while retaining the
|
||||
// same remote-style physical PUT counter for an in-binary A/B.
|
||||
"local"
|
||||
}
|
||||
}
|
||||
|
||||
fn local_blob_path(&self, _hash: &str) -> Option<PathBuf> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn payloads(seed: u128, count: usize, size: usize) -> Vec<Bytes> {
|
||||
(0..count)
|
||||
.map(|index| {
|
||||
let mut data = vec![0_u8; size];
|
||||
data[..16].copy_from_slice(&seed.to_le_bytes());
|
||||
data[16..24].copy_from_slice(&(index as u64).to_le_bytes());
|
||||
Bytes::from(data)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn run_case(
|
||||
name: &str,
|
||||
service: &DedupService,
|
||||
backend: &CountingRemote,
|
||||
pool: &sqlx::PgPool,
|
||||
frames: &[Bytes],
|
||||
) -> Vec<String> {
|
||||
let hashes: Vec<String> = frames
|
||||
.iter()
|
||||
.map(|frame| blake3::hash(frame).to_hex().to_string())
|
||||
.collect();
|
||||
let existing_before: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs WHERE hash = ANY($1::text[])")
|
||||
.bind(&hashes)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset();
|
||||
let input = stream::iter(frames.iter().cloned().map(Ok::<_, DomainError>));
|
||||
let started = Instant::now();
|
||||
let received = service.store_loose_chunks(input).await.unwrap();
|
||||
let elapsed = started.elapsed();
|
||||
println!(
|
||||
"{name}: frames={} existing_before={} logical_bytes={} heads={} puts={} physical_put_bytes={} sync_calls={} sync_hashes={} elapsed_ms={:.3}",
|
||||
frames.len(),
|
||||
existing_before,
|
||||
frames.iter().map(Bytes::len).sum::<usize>(),
|
||||
backend.exists_calls.load(Ordering::Relaxed),
|
||||
backend.puts.load(Ordering::Relaxed),
|
||||
backend.put_bytes.load(Ordering::Relaxed),
|
||||
backend.sync_calls.load(Ordering::Relaxed),
|
||||
backend.sync_hashes.load(Ordering::Relaxed),
|
||||
elapsed.as_secs_f64() * 1000.0,
|
||||
);
|
||||
received.into_iter().map(|(hash, _)| hash).collect()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL is required");
|
||||
let pool = Arc::new(
|
||||
PgPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let enable_prefilter = std::env::var("PROBE_PREFILTER").map_or(true, |v| v != "0");
|
||||
println!("prefilter={enable_prefilter}");
|
||||
let backend = Arc::new(CountingRemote::new(enable_prefilter));
|
||||
let service = DedupService::new(backend.clone(), pool.clone(), pool.clone());
|
||||
|
||||
// 400 × 256 KiB = exactly 100 MiB, the default per-request byte budget.
|
||||
let seed = Uuid::new_v4().as_u128();
|
||||
let known = payloads(seed, 400, 256 * 1024);
|
||||
let fresh = payloads(seed.wrapping_add(1), 400, 256 * 1024);
|
||||
let half_fresh = payloads(seed.wrapping_add(2), 200, 256 * 1024);
|
||||
|
||||
let mut cleanup = run_case("seed", &service, &backend, pool.as_ref(), &known).await;
|
||||
cleanup.extend(run_case("all_hit", &service, &backend, pool.as_ref(), &known).await);
|
||||
cleanup.extend(run_case("all_miss", &service, &backend, pool.as_ref(), &fresh).await);
|
||||
|
||||
let mixed: Vec<Bytes> = known[..200].iter().chain(&half_fresh).cloned().collect();
|
||||
cleanup.extend(run_case("half_hit", &service, &backend, pool.as_ref(), &mixed).await);
|
||||
|
||||
cleanup.sort_unstable();
|
||||
cleanup.dedup();
|
||||
sqlx::query("DELETE FROM storage.blobs WHERE hash = ANY($1::text[])")
|
||||
.bind(&cleanup)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
//! Rejected diagnostic A/B for identical-content overwrite reference accounting.
|
||||
//!
|
||||
//! The caller supplies a disposable PostgreSQL database containing the minimal
|
||||
//! storage schema used below. This exercises the public FileWritePort method,
|
||||
//! not a copy of `swap_blob_hash`.
|
||||
|
||||
use moka::sync::Cache;
|
||||
use oxicloud::application::ports::blob_lifecycle::BlobLifecycleHook;
|
||||
use oxicloud::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use oxicloud::application::ports::storage_ports::FileWritePort;
|
||||
use oxicloud::application::services::blob_lifecycle_service::BlobLifecycleService;
|
||||
use oxicloud::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
||||
use oxicloud::infrastructure::services::dedup_service::DedupService;
|
||||
use oxicloud::infrastructure::services::local_blob_backend::LocalBlobBackend;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingBlobHook {
|
||||
deleted: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RecordingBlobHook {
|
||||
fn deleted(&self) -> Vec<String> {
|
||||
self.deleted.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn clear(&self) {
|
||||
self.deleted.lock().unwrap().clear();
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobLifecycleHook for RecordingBlobHook {
|
||||
fn on_blob_created(&self, _blob_hash: &str, _content_type: Option<&str>) {}
|
||||
|
||||
fn on_blob_deleted(&self, blob_hash: &str) {
|
||||
self.deleted.lock().unwrap().push(blob_hash.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
async fn reset(pool: &sqlx::PgPool) {
|
||||
sqlx::query("TRUNCATE storage.files, storage.chunk_manifests, storage.blobs")
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn ref_count(pool: &sqlx::PgPool, hash: &str) -> Option<i32> {
|
||||
sqlx::query_scalar("SELECT ref_count FROM storage.blobs WHERE hash = $1")
|
||||
.bind(hash)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn manifest_ref_count(pool: &sqlx::PgPool, hash: &str) -> Option<i32> {
|
||||
sqlx::query_scalar("SELECT ref_count FROM storage.chunk_manifests WHERE file_hash = $1")
|
||||
.bind(hash)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn percentile_ms(samples_ns: &mut [u128], percentile: usize) -> f64 {
|
||||
samples_ns.sort_unstable();
|
||||
let index = (samples_ns.len() - 1) * percentile / 100;
|
||||
samples_ns[index] as f64 / 1_000_000.0
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL is required");
|
||||
let iterations = std::env::var("ITERATIONS")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or(1_000);
|
||||
let different_iterations = std::env::var("DIFFERENT_ITERATIONS")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or(100);
|
||||
let pool = Arc::new(
|
||||
PgPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let backend = Arc::new(LocalBlobBackend::new(Path::new(temp.path())));
|
||||
backend.initialize().await.unwrap();
|
||||
let recording_hook = Arc::new(RecordingBlobHook::default());
|
||||
let lifecycle = Arc::new(
|
||||
BlobLifecycleService::new().with_hook(recording_hook.clone() as Arc<dyn BlobLifecycleHook>),
|
||||
);
|
||||
let dedup = Arc::new(
|
||||
DedupService::new(backend, pool.clone(), pool.clone()).with_blob_lifecycle(lifecycle),
|
||||
);
|
||||
let repo = FileBlobWriteRepository::new(
|
||||
pool.clone(),
|
||||
dedup.clone(),
|
||||
Cache::builder().max_capacity(16).build(),
|
||||
);
|
||||
let caller = Uuid::new_v4();
|
||||
let file_id = Uuid::new_v4();
|
||||
let hash_a = blake3::hash(b"same-content").to_hex().to_string();
|
||||
|
||||
reset(pool.as_ref()).await;
|
||||
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 12, 1)")
|
||||
.bind(&hash_a)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.files (id, blob_hash, size, updated_by) VALUES ($1, $2, 12, $3)",
|
||||
)
|
||||
.bind(file_id)
|
||||
.bind(&hash_a)
|
||||
.bind(caller)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let started = Instant::now();
|
||||
let mut identical_samples = Vec::with_capacity(iterations);
|
||||
for _ in 0..iterations {
|
||||
let iteration_started = Instant::now();
|
||||
// Models the reference acquired by the ingest layer immediately before
|
||||
// FileWritePort consumes it.
|
||||
dedup.add_reference(&hash_a).await.unwrap();
|
||||
repo.update_file_content_with_blob(&file_id.to_string(), &hash_a, 12, None, caller)
|
||||
.await
|
||||
.unwrap();
|
||||
identical_samples.push(iteration_started.elapsed().as_nanos());
|
||||
}
|
||||
let identical_ref = ref_count(pool.as_ref(), &hash_a).await;
|
||||
let identical_p50 = percentile_ms(&mut identical_samples, 50);
|
||||
let identical_p95 = percentile_ms(&mut identical_samples, 95);
|
||||
println!(
|
||||
"identical: iterations={iterations} final_ref_count={:?} elapsed_ms={:.3} \
|
||||
p50_ms={identical_p50:.3} p95_ms={identical_p95:.3} roundtrips_per_iteration=3",
|
||||
identical_ref,
|
||||
started.elapsed().as_secs_f64() * 1_000.0
|
||||
);
|
||||
assert_eq!(identical_ref, Some(1));
|
||||
assert!(recording_hook.deleted().is_empty());
|
||||
|
||||
// Same-hash CDC manifest control. A single-chunk file deliberately has a
|
||||
// row in both tables under the same hash: only the manifest reference is
|
||||
// file-level and the chunk row must remain unchanged.
|
||||
reset(pool.as_ref()).await;
|
||||
recording_hook.clear();
|
||||
let manifest_iterations = iterations.min(100);
|
||||
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 12, 1)")
|
||||
.bind(&hash_a)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.chunk_manifests
|
||||
(file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, ref_count)
|
||||
VALUES ($1, ARRAY[$1], ARRAY[12::bigint], 12, 1, 1)",
|
||||
)
|
||||
.bind(&hash_a)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.files (id, blob_hash, size, updated_by) VALUES ($1, $2, 12, $3)",
|
||||
)
|
||||
.bind(file_id)
|
||||
.bind(&hash_a)
|
||||
.bind(caller)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
let started = Instant::now();
|
||||
let mut manifest_samples = Vec::with_capacity(manifest_iterations);
|
||||
for _ in 0..manifest_iterations {
|
||||
let iteration_started = Instant::now();
|
||||
dedup.add_reference(&hash_a).await.unwrap();
|
||||
repo.update_file_content_with_blob(&file_id.to_string(), &hash_a, 12, None, caller)
|
||||
.await
|
||||
.unwrap();
|
||||
manifest_samples.push(iteration_started.elapsed().as_nanos());
|
||||
}
|
||||
let manifest_ref = manifest_ref_count(pool.as_ref(), &hash_a).await;
|
||||
let chunk_ref = ref_count(pool.as_ref(), &hash_a).await;
|
||||
let manifest_p50 = percentile_ms(&mut manifest_samples, 50);
|
||||
let manifest_p95 = percentile_ms(&mut manifest_samples, 95);
|
||||
println!(
|
||||
"identical_manifest: iterations={manifest_iterations} manifest_ref={manifest_ref:?} \
|
||||
chunk_ref={chunk_ref:?} elapsed_ms={:.3} p50_ms={manifest_p50:.3} \
|
||||
p95_ms={manifest_p95:.3} roundtrips_per_iteration=2",
|
||||
started.elapsed().as_secs_f64() * 1_000.0
|
||||
);
|
||||
assert_eq!(manifest_ref, Some(1));
|
||||
assert_eq!(chunk_ref, Some(1));
|
||||
assert!(recording_hook.deleted().is_empty());
|
||||
|
||||
// Alternating-content latency control. A permanent base reference keeps
|
||||
// both blobs alive, isolating the normal different-hash decrement path.
|
||||
reset(pool.as_ref()).await;
|
||||
recording_hook.clear();
|
||||
let hash_b = blake3::hash(b"different-content").to_hex().to_string();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.blobs (hash, size, ref_count)
|
||||
VALUES ($1, 12, 2), ($2, 17, 1)",
|
||||
)
|
||||
.bind(&hash_a)
|
||||
.bind(&hash_b)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.files (id, blob_hash, size, updated_by) VALUES ($1, $2, 12, $3)",
|
||||
)
|
||||
.bind(file_id)
|
||||
.bind(&hash_a)
|
||||
.bind(caller)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
let started = Instant::now();
|
||||
let mut alternating_samples = Vec::with_capacity(different_iterations);
|
||||
for i in 0..different_iterations {
|
||||
let iteration_started = Instant::now();
|
||||
let (target, size) = if i % 2 == 0 {
|
||||
(&hash_b, 17)
|
||||
} else {
|
||||
(&hash_a, 12)
|
||||
};
|
||||
dedup.add_reference(target).await.unwrap();
|
||||
repo.update_file_content_with_blob(&file_id.to_string(), target, size, None, caller)
|
||||
.await
|
||||
.unwrap();
|
||||
alternating_samples.push(iteration_started.elapsed().as_nanos());
|
||||
}
|
||||
let alternating_a_ref = ref_count(pool.as_ref(), &hash_a).await;
|
||||
let alternating_b_ref = ref_count(pool.as_ref(), &hash_b).await;
|
||||
let expected_a = if different_iterations % 2 == 0 { 2 } else { 1 };
|
||||
let expected_b = if different_iterations % 2 == 0 { 1 } else { 2 };
|
||||
let alternating_p50 = percentile_ms(&mut alternating_samples, 50);
|
||||
let alternating_p95 = percentile_ms(&mut alternating_samples, 95);
|
||||
println!(
|
||||
"alternating: iterations={different_iterations} a_ref={alternating_a_ref:?} \
|
||||
b_ref={alternating_b_ref:?} elapsed_ms={:.3} p50_ms={alternating_p50:.3} \
|
||||
p95_ms={alternating_p95:.3} roundtrips_per_iteration=8",
|
||||
started.elapsed().as_secs_f64() * 1_000.0
|
||||
);
|
||||
assert_eq!(alternating_a_ref, Some(expected_a));
|
||||
assert_eq!(alternating_b_ref, Some(expected_b));
|
||||
assert!(recording_hook.deleted().is_empty());
|
||||
|
||||
// Different-content control: the old reference must disappear, the new
|
||||
// reference must remain exactly once.
|
||||
reset(pool.as_ref()).await;
|
||||
recording_hook.clear();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.blobs (hash, size, ref_count)
|
||||
VALUES ($1, 12, 1), ($2, 17, 1)",
|
||||
)
|
||||
.bind(&hash_a)
|
||||
.bind(&hash_b)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.files (id, blob_hash, size, updated_by) VALUES ($1, $2, 12, $3)",
|
||||
)
|
||||
.bind(file_id)
|
||||
.bind(&hash_a)
|
||||
.bind(caller)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
repo.update_file_content_with_blob(&file_id.to_string(), &hash_b, 17, None, caller)
|
||||
.await
|
||||
.unwrap();
|
||||
let old_ref = ref_count(pool.as_ref(), &hash_a).await;
|
||||
let new_ref = ref_count(pool.as_ref(), &hash_b).await;
|
||||
println!("different: old_ref={:?} new_ref={:?}", old_ref, new_ref);
|
||||
assert_eq!(old_ref, None);
|
||||
assert_eq!(new_ref, Some(1));
|
||||
assert_eq!(recording_hook.deleted(), vec![hash_a.clone()]);
|
||||
|
||||
repo.delete_file(&file_id.to_string()).await.unwrap();
|
||||
let (deleted, _) = dedup.garbage_collect_force().await.unwrap();
|
||||
let final_new_ref = ref_count(pool.as_ref(), &hash_b).await;
|
||||
println!(
|
||||
"delete_gc: deleted={deleted} final_new_ref={:?}",
|
||||
final_new_ref
|
||||
);
|
||||
assert_eq!(deleted, 1);
|
||||
assert_eq!(final_new_ref, None);
|
||||
assert_eq!(
|
||||
recording_hook.deleted(),
|
||||
vec![hash_a.clone(), hash_b.clone()]
|
||||
);
|
||||
|
||||
// Missing-file compensation consumes the incoming reference and fires the
|
||||
// deletion hook when it was the only one.
|
||||
reset(pool.as_ref()).await;
|
||||
recording_hook.clear();
|
||||
let missing_hash = blake3::hash(b"missing-target").to_hex().to_string();
|
||||
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 14, 1)")
|
||||
.bind(&missing_hash)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
repo.update_file_content_with_blob(
|
||||
&Uuid::new_v4().to_string(),
|
||||
&missing_hash,
|
||||
14,
|
||||
None,
|
||||
caller,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(ref_count(pool.as_ref(), &missing_hash).await, None);
|
||||
assert_eq!(recording_hook.deleted(), vec![missing_hash.clone()]);
|
||||
println!("missing_compensation: ref=None hook=1");
|
||||
|
||||
// SQL-error compensation (invalid UUID cast) follows the distinct Err
|
||||
// branch and must likewise consume the incoming reference exactly once.
|
||||
reset(pool.as_ref()).await;
|
||||
recording_hook.clear();
|
||||
let error_hash = blake3::hash(b"sql-error").to_hex().to_string();
|
||||
sqlx::query("INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 9, 1)")
|
||||
.bind(&error_hash)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
repo.update_file_content_with_blob("not-a-uuid", &error_hash, 9, None, caller)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(ref_count(pool.as_ref(), &error_hash).await, None);
|
||||
assert_eq!(recording_hook.deleted(), vec![error_hash]);
|
||||
println!("error_compensation: ref=None hook=1");
|
||||
reset(pool.as_ref()).await;
|
||||
}
|
||||
@@ -0,0 +1,771 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-22",
|
||||
"benchmark": "admin user created_at index representative cost gate",
|
||||
"environment": {
|
||||
"platform": "darwin",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"postgresImage": "postgres:18.2-alpine3.23",
|
||||
"container": "oxicloud-postgres-1",
|
||||
"client": "psql through docker exec"
|
||||
},
|
||||
"method": {
|
||||
"rowsPerDistribution": 500000,
|
||||
"independentTransactions": 3,
|
||||
"samplesPerTransactionAndShape": 5,
|
||||
"totalSamplesPerShape": 15,
|
||||
"warmupsPerTransactionAndShape": 1,
|
||||
"order": "A/B for the accepted timestamp-only result; a later A/B/C gate rotated no-index/simple/compound order inside every transaction",
|
||||
"harnessEvolution": "the current representative SQL is the expanded A/B/C version; the top-level distribution samples retain the earlier A/B authorization run, while compoundRepresentativeABC retains the later isolated rerun",
|
||||
"cleanup": "every transaction ended in ROLLBACK",
|
||||
"commonIndexes": "both baseline and candidate tables have the same UUID primary-key index",
|
||||
"insertBatchRows": 10000,
|
||||
"insertValues": "prebuilt identical A/B batches with new timestamps and an indexed sample selector"
|
||||
},
|
||||
"previousTiedFixtureComparison": {
|
||||
"timestampMultiplicity": 100,
|
||||
"indexBytes": 3448832,
|
||||
"uniqueToTiedIndexByteRatio": 3.263,
|
||||
"reasonForFollowup": "PostgreSQL posting-list deduplication compressed repeated created_at keys and understated a normal mostly-unique registration workload"
|
||||
},
|
||||
"uniqueTimestamps": {
|
||||
"distinctTimestamps": 500000,
|
||||
"indexBytesEachRound": [
|
||||
11255808,
|
||||
11255808,
|
||||
11255808
|
||||
],
|
||||
"indexBytesPerInitialUser": 22.512,
|
||||
"indexBytesAfter50000InsertsEachRound": [
|
||||
13271040,
|
||||
13271040,
|
||||
13271040
|
||||
],
|
||||
"firstPage": {
|
||||
"noIndexSamplesMs": [
|
||||
98.496,
|
||||
121.451,
|
||||
98.425,
|
||||
81.049,
|
||||
94.833,
|
||||
57.602,
|
||||
62.54,
|
||||
56.785,
|
||||
56.918,
|
||||
55.854,
|
||||
57.217,
|
||||
57.131,
|
||||
54.457,
|
||||
56.319,
|
||||
56.651
|
||||
],
|
||||
"indexSamplesMs": [
|
||||
0.459,
|
||||
0.454,
|
||||
0.242,
|
||||
0.207,
|
||||
0.215,
|
||||
0.287,
|
||||
0.208,
|
||||
0.225,
|
||||
0.14,
|
||||
0.2,
|
||||
0.232,
|
||||
0.366,
|
||||
0.146,
|
||||
0.207,
|
||||
0.203
|
||||
],
|
||||
"noIndexMedianMs": 57.217,
|
||||
"indexMedianMs": 0.215,
|
||||
"speedup": 266.13
|
||||
},
|
||||
"offset50000": {
|
||||
"noIndexSamplesMs": [
|
||||
230.025,
|
||||
227.108,
|
||||
191.025,
|
||||
183.211,
|
||||
178.795,
|
||||
126.075,
|
||||
132.155,
|
||||
127.478,
|
||||
120.407,
|
||||
126.377,
|
||||
123.691,
|
||||
115.336,
|
||||
119.425,
|
||||
119.952,
|
||||
120.519
|
||||
],
|
||||
"indexSamplesMs": [
|
||||
6.437,
|
||||
10.281,
|
||||
11.446,
|
||||
7.55,
|
||||
8.74,
|
||||
5.219,
|
||||
5.244,
|
||||
5.063,
|
||||
5.215,
|
||||
4.835,
|
||||
5.165,
|
||||
5.278,
|
||||
5.169,
|
||||
5.181,
|
||||
4.767
|
||||
],
|
||||
"noIndexMedianMs": 126.377,
|
||||
"indexMedianMs": 5.219,
|
||||
"speedup": 24.21
|
||||
},
|
||||
"insert10000Rows": {
|
||||
"noIndexSamplesMs": [
|
||||
47.392,
|
||||
56.38,
|
||||
58.991,
|
||||
80.918,
|
||||
46.37,
|
||||
34.132,
|
||||
36.737,
|
||||
37.678,
|
||||
37.218,
|
||||
38.92,
|
||||
32.183,
|
||||
35.237,
|
||||
34.152,
|
||||
36.031,
|
||||
34.256
|
||||
],
|
||||
"indexSamplesMs": [
|
||||
57.039,
|
||||
65.393,
|
||||
90.559,
|
||||
54.394,
|
||||
67.488,
|
||||
42.624,
|
||||
42.795,
|
||||
44.224,
|
||||
43.115,
|
||||
47.013,
|
||||
44.017,
|
||||
43.637,
|
||||
42.657,
|
||||
43.835,
|
||||
43.25
|
||||
],
|
||||
"noIndexMedianMs": 37.218,
|
||||
"indexMedianMs": 44.017,
|
||||
"extraMicrosecondsPerUser": 0.68
|
||||
},
|
||||
"correctness": {
|
||||
"initialOrderMatchesEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"finalCountsMatchEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"finalOrderMatchesEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
}
|
||||
},
|
||||
"tenUserBursts": {
|
||||
"distinctTimestamps": 50000,
|
||||
"indexBytesEachRound": [
|
||||
4751360,
|
||||
4751360,
|
||||
4751360
|
||||
],
|
||||
"indexBytesPerInitialUser": 9.503,
|
||||
"indexBytesAfter50000InsertsEachRound": [
|
||||
5603328,
|
||||
5603328,
|
||||
5603328
|
||||
],
|
||||
"firstPage": {
|
||||
"noIndexSamplesMs": [
|
||||
113.795,
|
||||
129.772,
|
||||
121.759,
|
||||
114.103,
|
||||
107.905,
|
||||
76.321,
|
||||
76.417,
|
||||
74.491,
|
||||
73.517,
|
||||
75.56,
|
||||
75.307,
|
||||
74.247,
|
||||
73.836,
|
||||
74.131,
|
||||
76.213
|
||||
],
|
||||
"indexSamplesMs": [
|
||||
1.134,
|
||||
0.219,
|
||||
0.24,
|
||||
0.7,
|
||||
0.219,
|
||||
0.286,
|
||||
0.19,
|
||||
0.269,
|
||||
0.244,
|
||||
0.212,
|
||||
0.228,
|
||||
0.196,
|
||||
0.237,
|
||||
0.237,
|
||||
0.191
|
||||
],
|
||||
"noIndexMedianMs": 76.213,
|
||||
"indexMedianMs": 0.237,
|
||||
"speedup": 321.57
|
||||
},
|
||||
"offset50000": {
|
||||
"noIndexSamplesMs": [
|
||||
221.554,
|
||||
235.467,
|
||||
327.723,
|
||||
220.735,
|
||||
219.383,
|
||||
139.801,
|
||||
142.349,
|
||||
136.331,
|
||||
143.815,
|
||||
149.221,
|
||||
137.129,
|
||||
133.334,
|
||||
133.841,
|
||||
132.927,
|
||||
139.428
|
||||
],
|
||||
"indexSamplesMs": [
|
||||
14.06,
|
||||
15.705,
|
||||
12.723,
|
||||
12.931,
|
||||
12.787,
|
||||
7.385,
|
||||
7.884,
|
||||
7.219,
|
||||
7.374,
|
||||
7.475,
|
||||
7.951,
|
||||
7.783,
|
||||
7.626,
|
||||
7.465,
|
||||
7.622
|
||||
],
|
||||
"noIndexMedianMs": 142.349,
|
||||
"indexMedianMs": 7.783,
|
||||
"speedup": 18.29
|
||||
},
|
||||
"insert10000Rows": {
|
||||
"noIndexSamplesMs": [
|
||||
52.854,
|
||||
59.003,
|
||||
62.892,
|
||||
70.446,
|
||||
70.98,
|
||||
32.716,
|
||||
33.745,
|
||||
32.633,
|
||||
34.996,
|
||||
34.804,
|
||||
43.5,
|
||||
33.932,
|
||||
32.596,
|
||||
35.148,
|
||||
34.663
|
||||
],
|
||||
"indexSamplesMs": [
|
||||
61.26,
|
||||
68.779,
|
||||
58.442,
|
||||
60.291,
|
||||
100.505,
|
||||
38.469,
|
||||
39.399,
|
||||
39.874,
|
||||
39.556,
|
||||
40.619,
|
||||
40.819,
|
||||
42.759,
|
||||
40.281,
|
||||
39.762,
|
||||
41.015
|
||||
],
|
||||
"noIndexMedianMs": 34.996,
|
||||
"indexMedianMs": 40.819,
|
||||
"extraMicrosecondsPerUser": 0.582
|
||||
},
|
||||
"correctness": {
|
||||
"initialOrderMatchesEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"finalCountsMatchEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"finalOrderMatchesEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
}
|
||||
},
|
||||
"decision": {
|
||||
"status": "accepted_by_explicit_user_tradeoff",
|
||||
"productionVariant": "timestamp-only created_at DESC index accepted; compound index rejected and not present in production",
|
||||
"reason": "After seeing the corrected representative cost, the user explicitly accepted 11,255,808 bytes and +0.680 us per inserted user for unique timestamps (24.21x-266.13x read gains), plus 4,751,360 bytes and +0.582 us per user for ten-user bursts (18.29x-321.57x read gains). This decision does not cover the compound index. The isolated representative A/B/C follow-up rejected the compound index: it regressed the unique-timestamp first page and materially increased disk/write cost."
|
||||
},
|
||||
"compoundRepresentativeABC": {
|
||||
"status": "rejected",
|
||||
"candidate": "created_at DESC, id DESC compound index versus the accepted created_at DESC index",
|
||||
"method": {
|
||||
"rowsPerDistribution": 500000,
|
||||
"independentTransactions": 3,
|
||||
"samplesPerTransactionAndShape": 5,
|
||||
"totalSamplesPerShape": 15,
|
||||
"warmupsPerTransactionAndShape": 1,
|
||||
"order": "A/B/C order rotated inside every transaction",
|
||||
"cleanup": "every transaction ended in ROLLBACK",
|
||||
"commonIndexes": "all tables have the same UUID primary-key index",
|
||||
"insertBatchRows": 10000,
|
||||
"timingIsolation": "canonical rerun after other CPU/PostgreSQL benchmark agents stopped"
|
||||
},
|
||||
"indexStorage": {
|
||||
"uniqueTimestamps": {
|
||||
"simpleBytesEachRound": [
|
||||
11255808,
|
||||
11255808,
|
||||
11255808
|
||||
],
|
||||
"compoundBytesEachRound": [
|
||||
20275200,
|
||||
20275200,
|
||||
20275200
|
||||
],
|
||||
"compoundMinusSimpleBytes": 9019392,
|
||||
"compoundToSimpleRatio": 1.8013,
|
||||
"compoundIncreasePercent": 80.131,
|
||||
"simpleBytesAfter50000InsertsEachRound": [
|
||||
13271040,
|
||||
13271040,
|
||||
13271040
|
||||
],
|
||||
"compoundBytesAfter50000InsertsEachRound": [
|
||||
23920640,
|
||||
23920640,
|
||||
23920640
|
||||
],
|
||||
"afterInsertCompoundIncreasePercent": 80.247
|
||||
},
|
||||
"tenUserBursts": {
|
||||
"simpleBytesEachRound": [
|
||||
4751360,
|
||||
4751360,
|
||||
4751360
|
||||
],
|
||||
"compoundBytesEachRound": [
|
||||
20324352,
|
||||
20324352,
|
||||
20324352
|
||||
],
|
||||
"compoundMinusSimpleBytes": 15572992,
|
||||
"compoundToSimpleRatio": 4.2776,
|
||||
"compoundIncreasePercent": 327.759,
|
||||
"simpleBytesAfter50000InsertsEachRound": [
|
||||
5603328,
|
||||
5603328,
|
||||
5603328
|
||||
],
|
||||
"compoundBytesAfter50000InsertsEachRound": [
|
||||
24068096,
|
||||
24068096,
|
||||
24068096
|
||||
],
|
||||
"afterInsertCompoundIncreasePercent": 329.532
|
||||
}
|
||||
},
|
||||
"uniqueTimestamps": {
|
||||
"firstPage": {
|
||||
"noIndexSamplesMs": [
|
||||
79.055,
|
||||
63.205,
|
||||
69.033,
|
||||
64.688,
|
||||
66.893,
|
||||
57.082,
|
||||
56,
|
||||
64.918,
|
||||
70.986,
|
||||
74.456,
|
||||
88.976,
|
||||
61.345,
|
||||
56.376,
|
||||
57.135,
|
||||
59.125
|
||||
],
|
||||
"simpleSamplesMs": [
|
||||
0.237,
|
||||
0.124,
|
||||
0.231,
|
||||
0.256,
|
||||
0.15,
|
||||
0.37,
|
||||
0.138,
|
||||
0.219,
|
||||
0.186,
|
||||
0.141,
|
||||
0.509,
|
||||
0.167,
|
||||
0.279,
|
||||
0.175,
|
||||
0.138
|
||||
],
|
||||
"compoundSamplesMs": [
|
||||
0.121,
|
||||
0.198,
|
||||
0.164,
|
||||
0.291,
|
||||
0.204,
|
||||
0.151,
|
||||
0.216,
|
||||
0.214,
|
||||
0.118,
|
||||
0.221,
|
||||
0.477,
|
||||
0.214,
|
||||
0.164,
|
||||
0.106,
|
||||
0.194
|
||||
],
|
||||
"noIndexMedianMs": 64.688,
|
||||
"simpleMedianMs": 0.186,
|
||||
"compoundMedianMs": 0.198,
|
||||
"compoundRegressionVsSimplePercent": 6.452
|
||||
},
|
||||
"offset50000": {
|
||||
"noIndexSamplesMs": [
|
||||
133.653,
|
||||
122.937,
|
||||
165.654,
|
||||
154.255,
|
||||
255.245,
|
||||
124.888,
|
||||
125.983,
|
||||
139.393,
|
||||
140.968,
|
||||
137.235,
|
||||
151.742,
|
||||
143.292,
|
||||
137.627,
|
||||
147.725,
|
||||
137.94
|
||||
],
|
||||
"simpleSamplesMs": [
|
||||
6.273,
|
||||
5.542,
|
||||
5.288,
|
||||
10.254,
|
||||
6.294,
|
||||
4.821,
|
||||
5.034,
|
||||
5.457,
|
||||
5.628,
|
||||
5.614,
|
||||
15.465,
|
||||
5.859,
|
||||
5.174,
|
||||
5.198,
|
||||
5.096
|
||||
],
|
||||
"compoundSamplesMs": [
|
||||
5.575,
|
||||
3.119,
|
||||
3.254,
|
||||
8.087,
|
||||
3.706,
|
||||
3.137,
|
||||
3.406,
|
||||
3.505,
|
||||
3.371,
|
||||
4.983,
|
||||
4.168,
|
||||
4.069,
|
||||
2.931,
|
||||
3.018,
|
||||
3.219
|
||||
],
|
||||
"noIndexMedianMs": 139.393,
|
||||
"simpleMedianMs": 5.542,
|
||||
"compoundMedianMs": 3.406,
|
||||
"compoundSpeedupVsSimple": 1.627
|
||||
},
|
||||
"insert10000Rows": {
|
||||
"noIndexSamplesMs": [
|
||||
41.324,
|
||||
42.365,
|
||||
34.69,
|
||||
37.787,
|
||||
38.282,
|
||||
49.078,
|
||||
54.823,
|
||||
58.893,
|
||||
40.822,
|
||||
56.537,
|
||||
45.011,
|
||||
38.21,
|
||||
42.21,
|
||||
35.842,
|
||||
36.346
|
||||
],
|
||||
"simpleSamplesMs": [
|
||||
54.828,
|
||||
48.465,
|
||||
42.161,
|
||||
45.736,
|
||||
58.347,
|
||||
102.65,
|
||||
71.567,
|
||||
64.681,
|
||||
56.949,
|
||||
55.432,
|
||||
55.426,
|
||||
46.866,
|
||||
47.472,
|
||||
46.171,
|
||||
46.097
|
||||
],
|
||||
"compoundSamplesMs": [
|
||||
53.643,
|
||||
51.835,
|
||||
69.438,
|
||||
44.606,
|
||||
45.863,
|
||||
65.229,
|
||||
55.378,
|
||||
56.733,
|
||||
48.836,
|
||||
47.771,
|
||||
66.354,
|
||||
47.254,
|
||||
58.389,
|
||||
44.03,
|
||||
43.741
|
||||
],
|
||||
"noIndexMedianMs": 41.324,
|
||||
"simpleMedianMs": 54.828,
|
||||
"compoundMedianMs": 51.835,
|
||||
"simpleExtraMicrosecondsPerUserVsNoIndex": 1.3504,
|
||||
"compoundExtraMicrosecondsPerUserVsNoIndex": 1.0511,
|
||||
"compoundDeltaMicrosecondsPerUserVsSimple": -0.2993
|
||||
}
|
||||
},
|
||||
"tenUserBursts": {
|
||||
"firstPage": {
|
||||
"noIndexSamplesMs": [
|
||||
97.536,
|
||||
83.007,
|
||||
82.563,
|
||||
77.801,
|
||||
81.854,
|
||||
80.281,
|
||||
84.272,
|
||||
86.625,
|
||||
117.336,
|
||||
91.597,
|
||||
78.241,
|
||||
100.035,
|
||||
73.378,
|
||||
76.37,
|
||||
73.884
|
||||
],
|
||||
"simpleSamplesMs": [
|
||||
0.22,
|
||||
0.124,
|
||||
0.238,
|
||||
0.371,
|
||||
0.146,
|
||||
0.231,
|
||||
0.157,
|
||||
0.612,
|
||||
0.368,
|
||||
0.436,
|
||||
0.276,
|
||||
0.133,
|
||||
0.545,
|
||||
0.224,
|
||||
0.154
|
||||
],
|
||||
"compoundSamplesMs": [
|
||||
0.11,
|
||||
0.163,
|
||||
0.343,
|
||||
0.156,
|
||||
0.192,
|
||||
0.132,
|
||||
0.206,
|
||||
0.24,
|
||||
0.175,
|
||||
0.486,
|
||||
0.134,
|
||||
0.158,
|
||||
0.137,
|
||||
0.106,
|
||||
0.235
|
||||
],
|
||||
"noIndexMedianMs": 82.563,
|
||||
"simpleMedianMs": 0.231,
|
||||
"compoundMedianMs": 0.163,
|
||||
"compoundSpeedupVsSimple": 1.417
|
||||
},
|
||||
"offset50000": {
|
||||
"noIndexSamplesMs": [
|
||||
181.149,
|
||||
185.834,
|
||||
155.003,
|
||||
151.478,
|
||||
153.175,
|
||||
139.144,
|
||||
140.267,
|
||||
213.158,
|
||||
188.745,
|
||||
190.545,
|
||||
184.289,
|
||||
159.779,
|
||||
156.309,
|
||||
160.642,
|
||||
157.767
|
||||
],
|
||||
"simpleSamplesMs": [
|
||||
8.038,
|
||||
8.418,
|
||||
7.489,
|
||||
7.892,
|
||||
9.026,
|
||||
7.378,
|
||||
7.724,
|
||||
8.3,
|
||||
8.486,
|
||||
11.09,
|
||||
10.755,
|
||||
7.569,
|
||||
7.527,
|
||||
7.927,
|
||||
7.424
|
||||
],
|
||||
"compoundSamplesMs": [
|
||||
4.191,
|
||||
3.245,
|
||||
3.386,
|
||||
3.44,
|
||||
3.327,
|
||||
3.745,
|
||||
3.893,
|
||||
4.017,
|
||||
4.171,
|
||||
4.551,
|
||||
7.326,
|
||||
3.198,
|
||||
3.402,
|
||||
3.314,
|
||||
3.317
|
||||
],
|
||||
"noIndexMedianMs": 159.779,
|
||||
"simpleMedianMs": 7.927,
|
||||
"compoundMedianMs": 3.44,
|
||||
"compoundSpeedupVsSimple": 2.304
|
||||
},
|
||||
"insert10000Rows": {
|
||||
"noIndexSamplesMs": [
|
||||
56.852,
|
||||
67.311,
|
||||
35.277,
|
||||
41.769,
|
||||
38.984,
|
||||
43.076,
|
||||
40.612,
|
||||
46.572,
|
||||
41.514,
|
||||
38.331,
|
||||
56.728,
|
||||
39.23,
|
||||
36.724,
|
||||
36.912,
|
||||
36.015
|
||||
],
|
||||
"simpleSamplesMs": [
|
||||
75.665,
|
||||
73.424,
|
||||
43.871,
|
||||
43.275,
|
||||
43.638,
|
||||
59.345,
|
||||
50.214,
|
||||
44.475,
|
||||
40.879,
|
||||
46.694,
|
||||
65.48,
|
||||
42.844,
|
||||
44.616,
|
||||
44.221,
|
||||
42.984
|
||||
],
|
||||
"compoundSamplesMs": [
|
||||
48.496,
|
||||
59.438,
|
||||
42.891,
|
||||
43.67,
|
||||
45.941,
|
||||
70.587,
|
||||
57.743,
|
||||
49.954,
|
||||
48.921,
|
||||
57.737,
|
||||
58.111,
|
||||
45.346,
|
||||
49.969,
|
||||
43.602,
|
||||
45.851
|
||||
],
|
||||
"noIndexMedianMs": 40.612,
|
||||
"simpleMedianMs": 44.475,
|
||||
"compoundMedianMs": 48.921,
|
||||
"simpleExtraMicrosecondsPerUserVsNoIndex": 0.3863,
|
||||
"compoundExtraMicrosecondsPerUserVsNoIndex": 0.8309,
|
||||
"compoundDeltaMicrosecondsPerUserVsSimple": 0.4446
|
||||
}
|
||||
},
|
||||
"correctness": {
|
||||
"initialOrderMatchesEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"finalCountsMatchEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"finalOrderMatchesEachRound": [
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"decision": "rejected: versus the accepted narrow index, the compound index regressed the common unique-timestamp first page by 6.45%, enlarged the btree by 80.13%-327.76%, and added 0.445 us/user to burst inserts. Its 1.63x-2.30x deep-page gains do not pass the no-regression resource gate."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-22",
|
||||
"benchmark": "admin user listing full component-path A/B",
|
||||
"environment": {
|
||||
"platform": "darwin",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"postgresImage": "postgres:18.2-alpine3.23",
|
||||
"container": "oxicloud-postgres-1",
|
||||
"client": "release Rust/sqlx harness over the local container bridge"
|
||||
},
|
||||
"scope": {
|
||||
"historical": "full-row SQL -> full DTO -> COUNT SQL -> Serde JSON",
|
||||
"candidate": "hot Moka user-flags lookup -> system-admin policy check -> narrow summary SQL -> summary DTO -> COUNT SQL -> Serde JSON",
|
||||
"commonWork": "one sqlx connection, identical 100-user ordering/count/pagination, identical temporary fixture and JSON page envelope",
|
||||
"excluded": "Axum routing, JWT parsing, socket-level HTTP framing and browser parsing; the historical side also omits the old handler's intermediate serde_json::Value materialization, making its baseline optimistic. This is a conservative component-path gate, not a whole HTTP-stack claim"
|
||||
},
|
||||
"method": {
|
||||
"users": 100,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"minimalProfile": "small ordinary account-detail fields",
|
||||
"heavyProfile": "512 KiB image and 8 KiB UI-preference payload per user",
|
||||
"timing": "interleaved/alternating paths after three warmups; isolated from other benchmark agents",
|
||||
"rss": "three fresh client processes per path/profile using /usr/bin/time -l; process setup is common and included in max RSS",
|
||||
"databaseState": "per-process PostgreSQL TEMP table; no persistent rows or indexes"
|
||||
},
|
||||
"minimal": {
|
||||
"timing": {
|
||||
"samples": 31,
|
||||
"warmups": 3,
|
||||
"order": "interleaved and alternated",
|
||||
"historicalFullSamplesMs": [
|
||||
4.4581669999999995,
|
||||
0.8505,
|
||||
0.9585,
|
||||
0.808334,
|
||||
0.865125,
|
||||
0.867916,
|
||||
0.8726670000000001,
|
||||
0.937375,
|
||||
1.013833,
|
||||
1.114166,
|
||||
5.163166,
|
||||
206.010333,
|
||||
208.088916,
|
||||
4.949625,
|
||||
1.5272919999999999,
|
||||
2.3782080000000003,
|
||||
0.9502090000000001,
|
||||
8.851790999999999,
|
||||
3.048167,
|
||||
1.546792,
|
||||
1.5305410000000002,
|
||||
1.380125,
|
||||
1.295833,
|
||||
1.279625,
|
||||
1.33425,
|
||||
0.9305410000000001,
|
||||
8.786916,
|
||||
1.217792,
|
||||
5.245125,
|
||||
0.967917,
|
||||
1.1993749999999999
|
||||
],
|
||||
"candidateSummaryHotAuthzSamplesMs": [
|
||||
1.0223330000000002,
|
||||
0.7933330000000001,
|
||||
0.829875,
|
||||
0.793083,
|
||||
0.86025,
|
||||
0.7274160000000001,
|
||||
5.1122499999999995,
|
||||
0.9088339999999999,
|
||||
0.707833,
|
||||
0.746458,
|
||||
0.8647079999999999,
|
||||
10.13475,
|
||||
1.094375,
|
||||
206.91925,
|
||||
1.455792,
|
||||
1.13725,
|
||||
0.791542,
|
||||
4.578875,
|
||||
1.0007499999999998,
|
||||
1.060708,
|
||||
1.033833,
|
||||
0.9833330000000001,
|
||||
1.194166,
|
||||
0.917292,
|
||||
0.966334,
|
||||
5.308375,
|
||||
0.965333,
|
||||
1.058333,
|
||||
0.804084,
|
||||
0.829,
|
||||
0.885375
|
||||
],
|
||||
"historicalFullMedianMs": 1.295833,
|
||||
"candidateSummaryHotAuthzMedianMs": 0.966334,
|
||||
"medianSpeedup": 1.3409783780763174,
|
||||
"historicalFullP95Ms": 206.010333,
|
||||
"candidateSummaryHotAuthzP95Ms": 10.13475,
|
||||
"p95Speedup": 20.327125286760896,
|
||||
"historicalJsonBytes": 43759,
|
||||
"candidateJsonBytes": 28726,
|
||||
"byteReductionPercent": 34.35407573299207,
|
||||
"summaryProjectionEqual": true,
|
||||
"totalEqual": true
|
||||
},
|
||||
"freshProcessMemory": {
|
||||
"runsPerMode": 3,
|
||||
"historical": [
|
||||
{
|
||||
"maxRssBytes": 7782400,
|
||||
"responseElapsedMs": 1.56875,
|
||||
"jsonBytes": 43759,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 7782400,
|
||||
"responseElapsedMs": 204.553417,
|
||||
"jsonBytes": 43759,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 7782400,
|
||||
"responseElapsedMs": 2.007917,
|
||||
"jsonBytes": 43759,
|
||||
"exitCode": 0
|
||||
}
|
||||
],
|
||||
"candidate": [
|
||||
{
|
||||
"maxRssBytes": 7651328,
|
||||
"responseElapsedMs": 3.949834,
|
||||
"jsonBytes": 28726,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 7667712,
|
||||
"responseElapsedMs": 1.091208,
|
||||
"jsonBytes": 28726,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 7684096,
|
||||
"responseElapsedMs": 0.982916,
|
||||
"jsonBytes": 28726,
|
||||
"exitCode": 0
|
||||
}
|
||||
],
|
||||
"historicalMedianMaxRssBytes": 7782400,
|
||||
"candidateMedianMaxRssBytes": 7667712,
|
||||
"candidateMinusHistoricalRssBytes": -114688,
|
||||
"rssReductionPercent": 1.473684210526316,
|
||||
"historicalColdMedianElapsedMs": 2.007917,
|
||||
"candidateColdMedianElapsedMs": 1.091208,
|
||||
"coldMedianSpeedup": 1.8400863996598267
|
||||
}
|
||||
},
|
||||
"heavy": {
|
||||
"timing": {
|
||||
"samples": 11,
|
||||
"warmups": 3,
|
||||
"order": "interleaved and alternated",
|
||||
"historicalFullSamplesMs": [
|
||||
3519.616458,
|
||||
4157.738708,
|
||||
870.990208,
|
||||
301.8805,
|
||||
1296.212125,
|
||||
1141.4217500000002,
|
||||
60.022708,
|
||||
297.898916,
|
||||
2261.6492089999997,
|
||||
1438.1172920000001,
|
||||
398.208167
|
||||
],
|
||||
"candidateSummaryHotAuthzSamplesMs": [
|
||||
4.283292,
|
||||
0.9297500000000001,
|
||||
0.933166,
|
||||
0.719,
|
||||
219.19591699999998,
|
||||
16.399291,
|
||||
0.919209,
|
||||
0.654208,
|
||||
0.965792,
|
||||
5.121874999999999,
|
||||
1.2095
|
||||
],
|
||||
"historicalFullMedianMs": 1141.4217500000002,
|
||||
"candidateSummaryHotAuthzMedianMs": 0.965792,
|
||||
"medianSpeedup": 1181.8504916172428,
|
||||
"historicalFullP95Ms": 4157.738708,
|
||||
"candidateSummaryHotAuthzP95Ms": 219.19591699999998,
|
||||
"p95Speedup": 18.96813939285192,
|
||||
"historicalJsonBytes": 53306743,
|
||||
"candidateJsonBytes": 28726,
|
||||
"byteReductionPercent": 99.9461118830689,
|
||||
"summaryProjectionEqual": true,
|
||||
"totalEqual": true
|
||||
},
|
||||
"freshProcessMemory": {
|
||||
"runsPerMode": 3,
|
||||
"historical": [
|
||||
{
|
||||
"maxRssBytes": 151683072,
|
||||
"responseElapsedMs": 4506.601708,
|
||||
"jsonBytes": 53306743,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 144162816,
|
||||
"responseElapsedMs": 2585.629333,
|
||||
"jsonBytes": 53306743,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 146292736,
|
||||
"responseElapsedMs": 6410.820542,
|
||||
"jsonBytes": 53306743,
|
||||
"exitCode": 0
|
||||
}
|
||||
],
|
||||
"candidate": [
|
||||
{
|
||||
"maxRssBytes": 7667712,
|
||||
"responseElapsedMs": 1.080875,
|
||||
"jsonBytes": 28726,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 7651328,
|
||||
"responseElapsedMs": 1.061042,
|
||||
"jsonBytes": 28726,
|
||||
"exitCode": 0
|
||||
},
|
||||
{
|
||||
"maxRssBytes": 7667712,
|
||||
"responseElapsedMs": 2.09175,
|
||||
"jsonBytes": 28726,
|
||||
"exitCode": 0
|
||||
}
|
||||
],
|
||||
"historicalMedianMaxRssBytes": 146292736,
|
||||
"candidateMedianMaxRssBytes": 7667712,
|
||||
"candidateMinusHistoricalRssBytes": -138625024,
|
||||
"rssReductionPercent": 94.75865158472394,
|
||||
"historicalColdMedianElapsedMs": 4506.601708,
|
||||
"candidateColdMedianElapsedMs": 1.080875,
|
||||
"coldMedianSpeedup": 4169.401372036545
|
||||
}
|
||||
},
|
||||
"correctness": {
|
||||
"exactRenderedFieldProjectionAndOrder": true,
|
||||
"exactTotalCount": true,
|
||||
"hotAuthorizationFlagsAsserted": {
|
||||
"role": "admin",
|
||||
"isExternal": false,
|
||||
"active": true
|
||||
},
|
||||
"allProcessesExitedSuccessfully": true
|
||||
},
|
||||
"decision": {
|
||||
"status": "accepted",
|
||||
"productionVariant": "summary=true compact admin listing with authoritative service-layer AuthZ",
|
||||
"reason": "The representative minimal profile improved median component latency 1.296 -> 0.966 ms, reduced JSON 34.35%, and saved 112 KiB median max RSS. The heavy profile improved median latency 1141.422 -> 0.966 ms, reduced JSON 99.946%, and saved 138,625,024 bytes median max RSS. Exact projected fields, order, and counts matched."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-22",
|
||||
"decisions": {
|
||||
"adminUserProjection": "accepted: conservative component-path gate passed latency, JSON bytes, exact fields/counts, and fresh-process RSS for minimal and heavy profiles",
|
||||
"adminCreatedAtIndex": "accepted_by_explicit_user_tradeoff: after the tied fixture was superseded, the user explicitly accepted representative unique timestamps at 11,255,808 bytes and +0.680 us per inserted user (24.21x-266.13x read gains), plus burst10 at 4,751,360 bytes and +0.582 us per user; the later representative compound candidate was rejected",
|
||||
"adminCountWindowFusion": "rejected; no production change",
|
||||
"localSyncPreparation": "accepted; empty fast return and every measured non-empty size improved; path/directory equivalence passed"
|
||||
},
|
||||
"environment": {
|
||||
"platform": "darwin",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"rustc": "1.93.0",
|
||||
"postgres": "oxicloud-postgres-1 local Docker container"
|
||||
},
|
||||
"adminUserProjection": {
|
||||
"timingScope": "psql query execution, row transfer, and client decoding; excludes HTTP/Serde and the service-layer authorization check",
|
||||
"fixture": {
|
||||
"users": 100,
|
||||
"avatarBytesPerUser": 524288,
|
||||
"preferencesBytesPerUser": 8192
|
||||
},
|
||||
"current": {
|
||||
"jsonBytes": 53315361,
|
||||
"samplesMs": [79.871, 64.878, 42.010, 63.094, 59.987],
|
||||
"medianMs": 63.094
|
||||
},
|
||||
"compactSummary": {
|
||||
"jsonBytes": 31671,
|
||||
"samplesMs": [0.764, 0.535, 0.660, 0.304, 0.524],
|
||||
"medianMs": 0.535
|
||||
},
|
||||
"speedup": 117.93,
|
||||
"byteReductionPercent": 99.9406,
|
||||
"decision": "accepted"
|
||||
},
|
||||
"adminCountWindowFusion": {
|
||||
"fixtureUsers": 100000,
|
||||
"currentPagePlusCountSamplesMs": [11.081, 5.255, 5.604, 10.597, 5.237],
|
||||
"currentMedianMs": 5.604,
|
||||
"countWindowSamplesMs": [61.966, 39.277, 36.788, 42.287, 64.007],
|
||||
"countWindowMedianMs": 42.287,
|
||||
"candidateSlowdown": 7.55,
|
||||
"decision": "rejected; no production change"
|
||||
},
|
||||
"adminCreatedAtIndex": {
|
||||
"fixtureUsers": 500000,
|
||||
"tiedTimestampGroups": 5000,
|
||||
"allOrdersMatch": true,
|
||||
"timestampIndexBytes": 3448832,
|
||||
"timestampIndexBytesPerUser": 6.90,
|
||||
"compoundIndexBytes": 20332544,
|
||||
"compoundIndexBytesPerUser": 40.67,
|
||||
"firstPage": {
|
||||
"noIndexSamplesMs": [162.111, 181.379, 185.911, 193.284, 193.223],
|
||||
"noIndexMedianMs": 185.911,
|
||||
"timestampSamplesMs": [0.368, 0.316, 0.397, 0.173, 0.438],
|
||||
"timestampMedianMs": 0.368,
|
||||
"timestampSpeedup": 505.19,
|
||||
"compoundSamplesMs": [0.178, 0.397, 0.234, 0.245, 0.240],
|
||||
"compoundMedianMs": 0.240,
|
||||
"compoundSpeedup": 774.63
|
||||
},
|
||||
"offset50000": {
|
||||
"noIndexSamplesMs": [343.654, 333.319, 343.129, 363.514, 347.188],
|
||||
"noIndexMedianMs": 343.654,
|
||||
"timestampSamplesMs": [21.305, 24.210, 26.183, 24.117, 20.882],
|
||||
"timestampMedianMs": 24.117,
|
||||
"timestampSpeedup": 14.25,
|
||||
"compoundSamplesMs": [6.530, 9.761, 5.968, 8.284, 8.557],
|
||||
"compoundMedianMs": 8.284,
|
||||
"compoundSpeedup": 41.48
|
||||
},
|
||||
"insert10000Rows": {
|
||||
"noIndexSamplesMs": [5.916, 8.354, 10.674, 5.658, 9.169],
|
||||
"noIndexMedianMs": 8.354,
|
||||
"timestampSamplesMs": [16.209, 14.310, 12.504, 11.911, 16.665],
|
||||
"timestampMedianMs": 14.310,
|
||||
"timestampExtraMicrosecondsPerUser": 0.596,
|
||||
"compoundSamplesMs": [18.748, 19.204, 19.819, 17.212, 21.492],
|
||||
"compoundMedianMs": 19.204,
|
||||
"compoundExtraMicrosecondsPerUser": 1.085
|
||||
},
|
||||
"decision": "evidence_only_superseded_by_representative_gate: tied-timestamp read results remain valid, but posting-list deduplication understated timestamp-only resource cost and invalidated the original compound comparison",
|
||||
"representativeGateResult": "admin-user-index-representative-postgres18-macos-arm64.json"
|
||||
},
|
||||
"localSyncEmpty": {
|
||||
"paths": 0,
|
||||
"processes": 11,
|
||||
"samplesPerProcess": 31,
|
||||
"repetitionsPerSample": 100000,
|
||||
"historicalProcessMediansNs": [26.501, 25.283, 26.715, 30.093, 29.759, 24.388, 25.102, 25.163, 32.574, 24.982, 25.607],
|
||||
"fastReturnProcessMediansNs": [21.679, 23.283, 24.260, 25.231, 22.946, 21.540, 20.316, 20.186, 24.095, 25.565, 21.287],
|
||||
"historicalMedianAcrossProcessesNs": 25.607,
|
||||
"fastReturnMedianAcrossProcessesNs": 22.946,
|
||||
"candidateWins": 10,
|
||||
"speedup": 1.116,
|
||||
"decision": "accepted"
|
||||
},
|
||||
"localSyncPreparation": [
|
||||
{
|
||||
"paths": 1,
|
||||
"cloneGroupsUs": 0.157,
|
||||
"moveGroupsUs": 0.128,
|
||||
"groupSpeedup": 1.22,
|
||||
"sortParentDirsUs": 0.119,
|
||||
"prefixBitmapDirsUs": 0.088,
|
||||
"directorySpeedup": 1.36
|
||||
},
|
||||
{
|
||||
"paths": 8,
|
||||
"cloneGroupsUs": 0.673,
|
||||
"moveGroupsUs": 0.409,
|
||||
"groupSpeedup": 1.64,
|
||||
"sortParentDirsUs": 1.599,
|
||||
"prefixBitmapDirsUs": 0.902,
|
||||
"directorySpeedup": 1.77
|
||||
},
|
||||
{
|
||||
"paths": 32,
|
||||
"cloneGroupsUs": 3.424,
|
||||
"moveGroupsUs": 2.166,
|
||||
"groupSpeedup": 1.58,
|
||||
"sortParentDirsUs": 4.078,
|
||||
"prefixBitmapDirsUs": 3.318,
|
||||
"directorySpeedup": 1.23
|
||||
},
|
||||
{
|
||||
"paths": 128,
|
||||
"cloneGroupsUs": 5.916,
|
||||
"moveGroupsUs": 3.250,
|
||||
"groupSpeedup": 1.82,
|
||||
"sortParentDirsUs": 12.291,
|
||||
"prefixBitmapDirsUs": 8.333,
|
||||
"directorySpeedup": 1.47
|
||||
},
|
||||
{
|
||||
"paths": 400,
|
||||
"cloneGroupsUs": 17.000,
|
||||
"moveGroupsUs": 8.958,
|
||||
"groupSpeedup": 1.90,
|
||||
"sortParentDirsUs": 97.500,
|
||||
"prefixBitmapDirsUs": 16.958,
|
||||
"directorySpeedup": 5.75
|
||||
},
|
||||
{
|
||||
"paths": 1600,
|
||||
"cloneGroupsUs": 64.541,
|
||||
"moveGroupsUs": 33.083,
|
||||
"groupSpeedup": 1.95,
|
||||
"sortParentDirsUs": 800.708,
|
||||
"prefixBitmapDirsUs": 22.584,
|
||||
"directorySpeedup": 35.45
|
||||
},
|
||||
{
|
||||
"paths": 10000,
|
||||
"cloneGroupsUs": 622.000,
|
||||
"moveGroupsUs": 260.458,
|
||||
"groupSpeedup": 2.39,
|
||||
"sortParentDirsUs": 4006.875,
|
||||
"prefixBitmapDirsUs": 38.125,
|
||||
"directorySpeedup": 105.10
|
||||
},
|
||||
{
|
||||
"paths": 100000,
|
||||
"cloneGroupsUs": 5893.958,
|
||||
"moveGroupsUs": 2856.792,
|
||||
"groupSpeedup": 2.06,
|
||||
"sortParentDirsUs": 45596.041,
|
||||
"prefixBitmapDirsUs": 214.750,
|
||||
"directorySpeedup": 212.32
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"date": "2026-07-22",
|
||||
"status": "accepted",
|
||||
"scope": "CachedBlobBackend range reads",
|
||||
"contract": {
|
||||
"range_semantics": "[start, end)",
|
||||
"production_change": "Replace end - start + 1 with end.saturating_sub(start) on cold and hot cached reads."
|
||||
},
|
||||
"baseline_correctness_probe": {
|
||||
"ranges": [
|
||||
"[0, 1)",
|
||||
"[1, 3)",
|
||||
"[3, 3)",
|
||||
"[2, EOF)"
|
||||
],
|
||||
"expected_lengths": [
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
4
|
||||
],
|
||||
"local_backend_lengths": [
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
4
|
||||
],
|
||||
"historical_cached_backend_lengths": [
|
||||
2,
|
||||
3,
|
||||
1,
|
||||
4
|
||||
],
|
||||
"cold_fill": {
|
||||
"remote_gets": 1,
|
||||
"remote_bytes": 6,
|
||||
"elapsed_us": 408.5
|
||||
},
|
||||
"hot_read_loop": {
|
||||
"iterations": 500,
|
||||
"local_bytes": 1000,
|
||||
"historical_cached_bytes": 1500,
|
||||
"local_average_us": 32.16,
|
||||
"historical_cached_average_us": 30.839
|
||||
},
|
||||
"outcome": "failed: the cached backend returned one extra byte whenever end was present"
|
||||
},
|
||||
"candidate_correctness": {
|
||||
"cached_backend_lengths": [
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
4
|
||||
],
|
||||
"cold_fill_remote_gets": 1,
|
||||
"cold_fill_remote_bytes": 6,
|
||||
"hot_read_loop_bytes": 1000,
|
||||
"focused_test": "cargo test --lib cached_blob_backend::tests::range_end_is_exclusive_on_cold_and_hot_cache_reads",
|
||||
"focused_test_result": "1 passed; 0 failed"
|
||||
},
|
||||
"standalone_ab": {
|
||||
"source": "tools/perf-audit/cached_range_ab.rs",
|
||||
"command": "cargo run --release --manifest-path tools/perf-audit/Cargo.toml --bin cached_range_ab",
|
||||
"environment": {
|
||||
"os": "macos",
|
||||
"arch": "aarch64"
|
||||
},
|
||||
"range": {
|
||||
"start": 1,
|
||||
"end_exclusive": 3
|
||||
},
|
||||
"warmups_per_variant": 1000,
|
||||
"iterations_per_variant": 10000,
|
||||
"historical_inclusive": {
|
||||
"bytes": 30000,
|
||||
"bytes_per_read": 3,
|
||||
"p50_us": 8.459,
|
||||
"p95_us": 28.292,
|
||||
"checksum": 2970000
|
||||
},
|
||||
"corrected_exclusive": {
|
||||
"bytes": 20000,
|
||||
"bytes_per_read": 2,
|
||||
"p50_us": 8.459,
|
||||
"p95_us": 26.917,
|
||||
"checksum": 1970000
|
||||
},
|
||||
"delta_percent": {
|
||||
"bytes": -33.333,
|
||||
"p50_latency": 0.0,
|
||||
"p95_latency": -4.86
|
||||
}
|
||||
},
|
||||
"decision": {
|
||||
"status": "accepted",
|
||||
"reason": "The candidate restores the exclusive-end contract, removes exactly one surplus byte per bounded cached range, adds no remote request or round trip, and is latency-neutral at p50 while improving this sample's p95."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:11:44.735Z",
|
||||
"decision": "evidence only; the above-10000 batching candidate was rejected by the full workflow gate and reverted",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "dedup",
|
||||
"warmup": 3,
|
||||
"samples": 15,
|
||||
"queueCounts": [
|
||||
10000,
|
||||
50000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000
|
||||
},
|
||||
{
|
||||
"items": 20000,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 10000,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"dedup": [
|
||||
{
|
||||
"hashCount": 25000,
|
||||
"current": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 13.010667000000012,
|
||||
"p95Ms": 319.40154200000006,
|
||||
"minMs": 5.714707999999973,
|
||||
"maxMs": 319.40154200000006,
|
||||
"medianHeapDeltaBytes": 4069400,
|
||||
"medianRssDeltaBytes": 1687552
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 25.275750000000016,
|
||||
"p95Ms": 310.1661670000001,
|
||||
"minMs": 9.728249999999662,
|
||||
"maxMs": 310.1661670000001,
|
||||
"medianHeapDeltaBytes": 12869632,
|
||||
"medianRssDeltaBytes": 344064
|
||||
},
|
||||
"speedup": 0.5147489985460374,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 0,
|
||||
"ownedCount": 0,
|
||||
"contentBytesAvoided": 0,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 0,
|
||||
"rejectedRequests": 1,
|
||||
"requestBytes": 1675012,
|
||||
"responseBytes": 27,
|
||||
"maxBatchHashes": 25000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 156237500,
|
||||
"ownedCount": 12500,
|
||||
"contentBytesAvoided": 819200000,
|
||||
"requests": 3,
|
||||
"acceptedRequests": 3,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 1675036,
|
||||
"responseBytes": 837533,
|
||||
"maxBatchHashes": 10000
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:11:34.135Z",
|
||||
"decision": "evidence only; the above-10000 batching candidate was rejected by the full workflow gate and reverted",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "dedup",
|
||||
"warmup": 3,
|
||||
"samples": 15,
|
||||
"queueCounts": [
|
||||
10000,
|
||||
50000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000
|
||||
},
|
||||
{
|
||||
"items": 20000,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 512,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"dedup": [
|
||||
{
|
||||
"hashCount": 25000,
|
||||
"current": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 21.21845900000062,
|
||||
"p95Ms": 263.2436670000002,
|
||||
"minMs": 5.763707999999951,
|
||||
"maxMs": 263.2436670000002,
|
||||
"medianHeapDeltaBytes": 4066384,
|
||||
"medianRssDeltaBytes": 1687552
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 294.9569169999995,
|
||||
"p95Ms": 607.8072920000004,
|
||||
"minMs": 39.99287500000037,
|
||||
"maxMs": 607.8072920000004,
|
||||
"medianHeapDeltaBytes": 16555112,
|
||||
"medianRssDeltaBytes": 49152
|
||||
},
|
||||
"speedup": 0.07193748570405845,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 0,
|
||||
"ownedCount": 0,
|
||||
"contentBytesAvoided": 0,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 0,
|
||||
"rejectedRequests": 1,
|
||||
"requestBytes": 1675012,
|
||||
"responseBytes": 27,
|
||||
"maxBatchHashes": 25000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 156237500,
|
||||
"ownedCount": 12500,
|
||||
"contentBytesAvoided": 819200000,
|
||||
"requests": 49,
|
||||
"acceptedRequests": 49,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 1675588,
|
||||
"responseBytes": 838039,
|
||||
"maxBatchHashes": 512
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:31:30.847Z",
|
||||
"decision": "evidence only; confirms the unchanged <=10000 fast path, while the above-10000 batching candidate was rejected and reverted",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "dedup",
|
||||
"warmup": 7,
|
||||
"samples": 41,
|
||||
"queueCounts": [
|
||||
64,
|
||||
256,
|
||||
1024,
|
||||
10000,
|
||||
50000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100
|
||||
},
|
||||
{
|
||||
"items": 10,
|
||||
"updates": 500
|
||||
},
|
||||
{
|
||||
"items": 100,
|
||||
"updates": 5000
|
||||
},
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
1000
|
||||
],
|
||||
"dedupBatchSize": 10000,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"dedup": [
|
||||
{
|
||||
"hashCount": 1000,
|
||||
"current": {
|
||||
"sampleCount": 41,
|
||||
"medianMs": 5.374666999999988,
|
||||
"p95Ms": 30.969750000000204,
|
||||
"minMs": 1.3017919999999776,
|
||||
"maxMs": 59.26037499999984,
|
||||
"medianHeapDeltaBytes": 575104,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 41,
|
||||
"medianMs": 4.9794579999999655,
|
||||
"p95Ms": 42.28670899999997,
|
||||
"minMs": 1.1001249999999345,
|
||||
"maxMs": 125.80937499999999,
|
||||
"medianHeapDeltaBytes": 574760,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 1.0793678749775628,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 249500,
|
||||
"ownedCount": 500,
|
||||
"contentBytesAvoided": 32768000,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 1,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 67012,
|
||||
"responseBytes": 33511,
|
||||
"maxBatchHashes": 1000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 249500,
|
||||
"ownedCount": 500,
|
||||
"contentBytesAvoided": 32768000,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 1,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 67012,
|
||||
"responseBytes": 33511,
|
||||
"maxBatchHashes": 1000
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:31:04.178Z",
|
||||
"decision": "evidence only; the above-10000 batching candidate was rejected by the full workflow gate and reverted",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "dedup",
|
||||
"warmup": 5,
|
||||
"samples": 15,
|
||||
"queueCounts": [
|
||||
64,
|
||||
256,
|
||||
1024,
|
||||
10000,
|
||||
50000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100
|
||||
},
|
||||
{
|
||||
"items": 10,
|
||||
"updates": 500
|
||||
},
|
||||
{
|
||||
"items": 100,
|
||||
"updates": 5000
|
||||
},
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
1000,
|
||||
10000,
|
||||
10001,
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 10000,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"dedup": [
|
||||
{
|
||||
"hashCount": 1000,
|
||||
"current": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 4.66216600000007,
|
||||
"p95Ms": 23.915542000000016,
|
||||
"minMs": 1.9842080000000237,
|
||||
"maxMs": 23.915542000000016,
|
||||
"medianHeapDeltaBytes": 583320,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 5.586167000000046,
|
||||
"p95Ms": 22.62520799999993,
|
||||
"minMs": 3.662083999999993,
|
||||
"maxMs": 22.62520799999993,
|
||||
"medianHeapDeltaBytes": 578504,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 0.8345912322349174,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 249500,
|
||||
"ownedCount": 500,
|
||||
"contentBytesAvoided": 32768000,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 1,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 67012,
|
||||
"responseBytes": 33511,
|
||||
"maxBatchHashes": 1000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 249500,
|
||||
"ownedCount": 500,
|
||||
"contentBytesAvoided": 32768000,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 1,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 67012,
|
||||
"responseBytes": 33511,
|
||||
"maxBatchHashes": 1000
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hashCount": 10000,
|
||||
"current": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 12.211541000000125,
|
||||
"p95Ms": 138.27816600000006,
|
||||
"minMs": 5.708917000000156,
|
||||
"maxMs": 138.27816600000006,
|
||||
"medianHeapDeltaBytes": 4629728,
|
||||
"medianRssDeltaBytes": 671744
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 10.960124999999834,
|
||||
"p95Ms": 68.23608400000012,
|
||||
"minMs": 7.660417000000052,
|
||||
"maxMs": 68.23608400000012,
|
||||
"medianHeapDeltaBytes": 4629608,
|
||||
"medianRssDeltaBytes": 671744
|
||||
},
|
||||
"speedup": 1.1141789897469516,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 24995000,
|
||||
"ownedCount": 5000,
|
||||
"contentBytesAvoided": 327680000,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 1,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 670012,
|
||||
"responseBytes": 335011,
|
||||
"maxBatchHashes": 10000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 24995000,
|
||||
"ownedCount": 5000,
|
||||
"contentBytesAvoided": 327680000,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 1,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 670012,
|
||||
"responseBytes": 335011,
|
||||
"maxBatchHashes": 10000
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hashCount": 10001,
|
||||
"current": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 5.437792000000172,
|
||||
"p95Ms": 10.699207999999999,
|
||||
"minMs": 2.1932090000000244,
|
||||
"maxMs": 10.699207999999999,
|
||||
"medianHeapDeltaBytes": 2370176,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 10.77975000000015,
|
||||
"p95Ms": 13.00845900000013,
|
||||
"minMs": 7.553665999999794,
|
||||
"maxMs": 13.00845900000013,
|
||||
"medianHeapDeltaBytes": 5290256,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 0.5044450938101623,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 0,
|
||||
"ownedCount": 0,
|
||||
"contentBytesAvoided": 0,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 0,
|
||||
"rejectedRequests": 1,
|
||||
"requestBytes": 670079,
|
||||
"responseBytes": 27,
|
||||
"maxBatchHashes": 10001
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 25005000,
|
||||
"ownedCount": 5001,
|
||||
"contentBytesAvoided": 327745536,
|
||||
"requests": 2,
|
||||
"acceptedRequests": 2,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 670091,
|
||||
"responseBytes": 335089,
|
||||
"maxBatchHashes": 10000
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hashCount": 25000,
|
||||
"current": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 9.382499999999709,
|
||||
"p95Ms": 19.153124999999818,
|
||||
"minMs": 5.52666599999975,
|
||||
"maxMs": 19.153124999999818,
|
||||
"medianHeapDeltaBytes": 4065728,
|
||||
"medianRssDeltaBytes": 1687552
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 15,
|
||||
"medianMs": 31.40979100000004,
|
||||
"p95Ms": 122.20316700000058,
|
||||
"minMs": 10.393082999999933,
|
||||
"maxMs": 122.20316700000058,
|
||||
"medianHeapDeltaBytes": 12853840,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 0.29871258933240585,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 0,
|
||||
"ownedCount": 0,
|
||||
"contentBytesAvoided": 0,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 0,
|
||||
"rejectedRequests": 1,
|
||||
"requestBytes": 1675012,
|
||||
"responseBytes": 27,
|
||||
"maxBatchHashes": 25000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 156237500,
|
||||
"ownedCount": 12500,
|
||||
"contentBytesAvoided": 819200000,
|
||||
"requests": 3,
|
||||
"acceptedRequests": 3,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 1675036,
|
||||
"responseBytes": 837533,
|
||||
"maxBatchHashes": 10000
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T21:22:59.466Z",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"arch": "arm64"
|
||||
},
|
||||
"fixture": {
|
||||
"hashes": 10001,
|
||||
"bytesPerFile": 4096,
|
||||
"uploadConcurrency": 2
|
||||
},
|
||||
"note": "Loopback mock includes every dedup, by-hash and content request. Hashing is excluded. Backend SQL is unmodeled: current rejects before ownership lookup while candidate would execute two accepted queries, so candidate wall time is optimistic.",
|
||||
"cases": [
|
||||
{
|
||||
"hitPercent": 0,
|
||||
"current": {
|
||||
"wallSamplesMs": [
|
||||
13455.567,
|
||||
7274.844,
|
||||
11384.656
|
||||
],
|
||||
"wallMedianMs": 11384.656,
|
||||
"peakHeapDeltaBytesMedian": 26577840,
|
||||
"peakRssDeltaBytesMedian": 1376256,
|
||||
"protocol": {
|
||||
"ownedCount": 0,
|
||||
"hitPercent": 0,
|
||||
"dedupAccepted": 0,
|
||||
"dedupRejected": 1,
|
||||
"dedupRequestBytes": 670079,
|
||||
"dedupResponseBytes": 27,
|
||||
"uploadRequests": 10001,
|
||||
"uploadContentBytes": 40964096,
|
||||
"byHashRequests": 0,
|
||||
"byHashRequestBytes": 0
|
||||
}
|
||||
},
|
||||
"candidate": {
|
||||
"wallSamplesMs": [
|
||||
12545.523,
|
||||
9374.644,
|
||||
14373.866
|
||||
],
|
||||
"wallMedianMs": 12545.523,
|
||||
"peakHeapDeltaBytesMedian": 27057808,
|
||||
"peakRssDeltaBytesMedian": 2408448,
|
||||
"protocol": {
|
||||
"ownedCount": 0,
|
||||
"hitPercent": 0,
|
||||
"dedupAccepted": 2,
|
||||
"dedupRejected": 0,
|
||||
"dedupRequestBytes": 670091,
|
||||
"dedupResponseBytes": 24,
|
||||
"uploadRequests": 10001,
|
||||
"uploadContentBytes": 40964096,
|
||||
"byHashRequests": 0,
|
||||
"byHashRequestBytes": 0
|
||||
}
|
||||
},
|
||||
"wallSpeedup": 0.907,
|
||||
"uploadByteReductionPercent": 0
|
||||
},
|
||||
{
|
||||
"hitPercent": 50,
|
||||
"current": {
|
||||
"wallSamplesMs": [
|
||||
8511.511,
|
||||
10370.193,
|
||||
7271.146
|
||||
],
|
||||
"wallMedianMs": 8511.511,
|
||||
"peakHeapDeltaBytesMedian": 25917352,
|
||||
"peakRssDeltaBytesMedian": 1753088,
|
||||
"protocol": {
|
||||
"ownedCount": 0,
|
||||
"hitPercent": 50,
|
||||
"dedupAccepted": 0,
|
||||
"dedupRejected": 1,
|
||||
"dedupRequestBytes": 670079,
|
||||
"dedupResponseBytes": 27,
|
||||
"uploadRequests": 10001,
|
||||
"uploadContentBytes": 40964096,
|
||||
"byHashRequests": 0,
|
||||
"byHashRequestBytes": 0
|
||||
}
|
||||
},
|
||||
"candidate": {
|
||||
"wallSamplesMs": [
|
||||
8095.953,
|
||||
10204.387,
|
||||
7060.576
|
||||
],
|
||||
"wallMedianMs": 8095.953,
|
||||
"peakHeapDeltaBytesMedian": 50966480,
|
||||
"peakRssDeltaBytesMedian": 29622272,
|
||||
"protocol": {
|
||||
"ownedCount": 5001,
|
||||
"hitPercent": 50,
|
||||
"dedupAccepted": 2,
|
||||
"dedupRejected": 0,
|
||||
"dedupRequestBytes": 670091,
|
||||
"dedupResponseBytes": 335089,
|
||||
"uploadRequests": 5000,
|
||||
"uploadContentBytes": 20480000,
|
||||
"byHashRequests": 5001,
|
||||
"byHashRequestBytes": 574561
|
||||
}
|
||||
},
|
||||
"wallSpeedup": 1.051,
|
||||
"uploadByteReductionPercent": 50.005
|
||||
}
|
||||
],
|
||||
"decision": "rejected; production reverted because the all-miss case was 10.2% slower with higher heap/RSS, while the 50%-hit case doubled heap and added about 27.9 MiB RSS"
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:09:18.980Z",
|
||||
"decisions": {
|
||||
"progressAccumulator": "accepted",
|
||||
"queueCursor": "superseded by the process-isolated memory gate; accepted only by explicit user trade-off",
|
||||
"dedupAbove10000": "rejected by the full workflow gate; production reverted"
|
||||
},
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "all",
|
||||
"warmup": 3,
|
||||
"samples": 9,
|
||||
"queueCounts": [
|
||||
10000,
|
||||
50000,
|
||||
100000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000
|
||||
},
|
||||
{
|
||||
"items": 20000,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
10001,
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 2048,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"queue": [
|
||||
{
|
||||
"chunkCount": 10000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 0.9245830000000126,
|
||||
"p95Ms": 1.8680839999999819,
|
||||
"minMs": 0.25729200000000674,
|
||||
"maxMs": 1.8680839999999819,
|
||||
"medianHeapDeltaBytes": 292640,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 0.18579199999999219,
|
||||
"p95Ms": 0.49079100000000153,
|
||||
"minMs": 0.17525000000000546,
|
||||
"maxMs": 0.49079100000000153,
|
||||
"medianHeapDeltaBytes": 295520,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 4.976441396831142,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 4052813061,
|
||||
"chunkCount": 10000,
|
||||
"totalBytes": 1474232320,
|
||||
"batchCount": 175
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 4052813061,
|
||||
"chunkCount": 10000,
|
||||
"totalBytes": 1474232320,
|
||||
"batchCount": 175,
|
||||
"compactions": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunkCount": 50000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 476.60450000000037,
|
||||
"p95Ms": 780.6920419999997,
|
||||
"minMs": 303.71608300000025,
|
||||
"maxMs": 780.6920419999997,
|
||||
"medianHeapDeltaBytes": 1459600,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 2.605415999998513,
|
||||
"p95Ms": 9.112084000000323,
|
||||
"minMs": 1.348041999999623,
|
||||
"maxMs": 9.112084000000323,
|
||||
"medianHeapDeltaBytes": 1459608,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 182.92836921254508,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 3209478661,
|
||||
"chunkCount": 50000,
|
||||
"totalBytes": 7372603392,
|
||||
"batchCount": 871
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 3209478661,
|
||||
"chunkCount": 50000,
|
||||
"totalBytes": 7372603392,
|
||||
"batchCount": 871,
|
||||
"compactions": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunkCount": 100000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 3615.438249999992,
|
||||
"p95Ms": 6469.931125000003,
|
||||
"minMs": 1282.712916999997,
|
||||
"maxMs": 6469.931125000003,
|
||||
"medianHeapDeltaBytes": 68144,
|
||||
"medianRssDeltaBytes": 622592
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 3.6333750000048894,
|
||||
"p95Ms": 5.698041999989073,
|
||||
"minMs": 2.7038749999919673,
|
||||
"maxMs": 5.698041999989073,
|
||||
"medianHeapDeltaBytes": 854600,
|
||||
"medianRssDeltaBytes": 983040
|
||||
},
|
||||
"speedup": 995.0633364282868,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 725464645,
|
||||
"chunkCount": 100000,
|
||||
"totalBytes": 14745501696,
|
||||
"batchCount": 1741
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 725464645,
|
||||
"chunkCount": 100000,
|
||||
"totalBytes": 14745501696,
|
||||
"batchCount": 1741,
|
||||
"compactions": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"progress": [
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 27.865125000011176,
|
||||
"p95Ms": 108.79308299999684,
|
||||
"minMs": 17.72166599999764,
|
||||
"maxMs": 108.79308299999684,
|
||||
"medianHeapDeltaBytes": 1540496,
|
||||
"medianRssDeltaBytes": 16384
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 0.04041699999652337,
|
||||
"p95Ms": 8.611833000002662,
|
||||
"minMs": 0.031541999996989034,
|
||||
"maxMs": 8.611833000002662,
|
||||
"medianHeapDeltaBytes": 24872,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 689.4407057032463,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 220783770,
|
||||
"lastPercent": 51,
|
||||
"finalSum": 509.25390625
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 220783770,
|
||||
"lastPercent": 51,
|
||||
"finalSum": 509.25390625
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 104.48600000000442,
|
||||
"p95Ms": 187.45629200000258,
|
||||
"minMs": 73.81195800000569,
|
||||
"maxMs": 187.45629200000258,
|
||||
"medianHeapDeltaBytes": 1456624,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 0.033750000002328306,
|
||||
"p95Ms": 0.18004200000723358,
|
||||
"minMs": 0.03200000000651926,
|
||||
"maxMs": 0.18004200000723358,
|
||||
"medianHeapDeltaBytes": 240872,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 3095.8814812680375,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 74165188,
|
||||
"lastPercent": 19,
|
||||
"finalSum": 1915.6953125
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 74165188,
|
||||
"lastPercent": 19,
|
||||
"finalSum": 1915.6953125
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"items": 20000,
|
||||
"updates": 5000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 282.27433400000155,
|
||||
"p95Ms": 465.90625,
|
||||
"minMs": 188.45816700000432,
|
||||
"maxMs": 465.90625,
|
||||
"medianHeapDeltaBytes": 4189728,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 0.08487500000046566,
|
||||
"p95Ms": 0.19566700000723358,
|
||||
"minMs": 0.06799999999930151,
|
||||
"maxMs": 0.19566700000723358,
|
||||
"medianHeapDeltaBytes": 680872,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 3325.7653490244816,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 47455232,
|
||||
"lastPercent": 11,
|
||||
"finalSum": 2175.87109375
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 47455232,
|
||||
"lastPercent": 11,
|
||||
"finalSum": 2175.87109375
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"dedup": [
|
||||
{
|
||||
"hashCount": 10001,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 4.849792000008165,
|
||||
"p95Ms": 17.9151669999992,
|
||||
"minMs": 1.8024589999986347,
|
||||
"maxMs": 17.9151669999992,
|
||||
"medianHeapDeltaBytes": 2372344,
|
||||
"medianRssDeltaBytes": 671744
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 12.786791999998968,
|
||||
"p95Ms": 75.20012499998847,
|
||||
"minMs": 7.695166999998037,
|
||||
"maxMs": 75.20012499998847,
|
||||
"medianHeapDeltaBytes": 5584440,
|
||||
"medianRssDeltaBytes": 16384
|
||||
},
|
||||
"speedup": 0.3792813709653333,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 0,
|
||||
"ownedCount": 0,
|
||||
"contentBytesAvoided": 0,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 0,
|
||||
"rejectedRequests": 1,
|
||||
"requestBytes": 670079,
|
||||
"responseBytes": 27,
|
||||
"maxBatchHashes": 10001
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 25005000,
|
||||
"ownedCount": 5001,
|
||||
"contentBytesAvoided": 327745536,
|
||||
"requests": 5,
|
||||
"acceptedRequests": 5,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 670127,
|
||||
"responseBytes": 335122,
|
||||
"maxBatchHashes": 2048
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hashCount": 25000,
|
||||
"current": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 5.98787500000617,
|
||||
"p95Ms": 27.3851250000007,
|
||||
"minMs": 3.5451669999893056,
|
||||
"maxMs": 27.3851250000007,
|
||||
"medianHeapDeltaBytes": 4067864,
|
||||
"medianRssDeltaBytes": 1687552
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 9,
|
||||
"medianMs": 19.696374999999534,
|
||||
"p95Ms": 56.47870900000271,
|
||||
"minMs": 15.562999999994645,
|
||||
"maxMs": 56.47870900000271,
|
||||
"medianHeapDeltaBytes": 13599800,
|
||||
"medianRssDeltaBytes": 16384
|
||||
},
|
||||
"speedup": 0.3040089864254875,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 0,
|
||||
"ownedCount": 0,
|
||||
"contentBytesAvoided": 0,
|
||||
"requests": 1,
|
||||
"acceptedRequests": 0,
|
||||
"rejectedRequests": 1,
|
||||
"requestBytes": 1675012,
|
||||
"responseBytes": 27,
|
||||
"maxBatchHashes": 25000
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 156237500,
|
||||
"ownedCount": 12500,
|
||||
"contentBytesAvoided": 819200000,
|
||||
"requests": 13,
|
||||
"acceptedRequests": 13,
|
||||
"rejectedRequests": 0,
|
||||
"requestBytes": 1675156,
|
||||
"responseBytes": 837643,
|
||||
"maxBatchHashes": 2048
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"benchmark": "dedup_gc_phase1_manifest_batching",
|
||||
"date": "2026-07-22",
|
||||
"baseline_source": "src/infrastructure/services/dedup_service.rs::garbage_collect_with_grace phase 1",
|
||||
"environment": {
|
||||
"postgres_image": "postgres:18.2-alpine3.23",
|
||||
"container": "oxicloud-postgres-1",
|
||||
"database_isolation": "random throw-away database, dropped by runner trap",
|
||||
"harness_profile": "release",
|
||||
"transport_note": "SQLx used the container private IP because the sandbox could not reach the published host port. Statement-count reduction is transport-independent; medians are specific to this host path.",
|
||||
"ordering": "candidate order rotated every sample; fixture reset and correctness validation excluded from timing"
|
||||
},
|
||||
"fixture": {
|
||||
"batch_size": 500,
|
||||
"chunks_per_manifest": 16,
|
||||
"shared_chunk_percent": 50,
|
||||
"shared_chunk_pool": 512,
|
||||
"duplicate_control": "every 10th manifest repeats one chunk; accounting is one reference per distinct hash per manifest"
|
||||
},
|
||||
"rejected_atomic_cte_idle_gate": [
|
||||
{
|
||||
"orphan_manifests": 0,
|
||||
"live_manifests": 10,
|
||||
"samples": 31,
|
||||
"current_median_ms": 1.950,
|
||||
"cte_median_ms": 2.463,
|
||||
"cte_regression_percent": 26.28,
|
||||
"statements_current_to_cte": "1 -> 1"
|
||||
},
|
||||
{
|
||||
"orphan_manifests": 0,
|
||||
"live_manifests": 500,
|
||||
"samples": 41,
|
||||
"current_median_ms": 1.704,
|
||||
"cte_median_ms": 2.468,
|
||||
"cte_regression_percent": 44.84,
|
||||
"statements_current_to_cte": "1 -> 1"
|
||||
},
|
||||
{
|
||||
"orphan_manifests": 0,
|
||||
"live_manifests": 5000,
|
||||
"samples": 31,
|
||||
"current_median_ms": 1.951,
|
||||
"cte_median_ms": 2.256,
|
||||
"cte_regression_percent": 15.59,
|
||||
"statements_current_to_cte": "1 -> 1"
|
||||
}
|
||||
],
|
||||
"accepted_hybrid": {
|
||||
"status": "accepted_by_explicit_user_tradeoff",
|
||||
"shape": "simple DELETE/RETURNING always; exact historical serial UPDATE for one manifest; Rust distinct-per-manifest aggregation plus one UPDATE FROM unnest for batches of two or more",
|
||||
"aggregate_threshold": 2,
|
||||
"thresholds_swept": [2, 4, 8, 32, 500],
|
||||
"idle_and_single_no_regression_gate": [
|
||||
{
|
||||
"orphan_manifests": 0,
|
||||
"live_manifests": 500,
|
||||
"warmups": 5,
|
||||
"samples": 31,
|
||||
"current_elapsed_ms": [0.639, 0.611, 0.619, 2.360, 0.770, 0.850, 0.934, 0.985, 0.655, 0.791, 0.708, 0.727, 0.740, 0.674, 1.260, 0.881, 0.877, 0.738, 0.674, 0.753, 0.769, 0.686, 0.652, 0.661, 0.531, 0.876, 0.653, 0.668, 0.674, 0.739, 0.740],
|
||||
"hybrid_elapsed_ms": [0.752, 0.602, 0.711, 1.061, 0.754, 0.868, 0.967, 0.815, 0.741, 0.777, 0.710, 0.869, 0.639, 0.547, 1.141, 1.206, 1.105, 0.692, 0.824, 0.794, 0.721, 0.656, 0.610, 0.632, 0.681, 0.726, 0.519, 0.623, 0.596, 0.679, 0.737],
|
||||
"current_median_ms": 0.738,
|
||||
"hybrid_median_ms": 0.726,
|
||||
"median_speedup_x": 1.02,
|
||||
"statements_current_to_hybrid": "1 -> 1",
|
||||
"correctness": "PASS"
|
||||
},
|
||||
{
|
||||
"orphan_manifests": 1,
|
||||
"live_manifests": 499,
|
||||
"warmups": 5,
|
||||
"samples": 31,
|
||||
"current_elapsed_ms": [5.058, 5.156, 4.205, 223.094, 4.718, 4.605, 3.509, 2.939, 4.250, 4.591, 5.255, 4.127, 5.163, 3.950, 6.076, 5.642, 4.428, 3.392, 4.246, 3.913, 4.708, 4.228, 3.927, 5.702, 4.062, 4.862, 4.540, 3.988, 3.854, 6.148, 4.436],
|
||||
"hybrid_elapsed_ms": [3.713, 4.915, 4.106, 5.162, 5.328, 3.439, 4.279, 3.096, 4.479, 4.595, 5.153, 5.766, 3.427, 7.188, 4.423, 3.684, 3.561, 4.434, 3.777, 3.453, 4.016, 6.334, 4.455, 4.692, 3.840, 3.406, 3.860, 4.336, 4.252, 6.061, 3.809],
|
||||
"current_median_ms": 4.436,
|
||||
"hybrid_median_ms": 4.279,
|
||||
"median_speedup_x": 1.04,
|
||||
"statements_current_to_hybrid": "3 -> 3",
|
||||
"correctness": "PASS"
|
||||
}
|
||||
],
|
||||
"crossover_gate": [
|
||||
{"orphan_manifests": 2, "samples": 9, "current_median_ms": 6.323, "hybrid_median_ms": 5.806, "median_speedup_x": 1.09, "statements_current_to_hybrid": "4 -> 3", "correctness": "PASS"},
|
||||
{"orphan_manifests": 4, "samples": 9, "current_median_ms": 11.719, "hybrid_median_ms": 5.761, "median_speedup_x": 2.03, "statements_current_to_hybrid": "6 -> 3", "correctness": "PASS"},
|
||||
{"orphan_manifests": 8, "samples": 9, "current_median_ms": 40.489, "hybrid_median_ms": 5.662, "median_speedup_x": 7.15, "statements_current_to_hybrid": "10 -> 3", "correctness": "PASS"},
|
||||
{"orphan_manifests": 32, "samples": 9, "current_median_ms": 212.165, "hybrid_median_ms": 8.117, "median_speedup_x": 26.14, "statements_current_to_hybrid": "34 -> 3", "correctness": "PASS"}
|
||||
],
|
||||
"large_batch_gate": [
|
||||
{
|
||||
"orphan_manifests": 500,
|
||||
"samples": 5,
|
||||
"current_elapsed_ms": [907.975, 458.300, 1456.096, 1795.532, 1568.453],
|
||||
"hybrid_elapsed_ms": [25.468, 18.753, 20.155, 191.159, 24.255],
|
||||
"current_median_ms": 1456.096,
|
||||
"hybrid_median_ms": 24.255,
|
||||
"median_speedup_x": 60.03,
|
||||
"statements_current_to_hybrid": "502 -> 3",
|
||||
"correctness": "PASS"
|
||||
},
|
||||
{
|
||||
"orphan_manifests": 1000,
|
||||
"samples": 5,
|
||||
"current_elapsed_ms": [4280.878, 3925.644, 3163.190, 1364.115, 1738.119],
|
||||
"hybrid_elapsed_ms": [61.829, 310.179, 49.885, 51.440, 65.973],
|
||||
"current_median_ms": 3163.190,
|
||||
"hybrid_median_ms": 61.829,
|
||||
"median_speedup_x": 51.16,
|
||||
"statements_current_to_hybrid": "1003 -> 5",
|
||||
"correctness": "PASS"
|
||||
}
|
||||
],
|
||||
"fresh_process_max_rss_vs_serial": [
|
||||
{
|
||||
"orphan_manifests": 2,
|
||||
"runs_per_mode": 5,
|
||||
"serial_bytes": [7733248, 7684096, 7749632, 7766016, 7749632],
|
||||
"hybrid_bytes": [7749632, 7798784, 7782400, 7733248, 7700480],
|
||||
"serial_median_bytes": 7749632,
|
||||
"hybrid_median_bytes": 7749632,
|
||||
"hybrid_delta_bytes": 0
|
||||
},
|
||||
{
|
||||
"orphan_manifests": 500,
|
||||
"runs_per_mode": 5,
|
||||
"serial_bytes": [8503296, 8404992, 8437760, 8519680, 8388608],
|
||||
"hybrid_bytes": [9093120, 9109504, 9175040, 9175040, 9175040],
|
||||
"serial_median_bytes": 8437760,
|
||||
"hybrid_median_bytes": 9175040,
|
||||
"hybrid_delta_bytes": 737280,
|
||||
"hybrid_delta_kib": 720,
|
||||
"hybrid_delta_percent": 8.74
|
||||
},
|
||||
{
|
||||
"orphan_manifests": 1000,
|
||||
"runs_per_mode": 5,
|
||||
"serial_bytes": [8568832, 8585216, 8568832, 8503296, 8536064],
|
||||
"hybrid_bytes": [9486336, 9650176, 9650176, 9601024, 9306112],
|
||||
"serial_median_bytes": 8568832,
|
||||
"hybrid_median_bytes": 9601024,
|
||||
"hybrid_delta_bytes": 1032192,
|
||||
"hybrid_delta_kib": 1008,
|
||||
"hybrid_delta_percent": 12.05
|
||||
}
|
||||
],
|
||||
"tradeoff": "explicitly accepted by the user: retain 60.03x/51.16x median speedups for +720 KiB/+1008 KiB max RSS at 500/1000 orphan manifests"
|
||||
},
|
||||
"rejected_borrowed_sqlx_bind": {
|
||||
"candidate": "bind Vec<&str> borrowed from the returned manifest batch instead of cloning each unique hash into Vec<String>",
|
||||
"latency_gate": [
|
||||
{"orphan_manifests": 2, "samples": 15, "owned_median_ms": 4.843, "borrowed_median_ms": 4.800, "borrowed_delta_percent": 0.89},
|
||||
{"orphan_manifests": 500, "samples": 15, "owned_median_ms": 24.709, "borrowed_median_ms": 27.459, "borrowed_regression_percent": 11.13},
|
||||
{"orphan_manifests": 1000, "samples": 15, "owned_median_ms": 55.042, "borrowed_median_ms": 56.439, "borrowed_regression_percent": 2.54}
|
||||
],
|
||||
"fresh_process_max_rss_gate": [
|
||||
{"orphan_manifests": 2, "runs_per_mode": 3, "owned_bytes": [7880704, 7798784, 7897088], "borrowed_bytes": [7700480, 7766016, 7864320], "owned_median_bytes": 7880704, "borrowed_median_bytes": 7766016, "borrowed_saves_bytes": 114688},
|
||||
{"orphan_manifests": 500, "runs_per_mode": 3, "owned_bytes": [9224192, 9191424, 9224192], "borrowed_bytes": [9093120, 9043968, 9011200], "owned_median_bytes": 9224192, "borrowed_median_bytes": 9043968, "borrowed_saves_bytes": 180224},
|
||||
{"orphan_manifests": 1000, "runs_per_mode": 3, "owned_bytes": [9912320, 9584640, 9797632], "borrowed_bytes": [9388032, 9469952, 9240576], "owned_median_bytes": 9797632, "borrowed_median_bytes": 9388032, "borrowed_saves_bytes": 409600}
|
||||
],
|
||||
"decision": "rejected; it saved 112-400 KiB max RSS but regressed median latency by 11.13% at 500 and 2.54% at 1000"
|
||||
},
|
||||
"rejected_sorted_rle": {
|
||||
"candidate": "borrowed hash references sorted/deduplicated per manifest, then globally sorted and run-length counted",
|
||||
"latency_vs_owned_hashmap": [
|
||||
{"orphan_manifests": 2, "samples": 15, "owned_median_ms": 37.077, "sorted_median_ms": 61.889, "sorted_regression_percent": 66.93},
|
||||
{"orphan_manifests": 500, "samples": 15, "owned_median_ms": 63.912, "sorted_median_ms": 64.707, "sorted_regression_percent": 1.24},
|
||||
{"orphan_manifests": 1000, "samples": 15, "owned_median_ms": 151.758, "sorted_median_ms": 140.429, "sorted_improvement_percent": 7.47}
|
||||
],
|
||||
"fresh_process_max_rss_vs_serial": [
|
||||
{"orphan_manifests": 2, "runs_per_mode": 5, "serial_median_bytes": 7766016, "sorted_median_bytes": 7782400, "sorted_delta_bytes": 16384},
|
||||
{"orphan_manifests": 500, "runs_per_mode": 5, "serial_median_bytes": 8421376, "sorted_median_bytes": 8978432, "sorted_delta_bytes": 557056},
|
||||
{"orphan_manifests": 1000, "runs_per_mode": 5, "serial_median_bytes": 8503296, "sorted_median_bytes": 9273344, "sorted_delta_bytes": 770048}
|
||||
],
|
||||
"bounded_followup": "exploratory only and interrupted after the user explicitly selected the faster threshold-2 HashMap implementation; never applied to production",
|
||||
"decision": "rejected; it was not Pareto on latency and still raised max RSS"
|
||||
},
|
||||
"correctness_checks": [
|
||||
"all collectible manifests removed",
|
||||
"file-backed live manifests preserved",
|
||||
"blob ref_count equals the distinct-per-manifest reference model",
|
||||
"repeated chunk hashes within one manifest decrement once",
|
||||
"no ref_count underflow",
|
||||
"new zero-ref blobs receive orphaned_at",
|
||||
"live referenced blobs keep orphaned_at NULL",
|
||||
"statement counts equal the selected algorithm exactly"
|
||||
],
|
||||
"failure_semantics": "The accepted hybrid retains the historical separate DELETE then UPDATE failure semantics. A failure between them can leave a conservative high ref_count/leak; no transaction was added because BEGIN/COMMIT would require a separate latency gate.",
|
||||
"decision": "accepted_by_explicit_user_tradeoff: keep owned-HashMap threshold 2; 60.03x/51.16x median speedups cost +720 KiB/+1008 KiB max RSS at 500/1000 manifests. Atomic-CTE, borrowed-bind, and sorted/RLE alternatives remain rejected."
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21",
|
||||
"environment": {
|
||||
"platform": "darwin",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"postgres": "18.2 local Docker container"
|
||||
},
|
||||
"fixture": {
|
||||
"blobs": 1000000,
|
||||
"sampleSize": 100,
|
||||
"hashShape": "two concatenated MD5 hex digests",
|
||||
"middleSampleCount": 100,
|
||||
"wraparoundSampleCount": 100
|
||||
},
|
||||
"orderByRandom": {
|
||||
"samplesMs": [111.796, 103.906, 93.862, 88.090, 100.442],
|
||||
"medianMs": 100.442
|
||||
},
|
||||
"indexedHashRing": {
|
||||
"samplesMs": [0.474, 0.323, 0.376, 0.267, 0.491],
|
||||
"medianMs": 0.376
|
||||
},
|
||||
"speedup": 267.13,
|
||||
"statisticalReview": {
|
||||
"orderedHashFailureRangePercent": 1,
|
||||
"contiguousWindowDetectionProbabilityPercent": 1,
|
||||
"independentSamplesDetectionProbabilityPercent": 63.4,
|
||||
"reason": "A single random-pivot successor window is gap-biased and its 100 rows are correlated. It does not preserve ORDER BY random() detection power for localized or prefix/backend failures."
|
||||
},
|
||||
"decision": "rejected; production reverted despite 267.13x query speed because verification semantics regressed"
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21",
|
||||
"benchmark": "run_migration_workset_materialization",
|
||||
"baseline": "one ordered SQL stream collected into Vec before backend work",
|
||||
"candidates": [
|
||||
"keyset pages of 65536 rows, each page released after consumption",
|
||||
"keyset pages of 262144 rows, each page released after consumption"
|
||||
],
|
||||
"environment": {
|
||||
"clientPlatform": "macOS 26.5.2 arm64, Apple M4 Pro",
|
||||
"postgres": "18.2-alpine3.23 in oxicloud-postgres-1",
|
||||
"transport": "SQLx over the container private IP because the published localhost port refused connections",
|
||||
"profile": "release",
|
||||
"processIsolation": "one fresh client process per sample; connection and COUNT warm-up precede the timer"
|
||||
},
|
||||
"fixture": {
|
||||
"rows": 1000000,
|
||||
"rowShape": "64-byte lowercase hexadecimal hash plus bigint size",
|
||||
"expectedChecksum": 4156242879243796640,
|
||||
"order": "ORDER BY hash",
|
||||
"measuredCompleteRounds": 5,
|
||||
"plannedRounds": 7,
|
||||
"executionOrder": [
|
||||
["current", "paged65536", "paged262144"],
|
||||
["paged262144", "paged65536", "current"],
|
||||
["paged65536", "current", "paged262144"],
|
||||
["current", "paged65536", "paged262144"],
|
||||
["paged262144", "paged65536", "current"]
|
||||
]
|
||||
},
|
||||
"results": {
|
||||
"current": {
|
||||
"elapsedMs": [644.398, 964.231, 261.508, 1175.821, 265.14],
|
||||
"medianElapsedMs": 644.398,
|
||||
"maxToMinElapsedRatio": 4.5,
|
||||
"rssBytes": [106020864, 106070016, 106004480, 106053632, 106053632],
|
||||
"medianRssBytes": 106053632,
|
||||
"peakRows": 1000000
|
||||
},
|
||||
"paged65536": {
|
||||
"elapsedMs": [1198.549, 4080.427, 1622.317, 2558.891, 1508.613],
|
||||
"medianElapsedMs": 1622.317,
|
||||
"elapsedRatioVsCurrent": 2.5176,
|
||||
"medianElapsedRegressionPercent": 151.76,
|
||||
"maxToMinElapsedRatio": 3.4,
|
||||
"rssBytes": [18366464, 16236544, 16236544, 18350080, 18300928],
|
||||
"medianRssBytes": 18300928,
|
||||
"medianRssReductionPercent": 82.74,
|
||||
"peakRows": 65536
|
||||
},
|
||||
"paged262144": {
|
||||
"elapsedMs": [320.191, 1297.052, 2871.904, 2333.342, 1928.731],
|
||||
"medianElapsedMs": 1928.731,
|
||||
"elapsedRatioVsCurrent": 2.9931,
|
||||
"medianElapsedRegressionPercent": 199.31,
|
||||
"maxToMinElapsedRatio": 8.97,
|
||||
"rssBytes": [60833792, 60866560, 58703872, 63012864, 58654720],
|
||||
"medianRssBytes": 60833792,
|
||||
"medianRssReductionPercent": 42.64,
|
||||
"peakRows": 262144
|
||||
}
|
||||
},
|
||||
"correctness": {
|
||||
"allMeasuredSamplesReturnedRows": 1000000,
|
||||
"allMeasuredSamplesReturnedChecksum": 4156242879243796640,
|
||||
"orderedRowEquivalence": "PASS"
|
||||
},
|
||||
"invalidAttempts": [
|
||||
"The first host-private-IP run stopped after four samples with PoolTimedOut and was discarded.",
|
||||
"The recorded run stopped after five complete three-way rounds when the Docker bridge failed again; its complete rounds are retained because every mode is present in each round.",
|
||||
"A final attempt compiled the harness inside a preinstalled Rust container sharing PostgreSQL's network namespace, but the build container exited before producing any benchmark sample."
|
||||
],
|
||||
"decision": "rejected; production unchanged",
|
||||
"reason": "Both bounded-memory candidates reduced RSS, but neither passed the no-material-latency-regression gate. Production run_migration remains unchanged."
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:30:26.794Z",
|
||||
"decision": "evidence_only_superseded_by_focused_gate; exact output and all medians improved, but the one-file p95 was noisy and regressed, so acceptance relies on the later 41-sample focused gate",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "progress",
|
||||
"warmup": 5,
|
||||
"samples": 25,
|
||||
"queueCounts": [
|
||||
64,
|
||||
256,
|
||||
1024,
|
||||
10000,
|
||||
50000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100
|
||||
},
|
||||
{
|
||||
"items": 10,
|
||||
"updates": 500
|
||||
},
|
||||
{
|
||||
"items": 100,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
1000,
|
||||
10000,
|
||||
10001,
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 10000,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"progress": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100,
|
||||
"repetitions": 1000,
|
||||
"normalizedMedianUsPerRun": {
|
||||
"current": 1.069250000000011,
|
||||
"candidate": 0.32845800000006875
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 1.069250000000011,
|
||||
"p95Ms": 2.1355839999999944,
|
||||
"minMs": 0.47479199999997945,
|
||||
"maxMs": 2.813957999999957,
|
||||
"medianHeapDeltaBytes": 1768864,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 0.32845800000006875,
|
||||
"p95Ms": 11.240291999999954,
|
||||
"minMs": 0.2692909999999529,
|
||||
"maxMs": 35.88008300000001,
|
||||
"medianHeapDeltaBytes": 168848,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 3.255362938335456,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 8562000,
|
||||
"lastPercent": 22,
|
||||
"finalSum": 215.8203125
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 8562000,
|
||||
"lastPercent": 22,
|
||||
"finalSum": 215.8203125
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"items": 10,
|
||||
"updates": 500,
|
||||
"repetitions": 200,
|
||||
"normalizedMedianUsPerRun": {
|
||||
"current": 17.718959999999697,
|
||||
"candidate": 3.242289999999457
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 3.5437919999999394,
|
||||
"p95Ms": 15.97524999999996,
|
||||
"minMs": 1.7652080000000296,
|
||||
"maxMs": 239.03012499999977,
|
||||
"medianHeapDeltaBytes": 1099512,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 0.6484579999998914,
|
||||
"p95Ms": 3.759457999999995,
|
||||
"minMs": 0.22216699999989942,
|
||||
"maxMs": 35.765333000000055,
|
||||
"medianHeapDeltaBytes": 77648,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 5.464952240546856,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 27254400,
|
||||
"lastPercent": 44,
|
||||
"finalSum": 871.2890625
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 27254400,
|
||||
"lastPercent": 44,
|
||||
"finalSum": 871.2890625
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"items": 100,
|
||||
"updates": 5000,
|
||||
"repetitions": 20,
|
||||
"normalizedMedianUsPerRun": {
|
||||
"current": 6256.843750000007,
|
||||
"candidate": 13.327049999998053
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 125.13687500000015,
|
||||
"p95Ms": 234.41116699999975,
|
||||
"minMs": 33.03545800000029,
|
||||
"maxMs": 246.10770900000034,
|
||||
"medianHeapDeltaBytes": 1590568,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 0.26654099999996106,
|
||||
"p95Ms": 2.754667000000154,
|
||||
"minMs": 0.2102919999997539,
|
||||
"maxMs": 4.791457999999693,
|
||||
"medianHeapDeltaBytes": 51728,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 469.48452583286786,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 240082660,
|
||||
"lastPercent": 51,
|
||||
"finalSum": 1024.8828125
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 240082660,
|
||||
"lastPercent": 51,
|
||||
"finalSum": 1024.8828125
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:30:43.534Z",
|
||||
"decision": "accepted; repeated tiny-case gate improved with identical output",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "progress",
|
||||
"warmup": 7,
|
||||
"samples": 41,
|
||||
"queueCounts": [
|
||||
64,
|
||||
256,
|
||||
1024,
|
||||
10000,
|
||||
50000
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
1000,
|
||||
10000,
|
||||
10001,
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 10000,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"progress": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100,
|
||||
"repetitions": 1000,
|
||||
"normalizedMedianUsPerRun": {
|
||||
"current": 0.7972500000000764,
|
||||
"candidate": 0.28862500000002456
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 41,
|
||||
"medianMs": 0.7972500000000764,
|
||||
"p95Ms": 14.158166000000165,
|
||||
"minMs": 0.4542079999998805,
|
||||
"maxMs": 34.321249999999964,
|
||||
"medianHeapDeltaBytes": 1768864,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 41,
|
||||
"medianMs": 0.28862500000002456,
|
||||
"p95Ms": 6.337833000000046,
|
||||
"minMs": 0.2605829999999969,
|
||||
"maxMs": 20.904042000000004,
|
||||
"medianHeapDeltaBytes": 168848,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 2.7622347336509607,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 8562000,
|
||||
"lastPercent": 22,
|
||||
"finalSum": 215.8203125
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 8562000,
|
||||
"lastPercent": 22,
|
||||
"finalSum": 215.8203125
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T20:30:20.429Z",
|
||||
"decision": "superseded by queue-memory-process-node26-macos-arm64.json; cursor accepted only by explicit user trade-off after the isolated RSS gate",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"release": "25.5.0",
|
||||
"arch": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"logicalCpus": 14,
|
||||
"gcExposed": true
|
||||
},
|
||||
"config": {
|
||||
"suite": "queue",
|
||||
"warmup": 5,
|
||||
"samples": 25,
|
||||
"queueCounts": [
|
||||
64,
|
||||
256,
|
||||
1024
|
||||
],
|
||||
"progressCases": [
|
||||
{
|
||||
"items": 1,
|
||||
"updates": 100
|
||||
},
|
||||
{
|
||||
"items": 10,
|
||||
"updates": 500
|
||||
},
|
||||
{
|
||||
"items": 100,
|
||||
"updates": 5000
|
||||
},
|
||||
{
|
||||
"items": 1000,
|
||||
"updates": 10000
|
||||
},
|
||||
{
|
||||
"items": 10000,
|
||||
"updates": 5000
|
||||
}
|
||||
],
|
||||
"hashCounts": [
|
||||
1000,
|
||||
10000,
|
||||
10001,
|
||||
25000
|
||||
],
|
||||
"dedupBatchSize": 10000,
|
||||
"dedupConcurrency": 4,
|
||||
"serverLatencyMs": 0,
|
||||
"modeledFileBytes": 65536
|
||||
},
|
||||
"notes": {
|
||||
"heap": "Median heap delta is indicative only; timing is the primary microbenchmark metric.",
|
||||
"dedup": "The current >10k path is faster only because it is rejected and returns no owned hashes."
|
||||
},
|
||||
"suites": {
|
||||
"queue": [
|
||||
{
|
||||
"chunkCount": 64,
|
||||
"repetitions": 3125,
|
||||
"normalizedMedianUsPerDrain": {
|
||||
"current": 3.4471731200000066,
|
||||
"candidate": 0.8570931200000632
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 10.772416000000021,
|
||||
"p95Ms": 106.93650000000002,
|
||||
"minMs": 3.077792000000045,
|
||||
"maxMs": 155.11179199999992,
|
||||
"medianHeapDeltaBytes": 296368,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 2.6784160000001975,
|
||||
"p95Ms": 14.533292000000074,
|
||||
"minMs": 1.02737500000012,
|
||||
"maxMs": 21.65000000000009,
|
||||
"medianHeapDeltaBytes": 159376,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 4.0219353528351185,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 2315114185,
|
||||
"chunkCount": 200000,
|
||||
"totalBytes": 29184000000,
|
||||
"batchCount": 6250
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 2315114185,
|
||||
"chunkCount": 200000,
|
||||
"totalBytes": 29184000000,
|
||||
"batchCount": 6250
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunkCount": 256,
|
||||
"repetitions": 782,
|
||||
"normalizedMedianUsPerDrain": {
|
||||
"current": 8.675297953964206,
|
||||
"candidate": 2.503888746802898
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 6.78408300000001,
|
||||
"p95Ms": 32.81195799999978,
|
||||
"minMs": 4.367833000000246,
|
||||
"maxMs": 164.17550000000028,
|
||||
"medianHeapDeltaBytes": 1858520,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 1.9580409999998665,
|
||||
"p95Ms": 5.698667000000114,
|
||||
"minMs": 0.9418750000004366,
|
||||
"maxMs": 47.76333300000033,
|
||||
"medianHeapDeltaBytes": 1816160,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 3.464729798814464,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 1196738502,
|
||||
"chunkCount": 200192,
|
||||
"totalBytes": 29365764096,
|
||||
"batchCount": 3910
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 1196738502,
|
||||
"chunkCount": 200192,
|
||||
"totalBytes": 29365764096,
|
||||
"batchCount": 3910
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"chunkCount": 1024,
|
||||
"repetitions": 196,
|
||||
"normalizedMedianUsPerDrain": {
|
||||
"current": 52.38349999999813,
|
||||
"candidate": 9.426867346937055
|
||||
},
|
||||
"current": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 10.267165999999634,
|
||||
"p95Ms": 142.3090830000001,
|
||||
"minMs": 4.935207999999875,
|
||||
"maxMs": 207.7039160000004,
|
||||
"medianHeapDeltaBytes": 1796184,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"candidate": {
|
||||
"sampleCount": 25,
|
||||
"medianMs": 1.8476659999996627,
|
||||
"p95Ms": 13.679167000000234,
|
||||
"minMs": 0.9290839999994205,
|
||||
"maxMs": 14.767958999999792,
|
||||
"medianHeapDeltaBytes": 1775256,
|
||||
"medianRssDeltaBytes": 0
|
||||
},
|
||||
"speedup": 5.556830076432379,
|
||||
"representative": {
|
||||
"current": {
|
||||
"checksum": 1444678356,
|
||||
"chunkCount": 200704,
|
||||
"totalBytes": 29543628800,
|
||||
"batchCount": 3528
|
||||
},
|
||||
"candidate": {
|
||||
"checksum": 1444678356,
|
||||
"chunkCount": 200704,
|
||||
"totalBytes": 29543628800,
|
||||
"batchCount": 3528
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"blackhole": 0
|
||||
}
|
||||
@@ -0,0 +1,693 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": "2026-07-21T21:43:54.899Z",
|
||||
"environment": {
|
||||
"node": "v26.5.0",
|
||||
"platform": "darwin",
|
||||
"arch": "arm64"
|
||||
},
|
||||
"fixture": {
|
||||
"chunks": 100000,
|
||||
"samples": 5,
|
||||
"uploadBatchBytes": 8388608
|
||||
},
|
||||
"note": "Each row is a fresh process; maxRSS is process.resourceUsage().maxRSS. The ordered chunks table remains live through final GC.",
|
||||
"decision": {
|
||||
"status": "accepted_by_explicit_user_tradeoff",
|
||||
"productionMode": "cursor-clear-4096",
|
||||
"reason": "The candidate is not Pareto-superior: the user explicitly accepted the measured RSS cost in exchange for the CPU/wall-time reduction.",
|
||||
"currentVsProduction": {
|
||||
"prefilled": {
|
||||
"wallSpeedup": 246.783,
|
||||
"maxRssDeltaBytes": 458752,
|
||||
"peakRssDeltaDeltaBytes": 131072,
|
||||
"retainedRssDeltaDeltaBytes": 458752,
|
||||
"retainedHeapDeltaDeltaBytes": -3080
|
||||
},
|
||||
"streamingAhead": {
|
||||
"wallSpeedup": 37.137,
|
||||
"maxRssDeltaBytes": 491520,
|
||||
"peakRssDeltaDeltaBytes": 507904,
|
||||
"retainedRssDeltaDeltaBytes": 1032192,
|
||||
"retainedHeapDeltaDeltaBytes": -3576
|
||||
},
|
||||
"streamingBalanced": {
|
||||
"wallSpeedup": 1.184,
|
||||
"maxRssDeltaBytes": 114688,
|
||||
"peakRssDeltaDeltaBytes": 163840,
|
||||
"retainedRssDeltaDeltaBytes": 0,
|
||||
"retainedHeapDeltaDeltaBytes": 3168
|
||||
}
|
||||
}
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "current-shift",
|
||||
"wallMedianMs": 3526.774,
|
||||
"maxRssBytesMedian": 71516160,
|
||||
"peakRssDeltaBytesMedian": 606208,
|
||||
"retainedRssDeltaBytesMedian": 180224,
|
||||
"retainedHeapDeltaBytesMedian": 265536,
|
||||
"maxRssSamplesBytes": [
|
||||
71335936,
|
||||
71516160,
|
||||
71974912,
|
||||
71368704,
|
||||
72220672
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
557056,
|
||||
81920,
|
||||
606208,
|
||||
622592,
|
||||
622592
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-clear-1024",
|
||||
"wallMedianMs": 11.252,
|
||||
"maxRssBytesMedian": 71745536,
|
||||
"peakRssDeltaBytesMedian": 606208,
|
||||
"retainedRssDeltaBytesMedian": 475136,
|
||||
"retainedHeapDeltaBytesMedian": 262240,
|
||||
"maxRssSamplesBytes": [
|
||||
71172096,
|
||||
72105984,
|
||||
71974912,
|
||||
71745536,
|
||||
71680000
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
671744,
|
||||
1015808,
|
||||
540672,
|
||||
606208,
|
||||
540672
|
||||
],
|
||||
"compactions": 7
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-clear-4096",
|
||||
"wallMedianMs": 14.291,
|
||||
"maxRssBytesMedian": 71974912,
|
||||
"peakRssDeltaBytesMedian": 737280,
|
||||
"retainedRssDeltaBytesMedian": 638976,
|
||||
"retainedHeapDeltaBytesMedian": 262456,
|
||||
"maxRssSamplesBytes": [
|
||||
71991296,
|
||||
71516160,
|
||||
70926336,
|
||||
71974912,
|
||||
72007680
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
933888,
|
||||
622592,
|
||||
720896,
|
||||
737280,
|
||||
802816
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-clear-16384",
|
||||
"wallMedianMs": 13.354,
|
||||
"maxRssBytesMedian": 71794688,
|
||||
"peakRssDeltaBytesMedian": 573440,
|
||||
"retainedRssDeltaBytesMedian": 262144,
|
||||
"retainedHeapDeltaBytesMedian": 262264,
|
||||
"maxRssSamplesBytes": [
|
||||
71794688,
|
||||
72417280,
|
||||
71909376,
|
||||
71434240,
|
||||
71794688
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
573440,
|
||||
720896,
|
||||
524288,
|
||||
573440,
|
||||
638976
|
||||
],
|
||||
"compactions": 3
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-no-clear-4096",
|
||||
"wallMedianMs": 22.188,
|
||||
"maxRssBytesMedian": 71942144,
|
||||
"peakRssDeltaBytesMedian": 737280,
|
||||
"retainedRssDeltaBytesMedian": 589824,
|
||||
"retainedHeapDeltaBytesMedian": 262432,
|
||||
"maxRssSamplesBytes": [
|
||||
71942144,
|
||||
73351168,
|
||||
71335936,
|
||||
71647232,
|
||||
72433664
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
786432,
|
||||
802816,
|
||||
737280,
|
||||
622592,
|
||||
720896
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-splice-4096",
|
||||
"wallMedianMs": 11.975,
|
||||
"maxRssBytesMedian": 72417280,
|
||||
"peakRssDeltaBytesMedian": 1589248,
|
||||
"retainedRssDeltaBytesMedian": 1392640,
|
||||
"retainedHeapDeltaBytesMedian": 262456,
|
||||
"maxRssSamplesBytes": [
|
||||
72646656,
|
||||
72220672,
|
||||
72269824,
|
||||
72417280,
|
||||
72679424
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
1441792,
|
||||
1605632,
|
||||
1474560,
|
||||
1589248,
|
||||
1589248
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-splice-16384",
|
||||
"wallMedianMs": 9.151,
|
||||
"maxRssBytesMedian": 72286208,
|
||||
"peakRssDeltaBytesMedian": 1441792,
|
||||
"retainedRssDeltaBytesMedian": 1441792,
|
||||
"retainedHeapDeltaBytesMedian": 263584,
|
||||
"maxRssSamplesBytes": [
|
||||
72138752,
|
||||
75939840,
|
||||
71516160,
|
||||
72318976,
|
||||
72286208
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
1310720,
|
||||
1474560,
|
||||
1441792,
|
||||
1441792,
|
||||
1294336
|
||||
],
|
||||
"compactions": 3
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-slice-4096",
|
||||
"wallMedianMs": 9.405,
|
||||
"maxRssBytesMedian": 72548352,
|
||||
"peakRssDeltaBytesMedian": 1540096,
|
||||
"retainedRssDeltaBytesMedian": 1540096,
|
||||
"retainedHeapDeltaBytesMedian": 262456,
|
||||
"maxRssSamplesBytes": [
|
||||
72187904,
|
||||
72548352,
|
||||
72744960,
|
||||
72843264,
|
||||
72286208
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
1556480,
|
||||
1540096,
|
||||
1556480,
|
||||
1523712,
|
||||
1523712
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "prefilled",
|
||||
"mode": "cursor-reset-4096",
|
||||
"wallMedianMs": 14.182,
|
||||
"maxRssBytesMedian": 71729152,
|
||||
"peakRssDeltaBytesMedian": 704512,
|
||||
"retainedRssDeltaBytesMedian": 475136,
|
||||
"retainedHeapDeltaBytesMedian": 261768,
|
||||
"maxRssSamplesBytes": [
|
||||
71778304,
|
||||
71729152,
|
||||
71696384,
|
||||
71581696,
|
||||
71761920
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
999424,
|
||||
638976,
|
||||
737280,
|
||||
704512,
|
||||
655360
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "current-shift",
|
||||
"wallMedianMs": 616.473,
|
||||
"maxRssBytesMedian": 74153984,
|
||||
"peakRssDeltaBytesMedian": 3948544,
|
||||
"retainedRssDeltaBytesMedian": 4145152,
|
||||
"retainedHeapDeltaBytesMedian": 1066792,
|
||||
"maxRssSamplesBytes": [
|
||||
74219520,
|
||||
76283904,
|
||||
74153984,
|
||||
73383936,
|
||||
74006528
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
3997696,
|
||||
3964928,
|
||||
3850240,
|
||||
3948544,
|
||||
3932160
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-clear-1024",
|
||||
"wallMedianMs": 12.093,
|
||||
"maxRssBytesMedian": 74661888,
|
||||
"peakRssDeltaBytesMedian": 4374528,
|
||||
"retainedRssDeltaBytesMedian": 4915200,
|
||||
"retainedHeapDeltaBytesMedian": 1063520,
|
||||
"maxRssSamplesBytes": [
|
||||
73924608,
|
||||
74661888,
|
||||
74235904,
|
||||
74809344,
|
||||
74727424
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
4194304,
|
||||
4374528,
|
||||
4390912,
|
||||
4308992,
|
||||
4505600
|
||||
],
|
||||
"compactions": 7
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-clear-4096",
|
||||
"wallMedianMs": 16.6,
|
||||
"maxRssBytesMedian": 74645504,
|
||||
"peakRssDeltaBytesMedian": 4456448,
|
||||
"retainedRssDeltaBytesMedian": 5177344,
|
||||
"retainedHeapDeltaBytesMedian": 1063216,
|
||||
"maxRssSamplesBytes": [
|
||||
74645504,
|
||||
74334208,
|
||||
73891840,
|
||||
74678272,
|
||||
75120640
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
4440064,
|
||||
4505600,
|
||||
4456448,
|
||||
4407296,
|
||||
4489216
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-clear-16384",
|
||||
"wallMedianMs": 19.843,
|
||||
"maxRssBytesMedian": 74612736,
|
||||
"peakRssDeltaBytesMedian": 4358144,
|
||||
"retainedRssDeltaBytesMedian": 5242880,
|
||||
"retainedHeapDeltaBytesMedian": 1063784,
|
||||
"maxRssSamplesBytes": [
|
||||
75186176,
|
||||
74350592,
|
||||
74760192,
|
||||
74465280,
|
||||
74612736
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
4653056,
|
||||
4390912,
|
||||
4308992,
|
||||
4325376,
|
||||
4358144
|
||||
],
|
||||
"compactions": 3
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-no-clear-4096",
|
||||
"wallMedianMs": 16.406,
|
||||
"maxRssBytesMedian": 74629120,
|
||||
"peakRssDeltaBytesMedian": 4390912,
|
||||
"retainedRssDeltaBytesMedian": 5423104,
|
||||
"retainedHeapDeltaBytesMedian": 1063640,
|
||||
"maxRssSamplesBytes": [
|
||||
74530816,
|
||||
74940416,
|
||||
74629120,
|
||||
74645504,
|
||||
74383360
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
4489216,
|
||||
4489216,
|
||||
4325376,
|
||||
4374528,
|
||||
4390912
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-splice-4096",
|
||||
"wallMedianMs": 12.952,
|
||||
"maxRssBytesMedian": 75218944,
|
||||
"peakRssDeltaBytesMedian": 5292032,
|
||||
"retainedRssDeltaBytesMedian": 5718016,
|
||||
"retainedHeapDeltaBytesMedian": 1063216,
|
||||
"maxRssSamplesBytes": [
|
||||
75087872,
|
||||
75382784,
|
||||
75218944,
|
||||
75317248,
|
||||
75038720
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
5292032,
|
||||
5210112,
|
||||
5308416,
|
||||
5242880,
|
||||
5341184
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-splice-16384",
|
||||
"wallMedianMs": 10.851,
|
||||
"maxRssBytesMedian": 75661312,
|
||||
"peakRssDeltaBytesMedian": 5210112,
|
||||
"retainedRssDeltaBytesMedian": 5914624,
|
||||
"retainedHeapDeltaBytesMedian": 1063704,
|
||||
"maxRssSamplesBytes": [
|
||||
76627968,
|
||||
75399168,
|
||||
75661312,
|
||||
75726848,
|
||||
75022336
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
5357568,
|
||||
5210112,
|
||||
5193728,
|
||||
5193728,
|
||||
5275648
|
||||
],
|
||||
"compactions": 3
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-slice-4096",
|
||||
"wallMedianMs": 11.689,
|
||||
"maxRssBytesMedian": 75792384,
|
||||
"peakRssDeltaBytesMedian": 5242880,
|
||||
"retainedRssDeltaBytesMedian": 6111232,
|
||||
"retainedHeapDeltaBytesMedian": 1064056,
|
||||
"maxRssSamplesBytes": [
|
||||
75792384,
|
||||
75923456,
|
||||
75644928,
|
||||
75218944,
|
||||
76185600
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
5324800,
|
||||
5242880,
|
||||
5177344,
|
||||
5242880,
|
||||
5586944
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "streaming-ahead",
|
||||
"mode": "cursor-reset-4096",
|
||||
"wallMedianMs": 15.48,
|
||||
"maxRssBytesMedian": 75005952,
|
||||
"peakRssDeltaBytesMedian": 4390912,
|
||||
"retainedRssDeltaBytesMedian": 5095424,
|
||||
"retainedHeapDeltaBytesMedian": 1062848,
|
||||
"maxRssSamplesBytes": [
|
||||
75005952,
|
||||
75005952,
|
||||
75448320,
|
||||
74498048,
|
||||
74743808
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
4489216,
|
||||
4325376,
|
||||
4702208,
|
||||
4390912,
|
||||
4341760
|
||||
],
|
||||
"compactions": 5
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "current-shift",
|
||||
"wallMedianMs": 12.637,
|
||||
"maxRssBytesMedian": 72941568,
|
||||
"peakRssDeltaBytesMedian": 2752512,
|
||||
"retainedRssDeltaBytesMedian": 3571712,
|
||||
"retainedHeapDeltaBytesMedian": 1066272,
|
||||
"maxRssSamplesBytes": [
|
||||
72941568,
|
||||
72941568,
|
||||
72876032,
|
||||
76644352,
|
||||
73334784
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2752512,
|
||||
2785280,
|
||||
2736128,
|
||||
2752512,
|
||||
2834432
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-clear-1024",
|
||||
"wallMedianMs": 11.721,
|
||||
"maxRssBytesMedian": 73302016,
|
||||
"peakRssDeltaBytesMedian": 2932736,
|
||||
"retainedRssDeltaBytesMedian": 3506176,
|
||||
"retainedHeapDeltaBytesMedian": 1068736,
|
||||
"maxRssSamplesBytes": [
|
||||
74792960,
|
||||
73662464,
|
||||
73302016,
|
||||
73187328,
|
||||
73170944
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2818048,
|
||||
2949120,
|
||||
2899968,
|
||||
2932736,
|
||||
3047424
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-clear-4096",
|
||||
"wallMedianMs": 10.671,
|
||||
"maxRssBytesMedian": 73056256,
|
||||
"peakRssDeltaBytesMedian": 2916352,
|
||||
"retainedRssDeltaBytesMedian": 3571712,
|
||||
"retainedHeapDeltaBytesMedian": 1069440,
|
||||
"maxRssSamplesBytes": [
|
||||
73875456,
|
||||
72695808,
|
||||
73891840,
|
||||
73056256,
|
||||
72990720
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2850816,
|
||||
2916352,
|
||||
2932736,
|
||||
2981888,
|
||||
2834432
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-clear-16384",
|
||||
"wallMedianMs": 12.458,
|
||||
"maxRssBytesMedian": 73498624,
|
||||
"peakRssDeltaBytesMedian": 2998272,
|
||||
"retainedRssDeltaBytesMedian": 3637248,
|
||||
"retainedHeapDeltaBytesMedian": 1068720,
|
||||
"maxRssSamplesBytes": [
|
||||
73252864,
|
||||
74465280,
|
||||
73498624,
|
||||
74334208,
|
||||
72941568
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2981888,
|
||||
3047424,
|
||||
2998272,
|
||||
3129344,
|
||||
2736128
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-no-clear-4096",
|
||||
"wallMedianMs": 11.249,
|
||||
"maxRssBytesMedian": 72744960,
|
||||
"peakRssDeltaBytesMedian": 2916352,
|
||||
"retainedRssDeltaBytesMedian": 3457024,
|
||||
"retainedHeapDeltaBytesMedian": 1068272,
|
||||
"maxRssSamplesBytes": [
|
||||
73154560,
|
||||
72417280,
|
||||
73334784,
|
||||
72744960,
|
||||
72728576
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2932736,
|
||||
2785280,
|
||||
2850816,
|
||||
2916352,
|
||||
2916352
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-splice-4096",
|
||||
"wallMedianMs": 10.396,
|
||||
"maxRssBytesMedian": 73007104,
|
||||
"peakRssDeltaBytesMedian": 2932736,
|
||||
"retainedRssDeltaBytesMedian": 3670016,
|
||||
"retainedHeapDeltaBytesMedian": 1068080,
|
||||
"maxRssSamplesBytes": [
|
||||
73449472,
|
||||
72810496,
|
||||
73007104,
|
||||
72679424,
|
||||
73121792
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2981888,
|
||||
2932736,
|
||||
2834432,
|
||||
2867200,
|
||||
2932736
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-splice-16384",
|
||||
"wallMedianMs": 15.18,
|
||||
"maxRssBytesMedian": 72974336,
|
||||
"peakRssDeltaBytesMedian": 2965504,
|
||||
"retainedRssDeltaBytesMedian": 3473408,
|
||||
"retainedHeapDeltaBytesMedian": 1068248,
|
||||
"maxRssSamplesBytes": [
|
||||
72974336,
|
||||
72925184,
|
||||
73351168,
|
||||
73334784,
|
||||
72925184
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
2998272,
|
||||
2965504,
|
||||
2834432,
|
||||
2850816,
|
||||
3063808
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-slice-4096",
|
||||
"wallMedianMs": 14.668,
|
||||
"maxRssBytesMedian": 73400320,
|
||||
"peakRssDeltaBytesMedian": 2965504,
|
||||
"retainedRssDeltaBytesMedian": 3604480,
|
||||
"retainedHeapDeltaBytesMedian": 1069208,
|
||||
"maxRssSamplesBytes": [
|
||||
74842112,
|
||||
72531968,
|
||||
73646080,
|
||||
73400320,
|
||||
73367552
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
5406720,
|
||||
3063808,
|
||||
2916352,
|
||||
2965504,
|
||||
2883584
|
||||
],
|
||||
"compactions": 0
|
||||
},
|
||||
{
|
||||
"shape": "streaming-balanced",
|
||||
"mode": "cursor-reset-4096",
|
||||
"wallMedianMs": 10.535,
|
||||
"maxRssBytesMedian": 72941568,
|
||||
"peakRssDeltaBytesMedian": 3014656,
|
||||
"retainedRssDeltaBytesMedian": 3489792,
|
||||
"retainedHeapDeltaBytesMedian": 1068896,
|
||||
"maxRssSamplesBytes": [
|
||||
72892416,
|
||||
72941568,
|
||||
72810496,
|
||||
73515008,
|
||||
73269248
|
||||
],
|
||||
"peakRssDeltaSamplesBytes": [
|
||||
3031040,
|
||||
3014656,
|
||||
2719744,
|
||||
3031040,
|
||||
2916352
|
||||
],
|
||||
"compactions": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"date": "2026-07-21",
|
||||
"status": "rejected_all_production_changes_rolled_back",
|
||||
"environment": {
|
||||
"host": "macOS arm64",
|
||||
"database": "disposable PostgreSQL container databases",
|
||||
"note": "Timing samples include visible host/container jitter; correctness and exact operation counts are the decisive gates."
|
||||
},
|
||||
"delta_loose_chunk_prefilter": {
|
||||
"probe_source": "tools/perf-audit/rejected_delta_loose_hit_probe.rs",
|
||||
"fixture": {
|
||||
"frames": 400,
|
||||
"frame_bytes": 262144,
|
||||
"logical_bytes": 104857600
|
||||
},
|
||||
"current_path_measurements": [
|
||||
{
|
||||
"case": "seed",
|
||||
"existing_before": 0,
|
||||
"puts": 400,
|
||||
"physical_put_bytes": 104857600,
|
||||
"sync_hashes": 400,
|
||||
"elapsed_ms": 854.548
|
||||
},
|
||||
{
|
||||
"case": "all_hit",
|
||||
"existing_before": 400,
|
||||
"puts": 400,
|
||||
"physical_put_bytes": 104857600,
|
||||
"sync_hashes": 400,
|
||||
"elapsed_ms": 685.03
|
||||
},
|
||||
{
|
||||
"case": "all_miss",
|
||||
"existing_before": 0,
|
||||
"puts": 400,
|
||||
"physical_put_bytes": 104857600,
|
||||
"sync_hashes": 400,
|
||||
"elapsed_ms": 1050.956
|
||||
},
|
||||
{
|
||||
"case": "half_hit",
|
||||
"existing_before": 200,
|
||||
"puts": 400,
|
||||
"physical_put_bytes": 104857600,
|
||||
"sync_hashes": 400,
|
||||
"elapsed_ms": 236.611
|
||||
}
|
||||
],
|
||||
"result": "rejected",
|
||||
"rejection_reasons": [
|
||||
"The browser delta protocol already negotiates missing hashes, so all-miss is the normal receive path; a PostgreSQL prefilter would add queries and up to 8 MiB of request buffering there.",
|
||||
"A metadata row does not prove that the backing object still exists. Skipping PUT from PostgreSQL state alone removes the current self-healing behavior for backend-missing objects.",
|
||||
"No candidate established a Pareto improvement across the normal miss path, remote bytes, memory, and repair semantics."
|
||||
]
|
||||
},
|
||||
"identical_overwrite_refcount_cte": {
|
||||
"probe_source": "tools/perf-audit/rejected_refcount_overwrite_probe.rs",
|
||||
"baseline_correctness": {
|
||||
"legacy_iterations": 1000,
|
||||
"initial_ref_count": 1,
|
||||
"final_ref_count": 1001,
|
||||
"elapsed_ms": 25534.071,
|
||||
"different_hash_old_ref": null,
|
||||
"different_hash_new_ref": 1,
|
||||
"delete_gc_final_ref": null
|
||||
},
|
||||
"candidate_correctness_on_unambiguous_fixtures": {
|
||||
"legacy_same_hash_1000_final_ref": 1,
|
||||
"manifest_same_hash_100_final_manifest_ref": 1,
|
||||
"manifest_same_hash_100_final_chunk_ref": 1,
|
||||
"different_hash_ref_counts": "passed",
|
||||
"delete_and_force_gc": "passed",
|
||||
"missing_file_compensation": "passed",
|
||||
"sql_error_compensation": "passed",
|
||||
"blob_deletion_hooks": "passed"
|
||||
},
|
||||
"roundtrips_per_iteration": {
|
||||
"legacy_same_hash": 3,
|
||||
"manifest_same_hash": 2,
|
||||
"alternating_different_hash": 8,
|
||||
"candidate_changed_roundtrip_count": false
|
||||
},
|
||||
"interleaved_short_samples": {
|
||||
"iterations": {
|
||||
"same_hash": 200,
|
||||
"manifest_same_hash": 100,
|
||||
"alternating_different_hash": 50
|
||||
},
|
||||
"baseline": [
|
||||
{
|
||||
"same_p50_ms": 3.679,
|
||||
"manifest_p50_ms": 1.544,
|
||||
"different_p50_ms": 3.008
|
||||
},
|
||||
{
|
||||
"same_p50_ms": 5.822,
|
||||
"manifest_p50_ms": 7.668,
|
||||
"different_p50_ms": 10.711
|
||||
}
|
||||
],
|
||||
"candidate": [
|
||||
{
|
||||
"same_p50_ms": 3.7,
|
||||
"manifest_p50_ms": 5.616,
|
||||
"different_p50_ms": 7.565
|
||||
},
|
||||
{
|
||||
"same_p50_ms": 2.156,
|
||||
"manifest_p50_ms": 1.438,
|
||||
"different_p50_ms": 3.804
|
||||
}
|
||||
],
|
||||
"interpretation": "Large host/container jitter prevents a latency non-regression claim; the mixed-representation correctness failure independently rejects the candidate."
|
||||
},
|
||||
"result": "rejected",
|
||||
"rejection_reasons": [
|
||||
"storage.files stores only a content hash and cannot identify whether that file reference is owned by storage.blobs or storage.chunk_manifests when both rows coexist for the same hash.",
|
||||
"In a mixed legacy-to-CDC transition the CTE can decrement the manifest when the displaced reference was legacy, causing undercount, or preserve the manifest and leak the shadowed legacy reference and bytes.",
|
||||
"The candidate therefore cannot be made correct solely inside FileBlobWriteRepository; representation ownership must first be normalized or made explicit."
|
||||
],
|
||||
"remaining_baseline_issue": "A normal identical-content overwrite leaks the newly acquired reference (1 becomes N+1). This remains deliberately unfixed rather than replacing it with ambiguous undercount/data-loss risk."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"benchmark": "verify_integrity_phase1_and_full_method_simulation",
|
||||
"date": "2026-07-21",
|
||||
"decision": {
|
||||
"status": "rejected",
|
||||
"productionVariant": "superseded owned-key candidate: 256-occurrence windows, concurrency 16, unchanged serial path through 4 occurrences",
|
||||
"reason": "This owned-key candidate increased scratch max RSS by 180224 bytes and was not accepted. It was superseded by the sorted borrowed-key concurrency-8 implementation documented in verify_integrity_sorted_c8_2026-07-22.json."
|
||||
},
|
||||
"concurrency": 16,
|
||||
"occurrence_window": 256,
|
||||
"serial_fast_path_max_occurrences": 4,
|
||||
"local_simulated_metadata_latency_us": 250,
|
||||
"local_simulated_hash_latency_ms": 1,
|
||||
"remote_simulated_latency_ms": 4,
|
||||
"acceptance_thresholds": {
|
||||
"ordered_issues": "byte-for-byte equal",
|
||||
"backend_calls": "candidate <= current",
|
||||
"substantive_timing": "candidate strictly faster",
|
||||
"unchanged_serial_fast_path": "candidate/current >= 0.95",
|
||||
"sub_100ns_measurements": "candidate <= current + 20ns",
|
||||
"rss": "candidate <= current unless the user explicitly authorizes a measured regression"
|
||||
},
|
||||
"rejected_zero_latency_before_fast_path": [
|
||||
{ "scenario": "1_manifest_x_2", "current_ms": 0.000014, "candidate_ms": 0.000894, "speedup": 0.016 },
|
||||
{ "scenario": "2_manifests_x_1", "current_ms": 0.000016, "candidate_ms": 0.001071, "speedup": 0.015 },
|
||||
{ "scenario": "1_manifest_x_4", "current_ms": 0.000023, "candidate_ms": 0.001469, "speedup": 0.016 }
|
||||
],
|
||||
"timing_results": [
|
||||
{ "scenario": "immediate_one_manifest_two", "phase1_ms": [0.000013, 0.000013], "phase1_speedup": 1.000, "full_ms": [0.000144, 0.000144], "full_speedup": 1.000, "phase1_calls": [2, 2], "full_calls": [4, 4], "issues_equal": true },
|
||||
{ "scenario": "immediate_two_manifests_one", "phase1_ms": [0.000014, 0.000016], "phase1_speedup": 0.875, "full_ms": [0.000145, 0.000146], "full_speedup": 0.993, "phase1_calls": [2, 2], "full_calls": [4, 4], "issues_equal": true },
|
||||
{ "scenario": "immediate_one_manifest_four", "phase1_ms": [0.000020, 0.000020], "phase1_speedup": 1.000, "full_ms": [0.000227, 0.000226], "full_speedup": 1.004, "phase1_calls": [4, 4], "full_calls": [8, 8], "issues_equal": true },
|
||||
{ "scenario": "local_tiny_empty", "phase1_ms": [0.000004, 0.000004], "phase1_speedup": 1.000, "full_ms": [0.000054, 0.000054], "full_speedup": 1.000, "phase1_calls": [0, 0], "full_calls": [0, 0], "issues_equal": true },
|
||||
{ "scenario": "local_tiny_single", "phase1_ms": [1.573850, 1.565981], "phase1_speedup": 1.005, "full_ms": [5.215304, 5.224075], "full_speedup": 0.998, "phase1_calls": [1, 1], "full_calls": [2, 2], "issues_equal": true },
|
||||
{ "scenario": "local_small_unique", "phase1_ms": [6.173358, 6.062208], "phase1_speedup": 1.018, "full_ms": [9.635729, 9.577837], "full_speedup": 1.006, "phase1_calls": [4, 4], "full_calls": [8, 8], "issues_equal": true },
|
||||
{ "scenario": "local_semantics", "phase1_ms": [19.824417, 1.293583], "phase1_speedup": 15.325, "full_ms": [23.969709, 5.060584], "full_speedup": 4.737, "phase1_calls": [12, 6], "full_calls": [16, 10], "issues_equal": true },
|
||||
{ "scenario": "local_shared", "phase1_ms": [771.930083, 5.914542], "phase1_speedup": 130.514, "full_ms": [784.165500, 13.935208], "full_speedup": 56.272, "phase1_calls": [512, 64], "full_calls": [544, 96], "issues_equal": true },
|
||||
{ "scenario": "local_unique", "phase1_ms": [417.543458, 26.467917], "phase1_speedup": 15.775, "full_ms": [488.088375, 95.696541], "full_speedup": 5.100, "phase1_calls": [256, 256], "full_calls": [512, 512], "issues_equal": true },
|
||||
{ "scenario": "local_hash_dominated", "phase1_ms": [798.974041, 7.262542], "phase1_speedup": 110.013, "full_ms": [947.894292, 142.175417], "full_speedup": 6.667, "phase1_calls": [512, 64], "full_calls": [672, 224], "issues_equal": true },
|
||||
{ "scenario": "remote_tiny_single", "phase1_ms": [5.946651, 5.825770], "phase1_speedup": 1.021, "full_ms": [12.171682, 12.221104], "full_speedup": 0.996, "phase1_calls": [1, 1], "full_calls": [2, 2], "issues_equal": true },
|
||||
{ "scenario": "remote_shared", "phase1_ms": [1169.103458, 12.549250], "phase1_speedup": 93.161, "full_ms": [1180.397208, 24.700583], "full_speedup": 47.788, "phase1_calls": [192, 24], "full_calls": [216, 48], "issues_equal": true },
|
||||
{ "scenario": "remote_unique", "phase1_ms": [777.041833, 48.517459], "phase1_speedup": 16.016, "full_ms": [830.103333, 97.427792], "full_speedup": 8.520, "phase1_calls": [128, 128], "full_calls": [256, 256], "issues_equal": true }
|
||||
],
|
||||
"remote_tiny_raw_interleaved_samples_ms": {
|
||||
"phase_current": [5.972041, 5.818484, 5.462442, 5.435223, 5.665474, 5.987218, 5.200437, 6.051963, 6.210510, 5.726651, 5.959099, 6.149390, 5.495677, 6.679635, 5.627703, 6.144088, 6.113260, 6.298692, 6.135348, 5.891848, 5.987041, 6.247567, 5.896729, 5.898213, 6.311750, 5.817687, 5.981401, 6.121276, 5.881609, 5.471724, 5.346536],
|
||||
"phase_candidate": [6.001317, 5.968807, 5.718093, 5.810250, 6.000500, 5.637484, 5.424822, 5.745713, 5.803015, 5.814833, 5.826723, 5.668151, 5.499484, 9.238000, 5.809541, 5.574656, 6.025552, 6.049541, 6.057848, 6.021921, 5.848974, 5.975885, 7.049984, 6.100453, 5.605041, 7.276510, 6.775093, 5.816354, 6.165192, 5.580307, 7.048598],
|
||||
"full_current": [11.370567, 11.387697, 11.628140, 11.325437, 11.993604, 11.828640, 11.401380, 11.721067, 12.006802, 12.157614, 18.172244, 12.098177, 12.020255, 11.991437, 11.633375, 12.128099, 12.005609, 11.712333, 16.508307, 11.919135, 12.695625, 12.155364, 11.964682, 11.588708, 11.581765, 11.558296, 12.172546, 12.264661, 12.716224, 12.976911, 11.821093],
|
||||
"full_candidate": [11.236448, 11.087390, 11.242302, 11.993781, 11.755302, 12.211015, 11.805994, 12.096088, 11.946072, 14.495333, 11.964203, 20.616036, 11.523067, 12.436895, 11.837380, 11.973791, 11.760260, 11.998291, 11.962328, 12.284317, 13.261562, 12.181573, 11.943026, 11.878151, 11.673031, 12.429109, 14.424937, 12.314994, 12.130489, 12.200453, 12.130453]
|
||||
},
|
||||
"rss_gate": {
|
||||
"fixture": "1000 manifests, 250000 unique occurrences, separate processes",
|
||||
"rejected_unbounded": { "current_max_rss_bytes": 34783232, "candidate_max_rss_bytes": 69681152, "decision": "rejected and rolled back" },
|
||||
"windowed_owned_keys": { "current_max_rss_bytes": 34897920, "candidate_max_rss_bytes": 35078144, "delta_bytes": 180224, "delta_percent": 0.52, "decision": "rejected and superseded by the sorted borrowed-key concurrency-8 implementation" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"benchmark": "verify_integrity_sorted_borrowed_c8",
|
||||
"date": "2026-07-22",
|
||||
"decision": {
|
||||
"status": "accepted_by_explicit_user_tradeoff",
|
||||
"productionVariant": "fetch_all + exact serial path through 4 occurrences + 256-occurrence sorted borrowed windows + concurrency 8",
|
||||
"reason": "Exact issue order and backend-call gates passed. Real filesystem and remote full-method cases improved materially. After disclosure of a measured peak cost up to 112 KiB, the user explicitly reauthorized retaining the candidate; the final exact BoxFut RSS medians were +112 KiB phase 1 and +80 KiB full method."
|
||||
},
|
||||
"implementation": {
|
||||
"keys": "sorted Vec<&str>",
|
||||
"results": "parallel Vec<Option<u64>>",
|
||||
"lookup": "binary search, at most 8 comparisons for a 256-occurrence window",
|
||||
"scheduler": "bounded FuturesUnordered",
|
||||
"manifest_concurrency": 8,
|
||||
"phase_two_concurrency_unchanged": 16,
|
||||
"occurrence_window": 256,
|
||||
"serial_fast_path_max_occurrences": 4,
|
||||
"manifest_query": "unchanged fetch_all"
|
||||
},
|
||||
"correctness": {
|
||||
"ordered_issues_equal": true,
|
||||
"candidate_backend_calls_not_greater": true,
|
||||
"malformed_manifest_backend_calls": 0,
|
||||
"repeated_occurrences_replayed": true,
|
||||
"large_manifest_sliced": true,
|
||||
"tiny_path_uses_identical_serial_code": true
|
||||
},
|
||||
"real_filesystem_boxfut_c8_31_samples": [
|
||||
{
|
||||
"scenario": "shared",
|
||||
"phase_ms": {
|
||||
"historical": 4.869042,
|
||||
"sorted": 0.458167,
|
||||
"speedup": 10.627221
|
||||
},
|
||||
"full_ms": {
|
||||
"historical": 5.599625,
|
||||
"sorted": 0.971125,
|
||||
"speedup": 5.766122
|
||||
},
|
||||
"phase_calls": [512, 64],
|
||||
"full_calls": [544, 96]
|
||||
},
|
||||
{
|
||||
"scenario": "unique",
|
||||
"phase_ms": {
|
||||
"historical": 2.398167,
|
||||
"sorted": 1.831541,
|
||||
"speedup": 1.309371
|
||||
},
|
||||
"full_ms": {
|
||||
"historical": 6.022667,
|
||||
"sorted": 5.545917,
|
||||
"speedup": 1.085964
|
||||
},
|
||||
"phase_calls": [256, 256],
|
||||
"full_calls": [512, 512]
|
||||
},
|
||||
{
|
||||
"scenario": "mixed_existing_and_missing_unique",
|
||||
"phase_ms": {
|
||||
"historical": 2.369208,
|
||||
"sorted": 1.833333,
|
||||
"speedup": 1.292296
|
||||
},
|
||||
"full_ms": {
|
||||
"historical": 5.755125,
|
||||
"sorted": 5.253833,
|
||||
"speedup": 1.095415
|
||||
},
|
||||
"phase_calls": [256, 256],
|
||||
"full_calls": [496, 496]
|
||||
}
|
||||
],
|
||||
"remote_boxfut_c8": [
|
||||
{
|
||||
"scenario": "shared",
|
||||
"phase_ms": [1122.720666, 17.170333],
|
||||
"phase_speedup": 65.387239,
|
||||
"full_ms": [1138.407208, 29.148208],
|
||||
"full_speedup": 39.055821,
|
||||
"phase_calls": [192, 24],
|
||||
"full_calls": [216, 48]
|
||||
},
|
||||
{
|
||||
"scenario": "unique",
|
||||
"phase_ms": [769.160459, 96.641416],
|
||||
"phase_speedup": 7.958911,
|
||||
"full_ms": [824.404334, 144.857708],
|
||||
"full_speedup": 5.691132,
|
||||
"phase_calls": [128, 128],
|
||||
"full_calls": [256, 256]
|
||||
}
|
||||
],
|
||||
"rss_11_fresh_process_medians": {
|
||||
"fixture": "1000 manifests, 250000 unique occurrences, BoxFut backend model",
|
||||
"phase_one": {
|
||||
"historical_bytes": 26771456,
|
||||
"sorted_bytes": 26886144,
|
||||
"delta_bytes": 114688,
|
||||
"delta_kib": 112,
|
||||
"delta_percent": 0.4284
|
||||
},
|
||||
"full_method": {
|
||||
"historical_bytes": 26836992,
|
||||
"sorted_bytes": 26918912,
|
||||
"delta_bytes": 81920,
|
||||
"delta_kib": 80,
|
||||
"delta_percent": 0.3053
|
||||
},
|
||||
"requested_heap_scratch_bytes": {
|
||||
"historical": 144,
|
||||
"sorted": 9360
|
||||
}
|
||||
},
|
||||
"tiny_gate": {
|
||||
"scenarios": ["1x2", "2x1", "1x4"],
|
||||
"implementation": "same serial loop for historical and candidate",
|
||||
"calls_equal": true,
|
||||
"issues_equal": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"benchmark": "verify_integrity_sqlx_streaming",
|
||||
"date": "2026-07-22",
|
||||
"decision": {
|
||||
"status": "rejected",
|
||||
"reason": "The bounded producer/channel variant reduced RSS by 74.23% and accelerated phase 1, but the same-round full-method median was 4.17% slower than the historical implementation. No streaming code was applied to production."
|
||||
},
|
||||
"fixture": {
|
||||
"database": "fresh disposable PostgreSQL database",
|
||||
"manifest_rows": 1000,
|
||||
"manifest_occurrences": 250000,
|
||||
"blob_rows": 250000,
|
||||
"query_count": {
|
||||
"phase_one": 1,
|
||||
"full_method": 2
|
||||
},
|
||||
"prefetch_rows": 16,
|
||||
"normal_prefetch_occurrence_bound": 4000,
|
||||
"window_occurrences": 256,
|
||||
"large_manifest_occurrences": 1024,
|
||||
"samples_per_mode": 7,
|
||||
"fresh_processes": true,
|
||||
"rotated_order": true
|
||||
},
|
||||
"semantic_gates": [
|
||||
{
|
||||
"scenario": "empty",
|
||||
"rows": 0,
|
||||
"phase_calls": [0, 0, 0, 0],
|
||||
"full_calls": [0, 0, 0, 0],
|
||||
"issues_equal": true
|
||||
},
|
||||
{
|
||||
"scenario": "one",
|
||||
"rows": 1,
|
||||
"phase_calls": [1, 1, 1, 1],
|
||||
"full_calls": [2, 2, 2, 2],
|
||||
"issues_equal": true
|
||||
},
|
||||
{
|
||||
"scenario": "four",
|
||||
"rows": 1,
|
||||
"phase_calls": [4, 4, 4, 4],
|
||||
"full_calls": [8, 8, 8, 8],
|
||||
"issues_equal": true
|
||||
},
|
||||
{
|
||||
"scenario": "semantics_with_malformed",
|
||||
"rows": 4,
|
||||
"issues": 8,
|
||||
"checksum": 1417964566409558305,
|
||||
"phase_calls_historical_materialized_direct_prefetch": [6, 4, 4, 4],
|
||||
"full_calls_historical_materialized_direct_prefetch": [10, 8, 8, 8],
|
||||
"malformed_probe_suppressed": true,
|
||||
"issues_equal_and_ordered": true
|
||||
},
|
||||
{
|
||||
"scenario": "shared",
|
||||
"phase_calls_historical_materialized_direct_prefetch": [512, 64, 64, 64],
|
||||
"full_calls_historical_materialized_direct_prefetch": [544, 96, 96, 96],
|
||||
"issues_equal": true
|
||||
},
|
||||
{
|
||||
"scenario": "unique",
|
||||
"phase_calls": [256, 256, 256, 256],
|
||||
"full_calls": [512, 512, 512, 512],
|
||||
"issues_equal": true
|
||||
},
|
||||
{
|
||||
"scenario": "large_manifest_sliced",
|
||||
"phase_calls": [1024, 1024, 1024, 1024],
|
||||
"full_calls": [2048, 2048, 2048, 2048],
|
||||
"issues_equal": true
|
||||
}
|
||||
],
|
||||
"final_same_round_abc_medians": {
|
||||
"historical": {
|
||||
"phase_ms": 638.057167,
|
||||
"full_ms": 1146.865750,
|
||||
"max_rss_bytes": 32620544
|
||||
},
|
||||
"materialized_owned": {
|
||||
"phase_ms": 754.104000,
|
||||
"full_ms": 1507.712417,
|
||||
"max_rss_bytes": 32751616
|
||||
},
|
||||
"streaming_prefetch": {
|
||||
"phase_ms": 370.620500,
|
||||
"full_ms": 1194.661583,
|
||||
"max_rss_bytes": 8404992
|
||||
},
|
||||
"prefetch_vs_historical": {
|
||||
"phase_speedup": 1.721592,
|
||||
"full_speedup": 0.959992,
|
||||
"full_regression_percent": 4.1675,
|
||||
"rss_reduction_percent": 74.2341
|
||||
},
|
||||
"prefetch_vs_materialized_owned": {
|
||||
"phase_speedup": 2.034707,
|
||||
"full_speedup": 1.262041,
|
||||
"rss_reduction_percent": 74.3372
|
||||
}
|
||||
},
|
||||
"raw_final_abc_samples": {
|
||||
"historical": {
|
||||
"phase_ms": [638.057167, 302.385875, 473.900417, 774.089833, 632.479584, 1075.826334, 1435.878333],
|
||||
"full_ms": [1146.865750, 630.661958, 1089.854458, 1794.876250, 885.336625, 1857.877417, 1657.854292],
|
||||
"max_rss_bytes": [32669696, 32538624, 32620544, 32636928, 32620544, 32669696, 32604160]
|
||||
},
|
||||
"materialized_owned": {
|
||||
"phase_ms": [299.143625, 704.092708, 657.428250, 1017.416708, 1391.611125, 754.104000, 1051.222917],
|
||||
"full_ms": [774.324375, 1370.037708, 1230.500333, 1860.589583, 1635.081625, 1507.712417, 1900.343500],
|
||||
"max_rss_bytes": [32833536, 32751616, 32899072, 32800768, 32555008, 32555008, 32718848]
|
||||
},
|
||||
"streaming_prefetch": {
|
||||
"phase_ms": [276.988833, 370.693334, 1387.695584, 370.620500, 219.138375, 637.540250, 332.878083],
|
||||
"full_ms": [581.776666, 870.599875, 1666.224584, 1194.661583, 923.753250, 2379.057125, 1565.068625],
|
||||
"max_rss_bytes": [8339456, 8404992, 8617984, 8667136, 8552448, 8372224, 8290304]
|
||||
}
|
||||
},
|
||||
"connection_lifecycle": {
|
||||
"pool_max_connections": 1,
|
||||
"held_during_nonempty_stream": true,
|
||||
"released_before_phase_two": true,
|
||||
"extra_queries": 0,
|
||||
"temporary_database_removed": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the dedup-GC phase-1 benchmark against a fresh database in the existing
|
||||
# local PostgreSQL container. The trap drops the database even on interruption.
|
||||
set -euo pipefail
|
||||
|
||||
container="${OXICLOUD_POSTGRES_CONTAINER:-oxicloud-postgres-1}"
|
||||
db="oxicloud_perf_gc_${$}_${RANDOM}"
|
||||
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
pg_host="${OXICLOUD_POSTGRES_HOST:-127.0.0.1}"
|
||||
|
||||
cleanup() {
|
||||
docker exec "$container" dropdb --if-exists --force -U postgres "$db" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
docker exec "$container" createdb -U postgres "$db"
|
||||
|
||||
# Pin IPv4 and disable TLS explicitly. The local container exposes plain TCP;
|
||||
# avoiding localhost/SSL negotiation keeps the harness independent of host
|
||||
# resolver and optional SQLx TLS features.
|
||||
export DATABASE_URL="postgres://postgres:postgres@${pg_host}:5432/$db?sslmode=disable"
|
||||
export GC_MANIFEST_COUNTS="${GC_MANIFEST_COUNTS:-10000,50000}"
|
||||
export GC_CHUNKS_PER_MANIFEST="${GC_CHUNKS_PER_MANIFEST:-16}"
|
||||
export GC_SHARED_PERCENT="${GC_SHARED_PERCENT:-50}"
|
||||
export GC_SHARED_POOL="${GC_SHARED_POOL:-512}"
|
||||
export GC_WARMUPS="${GC_WARMUPS:-1}"
|
||||
export GC_SAMPLES="${GC_SAMPLES:-5}"
|
||||
|
||||
cargo run \
|
||||
--release \
|
||||
--manifest-path "$repo_root/tools/perf-audit/Cargo.toml" \
|
||||
--bin gc_manifest_batch
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fresh-process max-RSS gate for owned String vs borrowed &str SQLx array binds.
|
||||
# Each process runs one validated phase-1 sample; order alternates per repetition.
|
||||
set -euo pipefail
|
||||
|
||||
container="${OXICLOUD_POSTGRES_CONTAINER:-oxicloud-postgres-1}"
|
||||
pg_host="${OXICLOUD_POSTGRES_HOST:-127.0.0.1}"
|
||||
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
binary="$repo_root/tools/perf-audit/target/release/gc_manifest_batch"
|
||||
database="oxicloud_perf_gc_bind_${$}_${RANDOM}"
|
||||
runs="${GC_RSS_RUNS:-5}"
|
||||
|
||||
cleanup() {
|
||||
docker exec "$container" dropdb --if-exists --force -U postgres "$database" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
cargo build --release --manifest-path "$repo_root/tools/perf-audit/Cargo.toml" --bin gc_manifest_batch
|
||||
docker exec "$container" createdb -U postgres "$database"
|
||||
database_url="postgres://postgres:postgres@${pg_host}:5432/${database}?sslmode=disable"
|
||||
|
||||
for scenario in 2:498 500:10 1000:10; do
|
||||
for ((run = 1; run <= runs; run++)); do
|
||||
modes=(current owned borrowed sorted)
|
||||
rotation=$(((run - 1) % 4))
|
||||
modes=("${modes[@]:rotation}" "${modes[@]:0:rotation}")
|
||||
for mode in "${modes[@]}"; do
|
||||
exclude_baselines=1
|
||||
include_cte=0
|
||||
owned_threshold=""
|
||||
borrowed_threshold=""
|
||||
sorted_threshold=""
|
||||
case "$mode" in
|
||||
current) exclude_baselines=0 ;;
|
||||
owned) owned_threshold=2 ;;
|
||||
borrowed) borrowed_threshold=2 ;;
|
||||
sorted) sorted_threshold=2 ;;
|
||||
esac
|
||||
echo "scenario=$scenario run=$run mode=$mode"
|
||||
/usr/bin/time -l env \
|
||||
DATABASE_URL="$database_url" \
|
||||
GC_SCENARIOS="$scenario" \
|
||||
GC_EXCLUDE_BASELINES="$exclude_baselines" \
|
||||
GC_INCLUDE_CTE="$include_cte" \
|
||||
GC_HYBRID_THRESHOLDS="$owned_threshold" \
|
||||
GC_BORROWED_THRESHOLDS="$borrowed_threshold" \
|
||||
GC_SORTED_THRESHOLDS="$sorted_threshold" \
|
||||
GC_WARMUPS=0 \
|
||||
GC_SAMPLES=1 \
|
||||
"$binary" 2>&1 \
|
||||
| sed -n -e '/summary:/p' -e '/ median=/p' -e '/maximum resident set size/p'
|
||||
done
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# Short decisive gate for the bounded SQLx producer/channel variant.
|
||||
set -euo pipefail
|
||||
|
||||
container="${OXICLOUD_POSTGRES_CONTAINER:-oxicloud-postgres-1}"
|
||||
database="oxicloud_perf_integrity_prefetch_${$}_${RANDOM}"
|
||||
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
host="${OXICLOUD_POSTGRES_HOST:-$(
|
||||
docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container"
|
||||
)}"
|
||||
samples="${INTEGRITY_PREFETCH_SAMPLES:-7}"
|
||||
|
||||
cleanup() {
|
||||
docker exec "$container" dropdb --if-exists --force -U postgres "$database" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
docker exec "$container" createdb -U postgres "$database"
|
||||
export DATABASE_URL="postgres://postgres:postgres@${host}:5432/${database}?sslmode=disable"
|
||||
cargo build --release --manifest-path "$repo_root/tools/perf-audit/Cargo.toml" \
|
||||
--bin verify_integrity_streaming
|
||||
binary="$repo_root/tools/perf-audit/target/release/verify_integrity_streaming"
|
||||
|
||||
"$binary" seed
|
||||
for scenario in empty one four semantics shared unique large_manifest large; do
|
||||
"$binary" compare "$scenario"
|
||||
done
|
||||
|
||||
"$binary" run historical large full >/dev/null
|
||||
"$binary" run materialized large full >/dev/null
|
||||
"$binary" run prefetch large full >/dev/null
|
||||
for ((sample = 0; sample < samples; sample++)); do
|
||||
case $((sample % 3)) in
|
||||
0) modes=(historical materialized prefetch) ;;
|
||||
1) modes=(materialized prefetch historical) ;;
|
||||
2) modes=(prefetch historical materialized) ;;
|
||||
esac
|
||||
for mode in "${modes[@]}"; do
|
||||
{ /usr/bin/time -l "$binary" run "$mode" large full; } 2>&1 \
|
||||
| rg 'mode=|maximum resident set size'
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# PostgreSQL-backed integrity materialization/streaming A/B. The database is
|
||||
# disposable and is dropped on success, failure, or interruption.
|
||||
set -euo pipefail
|
||||
|
||||
container="${OXICLOUD_POSTGRES_CONTAINER:-oxicloud-postgres-1}"
|
||||
database="oxicloud_perf_integrity_${$}_${RANDOM}"
|
||||
repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
host="${OXICLOUD_POSTGRES_HOST:-$(
|
||||
docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container"
|
||||
)}"
|
||||
|
||||
cleanup() {
|
||||
docker exec "$container" dropdb --if-exists --force -U postgres "$database" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
docker exec "$container" createdb -U postgres "$database"
|
||||
export DATABASE_URL="postgres://postgres:postgres@${host}:5432/${database}?sslmode=disable"
|
||||
|
||||
cargo build --release --manifest-path "$repo_root/tools/perf-audit/Cargo.toml" \
|
||||
--bin verify_integrity_streaming
|
||||
binary="$repo_root/tools/perf-audit/target/release/verify_integrity_streaming"
|
||||
samples="${INTEGRITY_STREAM_SAMPLES:-7}"
|
||||
|
||||
"$binary" seed
|
||||
for scenario in empty one four semantics shared unique large_manifest large; do
|
||||
"$binary" compare "$scenario"
|
||||
done
|
||||
|
||||
# Warm PostgreSQL/OS caches once; warm-up output and RSS are not measurements.
|
||||
for mode in historical materialized streaming; do
|
||||
"$binary" run "$mode" large full >/dev/null
|
||||
done
|
||||
|
||||
for scenario in empty one four shared unique large_manifest large; do
|
||||
for ((sample = 0; sample < samples; sample++)); do
|
||||
case $((sample % 3)) in
|
||||
0) modes=(historical materialized streaming) ;;
|
||||
1) modes=(materialized streaming historical) ;;
|
||||
2) modes=(streaming historical materialized) ;;
|
||||
esac
|
||||
for mode in "${modes[@]}"; do
|
||||
{ /usr/bin/time -l "$binary" run "$mode" "$scenario" full; } 2>&1 \
|
||||
| rg 'mode=|maximum resident set size'
|
||||
done
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,902 @@
|
||||
//! A/B/C for the phase-1 integrity verifier's bounded result table.
|
||||
//!
|
||||
//! `historical` probes every manifest occurrence serially. `owned` models the
|
||||
//! current accepted 256-occurrence/concurrency-16 candidate exactly: each
|
||||
//! distinct key is cloned into the map and cloned again into the work vector.
|
||||
//! `borrowed` changes only those scratch keys to `&str`. Tiny stores retain the
|
||||
//! exact historical loop through four valid occurrences in both candidates.
|
||||
|
||||
use foldhash::quality::RandomState;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use std::alloc::{GlobalAlloc, Layout, System};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::hint::black_box;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const PRODUCTION_CONCURRENCY: usize = 16;
|
||||
const WINDOW: usize = 256;
|
||||
const SERIAL_FAST_PATH_OCCURRENCES: usize = 4;
|
||||
static CANDIDATE_CONCURRENCY: AtomicUsize = AtomicUsize::new(PRODUCTION_CONCURRENCY);
|
||||
type Manifest = (String, Vec<String>, Vec<i64>, i64);
|
||||
type OwnedSizes = HashMap<String, Option<u64>, RandomState>;
|
||||
type BorrowedSizes<'a> = HashMap<&'a str, Option<u64>, RandomState>;
|
||||
type AuditBoxFut<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
struct TrackingAllocator;
|
||||
|
||||
static LIVE_ALLOCATED: AtomicUsize = AtomicUsize::new(0);
|
||||
static PEAK_ALLOCATED: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[global_allocator]
|
||||
static ALLOCATOR: TrackingAllocator = TrackingAllocator;
|
||||
|
||||
#[inline]
|
||||
fn update_peak(candidate: usize) {
|
||||
let mut peak = PEAK_ALLOCATED.load(Ordering::Relaxed);
|
||||
while candidate > peak {
|
||||
match PEAK_ALLOCATED.compare_exchange_weak(
|
||||
peak,
|
||||
candidate,
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => break,
|
||||
Err(observed) => peak = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: every operation delegates to `System` with the original pointer and
|
||||
// layout; the counters are diagnostic and do not affect allocation semantics.
|
||||
unsafe impl GlobalAlloc for TrackingAllocator {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
let pointer = unsafe { System.alloc(layout) };
|
||||
if !pointer.is_null() {
|
||||
let live = LIVE_ALLOCATED.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
|
||||
update_peak(live);
|
||||
}
|
||||
pointer
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) {
|
||||
LIVE_ALLOCATED.fetch_sub(layout.size(), Ordering::Relaxed);
|
||||
unsafe { System.dealloc(pointer, layout) };
|
||||
}
|
||||
|
||||
unsafe fn realloc(&self, pointer: *mut u8, old: Layout, new_size: usize) -> *mut u8 {
|
||||
let new_pointer = unsafe { System.realloc(pointer, old, new_size) };
|
||||
if !new_pointer.is_null() {
|
||||
if new_size >= old.size() {
|
||||
let growth = new_size - old.size();
|
||||
let live = LIVE_ALLOCATED.fetch_add(growth, Ordering::Relaxed) + growth;
|
||||
update_peak(live);
|
||||
} else {
|
||||
LIVE_ALLOCATED.fetch_sub(old.size() - new_size, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
new_pointer
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum Mode {
|
||||
Historical,
|
||||
Owned,
|
||||
Borrowed,
|
||||
Sorted,
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
fn parse(value: &str) -> Self {
|
||||
match value {
|
||||
"historical" => Self::Historical,
|
||||
"owned" => Self::Owned,
|
||||
"borrowed" => Self::Borrowed,
|
||||
"sorted" => Self::Sorted,
|
||||
_ => panic!("mode must be historical, owned, borrowed, or sorted"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Latency {
|
||||
Immediate,
|
||||
Local { metadata: Duration, hash: Duration },
|
||||
Remote(Duration),
|
||||
RealFs(&'static Path),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SimBackend {
|
||||
latency: Latency,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl SimBackend {
|
||||
fn new(latency: Latency) -> Self {
|
||||
Self {
|
||||
latency,
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
fn blob_size<'a>(&'a self, hash: &'a str) -> AuditBoxFut<'a, Option<u64>> {
|
||||
Box::pin(async move {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
match self.latency {
|
||||
Latency::Immediate => {}
|
||||
Latency::Local { metadata, .. } => tokio::time::sleep(metadata).await,
|
||||
Latency::Remote(delay) => tokio::time::sleep(delay).await,
|
||||
Latency::RealFs(root) => {
|
||||
return tokio::fs::metadata(root.join(hash))
|
||||
.await
|
||||
.ok()
|
||||
.map(|metadata| metadata.len());
|
||||
}
|
||||
}
|
||||
if hash.starts_with("missing-") {
|
||||
None
|
||||
} else if hash.starts_with("wrong-") {
|
||||
Some(999)
|
||||
} else {
|
||||
Some(256)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn hash_local_blob(&self, blob_hash: &str) {
|
||||
match self.latency {
|
||||
Latency::Local { hash, .. } => tokio::time::sleep(hash).await,
|
||||
Latency::RealFs(root) => {
|
||||
black_box(tokio::fs::read(root.join(blob_hash)).await.ok());
|
||||
}
|
||||
Latency::Immediate | Latency::Remote(_) => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn calls(&self) -> usize {
|
||||
self.calls.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn label(value: &str) -> &str {
|
||||
&value[..value.len().min(12)]
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn uses_serial_fast_path(manifests: &[Manifest]) -> bool {
|
||||
if manifests.len() == 1 {
|
||||
let (_, hashes, sizes, _) = &manifests[0];
|
||||
return hashes.len() != sizes.len() || hashes.len() <= SERIAL_FAST_PATH_OCCURRENCES;
|
||||
}
|
||||
let mut occurrences = 0usize;
|
||||
for (_, hashes, sizes, _) in manifests {
|
||||
if hashes.len() == sizes.len() {
|
||||
occurrences = occurrences.saturating_add(hashes.len());
|
||||
if occurrences > SERIAL_FAST_PATH_OCCURRENCES {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn historical(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
for (file_hash, hashes, expected_sizes, total_size) in manifests {
|
||||
let file_label = label(file_hash);
|
||||
if hashes.len() != expected_sizes.len() {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let sum: i64 = expected_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for (index, hash) in hashes.iter().enumerate() {
|
||||
let chunk_label = label(hash);
|
||||
match backend.blob_size(hash).await {
|
||||
Some(actual) if actual != expected_sizes[index] as u64 => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch (expected {}, actual {actual})",
|
||||
expected_sizes[index]
|
||||
)),
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
fn replay_owned(manifests: &[Manifest], sizes: &OwnedSizes) -> Vec<String> {
|
||||
replay(manifests, |hash| sizes.get(hash).copied().flatten())
|
||||
}
|
||||
|
||||
fn replay_borrowed(manifests: &[Manifest], sizes: &BorrowedSizes<'_>) -> Vec<String> {
|
||||
replay(manifests, |hash| sizes.get(hash).copied().flatten())
|
||||
}
|
||||
|
||||
fn replay<F>(manifests: &[Manifest], mut size_of: F) -> Vec<String>
|
||||
where
|
||||
F: FnMut(&str) -> Option<u64>,
|
||||
{
|
||||
let mut issues = Vec::new();
|
||||
for (file_hash, hashes, expected_sizes, total_size) in manifests {
|
||||
let file_label = label(file_hash);
|
||||
if hashes.len() != expected_sizes.len() {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let sum: i64 = expected_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for (index, hash) in hashes.iter().enumerate() {
|
||||
let chunk_label = label(hash);
|
||||
match size_of(hash) {
|
||||
Some(actual) if actual != expected_sizes[index] as u64 => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch (expected {}, actual {actual})",
|
||||
expected_sizes[index]
|
||||
)),
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
async fn fill_owned(sizes: OwnedSizes, backend: &SimBackend) -> OwnedSizes {
|
||||
let hashes: Vec<String> = sizes.keys().cloned().collect();
|
||||
stream::iter(hashes)
|
||||
.map(|hash| async move {
|
||||
let size = backend.blob_size(&hash).await;
|
||||
(hash, size)
|
||||
})
|
||||
.buffer_unordered(CANDIDATE_CONCURRENCY.load(Ordering::Relaxed))
|
||||
.fold(sizes, |mut sizes, (hash, size)| async move {
|
||||
sizes.insert(hash, size);
|
||||
sizes
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn fill_borrowed<'a>(sizes: BorrowedSizes<'a>, backend: &SimBackend) -> BorrowedSizes<'a> {
|
||||
let hashes: Vec<&'a str> = sizes.keys().copied().collect();
|
||||
stream::iter(hashes)
|
||||
.map(|hash| async move {
|
||||
let size = backend.blob_size(hash).await;
|
||||
(hash, size)
|
||||
})
|
||||
.buffer_unordered(CANDIDATE_CONCURRENCY.load(Ordering::Relaxed))
|
||||
.fold(sizes, |mut sizes, (hash, size)| async move {
|
||||
sizes.insert(hash, size);
|
||||
sizes
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn owned_batch(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
let mut sizes = OwnedSizes::default();
|
||||
for (_, hashes, expected_sizes, _) in manifests {
|
||||
if hashes.len() == expected_sizes.len() {
|
||||
for hash in hashes {
|
||||
sizes.entry(hash.clone()).or_insert(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
let sizes = fill_owned(sizes, backend).await;
|
||||
replay_owned(manifests, &sizes)
|
||||
}
|
||||
|
||||
async fn borrowed_batch(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
let mut sizes = BorrowedSizes::default();
|
||||
for (_, hashes, expected_sizes, _) in manifests {
|
||||
if hashes.len() == expected_sizes.len() {
|
||||
for hash in hashes {
|
||||
sizes.entry(hash.as_str()).or_insert(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
let sizes = fill_borrowed(sizes, backend).await;
|
||||
replay_borrowed(manifests, &sizes)
|
||||
}
|
||||
|
||||
async fn sorted_batch(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
let mut hashes = Vec::new();
|
||||
for (_, manifest_hashes, expected_sizes, _) in manifests {
|
||||
if manifest_hashes.len() == expected_sizes.len() {
|
||||
hashes.extend(manifest_hashes.iter().map(String::as_str));
|
||||
}
|
||||
}
|
||||
hashes.sort_unstable();
|
||||
hashes.dedup();
|
||||
let mut values = vec![None; hashes.len()];
|
||||
let concurrency = CANDIDATE_CONCURRENCY.load(Ordering::Relaxed).max(1);
|
||||
let mut pending = futures::stream::FuturesUnordered::new();
|
||||
let mut next = 0usize;
|
||||
while next < hashes.len() || !pending.is_empty() {
|
||||
while next < hashes.len() && pending.len() < concurrency {
|
||||
let index = next;
|
||||
let hash = hashes[index];
|
||||
pending.push(async move {
|
||||
let size = backend.blob_size(hash).await;
|
||||
(index, size)
|
||||
});
|
||||
next += 1;
|
||||
}
|
||||
if let Some((index, size)) = pending.next().await {
|
||||
values[index] = size;
|
||||
}
|
||||
}
|
||||
replay(manifests, |hash| {
|
||||
hashes
|
||||
.binary_search(&hash)
|
||||
.ok()
|
||||
.and_then(|index| values[index])
|
||||
})
|
||||
}
|
||||
|
||||
async fn windowed(manifests: &[Manifest], backend: &SimBackend, mode: Mode) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
let mut start = 0;
|
||||
while start < manifests.len() {
|
||||
let (_, hashes, expected_sizes, _) = &manifests[start];
|
||||
if hashes.len() == expected_sizes.len() && hashes.len() > WINDOW {
|
||||
let (file_hash, hashes, expected_sizes, total_size) = &manifests[start];
|
||||
let file_label = label(file_hash);
|
||||
let sum: i64 = expected_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for offset in (0..hashes.len()).step_by(WINDOW) {
|
||||
let end = (offset + WINDOW).min(hashes.len());
|
||||
let synthetic = (
|
||||
file_hash.clone(),
|
||||
hashes[offset..end].to_vec(),
|
||||
expected_sizes[offset..end].to_vec(),
|
||||
expected_sizes[offset..end].iter().sum(),
|
||||
);
|
||||
let batch = std::slice::from_ref(&synthetic);
|
||||
let mut batch_issues = match mode {
|
||||
Mode::Owned => owned_batch(batch, backend).await,
|
||||
Mode::Borrowed => borrowed_batch(batch, backend).await,
|
||||
Mode::Sorted => sorted_batch(batch, backend).await,
|
||||
Mode::Historical => unreachable!(),
|
||||
};
|
||||
// Slice replay must not repeat a total-size issue already emitted.
|
||||
batch_issues.retain(|issue| !issue.contains("sum of chunk_sizes"));
|
||||
issues.extend(batch_issues);
|
||||
}
|
||||
start += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut occurrences = 0;
|
||||
let mut end = start;
|
||||
while end < manifests.len() {
|
||||
let (_, hashes, expected_sizes, _) = &manifests[end];
|
||||
let next = if hashes.len() == expected_sizes.len() {
|
||||
hashes.len()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if next > WINDOW || (occurrences > 0 && occurrences + next > WINDOW) {
|
||||
break;
|
||||
}
|
||||
occurrences += next;
|
||||
end += 1;
|
||||
}
|
||||
debug_assert!(end > start);
|
||||
let batch = &manifests[start..end];
|
||||
issues.extend(match mode {
|
||||
Mode::Owned => owned_batch(batch, backend).await,
|
||||
Mode::Borrowed => borrowed_batch(batch, backend).await,
|
||||
Mode::Sorted => sorted_batch(batch, backend).await,
|
||||
Mode::Historical => unreachable!(),
|
||||
});
|
||||
start = end;
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
async fn verify(manifests: &[Manifest], backend: &SimBackend, mode: Mode) -> Vec<String> {
|
||||
if mode == Mode::Historical || uses_serial_fast_path(manifests) {
|
||||
historical(manifests, backend).await
|
||||
} else {
|
||||
windowed(manifests, backend, mode).await
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(manifests: usize, chunks: usize, unique: usize, anomalies: bool) -> Vec<Manifest> {
|
||||
let unique = unique.max(1);
|
||||
let mut rows = Vec::with_capacity(manifests + usize::from(anomalies) * 3);
|
||||
for manifest in 0..manifests {
|
||||
let hashes = (0..chunks)
|
||||
.map(|chunk| audit_hash((manifest * chunks + chunk) % unique))
|
||||
.collect();
|
||||
rows.push((
|
||||
format!("file-{manifest:059}"),
|
||||
hashes,
|
||||
vec![256; chunks],
|
||||
(chunks * 256) as i64,
|
||||
));
|
||||
}
|
||||
if anomalies {
|
||||
rows.push((
|
||||
"sum-mismatch-file".into(),
|
||||
vec!["wrong-shared".into(), "wrong-shared".into()],
|
||||
vec![256, 257],
|
||||
1,
|
||||
));
|
||||
rows.push((
|
||||
"missing-file".into(),
|
||||
vec!["missing-shared".into(), "missing-shared".into()],
|
||||
vec![256, 256],
|
||||
512,
|
||||
));
|
||||
rows.push((
|
||||
"malformed-file".into(),
|
||||
vec!["missing-must-not-be-queried".into()],
|
||||
vec![],
|
||||
0,
|
||||
));
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
fn audit_hash(index: usize) -> String {
|
||||
fn mix(mut value: u64) -> u64 {
|
||||
value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
|
||||
value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
|
||||
value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
|
||||
value ^ (value >> 31)
|
||||
}
|
||||
let a = mix(index as u64);
|
||||
let b = mix(a);
|
||||
let c = mix(b);
|
||||
let d = mix(c);
|
||||
format!("{a:016x}{b:016x}{c:016x}{d:016x}")
|
||||
}
|
||||
|
||||
fn mark_missing(rows: &mut [Manifest], every: usize) {
|
||||
let mut occurrence = 0usize;
|
||||
for (_, hashes, _, _) in rows {
|
||||
for hash in hashes {
|
||||
if occurrence.is_multiple_of(every) {
|
||||
*hash = format!("missing-{hash}");
|
||||
}
|
||||
occurrence += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn populate_real_fs(root: &Path, scenarios: &[Vec<Manifest>]) {
|
||||
std::fs::create_dir_all(root).expect("create real-filesystem fixture directory");
|
||||
for rows in scenarios {
|
||||
for (_, hashes, expected_sizes, _) in rows {
|
||||
if hashes.len() != expected_sizes.len() {
|
||||
continue;
|
||||
}
|
||||
for hash in hashes {
|
||||
if hash.starts_with("missing-") {
|
||||
continue;
|
||||
}
|
||||
let file = std::fs::File::create(root.join(hash)).expect("create fixture blob");
|
||||
let length = if hash.starts_with("wrong-") { 999 } else { 256 };
|
||||
file.set_len(length).expect("size fixture blob");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_hashes(manifests: &[Manifest]) -> Vec<&str> {
|
||||
let mut unique: HashMap<&str, (), RandomState> = HashMap::default();
|
||||
for (_, hashes, expected_sizes, _) in manifests {
|
||||
if hashes.len() == expected_sizes.len() {
|
||||
for hash in hashes {
|
||||
if !hash.starts_with("missing-") && !hash.starts_with("wrong-") {
|
||||
unique.entry(hash).or_insert(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut hashes: Vec<&str> = unique.into_keys().collect();
|
||||
hashes.sort_unstable();
|
||||
hashes
|
||||
}
|
||||
|
||||
async fn phase_two(hashes: &[&str], backend: &SimBackend) {
|
||||
stream::iter(hashes.iter().copied())
|
||||
.map(|hash| async move {
|
||||
black_box(backend.blob_size(hash).await);
|
||||
backend.hash_local_blob(hash).await;
|
||||
})
|
||||
.buffer_unordered(PRODUCTION_CONCURRENCY)
|
||||
.collect::<Vec<()>>()
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn phase_two_generated(count: usize, backend: &SimBackend) {
|
||||
stream::iter(0..count)
|
||||
.map(|index| async move {
|
||||
let hash = audit_hash(index);
|
||||
black_box(backend.blob_size(&hash).await);
|
||||
backend.hash_local_blob(&hash).await;
|
||||
})
|
||||
.buffer_unordered(PRODUCTION_CONCURRENCY)
|
||||
.collect::<Vec<()>>()
|
||||
.await;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct Observation {
|
||||
phase: Duration,
|
||||
full: Duration,
|
||||
phase_calls: usize,
|
||||
full_calls: usize,
|
||||
issue_checksum: usize,
|
||||
}
|
||||
|
||||
async fn observe(
|
||||
mode: Mode,
|
||||
manifests: &[Manifest],
|
||||
latency: Latency,
|
||||
repetitions: usize,
|
||||
storage: &[&str],
|
||||
) -> Observation {
|
||||
let phase_backend = SimBackend::new(latency);
|
||||
let start = Instant::now();
|
||||
let mut issues = Vec::new();
|
||||
for _ in 0..repetitions {
|
||||
issues = verify(manifests, &phase_backend, mode).await;
|
||||
black_box(&issues);
|
||||
}
|
||||
let phase = start.elapsed() / repetitions as u32;
|
||||
|
||||
let full_backend = SimBackend::new(latency);
|
||||
let start = Instant::now();
|
||||
for _ in 0..repetitions {
|
||||
issues = verify(manifests, &full_backend, mode).await;
|
||||
phase_two(storage, &full_backend).await;
|
||||
black_box(&issues);
|
||||
}
|
||||
let full = start.elapsed() / repetitions as u32;
|
||||
Observation {
|
||||
phase,
|
||||
full,
|
||||
phase_calls: phase_backend.calls() / repetitions,
|
||||
full_calls: full_backend.calls() / repetitions,
|
||||
issue_checksum: issues.iter().map(String::len).sum(),
|
||||
}
|
||||
}
|
||||
|
||||
fn median(mut values: Vec<Duration>) -> Duration {
|
||||
values.sort_unstable();
|
||||
values[values.len() / 2]
|
||||
}
|
||||
|
||||
async fn measure(
|
||||
manifests: &[Manifest],
|
||||
latency: Latency,
|
||||
samples: usize,
|
||||
repetitions: usize,
|
||||
) -> [Observation; 4] {
|
||||
let storage = storage_hashes(manifests);
|
||||
let modes = [Mode::Historical, Mode::Owned, Mode::Borrowed, Mode::Sorted];
|
||||
let mut phase = [Vec::new(), Vec::new(), Vec::new(), Vec::new()];
|
||||
let mut full = [Vec::new(), Vec::new(), Vec::new(), Vec::new()];
|
||||
let mut last = [None, None, None, None];
|
||||
for sample in 0..=samples {
|
||||
for offset in 0..4 {
|
||||
let index = (sample + offset) % 4;
|
||||
let observation =
|
||||
observe(modes[index], manifests, latency, repetitions, &storage).await;
|
||||
if sample > 0 {
|
||||
phase[index].push(observation.phase);
|
||||
full[index].push(observation.full);
|
||||
last[index] = Some(observation);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::array::from_fn(|index| {
|
||||
let mut observation = last[index].expect("at least one measured sample");
|
||||
observation.phase = median(std::mem::take(&mut phase[index]));
|
||||
observation.full = median(std::mem::take(&mut full[index]));
|
||||
observation
|
||||
})
|
||||
}
|
||||
|
||||
fn print_header() {
|
||||
println!(
|
||||
"scenario,historical_phase_ms,owned_phase_ms,borrowed_phase_ms,sorted_phase_ms,\
|
||||
borrowed_vs_owned_phase,sorted_vs_owned_phase,historical_full_ms,owned_full_ms,\
|
||||
borrowed_full_ms,sorted_full_ms,borrowed_vs_owned_full,sorted_vs_owned_full,\
|
||||
calls_historical,calls_owned,calls_borrowed,calls_sorted,full_calls_historical,\
|
||||
full_calls_owned,full_calls_borrowed,full_calls_sorted,issues_equal"
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_scenario(
|
||||
name: &str,
|
||||
latency: Latency,
|
||||
rows: &[Manifest],
|
||||
samples: usize,
|
||||
repetitions: usize,
|
||||
) {
|
||||
let historical_gate =
|
||||
verify(rows, &SimBackend::new(Latency::Immediate), Mode::Historical).await;
|
||||
let owned_gate = verify(rows, &SimBackend::new(Latency::Immediate), Mode::Owned).await;
|
||||
let borrowed_gate = verify(rows, &SimBackend::new(Latency::Immediate), Mode::Borrowed).await;
|
||||
let sorted_gate = verify(rows, &SimBackend::new(Latency::Immediate), Mode::Sorted).await;
|
||||
assert_eq!(
|
||||
historical_gate, owned_gate,
|
||||
"owned issue gate failed for {name}"
|
||||
);
|
||||
assert_eq!(
|
||||
historical_gate, borrowed_gate,
|
||||
"borrowed issue gate failed for {name}"
|
||||
);
|
||||
assert_eq!(
|
||||
historical_gate, sorted_gate,
|
||||
"sorted issue gate failed for {name}"
|
||||
);
|
||||
let observations = measure(rows, latency, samples, repetitions).await;
|
||||
let [historical, owned, borrowed, sorted] = observations;
|
||||
let equal = historical.issue_checksum == owned.issue_checksum
|
||||
&& historical.issue_checksum == borrowed.issue_checksum
|
||||
&& historical.issue_checksum == sorted.issue_checksum;
|
||||
assert!(equal, "issue gate failed for {name}");
|
||||
assert_eq!(owned.phase_calls, borrowed.phase_calls);
|
||||
assert_eq!(owned.phase_calls, sorted.phase_calls);
|
||||
assert_eq!(owned.full_calls, borrowed.full_calls);
|
||||
assert_eq!(owned.full_calls, sorted.full_calls);
|
||||
assert!(owned.phase_calls <= historical.phase_calls);
|
||||
println!(
|
||||
"{name},{:.6},{:.6},{:.6},{:.6},{:.3},{:.3},{:.6},{:.6},{:.6},{:.6},{:.3},{:.3},{},{},{},{},{},{},{},{},{}",
|
||||
historical.phase.as_secs_f64() * 1e3,
|
||||
owned.phase.as_secs_f64() * 1e3,
|
||||
borrowed.phase.as_secs_f64() * 1e3,
|
||||
sorted.phase.as_secs_f64() * 1e3,
|
||||
owned.phase.as_secs_f64() / borrowed.phase.as_secs_f64(),
|
||||
owned.phase.as_secs_f64() / sorted.phase.as_secs_f64(),
|
||||
historical.full.as_secs_f64() * 1e3,
|
||||
owned.full.as_secs_f64() * 1e3,
|
||||
borrowed.full.as_secs_f64() * 1e3,
|
||||
sorted.full.as_secs_f64() * 1e3,
|
||||
owned.full.as_secs_f64() / borrowed.full.as_secs_f64(),
|
||||
owned.full.as_secs_f64() / sorted.full.as_secs_f64(),
|
||||
historical.phase_calls,
|
||||
owned.phase_calls,
|
||||
borrowed.phase_calls,
|
||||
sorted.phase_calls,
|
||||
historical.full_calls,
|
||||
owned.full_calls,
|
||||
borrowed.full_calls,
|
||||
sorted.full_calls,
|
||||
equal,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let candidate_concurrency = std::env::var("OXICLOUD_AUDIT_CONCURRENCY")
|
||||
.ok()
|
||||
.map(|value| value.parse::<usize>().expect("numeric concurrency"))
|
||||
.unwrap_or(PRODUCTION_CONCURRENCY);
|
||||
assert!(matches!(candidate_concurrency, 4 | 8 | 16));
|
||||
CANDIDATE_CONCURRENCY.store(candidate_concurrency, Ordering::Relaxed);
|
||||
if args.get(1).is_some_and(|arg| arg == "--memory") {
|
||||
let mode = Mode::parse(args.get(2).map(String::as_str).unwrap_or("borrowed"));
|
||||
let full_method = args.iter().any(|arg| arg == "full" || arg == "full-drop");
|
||||
let drop_manifests = args.iter().any(|arg| arg == "full-drop");
|
||||
let rows = fixture(1_000, 250, 250_000, false);
|
||||
let manifest_count = rows.len();
|
||||
let backend = SimBackend::new(Latency::Immediate);
|
||||
let live_before = LIVE_ALLOCATED.load(Ordering::Relaxed);
|
||||
PEAK_ALLOCATED.store(live_before, Ordering::Relaxed);
|
||||
let issues = verify(&rows, &backend, mode).await;
|
||||
let phase_peak = PEAK_ALLOCATED.load(Ordering::Relaxed);
|
||||
black_box(&issues);
|
||||
if drop_manifests {
|
||||
drop(rows);
|
||||
}
|
||||
let live_before_phase_two = LIVE_ALLOCATED.load(Ordering::Relaxed);
|
||||
if full_method {
|
||||
phase_two_generated(250_000, &backend).await;
|
||||
}
|
||||
let full_peak = PEAK_ALLOCATED.load(Ordering::Relaxed);
|
||||
println!(
|
||||
"mode={mode:?} concurrency={candidate_concurrency} full={full_method} drop_manifests={drop_manifests} manifests={manifest_count} occurrences=250000 calls={} issues={} live_before={} phase_peak={} phase_scratch_peak={} live_before_phase_two={} full_peak={}",
|
||||
backend.calls(),
|
||||
issues.len(),
|
||||
live_before,
|
||||
phase_peak,
|
||||
phase_peak.saturating_sub(live_before),
|
||||
live_before_phase_two,
|
||||
full_peak,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if args.iter().any(|arg| arg == "--real-fs") {
|
||||
let mut tiny_missing = fixture(2, 1, 2, false);
|
||||
mark_missing(&mut tiny_missing, 2);
|
||||
let shared = fixture(64, 8, 32, false);
|
||||
let unique = fixture(32, 8, 256, false);
|
||||
let mut mixed = fixture(32, 8, 256, false);
|
||||
mark_missing(&mut mixed, 17);
|
||||
let real_rows = vec![fixture(1, 2, 2, false), tiny_missing, shared, unique, mixed];
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"oxicloud-integrity-real-fs-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock")
|
||||
.as_nanos()
|
||||
));
|
||||
populate_real_fs(&path, &real_rows);
|
||||
let leaked_root: &'static Path = Box::leak(path.clone().into_boxed_path());
|
||||
print_header();
|
||||
let names = [
|
||||
"real_tiny_1x2",
|
||||
"real_tiny_missing_2x1",
|
||||
"real_shared",
|
||||
"real_unique",
|
||||
"real_mixed_unique",
|
||||
];
|
||||
for (name, rows) in names.into_iter().zip(real_rows.iter()) {
|
||||
let repetitions = if name.starts_with("real_tiny") {
|
||||
128
|
||||
} else {
|
||||
1
|
||||
};
|
||||
run_scenario(name, Latency::RealFs(leaked_root), rows, 31, repetitions).await;
|
||||
}
|
||||
std::fs::remove_dir_all(path).expect("remove real-filesystem fixture directory");
|
||||
return;
|
||||
}
|
||||
|
||||
let scenarios = [
|
||||
(
|
||||
"immediate_1x2",
|
||||
Latency::Immediate,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
false,
|
||||
51,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"immediate_2x1",
|
||||
Latency::Immediate,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
false,
|
||||
51,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"immediate_1x4",
|
||||
Latency::Immediate,
|
||||
1,
|
||||
4,
|
||||
4,
|
||||
false,
|
||||
51,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"cpu_shared",
|
||||
Latency::Immediate,
|
||||
64,
|
||||
8,
|
||||
32,
|
||||
false,
|
||||
31,
|
||||
1_000,
|
||||
),
|
||||
(
|
||||
"cpu_unique",
|
||||
Latency::Immediate,
|
||||
32,
|
||||
8,
|
||||
256,
|
||||
false,
|
||||
31,
|
||||
1_000,
|
||||
),
|
||||
(
|
||||
"local_semantics",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
2,
|
||||
4,
|
||||
4,
|
||||
true,
|
||||
15,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"local_shared",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
64,
|
||||
8,
|
||||
32,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"local_unique",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
32,
|
||||
8,
|
||||
256,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"remote_shared",
|
||||
Latency::Remote(Duration::from_millis(4)),
|
||||
24,
|
||||
8,
|
||||
24,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"remote_unique",
|
||||
Latency::Remote(Duration::from_millis(4)),
|
||||
16,
|
||||
8,
|
||||
128,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
];
|
||||
|
||||
print_header();
|
||||
let remote_only = args.iter().any(|arg| arg == "--remote-only");
|
||||
for (name, latency, manifests, chunks, unique, anomalies, samples, repetitions) in scenarios {
|
||||
if remote_only && !name.starts_with("remote_") {
|
||||
continue;
|
||||
}
|
||||
let rows = fixture(manifests, chunks, unique, anomalies);
|
||||
run_scenario(name, latency, &rows, samples, repetitions).await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,811 @@
|
||||
//! Independent A/B for `DedupService::verify_integrity` phase 1.
|
||||
//!
|
||||
//! This deliberately does not import the OxiCloud crate. It models the exact
|
||||
//! manifest validation/messages and a backend whose `blob_size` operation has
|
||||
//! either local-filesystem scheduling latency or remote request latency.
|
||||
|
||||
use foldhash::quality::RandomState;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use std::collections::HashMap;
|
||||
use std::hint::black_box;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const CONCURRENCY: usize = 16;
|
||||
const SERIAL_FAST_PATH_OCCURRENCES: usize = 4;
|
||||
type Manifest = (String, Vec<String>, Vec<i64>, i64);
|
||||
type SizeMap = HashMap<String, Option<u64>, RandomState>;
|
||||
|
||||
#[inline]
|
||||
fn uses_serial_fast_path(manifests: &[Manifest]) -> bool {
|
||||
if manifests.len() == 1 {
|
||||
let (_, hashes, sizes, _) = &manifests[0];
|
||||
return hashes.len() != sizes.len() || hashes.len() <= SERIAL_FAST_PATH_OCCURRENCES;
|
||||
}
|
||||
let mut occurrences = 0usize;
|
||||
for (_, hashes, sizes, _) in manifests {
|
||||
if hashes.len() == sizes.len() {
|
||||
occurrences = occurrences.saturating_add(hashes.len());
|
||||
if occurrences > SERIAL_FAST_PATH_OCCURRENCES {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Latency {
|
||||
/// No delay: used only by the separate-process peak-RSS probe.
|
||||
Immediate,
|
||||
/// Warm/cached local metadata latency as observed by an async caller.
|
||||
Local { metadata: Duration, hash: Duration },
|
||||
/// Object-store HEAD request: asynchronously wait for network latency.
|
||||
Remote(Duration),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SimBackend {
|
||||
latency: Latency,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl SimBackend {
|
||||
fn new(latency: Latency) -> Self {
|
||||
Self {
|
||||
latency,
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn blob_size(&self, hash: &str) -> Option<u64> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
match self.latency {
|
||||
Latency::Immediate => {}
|
||||
Latency::Local { metadata, .. } => tokio::time::sleep(metadata).await,
|
||||
Latency::Remote(delay) => tokio::time::sleep(delay).await,
|
||||
}
|
||||
if hash.starts_with("missing-") {
|
||||
None
|
||||
} else if hash.starts_with("wrong-") {
|
||||
Some(999)
|
||||
} else {
|
||||
Some(256)
|
||||
}
|
||||
}
|
||||
|
||||
async fn hash_local_blob(&self) {
|
||||
if let Latency::Local { hash, .. } = self.latency {
|
||||
// Equal phase-2 work: model mmap/BLAKE3 verification separately
|
||||
// from metadata. The exact value only dilutes the phase-1 win; it
|
||||
// does not differ between current and candidate.
|
||||
tokio::time::sleep(hash).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn calls(&self) -> usize {
|
||||
self.calls.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
fn label(value: &str) -> &str {
|
||||
&value[..value.len().min(12)]
|
||||
}
|
||||
|
||||
async fn current(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
for (file_hash, chunk_hashes, chunk_sizes, total_size) in manifests {
|
||||
let file_label = label(file_hash);
|
||||
if chunk_hashes.len() != chunk_sizes.len() {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let sum: i64 = chunk_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for (index, chunk_hash) in chunk_hashes.iter().enumerate() {
|
||||
let chunk_label = label(chunk_hash);
|
||||
match backend.blob_size(chunk_hash).await {
|
||||
Some(actual_size) if actual_size != chunk_sizes[index] as u64 => {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch \
|
||||
(expected {}, actual {actual_size})",
|
||||
chunk_sizes[index]
|
||||
));
|
||||
}
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
fn replay(manifests: &[Manifest], sizes: &SizeMap) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
for (file_hash, chunk_hashes, chunk_sizes, total_size) in manifests {
|
||||
let file_label = label(file_hash);
|
||||
if chunk_hashes.len() != chunk_sizes.len() {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let sum: i64 = chunk_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for (index, chunk_hash) in chunk_hashes.iter().enumerate() {
|
||||
let chunk_label = label(chunk_hash);
|
||||
match sizes.get(chunk_hash.as_str()).copied().flatten() {
|
||||
Some(actual_size) if actual_size != chunk_sizes[index] as u64 => {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch \
|
||||
(expected {}, actual {actual_size})",
|
||||
chunk_sizes[index]
|
||||
));
|
||||
}
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
const CANDIDATE_BATCH_OCCURRENCES: usize = 256;
|
||||
|
||||
async fn fill_sizes(size_by_hash: SizeMap, backend: &SimBackend) -> SizeMap {
|
||||
let hashes: Vec<String> = size_by_hash.keys().cloned().collect();
|
||||
stream::iter(hashes)
|
||||
.map(|hash| async move {
|
||||
let size = backend.blob_size(&hash).await;
|
||||
(hash, size)
|
||||
})
|
||||
.buffer_unordered(CONCURRENCY)
|
||||
.fold(size_by_hash, |mut sizes, (hash, size)| async move {
|
||||
sizes.insert(hash, size);
|
||||
sizes
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn candidate_batch(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
// Invalid manifests are skipped by the current implementation, so their
|
||||
// hashes must not become backend calls in the candidate either.
|
||||
let mut size_by_hash = SizeMap::default();
|
||||
for (_, hashes, chunk_sizes, _) in manifests {
|
||||
if hashes.len() == chunk_sizes.len() {
|
||||
for hash in hashes {
|
||||
size_by_hash.entry(hash.clone()).or_insert(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
let size_by_hash = fill_sizes(size_by_hash, backend).await;
|
||||
replay(manifests, &size_by_hash)
|
||||
}
|
||||
|
||||
async fn candidate_large_manifest(manifest: &Manifest, backend: &SimBackend) -> Vec<String> {
|
||||
let (file_hash, hashes, chunk_sizes, total_size) = manifest;
|
||||
let file_label = label(file_hash);
|
||||
if hashes.len() != chunk_sizes.len() {
|
||||
return vec![format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
)];
|
||||
}
|
||||
let mut issues = Vec::new();
|
||||
let sum: i64 = chunk_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for offset in (0..hashes.len()).step_by(CANDIDATE_BATCH_OCCURRENCES) {
|
||||
let end = (offset + CANDIDATE_BATCH_OCCURRENCES).min(hashes.len());
|
||||
let mut size_by_hash = SizeMap::default();
|
||||
for hash in &hashes[offset..end] {
|
||||
size_by_hash.entry(hash.clone()).or_insert(None);
|
||||
}
|
||||
let size_by_hash = fill_sizes(size_by_hash, backend).await;
|
||||
for (index, hash) in hashes[offset..end].iter().enumerate() {
|
||||
let expected = chunk_sizes[offset + index];
|
||||
let chunk_label = label(hash);
|
||||
match size_by_hash.get(hash.as_str()).copied().flatten() {
|
||||
Some(actual) if actual != expected as u64 => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch \
|
||||
(expected {expected}, actual {actual})"
|
||||
)),
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
async fn candidate(manifests: &[Manifest], backend: &SimBackend) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
let mut start = 0;
|
||||
while start < manifests.len() {
|
||||
let (_, hashes, chunk_sizes, _) = &manifests[start];
|
||||
if hashes.len() == chunk_sizes.len() && hashes.len() > CANDIDATE_BATCH_OCCURRENCES {
|
||||
issues.extend(candidate_large_manifest(&manifests[start], backend).await);
|
||||
start += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut occurrences = 0;
|
||||
let mut end = start;
|
||||
while end < manifests.len() {
|
||||
let (_, hashes, chunk_sizes, _) = &manifests[end];
|
||||
let next = if hashes.len() == chunk_sizes.len() {
|
||||
hashes.len()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if next > CANDIDATE_BATCH_OCCURRENCES
|
||||
|| (occurrences > 0 && occurrences + next > CANDIDATE_BATCH_OCCURRENCES)
|
||||
{
|
||||
break;
|
||||
}
|
||||
occurrences += next;
|
||||
end += 1;
|
||||
}
|
||||
debug_assert!(end > start);
|
||||
issues.extend(candidate_batch(&manifests[start..end], backend).await);
|
||||
start = end;
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
fn storage_hashes(manifests: &[Manifest]) -> Vec<&str> {
|
||||
let mut sizes: HashMap<&str, (), RandomState> = HashMap::default();
|
||||
for (_, hashes, chunk_sizes, _) in manifests {
|
||||
if hashes.len() == chunk_sizes.len() {
|
||||
for hash in hashes {
|
||||
// Keep phase 2 free of synthetic issues: both variants then
|
||||
// append exactly the same empty vector regardless of task
|
||||
// completion order. Phase-1 anomaly semantics remain gated.
|
||||
if !hash.starts_with("missing-") && !hash.starts_with("wrong-") {
|
||||
sizes.entry(hash.as_str()).or_insert(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut hashes: Vec<&str> = sizes.into_keys().collect();
|
||||
hashes.sort_unstable();
|
||||
hashes
|
||||
}
|
||||
|
||||
async fn phase_two(blob_hashes: &[&str], backend: &SimBackend) -> Vec<String> {
|
||||
stream::iter(blob_hashes.iter().copied())
|
||||
.map(|hash| async move {
|
||||
let mut issues = Vec::new();
|
||||
match backend.blob_size(hash).await {
|
||||
Some(actual) if actual != 256 => {
|
||||
issues.push(format!(
|
||||
"{hash}: size mismatch (expected: 256, actual: {actual})"
|
||||
));
|
||||
}
|
||||
None => {
|
||||
issues.push(format!("{hash}: blob missing in backend"));
|
||||
return issues;
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
backend.hash_local_blob().await;
|
||||
issues
|
||||
})
|
||||
.buffer_unordered(CONCURRENCY)
|
||||
.flat_map(stream::iter)
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
fn fixture(manifests: usize, chunks: usize, unique: usize, anomalies: bool) -> Vec<Manifest> {
|
||||
let unique = unique.max(1);
|
||||
let mut rows = Vec::with_capacity(manifests + 3);
|
||||
for manifest in 0..manifests {
|
||||
let hashes: Vec<String> = (0..chunks)
|
||||
.map(|chunk| format!("chunk-{:058}", (manifest * chunks + chunk) % unique))
|
||||
.collect();
|
||||
rows.push((
|
||||
format!("file-{manifest:059}"),
|
||||
hashes,
|
||||
vec![256; chunks],
|
||||
(chunks * 256) as i64,
|
||||
));
|
||||
}
|
||||
|
||||
if !anomalies {
|
||||
return rows;
|
||||
}
|
||||
|
||||
// Semantic gates: total mismatch, repeated wrong-size hash, repeated
|
||||
// missing hash, and a malformed manifest that must not trigger a call.
|
||||
rows.push((
|
||||
"sum-mismatch-file".into(),
|
||||
vec!["wrong-shared".into(), "wrong-shared".into()],
|
||||
vec![256, 257],
|
||||
1,
|
||||
));
|
||||
rows.push((
|
||||
"missing-file".into(),
|
||||
vec!["missing-shared".into(), "missing-shared".into()],
|
||||
vec![256, 256],
|
||||
512,
|
||||
));
|
||||
rows.push((
|
||||
"malformed-file".into(),
|
||||
vec!["missing-must-not-be-queried".into()],
|
||||
vec![],
|
||||
0,
|
||||
));
|
||||
rows
|
||||
}
|
||||
|
||||
fn median(mut values: Vec<Duration>) -> Duration {
|
||||
values.sort_unstable();
|
||||
values[values.len() / 2]
|
||||
}
|
||||
|
||||
async fn timed_once(
|
||||
candidate_mode: bool,
|
||||
manifests: &[Manifest],
|
||||
latency: Latency,
|
||||
repetitions: usize,
|
||||
full_method: bool,
|
||||
blob_hashes: &[&str],
|
||||
) -> (Duration, usize, Vec<String>) {
|
||||
let backend = SimBackend::new(latency);
|
||||
let start = Instant::now();
|
||||
let mut issues = Vec::new();
|
||||
for _ in 0..repetitions {
|
||||
issues = if candidate_mode && !uses_serial_fast_path(manifests) {
|
||||
candidate(manifests, &backend).await
|
||||
} else {
|
||||
current(manifests, &backend).await
|
||||
};
|
||||
if full_method {
|
||||
issues.extend(phase_two(blob_hashes, &backend).await);
|
||||
}
|
||||
black_box(&issues);
|
||||
}
|
||||
(
|
||||
start.elapsed() / repetitions as u32,
|
||||
backend.calls() / repetitions,
|
||||
issues,
|
||||
)
|
||||
}
|
||||
|
||||
async fn timed_pair(
|
||||
manifests: &[Manifest],
|
||||
latency: Latency,
|
||||
samples: usize,
|
||||
repetitions: usize,
|
||||
full_method: bool,
|
||||
blob_hashes: &[&str],
|
||||
) -> (
|
||||
(Duration, usize, Vec<String>),
|
||||
(Duration, usize, Vec<String>),
|
||||
) {
|
||||
let mut current_times = Vec::with_capacity(samples);
|
||||
let mut candidate_times = Vec::with_capacity(samples);
|
||||
let mut current_observation = None;
|
||||
let mut candidate_observation = None;
|
||||
for sample in 0..samples + 1 {
|
||||
let (current_run, candidate_run) = if sample % 2 == 0 {
|
||||
(
|
||||
timed_once(
|
||||
false,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await,
|
||||
timed_once(
|
||||
true,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
let candidate = timed_once(
|
||||
true,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await;
|
||||
let current = timed_once(
|
||||
false,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await;
|
||||
(current, candidate)
|
||||
};
|
||||
if sample > 0 {
|
||||
current_times.push(current_run.0);
|
||||
candidate_times.push(candidate_run.0);
|
||||
current_observation = Some((current_run.1, current_run.2));
|
||||
candidate_observation = Some((candidate_run.1, candidate_run.2));
|
||||
}
|
||||
}
|
||||
let (current_calls, current_issues) = current_observation.expect("measured current run");
|
||||
let (candidate_calls, candidate_issues) =
|
||||
candidate_observation.expect("measured candidate run");
|
||||
(
|
||||
(median(current_times), current_calls, current_issues),
|
||||
(median(candidate_times), candidate_calls, candidate_issues),
|
||||
)
|
||||
}
|
||||
|
||||
async fn raw_pair(
|
||||
manifests: &[Manifest],
|
||||
latency: Latency,
|
||||
samples: usize,
|
||||
repetitions: usize,
|
||||
full_method: bool,
|
||||
blob_hashes: &[&str],
|
||||
) -> (Vec<f64>, Vec<f64>) {
|
||||
let mut current_times = Vec::with_capacity(samples);
|
||||
let mut candidate_times = Vec::with_capacity(samples);
|
||||
for sample in 0..samples + 1 {
|
||||
let (current, candidate) = if sample % 2 == 0 {
|
||||
(
|
||||
timed_once(
|
||||
false,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await,
|
||||
timed_once(
|
||||
true,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
let candidate = timed_once(
|
||||
true,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await;
|
||||
let current = timed_once(
|
||||
false,
|
||||
manifests,
|
||||
latency,
|
||||
repetitions,
|
||||
full_method,
|
||||
blob_hashes,
|
||||
)
|
||||
.await;
|
||||
(current, candidate)
|
||||
};
|
||||
if sample > 0 {
|
||||
current_times.push(current.0.as_secs_f64() * 1e3);
|
||||
candidate_times.push(candidate.0.as_secs_f64() * 1e3);
|
||||
}
|
||||
}
|
||||
(current_times, candidate_times)
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.get(1).is_some_and(|value| value == "--tiny-raw") {
|
||||
let rows = fixture(1, 1, 1, false);
|
||||
let blob_hashes = storage_hashes(&rows);
|
||||
let latency = Latency::Remote(Duration::from_millis(4));
|
||||
let (phase_current, phase_candidate) =
|
||||
raw_pair(&rows, latency, 31, 8, false, &blob_hashes).await;
|
||||
let (full_current, full_candidate) =
|
||||
raw_pair(&rows, latency, 31, 8, true, &blob_hashes).await;
|
||||
println!("phase_current_ms={phase_current:?}");
|
||||
println!("phase_candidate_ms={phase_candidate:?}");
|
||||
println!("full_current_ms={full_current:?}");
|
||||
println!("full_candidate_ms={full_candidate:?}");
|
||||
return;
|
||||
}
|
||||
if args.get(1).is_some_and(|value| value == "--memory") {
|
||||
let mode = args.get(2).map(String::as_str).unwrap_or("candidate");
|
||||
assert!(matches!(mode, "current" | "candidate"));
|
||||
// 250k unique occurrences: large enough for process-level max RSS to
|
||||
// rise above allocator noise while keeping the probe quick.
|
||||
let rows = fixture(1_000, 250, 250_000, false);
|
||||
let backend = SimBackend::new(Latency::Immediate);
|
||||
let issues = if mode == "candidate" {
|
||||
candidate(&rows, &backend).await
|
||||
} else {
|
||||
current(&rows, &backend).await
|
||||
};
|
||||
black_box(&issues);
|
||||
println!(
|
||||
"mode={mode} manifests={} occurrences={} calls={} issues={}",
|
||||
rows.len(),
|
||||
250_000,
|
||||
backend.calls(),
|
||||
issues.len()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let scenarios = [
|
||||
(
|
||||
"immediate_one_manifest_two",
|
||||
Latency::Immediate,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
false,
|
||||
51,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"immediate_two_manifests_one",
|
||||
Latency::Immediate,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
false,
|
||||
51,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"immediate_one_manifest_four",
|
||||
Latency::Immediate,
|
||||
1,
|
||||
4,
|
||||
4,
|
||||
false,
|
||||
51,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"local_tiny_empty",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
false,
|
||||
101,
|
||||
10_000,
|
||||
),
|
||||
(
|
||||
"local_tiny_single",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
false,
|
||||
51,
|
||||
128,
|
||||
),
|
||||
(
|
||||
"local_small_unique",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
1,
|
||||
4,
|
||||
4,
|
||||
false,
|
||||
31,
|
||||
32,
|
||||
),
|
||||
(
|
||||
"local_semantics",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
2,
|
||||
4,
|
||||
4,
|
||||
true,
|
||||
15,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"local_shared",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
64,
|
||||
8,
|
||||
32,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"local_unique",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
hash: Duration::from_millis(1),
|
||||
},
|
||||
32,
|
||||
8,
|
||||
256,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"local_hash_dominated",
|
||||
Latency::Local {
|
||||
metadata: Duration::from_micros(250),
|
||||
// Models large legacy/local blobs in phase 2. Both variants
|
||||
// pay exactly the same bounded-concurrency rehash cost.
|
||||
hash: Duration::from_millis(10),
|
||||
},
|
||||
64,
|
||||
8,
|
||||
32,
|
||||
false,
|
||||
3,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"remote_tiny_single",
|
||||
Latency::Remote(Duration::from_millis(4)),
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
false,
|
||||
31,
|
||||
8,
|
||||
),
|
||||
(
|
||||
"remote_shared",
|
||||
Latency::Remote(Duration::from_millis(4)),
|
||||
24,
|
||||
8,
|
||||
24,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"remote_unique",
|
||||
Latency::Remote(Duration::from_millis(4)),
|
||||
16,
|
||||
8,
|
||||
128,
|
||||
false,
|
||||
7,
|
||||
1,
|
||||
),
|
||||
];
|
||||
|
||||
println!(
|
||||
"scenario,phase1_current_ms,phase1_candidate_ms,phase1_speedup,full_current_ms,\
|
||||
full_candidate_ms,full_speedup,phase1_current_calls,phase1_candidate_calls,\
|
||||
full_current_calls,full_candidate_calls,issues_equal"
|
||||
);
|
||||
let mut all_pass = true;
|
||||
for (name, latency, manifest_count, chunks, unique, anomalies, samples, repetitions) in
|
||||
scenarios
|
||||
{
|
||||
let rows = fixture(manifest_count, chunks, unique, anomalies);
|
||||
let extra_storage: Vec<String> = if name == "local_hash_dominated" {
|
||||
(0..128)
|
||||
.map(|index| format!("legacy-{index:057}"))
|
||||
.collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut blob_hashes = storage_hashes(&rows);
|
||||
blob_hashes.extend(extra_storage.iter().map(String::as_str));
|
||||
let tiny = name.contains("tiny");
|
||||
let serial_fast_path = uses_serial_fast_path(&rows);
|
||||
let (
|
||||
(phase_current, phase_current_calls, phase_current_issues),
|
||||
(phase_candidate, phase_candidate_calls, phase_candidate_issues),
|
||||
) = timed_pair(&rows, latency, samples, repetitions, false, &blob_hashes).await;
|
||||
let (
|
||||
(full_current, full_current_calls, full_current_issues),
|
||||
(full_candidate, full_candidate_calls, full_candidate_issues),
|
||||
) = timed_pair(
|
||||
&rows,
|
||||
latency,
|
||||
samples,
|
||||
if tiny {
|
||||
repetitions.min(16)
|
||||
} else {
|
||||
repetitions
|
||||
},
|
||||
true,
|
||||
&blob_hashes,
|
||||
)
|
||||
.await;
|
||||
let equal = phase_current_issues == phase_candidate_issues
|
||||
&& full_current_issues == full_candidate_issues;
|
||||
let phase_speedup = if phase_current.is_zero() && phase_candidate.is_zero() {
|
||||
1.0
|
||||
} else {
|
||||
phase_current.as_secs_f64() / phase_candidate.as_secs_f64()
|
||||
};
|
||||
let full_speedup = if full_current.is_zero() && full_candidate.is_zero() {
|
||||
1.0
|
||||
} else {
|
||||
full_current.as_secs_f64() / full_candidate.as_secs_f64()
|
||||
};
|
||||
// Tiny fast paths tolerate only timer noise; substantive cases must
|
||||
// be a strict win. Semantics and call-count reduction are hard gates.
|
||||
let phase_timing_pass = if phase_current < Duration::from_nanos(100) {
|
||||
// An empty Vec return is below the clock's useful resolution;
|
||||
// permit at most twenty nanoseconds of measurement noise.
|
||||
phase_candidate <= phase_current + Duration::from_nanos(20)
|
||||
} else if serial_fast_path {
|
||||
phase_speedup >= 0.95
|
||||
} else {
|
||||
phase_speedup > 1.0
|
||||
};
|
||||
let full_timing_pass = if full_current < Duration::from_nanos(100) {
|
||||
full_candidate <= full_current + Duration::from_nanos(20)
|
||||
} else if serial_fast_path {
|
||||
full_speedup >= 0.95
|
||||
} else {
|
||||
full_speedup > 1.0
|
||||
};
|
||||
let calls_pass = phase_candidate_calls <= phase_current_calls
|
||||
&& full_candidate_calls <= full_current_calls;
|
||||
all_pass &= equal && phase_timing_pass && full_timing_pass && calls_pass;
|
||||
println!(
|
||||
"{name},{:.6},{:.6},{phase_speedup:.3},{:.6},{:.6},{full_speedup:.3},\
|
||||
{phase_current_calls},{phase_candidate_calls},{full_current_calls},\
|
||||
{full_candidate_calls},{equal}",
|
||||
phase_current.as_secs_f64() * 1e3,
|
||||
phase_candidate.as_secs_f64() * 1e3,
|
||||
full_current.as_secs_f64() * 1e3,
|
||||
full_candidate.as_secs_f64() * 1e3,
|
||||
);
|
||||
}
|
||||
assert!(all_pass, "candidate failed a correctness/performance gate");
|
||||
}
|
||||
@@ -0,0 +1,819 @@
|
||||
//! PostgreSQL-backed A/B/C for online manifest integrity verification.
|
||||
//!
|
||||
//! This is an audit-only executable. It compares the historical serial
|
||||
//! `fetch_all`, the current owned-window `fetch_all`, and a bounded online
|
||||
//! SQLx `.fetch` design. All modes issue one manifest query and, when `full`
|
||||
//! is requested, the same one-query streamed phase 2.
|
||||
|
||||
use foldhash::quality::RandomState;
|
||||
use futures::TryStreamExt;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use sqlx::postgres::{PgConnection, PgPoolOptions};
|
||||
use sqlx::{Connection, PgPool, Row};
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::error::Error;
|
||||
use std::future::Future;
|
||||
use std::hint::black_box;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const WINDOW: usize = 256;
|
||||
const CURRENT_CONCURRENCY: usize = 16;
|
||||
const STREAMING_CONCURRENCY: usize = 8;
|
||||
const SERIAL_FAST_PATH_OCCURRENCES: usize = 4;
|
||||
const PHASE_TWO_CONCURRENCY: usize = 16;
|
||||
const PREFETCH_ROWS: usize = 16;
|
||||
type BoxFut<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
type OwnedSizes = HashMap<String, Option<u64>, RandomState>;
|
||||
type ManifestRow = (i64, String, Vec<String>, Vec<i64>, i64);
|
||||
|
||||
const MANIFEST_QUERY: &str = r#"
|
||||
SELECT ordinal, file_hash, chunk_hashes, chunk_sizes, total_size
|
||||
FROM perf_integrity.manifests
|
||||
WHERE scenario = $1
|
||||
ORDER BY ordinal
|
||||
"#;
|
||||
|
||||
const BLOB_QUERY: &str = r#"
|
||||
SELECT hash, size
|
||||
FROM perf_integrity.blobs
|
||||
WHERE scenario = $1
|
||||
ORDER BY ordinal
|
||||
"#;
|
||||
|
||||
const SEED_SQL: &str = r#"
|
||||
DROP SCHEMA IF EXISTS perf_integrity CASCADE;
|
||||
CREATE SCHEMA perf_integrity;
|
||||
CREATE TABLE perf_integrity.manifests (
|
||||
scenario text NOT NULL,
|
||||
ordinal bigint NOT NULL,
|
||||
file_hash text NOT NULL,
|
||||
chunk_hashes text[] NOT NULL,
|
||||
chunk_sizes bigint[] NOT NULL,
|
||||
total_size bigint NOT NULL,
|
||||
PRIMARY KEY (scenario, ordinal)
|
||||
);
|
||||
CREATE TABLE perf_integrity.blobs (
|
||||
scenario text NOT NULL,
|
||||
ordinal bigint NOT NULL,
|
||||
hash text NOT NULL,
|
||||
size bigint NOT NULL,
|
||||
PRIMARY KEY (scenario, ordinal)
|
||||
);
|
||||
|
||||
INSERT INTO perf_integrity.manifests VALUES
|
||||
('one', 0, 'file-one', ARRAY[md5('0') || md5('0x')], ARRAY[256::bigint], 256),
|
||||
('four', 0, 'file-four',
|
||||
ARRAY(SELECT md5(i::text) || md5(i::text || 'x') FROM generate_series(0, 3) AS g(i)),
|
||||
ARRAY[256::bigint, 256, 256, 256], 1024);
|
||||
|
||||
WITH chunks AS (
|
||||
SELECT scenario, manifest, chunk,
|
||||
md5(hash_index::text) || md5(hash_index::text || 'x') AS hash
|
||||
FROM (
|
||||
SELECT 'shared'::text AS scenario, m AS manifest, c AS chunk,
|
||||
((m * 8 + c) % 32)::bigint AS hash_index
|
||||
FROM generate_series(0, 63) AS manifests(m)
|
||||
CROSS JOIN generate_series(0, 7) AS chunks(c)
|
||||
UNION ALL
|
||||
SELECT 'unique'::text, m, c, (m * 8 + c)::bigint
|
||||
FROM generate_series(0, 31) AS manifests(m)
|
||||
CROSS JOIN generate_series(0, 7) AS chunks(c)
|
||||
UNION ALL
|
||||
SELECT 'large'::text, m, c, (m * 250 + c)::bigint
|
||||
FROM generate_series(0, 999) AS manifests(m)
|
||||
CROSS JOIN generate_series(0, 249) AS chunks(c)
|
||||
UNION ALL
|
||||
SELECT 'large_manifest'::text, 0, c, c::bigint
|
||||
FROM generate_series(0, 1023) AS chunks(c)
|
||||
) AS source
|
||||
), aggregated AS (
|
||||
SELECT scenario, manifest,
|
||||
array_agg(hash ORDER BY chunk) AS hashes,
|
||||
array_agg(256::bigint ORDER BY chunk) AS sizes,
|
||||
COUNT(*)::bigint * 256 AS total_size
|
||||
FROM chunks
|
||||
GROUP BY scenario, manifest
|
||||
)
|
||||
INSERT INTO perf_integrity.manifests
|
||||
SELECT scenario, manifest, 'file-' || scenario || '-' || manifest,
|
||||
hashes, sizes, total_size
|
||||
FROM aggregated;
|
||||
|
||||
INSERT INTO perf_integrity.manifests VALUES
|
||||
('semantics', 0, 'sum-mismatch-file', ARRAY['wrong-shared', 'wrong-shared'],
|
||||
ARRAY[256::bigint, 257], 1),
|
||||
('semantics', 1, 'missing-file', ARRAY['missing-shared', 'missing-shared'],
|
||||
ARRAY[256::bigint, 256], 512),
|
||||
('semantics', 2, 'valid-file',
|
||||
ARRAY[md5('semantics-0') || md5('semantics-0x'), md5('semantics-1') || md5('semantics-1x')],
|
||||
ARRAY[256::bigint, 256], 512),
|
||||
('semantics', 3, 'malformed-file', ARRAY['missing-must-not-be-queried'],
|
||||
ARRAY[]::bigint[], 0);
|
||||
|
||||
WITH distinct_hashes AS (
|
||||
SELECT scenario, hash
|
||||
FROM perf_integrity.manifests
|
||||
CROSS JOIN LATERAL unnest(chunk_hashes) AS u(hash)
|
||||
WHERE hash <> 'missing-must-not-be-queried'
|
||||
GROUP BY scenario, hash
|
||||
), numbered AS (
|
||||
SELECT scenario, hash,
|
||||
row_number() OVER (PARTITION BY scenario ORDER BY hash) - 1 AS ordinal
|
||||
FROM distinct_hashes
|
||||
)
|
||||
INSERT INTO perf_integrity.blobs
|
||||
SELECT scenario, ordinal, hash, 256 FROM numbered;
|
||||
|
||||
ANALYZE perf_integrity.manifests;
|
||||
ANALYZE perf_integrity.blobs;
|
||||
"#;
|
||||
|
||||
const SEED_SMOKE_SQL: &str = r#"
|
||||
DROP SCHEMA IF EXISTS perf_integrity CASCADE;
|
||||
CREATE SCHEMA perf_integrity;
|
||||
CREATE TABLE perf_integrity.manifests (
|
||||
scenario text NOT NULL,
|
||||
ordinal bigint NOT NULL,
|
||||
file_hash text NOT NULL,
|
||||
chunk_hashes text[] NOT NULL,
|
||||
chunk_sizes bigint[] NOT NULL,
|
||||
total_size bigint NOT NULL,
|
||||
PRIMARY KEY (scenario, ordinal)
|
||||
);
|
||||
CREATE TABLE perf_integrity.blobs (
|
||||
scenario text NOT NULL,
|
||||
ordinal bigint NOT NULL,
|
||||
hash text NOT NULL,
|
||||
size bigint NOT NULL,
|
||||
PRIMARY KEY (scenario, ordinal)
|
||||
);
|
||||
INSERT INTO perf_integrity.manifests VALUES
|
||||
('one', 0, 'file-one', ARRAY[md5('0') || md5('0x')], ARRAY[256::bigint], 256);
|
||||
INSERT INTO perf_integrity.blobs
|
||||
SELECT 'one', 0, md5('0') || md5('0x'), 256;
|
||||
"#;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum Mode {
|
||||
Historical,
|
||||
MaterializedOwned,
|
||||
StreamingSorted,
|
||||
StreamingPrefetch,
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
fn parse(value: &str) -> Self {
|
||||
match value {
|
||||
"historical" => Self::Historical,
|
||||
"materialized" => Self::MaterializedOwned,
|
||||
"streaming" => Self::StreamingSorted,
|
||||
"prefetch" => Self::StreamingPrefetch,
|
||||
_ => panic!("mode must be historical, materialized, streaming, or prefetch"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ModelBackend {
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl ModelBackend {
|
||||
fn blob_size<'a>(&'a self, hash: &'a str) -> BoxFut<'a, Option<u64>> {
|
||||
Box::pin(async move {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
if hash.starts_with("missing-") {
|
||||
None
|
||||
} else if hash.starts_with("wrong-") {
|
||||
Some(999)
|
||||
} else {
|
||||
Some(256)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn calls(&self) -> usize {
|
||||
self.calls.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Outcome {
|
||||
phase_elapsed: Duration,
|
||||
full_elapsed: Duration,
|
||||
issues: Vec<String>,
|
||||
phase_calls: usize,
|
||||
full_calls: usize,
|
||||
manifest_rows: usize,
|
||||
queries: usize,
|
||||
held_connection_while_streaming: bool,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn label(value: &str) -> &str {
|
||||
&value[..value.len().min(12)]
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn valid_occurrences(row: &ManifestRow) -> usize {
|
||||
if row.2.len() == row.3.len() {
|
||||
row.2.len()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
fn uses_serial_fast_path(rows: &[ManifestRow]) -> bool {
|
||||
if rows.len() == 1 {
|
||||
return rows[0].2.len() != rows[0].3.len()
|
||||
|| rows[0].2.len() <= SERIAL_FAST_PATH_OCCURRENCES;
|
||||
}
|
||||
let mut occurrences = 0usize;
|
||||
for row in rows {
|
||||
occurrences = occurrences.saturating_add(valid_occurrences(row));
|
||||
if occurrences > SERIAL_FAST_PATH_OCCURRENCES {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn serial_rows(rows: &[ManifestRow], backend: &ModelBackend) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
for (_, file_hash, hashes, expected_sizes, total_size) in rows {
|
||||
let file_label = label(file_hash);
|
||||
if hashes.len() != expected_sizes.len() {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let sum: i64 = expected_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for (index, hash) in hashes.iter().enumerate() {
|
||||
let chunk_label = label(hash);
|
||||
match backend.blob_size(hash).await {
|
||||
Some(actual) if actual != expected_sizes[index] as u64 => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch (expected {}, actual {actual})",
|
||||
expected_sizes[index]
|
||||
)),
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
fn replay_with<F>(rows: &[ManifestRow], mut size_of: F) -> Vec<String>
|
||||
where
|
||||
F: FnMut(&str) -> Option<u64>,
|
||||
{
|
||||
let mut issues = Vec::new();
|
||||
for (_, file_hash, hashes, expected_sizes, total_size) in rows {
|
||||
let file_label = label(file_hash);
|
||||
if hashes.len() != expected_sizes.len() {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: chunk_hashes/chunk_sizes length mismatch"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let sum: i64 = expected_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for (index, hash) in hashes.iter().enumerate() {
|
||||
let chunk_label = label(hash);
|
||||
match size_of(hash) {
|
||||
Some(actual) if actual != expected_sizes[index] as u64 => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: size mismatch (expected {}, actual {actual})",
|
||||
expected_sizes[index]
|
||||
)),
|
||||
None => issues.push(format!(
|
||||
"Manifest {file_label} chunk {chunk_label}: missing in backend"
|
||||
)),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
async fn owned_batch(rows: &[ManifestRow], backend: &ModelBackend) -> Vec<String> {
|
||||
let mut sizes = OwnedSizes::default();
|
||||
for row in rows {
|
||||
if row.2.len() == row.3.len() {
|
||||
for hash in &row.2 {
|
||||
sizes.entry(hash.clone()).or_insert(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
let hashes: Vec<String> = sizes.keys().cloned().collect();
|
||||
let sizes = stream::iter(hashes)
|
||||
.map(|hash| async move {
|
||||
let size = backend.blob_size(&hash).await;
|
||||
(hash, size)
|
||||
})
|
||||
.buffer_unordered(CURRENT_CONCURRENCY)
|
||||
.fold(sizes, |mut sizes, (hash, size)| async move {
|
||||
sizes.insert(hash, size);
|
||||
sizes
|
||||
})
|
||||
.await;
|
||||
replay_with(rows, |hash| sizes.get(hash).copied().flatten())
|
||||
}
|
||||
|
||||
async fn sorted_batch(rows: &[ManifestRow], backend: &ModelBackend) -> Vec<String> {
|
||||
let mut hashes = Vec::new();
|
||||
for row in rows {
|
||||
if row.2.len() == row.3.len() {
|
||||
hashes.extend(row.2.iter().map(String::as_str));
|
||||
}
|
||||
}
|
||||
hashes.sort_unstable();
|
||||
hashes.dedup();
|
||||
let values = vec![None; hashes.len()];
|
||||
let values = stream::iter(hashes.iter().copied().enumerate())
|
||||
.map(|(index, hash)| async move {
|
||||
let value = backend.blob_size(hash).await;
|
||||
(index, value)
|
||||
})
|
||||
.buffer_unordered(STREAMING_CONCURRENCY)
|
||||
.fold(values, |mut values, (index, value)| async move {
|
||||
values[index] = value;
|
||||
values
|
||||
})
|
||||
.await;
|
||||
replay_with(rows, |hash| {
|
||||
hashes
|
||||
.binary_search(&hash)
|
||||
.ok()
|
||||
.and_then(|index| values[index])
|
||||
})
|
||||
}
|
||||
|
||||
async fn large_row(row: &ManifestRow, backend: &ModelBackend, mode: Mode) -> Vec<String> {
|
||||
let (_, file_hash, hashes, expected_sizes, total_size) = row;
|
||||
let file_label = label(file_hash);
|
||||
let mut issues = Vec::new();
|
||||
let sum: i64 = expected_sizes.iter().sum();
|
||||
if sum != *total_size {
|
||||
issues.push(format!(
|
||||
"Manifest {file_label}: total_size {total_size} != sum of chunk_sizes {sum}"
|
||||
));
|
||||
}
|
||||
for offset in (0..hashes.len()).step_by(WINDOW) {
|
||||
let end = (offset + WINDOW).min(hashes.len());
|
||||
let slice_row = (
|
||||
row.0,
|
||||
file_hash.clone(),
|
||||
hashes[offset..end].to_vec(),
|
||||
expected_sizes[offset..end].to_vec(),
|
||||
expected_sizes[offset..end].iter().sum(),
|
||||
);
|
||||
let slice = std::slice::from_ref(&slice_row);
|
||||
let mut slice_issues = match mode {
|
||||
Mode::MaterializedOwned => owned_batch(slice, backend).await,
|
||||
Mode::StreamingSorted | Mode::StreamingPrefetch => sorted_batch(slice, backend).await,
|
||||
Mode::Historical => unreachable!(),
|
||||
};
|
||||
slice_issues.retain(|issue| !issue.contains("sum of chunk_sizes"));
|
||||
issues.extend(slice_issues);
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
async fn process_materialized_windowed(
|
||||
rows: &[ManifestRow],
|
||||
backend: &ModelBackend,
|
||||
mode: Mode,
|
||||
) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
let mut start = 0usize;
|
||||
while start < rows.len() {
|
||||
let next = valid_occurrences(&rows[start]);
|
||||
if next > WINDOW {
|
||||
issues.extend(large_row(&rows[start], backend, mode).await);
|
||||
start += 1;
|
||||
continue;
|
||||
}
|
||||
let mut occurrences = 0usize;
|
||||
let mut end = start;
|
||||
while end < rows.len() {
|
||||
let next = valid_occurrences(&rows[end]);
|
||||
if next > WINDOW || (occurrences > 0 && occurrences + next > WINDOW) {
|
||||
break;
|
||||
}
|
||||
occurrences += next;
|
||||
end += 1;
|
||||
}
|
||||
debug_assert!(end > start);
|
||||
issues.extend(match mode {
|
||||
Mode::MaterializedOwned => owned_batch(&rows[start..end], backend).await,
|
||||
Mode::StreamingSorted | Mode::StreamingPrefetch => {
|
||||
sorted_batch(&rows[start..end], backend).await
|
||||
}
|
||||
Mode::Historical => unreachable!(),
|
||||
});
|
||||
start = end;
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
struct WindowProcessor<'a> {
|
||||
backend: &'a ModelBackend,
|
||||
mode: Mode,
|
||||
rows: Vec<ManifestRow>,
|
||||
occurrences: usize,
|
||||
issues: Vec<String>,
|
||||
}
|
||||
|
||||
impl<'a> WindowProcessor<'a> {
|
||||
fn new(backend: &'a ModelBackend, mode: Mode) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
mode,
|
||||
rows: Vec::new(),
|
||||
occurrences: 0,
|
||||
issues: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
if self.rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
let rows = std::mem::take(&mut self.rows);
|
||||
let issues = match self.mode {
|
||||
Mode::MaterializedOwned => owned_batch(&rows, self.backend).await,
|
||||
Mode::StreamingSorted | Mode::StreamingPrefetch => {
|
||||
sorted_batch(&rows, self.backend).await
|
||||
}
|
||||
Mode::Historical => unreachable!(),
|
||||
};
|
||||
self.issues.extend(issues);
|
||||
self.occurrences = 0;
|
||||
}
|
||||
|
||||
async fn push(&mut self, row: ManifestRow) {
|
||||
let next = valid_occurrences(&row);
|
||||
if next > WINDOW {
|
||||
self.flush().await;
|
||||
self.issues
|
||||
.extend(large_row(&row, self.backend, self.mode).await);
|
||||
return;
|
||||
}
|
||||
if self.occurrences > 0 && self.occurrences + next > WINDOW {
|
||||
self.flush().await;
|
||||
}
|
||||
self.occurrences += next;
|
||||
self.rows.push(row);
|
||||
}
|
||||
|
||||
async fn finish(mut self) -> Vec<String> {
|
||||
self.flush().await;
|
||||
self.issues
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(row: sqlx::postgres::PgRow) -> Result<ManifestRow, sqlx::Error> {
|
||||
Ok((
|
||||
row.try_get("ordinal")?,
|
||||
row.try_get("file_hash")?,
|
||||
row.try_get("chunk_hashes")?,
|
||||
row.try_get("chunk_sizes")?,
|
||||
row.try_get("total_size")?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn phase_one_materialized(
|
||||
pool: &PgPool,
|
||||
scenario: &str,
|
||||
backend: &ModelBackend,
|
||||
mode: Mode,
|
||||
) -> Result<(Vec<String>, Vec<ManifestRow>), sqlx::Error> {
|
||||
let tuples: Vec<ManifestRow> = sqlx::query_as(MANIFEST_QUERY)
|
||||
.bind(scenario)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
let issues = if mode == Mode::Historical || uses_serial_fast_path(&tuples) {
|
||||
serial_rows(&tuples, backend).await
|
||||
} else {
|
||||
process_materialized_windowed(&tuples, backend, mode).await
|
||||
};
|
||||
Ok((issues, tuples))
|
||||
}
|
||||
|
||||
async fn phase_one_streaming(
|
||||
pool: &PgPool,
|
||||
scenario: &str,
|
||||
backend: &ModelBackend,
|
||||
) -> Result<(Vec<String>, usize, bool), sqlx::Error> {
|
||||
let mut rows = sqlx::query(MANIFEST_QUERY).bind(scenario).fetch(pool);
|
||||
let mut initial = Vec::new();
|
||||
let mut occurrences = 0usize;
|
||||
let mut row_count = 0usize;
|
||||
let mut held_connection = false;
|
||||
let mut reached_eof = false;
|
||||
|
||||
while occurrences <= SERIAL_FAST_PATH_OCCURRENCES {
|
||||
let Some(row) = rows.try_next().await? else {
|
||||
reached_eof = true;
|
||||
break;
|
||||
};
|
||||
held_connection |= pool.num_idle() == 0;
|
||||
let row = decode(row)?;
|
||||
occurrences = occurrences.saturating_add(valid_occurrences(&row));
|
||||
row_count += 1;
|
||||
initial.push(row);
|
||||
}
|
||||
|
||||
if reached_eof {
|
||||
debug_assert!(uses_serial_fast_path(&initial));
|
||||
drop(rows);
|
||||
return Ok((
|
||||
serial_rows(&initial, backend).await,
|
||||
row_count,
|
||||
held_connection,
|
||||
));
|
||||
}
|
||||
|
||||
let mut processor = WindowProcessor::new(backend, Mode::StreamingSorted);
|
||||
for row in initial {
|
||||
processor.push(row).await;
|
||||
}
|
||||
while let Some(row) = rows.try_next().await? {
|
||||
held_connection |= pool.num_idle() == 0;
|
||||
processor.push(decode(row)?).await;
|
||||
row_count += 1;
|
||||
}
|
||||
drop(rows);
|
||||
Ok((processor.finish().await, row_count, held_connection))
|
||||
}
|
||||
|
||||
async fn phase_one_streaming_prefetch(
|
||||
pool: &PgPool,
|
||||
scenario: &str,
|
||||
backend: &ModelBackend,
|
||||
) -> Result<(Vec<String>, usize, bool), sqlx::Error> {
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel(PREFETCH_ROWS);
|
||||
let producer_pool = pool.clone();
|
||||
let producer_scenario = scenario.to_owned();
|
||||
let connection_held = Arc::new(AtomicBool::new(false));
|
||||
let producer_held = connection_held.clone();
|
||||
let producer = tokio::spawn(async move {
|
||||
let mut rows = sqlx::query(MANIFEST_QUERY)
|
||||
.bind(producer_scenario)
|
||||
.fetch(&producer_pool);
|
||||
while let Some(row) = rows.try_next().await? {
|
||||
producer_held.fetch_or(producer_pool.num_idle() == 0, Ordering::Relaxed);
|
||||
if sender.send(decode(row)?).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok::<(), sqlx::Error>(())
|
||||
});
|
||||
|
||||
let mut initial = Vec::new();
|
||||
let mut occurrences = 0usize;
|
||||
let mut row_count = 0usize;
|
||||
let mut reached_eof = false;
|
||||
while occurrences <= SERIAL_FAST_PATH_OCCURRENCES {
|
||||
let Some(row) = receiver.recv().await else {
|
||||
reached_eof = true;
|
||||
break;
|
||||
};
|
||||
occurrences = occurrences.saturating_add(valid_occurrences(&row));
|
||||
row_count += 1;
|
||||
initial.push(row);
|
||||
}
|
||||
|
||||
if reached_eof {
|
||||
producer
|
||||
.await
|
||||
.expect("manifest prefetch producer panicked")?;
|
||||
debug_assert!(uses_serial_fast_path(&initial));
|
||||
return Ok((
|
||||
serial_rows(&initial, backend).await,
|
||||
row_count,
|
||||
connection_held.load(Ordering::Relaxed),
|
||||
));
|
||||
}
|
||||
|
||||
let mut processor = WindowProcessor::new(backend, Mode::StreamingPrefetch);
|
||||
for row in initial {
|
||||
processor.push(row).await;
|
||||
}
|
||||
while let Some(row) = receiver.recv().await {
|
||||
processor.push(row).await;
|
||||
row_count += 1;
|
||||
}
|
||||
producer
|
||||
.await
|
||||
.expect("manifest prefetch producer panicked")?;
|
||||
Ok((
|
||||
processor.finish().await,
|
||||
row_count,
|
||||
connection_held.load(Ordering::Relaxed),
|
||||
))
|
||||
}
|
||||
|
||||
async fn phase_two(
|
||||
pool: &PgPool,
|
||||
scenario: &str,
|
||||
backend: &ModelBackend,
|
||||
) -> Result<Vec<String>, sqlx::Error> {
|
||||
let mut rows = sqlx::query(BLOB_QUERY).bind(scenario).fetch(pool);
|
||||
let mut issues = Vec::new();
|
||||
let mut batch = Vec::with_capacity(PHASE_TWO_CONCURRENCY);
|
||||
loop {
|
||||
let next = rows.try_next().await?;
|
||||
let done = next.is_none();
|
||||
if let Some(row) = next {
|
||||
batch.push((
|
||||
row.try_get::<String, _>("hash")?,
|
||||
row.try_get::<i64, _>("size")?,
|
||||
));
|
||||
}
|
||||
if batch.len() == PHASE_TWO_CONCURRENCY || (done && !batch.is_empty()) {
|
||||
let current = std::mem::replace(&mut batch, Vec::with_capacity(PHASE_TWO_CONCURRENCY));
|
||||
let mut batch_issues: Vec<String> = stream::iter(current)
|
||||
.map(|(hash, expected)| async move {
|
||||
match backend.blob_size(&hash).await {
|
||||
Some(actual) if actual != expected as u64 => Some(format!(
|
||||
"{hash}: size mismatch (expected: {expected}, actual: {actual})"
|
||||
)),
|
||||
None => Some(format!("{hash}: blob missing in backend")),
|
||||
Some(_) => None,
|
||||
}
|
||||
})
|
||||
.buffer_unordered(PHASE_TWO_CONCURRENCY)
|
||||
.filter_map(async move |issue| issue)
|
||||
.collect()
|
||||
.await;
|
||||
issues.append(&mut batch_issues);
|
||||
}
|
||||
if done {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(issues)
|
||||
}
|
||||
|
||||
async fn run(
|
||||
pool: &PgPool,
|
||||
mode: Mode,
|
||||
scenario: &str,
|
||||
full: bool,
|
||||
) -> Result<Outcome, sqlx::Error> {
|
||||
let backend = Arc::new(ModelBackend::default());
|
||||
let start = Instant::now();
|
||||
let (mut issues, manifest_rows, held, retained_rows) = match mode {
|
||||
Mode::Historical | Mode::MaterializedOwned => {
|
||||
let (issues, rows) = phase_one_materialized(pool, scenario, &backend, mode).await?;
|
||||
let count = rows.len();
|
||||
(issues, count, false, Some(rows))
|
||||
}
|
||||
Mode::StreamingSorted => {
|
||||
let (issues, count, held) = phase_one_streaming(pool, scenario, &backend).await?;
|
||||
(issues, count, held, None)
|
||||
}
|
||||
Mode::StreamingPrefetch => {
|
||||
let (issues, count, held) =
|
||||
phase_one_streaming_prefetch(pool, scenario, &backend).await?;
|
||||
(issues, count, held, None)
|
||||
}
|
||||
};
|
||||
let phase_elapsed = start.elapsed();
|
||||
let phase_calls = backend.calls();
|
||||
if full {
|
||||
issues.extend(phase_two(pool, scenario, &backend).await?);
|
||||
}
|
||||
let full_elapsed = start.elapsed();
|
||||
let full_calls = backend.calls();
|
||||
black_box(&issues);
|
||||
black_box(&retained_rows);
|
||||
Ok(Outcome {
|
||||
phase_elapsed,
|
||||
full_elapsed,
|
||||
issues,
|
||||
phase_calls,
|
||||
full_calls,
|
||||
manifest_rows,
|
||||
queries: 1 + usize::from(full),
|
||||
held_connection_while_streaming: held,
|
||||
})
|
||||
}
|
||||
|
||||
fn checksum(issues: &[String]) -> u64 {
|
||||
issues
|
||||
.iter()
|
||||
.flat_map(|issue| issue.bytes())
|
||||
.fold(0xcbf2_9ce4_8422_2325, |hash, byte| {
|
||||
(hash ^ u64::from(byte)).wrapping_mul(0x1000_0000_01b3)
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn main() -> Result<(), Box<dyn Error>> {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let database_url = env::var("DATABASE_URL").expect("DATABASE_URL is required");
|
||||
|
||||
if args
|
||||
.get(1)
|
||||
.is_some_and(|value| value == "seed" || value == "seed-smoke")
|
||||
{
|
||||
let mut connection = PgConnection::connect(&database_url).await?;
|
||||
let sql = if args.get(1).is_some_and(|value| value == "seed-smoke") {
|
||||
SEED_SMOKE_SQL
|
||||
} else {
|
||||
SEED_SQL
|
||||
};
|
||||
sqlx::raw_sql(sql).execute(&mut connection).await?;
|
||||
let manifests: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM perf_integrity.manifests")
|
||||
.fetch_one(&mut connection)
|
||||
.await?;
|
||||
let blobs: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM perf_integrity.blobs")
|
||||
.fetch_one(&mut connection)
|
||||
.await?;
|
||||
println!("seeded manifests={manifests} blobs={blobs}");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await?;
|
||||
|
||||
match args.get(1).map(String::as_str) {
|
||||
Some("seed" | "seed-smoke") => unreachable!(),
|
||||
Some("compare") => {
|
||||
let scenario = args.get(2).map(String::as_str).unwrap_or("semantics");
|
||||
let historical = run(&pool, Mode::Historical, scenario, true).await?;
|
||||
let materialized = run(&pool, Mode::MaterializedOwned, scenario, true).await?;
|
||||
let streaming = run(&pool, Mode::StreamingSorted, scenario, true).await?;
|
||||
let prefetch = run(&pool, Mode::StreamingPrefetch, scenario, true).await?;
|
||||
assert_eq!(historical.issues, materialized.issues);
|
||||
assert_eq!(historical.issues, streaming.issues);
|
||||
assert_eq!(historical.issues, prefetch.issues);
|
||||
assert!(materialized.phase_calls <= historical.phase_calls);
|
||||
assert_eq!(materialized.phase_calls, streaming.phase_calls);
|
||||
assert_eq!(materialized.phase_calls, prefetch.phase_calls);
|
||||
assert_eq!(historical.manifest_rows, streaming.manifest_rows);
|
||||
assert_eq!(historical.manifest_rows, prefetch.manifest_rows);
|
||||
assert_eq!(historical.queries, streaming.queries);
|
||||
assert_eq!(historical.queries, prefetch.queries);
|
||||
println!(
|
||||
"scenario={scenario} issues={} checksum={} phase_calls={}/{}/{}/{} full_calls={}/{}/{}/{} rows={} queries={} streaming_held_connection={} prefetch_held_connection={}",
|
||||
historical.issues.len(),
|
||||
checksum(&historical.issues),
|
||||
historical.phase_calls,
|
||||
materialized.phase_calls,
|
||||
streaming.phase_calls,
|
||||
prefetch.phase_calls,
|
||||
historical.full_calls,
|
||||
materialized.full_calls,
|
||||
streaming.full_calls,
|
||||
prefetch.full_calls,
|
||||
historical.manifest_rows,
|
||||
streaming.queries,
|
||||
streaming.held_connection_while_streaming,
|
||||
prefetch.held_connection_while_streaming,
|
||||
);
|
||||
}
|
||||
Some("run") => {
|
||||
let mode = Mode::parse(args.get(2).map(String::as_str).unwrap_or("streaming"));
|
||||
let scenario = args.get(3).map(String::as_str).unwrap_or("large");
|
||||
let full = args.get(4).is_some_and(|value| value == "full");
|
||||
let outcome = run(&pool, mode, scenario, full).await?;
|
||||
println!(
|
||||
"mode={mode:?} scenario={scenario} full={full} phase_ms={:.6} full_ms={:.6} issues={} checksum={} phase_calls={} full_calls={} rows={} queries={} streaming_held_connection={}",
|
||||
outcome.phase_elapsed.as_secs_f64() * 1e3,
|
||||
outcome.full_elapsed.as_secs_f64() * 1e3,
|
||||
outcome.issues.len(),
|
||||
checksum(&outcome.issues),
|
||||
outcome.phase_calls,
|
||||
outcome.full_calls,
|
||||
outcome.manifest_rows,
|
||||
outcome.queries,
|
||||
outcome.held_connection_while_streaming,
|
||||
);
|
||||
}
|
||||
_ => panic!(
|
||||
"usage: verify_integrity_streaming seed|compare SCENARIO|run MODE SCENARIO [full]"
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { startVideoThumbnailServer } from "./video-thumbnail-server.mjs";
|
||||
|
||||
const fixturePath = process.argv[2] ?? "/tmp/oxicloud-thumbnail-perf.webm";
|
||||
const server = await startVideoThumbnailServer({ fixturePath });
|
||||
|
||||
console.log(
|
||||
JSON.stringify({ url: server.url, fixtureBytes: server.fixtureBytes }),
|
||||
);
|
||||
|
||||
let closing = false;
|
||||
async function close() {
|
||||
if (closing) return;
|
||||
closing = true;
|
||||
await server.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on("SIGINT", () => void close());
|
||||
process.on("SIGTERM", () => void close());
|
||||
await new Promise(() => {});
|
||||
@@ -0,0 +1,224 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
|
||||
function freshStats() {
|
||||
return {
|
||||
thumbnailMissRequests: 0,
|
||||
originalVideoRequests: 0,
|
||||
originalVideoBytes: 0,
|
||||
generatedThumbnailPuts: 0,
|
||||
generatedThumbnailBytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function pageHtml() {
|
||||
return [
|
||||
"<!doctype html>",
|
||||
"<html><head><meta charset='utf-8'><title>thumbnail perf</title></head>",
|
||||
"<body><main id='root'></main><script>",
|
||||
"const params = new URLSearchParams(location.search);",
|
||||
"const mode = params.get('mode') === 'candidate' ? 'candidate' : 'current';",
|
||||
"const count = Math.max(1, Number(params.get('videos') || 6));",
|
||||
"const sizes = [[150, 150, 'icon'], [300, 300, 'preview'], [900, 800, 'large']];",
|
||||
"const result = { mode, count, completed: 0, errors: 0, wallMs: 0 };",
|
||||
"window.__thumbnailPerf = result;",
|
||||
"window.__thumbnailPerfDone = false;",
|
||||
"const started = performance.now();",
|
||||
"let active = 0;",
|
||||
"const waiting = [];",
|
||||
"function complete(ok) {",
|
||||
" result.completed++;",
|
||||
" if (!ok) result.errors++;",
|
||||
" if (result.completed === count) {",
|
||||
" result.wallMs = performance.now() - started;",
|
||||
" window.__thumbnailPerfDone = true;",
|
||||
" document.title = 'done';",
|
||||
" }",
|
||||
"}",
|
||||
"function blobToDataUrl(blob) {",
|
||||
" return new Promise((resolve, reject) => {",
|
||||
" const reader = new FileReader();",
|
||||
" reader.onload = () => resolve(reader.result);",
|
||||
" reader.onerror = reject;",
|
||||
" reader.readAsDataURL(blob);",
|
||||
" });",
|
||||
"}",
|
||||
"function videoBitmap(id) {",
|
||||
" return new Promise((resolve, reject) => {",
|
||||
" const video = document.createElement('video');",
|
||||
" video.muted = true;",
|
||||
" video.preload = 'metadata';",
|
||||
" video.style.display = 'none';",
|
||||
" document.body.append(video);",
|
||||
" const clean = () => {",
|
||||
" video.pause();",
|
||||
" video.removeAttribute('src');",
|
||||
" video.load();",
|
||||
" video.remove();",
|
||||
" };",
|
||||
" video.onloadedmetadata = () => { video.currentTime = Math.max(0.1, video.duration / 3); };",
|
||||
" video.onseeked = async () => {",
|
||||
" try {",
|
||||
" const bitmap = await createImageBitmap(video);",
|
||||
" clean();",
|
||||
" resolve(bitmap);",
|
||||
" } catch (error) { clean(); reject(error); }",
|
||||
" };",
|
||||
" video.onerror = () => { clean(); reject(new Error('video decode failed')); };",
|
||||
" video.src = '/api/files/video-' + id;",
|
||||
" });",
|
||||
"}",
|
||||
"async function generate(id) {",
|
||||
" const bitmap = await videoBitmap(id);",
|
||||
" const blobs = await Promise.all(sizes.map(async ([targetWidth, targetHeight]) => {",
|
||||
" const ratio = Math.min(targetWidth / bitmap.width, targetHeight / bitmap.height);",
|
||||
" const canvas = new OffscreenCanvas(Math.round(bitmap.width * ratio), Math.round(bitmap.height * ratio));",
|
||||
" canvas.getContext('2d').drawImage(bitmap, 0, 0, canvas.width, canvas.height);",
|
||||
" return canvas.convertToBlob({ type: 'image/jpeg', quality: 0.8 });",
|
||||
" }));",
|
||||
" bitmap.close();",
|
||||
" await blobToDataUrl(blobs[0]);",
|
||||
" await Promise.all(blobs.map((blob, index) => fetch(",
|
||||
" '/api/files/video-' + id + '/thumbnail/' + sizes[index][2],",
|
||||
" { method: 'PUT', headers: { 'content-type': 'image/jpeg' }, body: blob }",
|
||||
" )));",
|
||||
"}",
|
||||
"async function onThumbnailError(id, image) {",
|
||||
" image.style.display = 'none';",
|
||||
" if (mode === 'candidate') { complete(true); return; }",
|
||||
" if (active >= 3) await new Promise((resolve) => waiting.push(resolve));",
|
||||
" active++;",
|
||||
" try { await generate(id); complete(true); } catch (error) { complete(false); }",
|
||||
" finally { active--; const next = waiting.shift(); if (next) next(); }",
|
||||
"}",
|
||||
"for (let id = 0; id < count; id++) {",
|
||||
" const image = document.createElement('img');",
|
||||
" image.alt = '';",
|
||||
" image.onerror = () => { void onThumbnailError(id, image); };",
|
||||
" image.src = '/thumbnail/video-' + id;",
|
||||
" document.getElementById('root').append(image);",
|
||||
"}",
|
||||
"</script></body></html>",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parseRange(header, size) {
|
||||
const match = /^bytes=(\d+)-(\d*)$/.exec(header ?? "");
|
||||
if (!match) return null;
|
||||
const start = Number(match[1]);
|
||||
const end = match[2] ? Math.min(Number(match[2]), size - 1) : size - 1;
|
||||
if (!Number.isSafeInteger(start) || start < 0 || start > end || start >= size)
|
||||
return null;
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
export async function startVideoThumbnailServer({ fixturePath }) {
|
||||
const video = await readFile(fixturePath);
|
||||
const html = Buffer.from(pageHtml());
|
||||
let stats = freshStats();
|
||||
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
if (request.method === "GET" && url.pathname === "/") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/html; charset=utf-8",
|
||||
"content-length": html.byteLength,
|
||||
"cache-control": "no-store",
|
||||
});
|
||||
response.end(html);
|
||||
return;
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === "/__reset") {
|
||||
stats = freshStats();
|
||||
response.writeHead(204, { "cache-control": "no-store" });
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
if (request.method === "GET" && url.pathname === "/__stats") {
|
||||
const body = Buffer.from(JSON.stringify(stats));
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"content-length": body.byteLength,
|
||||
"cache-control": "no-store",
|
||||
});
|
||||
response.end(body);
|
||||
return;
|
||||
}
|
||||
if (request.method === "GET" && url.pathname.startsWith("/thumbnail/")) {
|
||||
stats.thumbnailMissRequests++;
|
||||
response.writeHead(204, { "cache-control": "no-store" });
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
if (
|
||||
request.method === "GET" &&
|
||||
/^\/api\/files\/video-\d+$/.test(url.pathname)
|
||||
) {
|
||||
stats.originalVideoRequests++;
|
||||
const range = parseRange(request.headers.range, video.byteLength);
|
||||
const start = range?.start ?? 0;
|
||||
const end = range?.end ?? video.byteLength - 1;
|
||||
const body = video.subarray(start, end + 1);
|
||||
stats.originalVideoBytes += body.byteLength;
|
||||
response.writeHead(range ? 206 : 200, {
|
||||
"content-type": "video/webm",
|
||||
"content-length": body.byteLength,
|
||||
"accept-ranges": "bytes",
|
||||
"cache-control": "no-store",
|
||||
...(range
|
||||
? {
|
||||
"content-range":
|
||||
"bytes " + start + "-" + end + "/" + video.byteLength,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
response.end(body);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
request.method === "PUT" &&
|
||||
/^\/api\/files\/video-\d+\/thumbnail\/(icon|preview|large)$/.test(
|
||||
url.pathname,
|
||||
)
|
||||
) {
|
||||
let bytes = 0;
|
||||
for await (const chunk of request) bytes += chunk.length;
|
||||
stats.generatedThumbnailPuts++;
|
||||
stats.generatedThumbnailBytes += bytes;
|
||||
response.writeHead(201, { "content-length": "0" });
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
response.writeHead(404, { "content-length": "0" });
|
||||
response.end();
|
||||
} catch (error) {
|
||||
response.writeHead(500, { "content-type": "text/plain" });
|
||||
response.end(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string")
|
||||
throw new Error("No loopback server address");
|
||||
|
||||
return {
|
||||
url: "http://127.0.0.1:" + address.port + "/",
|
||||
fixtureBytes: video.byteLength,
|
||||
resetStats() {
|
||||
stats = freshStats();
|
||||
},
|
||||
snapshot() {
|
||||
return { ...stats };
|
||||
},
|
||||
async close() {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user