Merge pull request #599 from AtalayaLabs/claude/code-performance-optimization-bq9tkf
This commit is contained in:
+58
@@ -170,6 +170,64 @@ name = "bench_thumbnails_mem"
|
|||||||
path = "examples/bench_thumbnails_mem.rs"
|
path = "examples/bench_thumbnails_mem.rs"
|
||||||
required-features = ["bench"]
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# ZIP entry-compression benchmark — Deflate-always vs MIME-aware Stored for
|
||||||
|
# already-compressed media on the folder/batch ZIP download path. No Postgres.
|
||||||
|
[[example]]
|
||||||
|
name = "bench_zip_media"
|
||||||
|
path = "examples/bench_zip_media.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# WebDAV dead-properties fetch benchmark — PROPFIND's per-child N+1 (with a
|
||||||
|
# non-indexable IS NOT DISTINCT FROM predicate) vs batched = ANY($1) per page
|
||||||
|
# (needs the dev Postgres up).
|
||||||
|
[[example]]
|
||||||
|
name = "bench_dead_props"
|
||||||
|
path = "examples/bench_dead_props.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# NC chroot / default-drive resolution benchmark — the middleware's 2 uncached
|
||||||
|
# queries per request vs the moka caches (needs the dev Postgres up).
|
||||||
|
[[example]]
|
||||||
|
name = "bench_chroot_cache"
|
||||||
|
path = "examples/bench_chroot_cache.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# Quota-path benchmark — full auth.users row (incl. 512 KiB avatar) vs the
|
||||||
|
# narrow 2-column read, on every upload check / quota PROPFIND (needs Postgres).
|
||||||
|
[[example]]
|
||||||
|
name = "bench_quota_path"
|
||||||
|
path = "examples/bench_quota_path.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# People-tab benchmark — full faces scan (2 KiB embedding per row) vs grouped
|
||||||
|
# COUNT + batched cover lookup (needs Postgres).
|
||||||
|
[[example]]
|
||||||
|
name = "bench_people_list"
|
||||||
|
path = "examples/bench_people_list.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# PROPFIND folder-paging benchmark — LIMIT/OFFSET full-folder rescan per page
|
||||||
|
# vs keyset + (folder_id, name) index (needs Postgres).
|
||||||
|
[[example]]
|
||||||
|
name = "bench_propfind_paging"
|
||||||
|
path = "examples/bench_propfind_paging.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# Static-asset compression benchmark — per-request Brotli vs precompressed
|
||||||
|
# sibling read. No Postgres.
|
||||||
|
[[example]]
|
||||||
|
name = "bench_static_precompress"
|
||||||
|
path = "examples/bench_static_precompress.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
|
# Round-2 battery: range-from-cache, NC chunk gate, delta prefetch, ingest
|
||||||
|
# overlap (real store_from_stream; run with OXICLOUD_INGEST_OVERLAP=0/1),
|
||||||
|
# ZIP streaming TTFB. Sections 1 and 4 need Postgres.
|
||||||
|
[[example]]
|
||||||
|
name = "bench_round2"
|
||||||
|
path = "examples/bench_round2.rs"
|
||||||
|
required-features = ["bench"]
|
||||||
|
|
||||||
# Video thumbnail benchmark — Option B (server-side ffmpeg frame → WebP). Needs
|
# Video thumbnail benchmark — Option B (server-side ffmpeg frame → WebP). Needs
|
||||||
# `ffmpeg` on PATH (libx264/libx265/libvpx-vp9 to synthesize the test corpus).
|
# `ffmpeg` on PATH (libx264/libx265/libvpx-vp9 to synthesize the test corpus).
|
||||||
[[example]]
|
[[example]]
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# NC chroot / default-drive resolution — moka caches (vs 2 queries/request)
|
||||||
|
|
||||||
|
With app-password verification already cached (5 min) and user flags cached
|
||||||
|
(30 s), the NextCloud basic-auth middleware still resolved the chroot from
|
||||||
|
scratch on EVERY protected NC request: `find_default_for_user` (drives JOIN
|
||||||
|
folders) + `get_folder(root_id)` (folders by PK) — 2 uncached round-trips + 2
|
||||||
|
pool checkouts before the handler even ran, for values that change only on
|
||||||
|
provisioning / drive deletion / a root-folder rename. The native `/webdav`
|
||||||
|
surface repeated the drive lookup per request (Mode-B scope resolution, MOVE
|
||||||
|
and COPY twice), WOPI once per call.
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
|
||||||
|
1. `DrivePgRepository::find_default_for_user` memoised (moka, 30 s TTL —
|
||||||
|
same tier as `drive_role_cache`), invalidated on personal-drive creation,
|
||||||
|
drive deletion and policy updates. Only `Ok` is cached, so the
|
||||||
|
provisioning idempotency check still sees the live table.
|
||||||
|
2. NC middleware markerless-chroot `FolderDto` cached by root-folder id
|
||||||
|
(30 s TTL). Only the markerless branch — the drive-marker branch keeps
|
||||||
|
its per-request `get_folder_with_perms` authz.
|
||||||
|
|
||||||
|
Staleness: bounded at 30 s for a root-folder *rename* (doesn't pass through
|
||||||
|
the repo); every other mutation invalidates explicitly.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_chroot_cache
|
||||||
|
# tunables: BENCH_POOL=20 BENCH_SECONDS=4 BENCH_CONCURRENCIES=8,64
|
||||||
|
```
|
||||||
|
|
||||||
|
## Results (4 cores, local PG16, pool=20)
|
||||||
|
|
||||||
|
| conc | mode | req/s | p50 µs | p95 µs | p99 µs | queries |
|
||||||
|
|-----:|--------|----------:|--------:|--------:|--------:|--------:|
|
||||||
|
| 8 | BEFORE | 11,013 | 696.8 | 1,203.2 | 1,642.6 | 88,102 |
|
||||||
|
| 8 | AFTER | 2,011,191 | 0.69 | 1.97 | 8.47 | 0 |
|
||||||
|
| 64 | BEFORE | 16,952 | 3,633.3 | 5,617.6 | 7,189.1 | 135,618 |
|
||||||
|
| 64 | AFTER | 2,337,233 | 0.93 | 2.23 | 11.30 | 0 |
|
||||||
|
|
||||||
|
- The fixed per-request DB tax of the whole NC surface (sync PROPFIND storms,
|
||||||
|
per-chunk uploads, previews, OCS polls) drops from **0.7–3.6 ms p50 (and 2
|
||||||
|
pool checkouts)** to a **sub-µs moka hit**.
|
||||||
|
- Under sync-storm concurrency (64 in-flight) the BEFORE p99 was 7.2 ms of
|
||||||
|
pure chroot overhead per request — that whole term vanishes.
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# WebDAV dead-properties — batched per-page fetch (vs per-child N+1)
|
||||||
|
|
||||||
|
The streaming PROPFIND walkers (native `webdav_handler.rs`, NextCloud
|
||||||
|
`nextcloud/webdav_handler.rs`, plus both NC REPORT handlers) fetched dead
|
||||||
|
properties **one child at a time, sequentially** — one DB round-trip per file
|
||||||
|
and per subfolder of every Depth:1 listing. On top, every
|
||||||
|
`DeadPropertyStore` query filtered with `folder_id IS NOT DISTINCT FROM $1 AND
|
||||||
|
file_id IS NOT DISTINCT FROM $2`, which PostgreSQL cannot serve from a B-tree
|
||||||
|
index (`IS NOT DISTINCT FROM` is not an indexable operator) — so each of those
|
||||||
|
N round-trips also degraded to a **sequential scan** as the table grew.
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
|
||||||
|
1. `DeadPropertyStore::get_all_for_files / get_all_for_folders` — ONE
|
||||||
|
`file_id = ANY($1)` round-trip per 500-child PROPFIND page (indexable via
|
||||||
|
the partial unique indexes from migration 20260830000001).
|
||||||
|
2. All single-resource queries (`get`, `get_all`, `remove`) now filter on the
|
||||||
|
concrete column (`file_id = $1` / `folder_id = $1`) instead of the
|
||||||
|
NULL-tolerant pair — index scans instead of seq scans.
|
||||||
|
3. All four handler loops replaced with one batched map lookup per page.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_dead_props
|
||||||
|
# tunables: BENCH_CHILDREN=2000 BENCH_PAGE=500 BENCH_NOISE_ROWS=20000 BENCH_REPS=5
|
||||||
|
```
|
||||||
|
|
||||||
|
Measures exactly the dead-prop portion of one Depth:1 PROPFIND of a
|
||||||
|
2,000-child folder (what the walker adds on top of the listing queries).
|
||||||
|
|
||||||
|
## Results (4 cores, local PG16, this container)
|
||||||
|
|
||||||
|
**Table with only the 2,000 seeded rows:**
|
||||||
|
|
||||||
|
| mode | queries | total ms | vs OLD |
|
||||||
|
|-------------------------------|--------:|---------:|-------:|
|
||||||
|
| OLD — seq, IS NOT DISTINCT | 2000 | 1072.41 | 1.0× |
|
||||||
|
| EQ — seq, `file_id = $1` | 2000 | 509.89 | 2.1× |
|
||||||
|
| BATCH — `= ANY($1)` per page | 4 | 4.15 | **258×** |
|
||||||
|
|
||||||
|
**Table with 22,000 rows (realistic volume — seq scans hurt):**
|
||||||
|
|
||||||
|
| mode | queries | total ms | vs OLD |
|
||||||
|
|-------------------------------|--------:|---------:|-------:|
|
||||||
|
| OLD — seq, IS NOT DISTINCT | 2000 | 4543.74 | 1.0× |
|
||||||
|
| EQ — seq, `file_id = $1` | 2000 | 515.84 | 8.8× |
|
||||||
|
| BATCH — `= ANY($1)` per page | 4 | 5.88 | **773×** |
|
||||||
|
|
||||||
|
- A Depth:1 PROPFIND of a 2,000-child folder was spending **1.1–4.5 s** on
|
||||||
|
dead-prop chatter alone — now **~5 ms**. This is per folder per sync poll,
|
||||||
|
on the hottest path desktop sync clients have.
|
||||||
|
- The `EQ` row isolates the indexability fix (2.1–8.8×); the batching is the
|
||||||
|
rest. Both are applied.
|
||||||
|
- Same unit economics apply to the other N+1s fixed alongside (search ReBAC
|
||||||
|
batch, ZIP batch authz): each eliminated sequential point query is worth
|
||||||
|
~0.25–2.3 ms of the numbers above depending on table size.
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# Companion fixes — same measured unit economics, no dedicated harness
|
||||||
|
|
||||||
|
These changes share their cost model with benches that already exist, so
|
||||||
|
instead of near-duplicate harnesses each entry cites the bench that measured
|
||||||
|
its unit price. (The per-query unit prices below: sequential indexed point
|
||||||
|
SELECT ≈ 0.25–0.55 ms and `= ANY($1)` batch ≈ 1–1.5 ms/500 ids from
|
||||||
|
benches/DEAD-PROPS.md; manifest-row fetch p50 0.44–4.4 ms from
|
||||||
|
benches/BLOB-MANIFEST.md; moka hit ≈ 1 µs from benches/CHROOT-CACHE.md.)
|
||||||
|
|
||||||
|
## 1. Content-search ReBAC re-verification — batched (SEARCH-REBAC)
|
||||||
|
|
||||||
|
`SearchService::lookup_content_hits` re-verified up to `CONTENT_HITS_LIMIT =
|
||||||
|
200` Tantivy hits with sequential `authz.check(Read, File)` calls — each a
|
||||||
|
point SELECT on owner-cache miss (distinct file ids ⇒ ~always). New
|
||||||
|
`AuthorizationEngine::check_files_read_batch` (default = the old loop, so
|
||||||
|
mocks/other impls stay correct; `PgAclEngine` override): ONE
|
||||||
|
`id = ANY($1)` drive resolution + cached per-drive role + per-file cascade
|
||||||
|
only for drive-floor misses. Decision-equivalent; per 200-hit search:
|
||||||
|
**~200 sequential round-trips (≈ 50–110 ms of DB chatter) → 1–2 queries
|
||||||
|
(≈ 1–3 ms)**. Also primes the owner cache for the hits' follow-up requests.
|
||||||
|
|
||||||
|
## 2. Batch-ZIP downloads — no per-file authz/Recent (ZIP-BATCH-AUTHZ)
|
||||||
|
|
||||||
|
`BatchOperations::add_folder_subtree_to_zip` had already authorized the
|
||||||
|
subtree ROOT (`get_folder_with_perms`), yet every enumerated file still paid
|
||||||
|
`get_file_stream_with_perms` = 1 authz point SELECT + a Recent-hook spawn
|
||||||
|
issuing 2 writes (INSERT … ON CONFLICT + prune DELETE). A 2,000-file folder
|
||||||
|
ZIP ⇒ ~6,000 extra statements. Subtree entries now use the plain
|
||||||
|
`get_file_stream` — exactly what `ZipService::create_folder_zip` (the native
|
||||||
|
folder-download path) has always done. Explicitly-selected top-level files
|
||||||
|
keep per-file authz + Recent. Unit price: DEAD-PROPS.md sequential rows —
|
||||||
|
**~1.5–4.5 s of DB chatter removed** from a 2,000-file archive, plus the ZIP
|
||||||
|
no longer floods Recents with every archived file.
|
||||||
|
|
||||||
|
## 3. CDC manifest RAM cache (MANIFEST-CACHE)
|
||||||
|
|
||||||
|
Every stream / range / full read of a CDC blob paid one
|
||||||
|
`chunk_manifests` row fetch first — p50 0.44 ms (4.4 ms under pool pressure,
|
||||||
|
benches/BLOB-MANIFEST.md), on the hottest read paths there are (media
|
||||||
|
serving, thumbnails, range seeks). Manifests are immutable by content
|
||||||
|
address, so `DedupService` now memoises them (moka, weight-bounded 32 MiB,
|
||||||
|
60 s TTL, positive-only so background rechunking is honoured immediately;
|
||||||
|
invalidated post-commit on the two delete paths). Warm read: **0.44–4.4 ms →
|
||||||
|
~1 µs** (CHROOT-CACHE.md's moka row) and one fewer pool checkout per read —
|
||||||
|
range-seek storms (video scrubbing) hit this every request.
|
||||||
|
|
||||||
|
## 4. Public share landing — 3 round-trips → 1 atomic UPDATE (SHARE-ACCESS)
|
||||||
|
|
||||||
|
`GET /api/s/{token}` ran find_share_by_token (with a correlated
|
||||||
|
`MIN(expires_at)` subquery), a full-row UPDATE writing back a Rust-side
|
||||||
|
increment (racy: lost updates between concurrent visitors, and it rewrote
|
||||||
|
`item_name`/`password_hash` wholesale — clobbering concurrent owner edits),
|
||||||
|
then the handler's follow-up fetched the share a third time.
|
||||||
|
`ShareStoragePort::increment_access_count` is now one
|
||||||
|
`UPDATE … SET access_count = access_count + 1 WHERE token = $1 AND <expiry>`:
|
||||||
|
**3 subquery round-trips → 2** for the landing (register + fetch), no
|
||||||
|
read-modify-write race, no collateral column rewrites.
|
||||||
|
|
||||||
|
## 5. Trash — dead SELECT removed
|
||||||
|
|
||||||
|
`TrashService::move_to_trash` fetched the full file/folder entity to build a
|
||||||
|
`TrashedItem` consumed only by `TrashRepository::add_to_trash` — a documented
|
||||||
|
no-op in the soft-delete model. Both branches now go straight to the
|
||||||
|
`move_to_trash` UPDATE: **one uncached SELECT + entity hydration removed per
|
||||||
|
trash operation** (file and folder).
|
||||||
|
|
||||||
|
## 6. NFC normalization fast path
|
||||||
|
|
||||||
|
`normalize_storage_name` ran unicode-normalization's full
|
||||||
|
decompose/recompose state machine on every name of every row loaded from PG
|
||||||
|
(listings, PROPFIND, photos — 27 constructor call sites), even though the DB
|
||||||
|
invariant guarantees stored names are already NFC. `is_nfc_quick` (a
|
||||||
|
per-char table lookup) now short-circuits the ~100 % case to a plain copy;
|
||||||
|
`Maybe`/`No` still run the full pipeline, so semantics are unchanged.
|
||||||
|
|
||||||
|
## 7. Frontend — first-page render for large folders
|
||||||
|
|
||||||
|
`fetchFolderListing` paged the ENTIRE folder (sequential 200-item requests)
|
||||||
|
before returning anything — a 2,000-item folder waited ~10 round-trips
|
||||||
|
before first paint. The files route now paints page one immediately via the
|
||||||
|
new `onPage` hook and fills in as later pages land (skipped when a cached
|
||||||
|
listing is already on screen, so views never shrink). First-paint latency
|
||||||
|
for an N-item folder drops from ⌈N/200⌉ sequential RTTs to 1.
|
||||||
|
|
||||||
|
## Refuted by benchmark (reverted, kept for the record)
|
||||||
|
|
||||||
|
- **Cached `Intl.Collator` for name sorts (frontend):** sorting 5,000 names —
|
||||||
|
argument-less `localeCompare` 5.6 ms vs cached collator **12.1 ms (2×
|
||||||
|
slower)**. V8 fast-paths argument-less `localeCompare`; the "cache the
|
||||||
|
collator" folklore does not apply. Reverted, ordering untouched.
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# People tab — grouped COUNT (vs full faces scan with embeddings)
|
||||||
|
|
||||||
|
`PeopleService::list_people` (GET `/api/people`, fetched on every People-tab
|
||||||
|
mount) called `faces_for_user`, which SELECTs every face row for the caller —
|
||||||
|
each carrying a 2,048-byte embedding BYTEA that gets decoded into a fresh
|
||||||
|
`Vec<f32>` — only to (a) count faces per person and (b) resolve a handful of
|
||||||
|
cover faces to file ids. A 10k-face library moved ~21 MB of embeddings per
|
||||||
|
request. `merge()` had the same over-fetch plus one UPDATE per face.
|
||||||
|
|
||||||
|
Changes (`FaceRepository` + `PeopleService`):
|
||||||
|
|
||||||
|
- `person_face_stats`: `SELECT person_id, COUNT(*) … GROUP BY person_id`.
|
||||||
|
- `file_ids_for_faces`: one `id = ANY($1)` over just the cover face ids.
|
||||||
|
- `reassign_person_faces`: merge as ONE set-based UPDATE (was: load all
|
||||||
|
faces, filter in Rust, one UPDATE per face).
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_people_list
|
||||||
|
# tunables: BENCH_FACES=10000 BENCH_PERSONS=20 BENCH_REPS=5
|
||||||
|
```
|
||||||
|
|
||||||
|
## Results (4 cores, local PG16, 10,000 faces / 20 persons)
|
||||||
|
|
||||||
|
| mode | total ms | bytes moved |
|
||||||
|
|--------------------------|---------:|------------:|
|
||||||
|
| BEFORE — full face rows | 30.40 | 20,960,000 |
|
||||||
|
| AFTER — COUNT + covers | 3.76 | 1,280 |
|
||||||
|
|
||||||
|
- **8.1× faster** and **~16,000× fewer bytes** off the wire per People-tab
|
||||||
|
mount. The heap never materialises 10k embedding `Vec<f32>`s.
|
||||||
|
- The BEFORE row also allocated ~21 MB per request on the server; under a
|
||||||
|
handful of concurrent mounts that was tens of MB of transient RSS for a
|
||||||
|
page that shows 20 avatars.
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# PROPFIND folder paging — keyset cursor + (folder_id, name) index
|
||||||
|
|
||||||
|
`list_files_batch` walks a folder's children in name order, 500 per page
|
||||||
|
(native + NextCloud PROPFIND streamers). The old shape was `ORDER BY name
|
||||||
|
LIMIT 500 OFFSET k` with **no supporting index** — the initial schema's
|
||||||
|
`(folder_id, name, user_id)` index that served it was dropped by migration
|
||||||
|
20260902000000 (user_id → nullable), leaving only `idx_files_folder_id`. So
|
||||||
|
every page bitmap-scanned all N children and top-sorted them: a full listing
|
||||||
|
of an N-file folder cost O(N²/500) row visits + ⌈N/500⌉ sorts.
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
|
||||||
|
1. Migration `20260917000000_files_folder_name_index.sql`: partial composite
|
||||||
|
`idx_files_folder_name (folder_id, name) WHERE NOT is_trashed`.
|
||||||
|
2. `list_files_batch` cursor switched from OFFSET to keyset
|
||||||
|
(`name > $last`, names are unique per folder via the
|
||||||
|
`(drive_id, folder_id, name)` unique index) across the port trait, the
|
||||||
|
repository and both handler loops. The cursor predicate is only emitted
|
||||||
|
when a cursor exists — a `$2 IS NULL OR …` disjunction would block the
|
||||||
|
index condition under the extended protocol's generic plans.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_propfind_paging
|
||||||
|
# tunables: BENCH_FILES=20000 BENCH_PAGE=500 BENCH_REPS=3
|
||||||
|
```
|
||||||
|
|
||||||
|
Times the FULL page-by-page walk of a 20,000-file folder (the listing
|
||||||
|
portion of one Depth:1 PROPFIND).
|
||||||
|
|
||||||
|
## Results (4 cores, local PG16)
|
||||||
|
|
||||||
|
| mode | total ms | vs OLD |
|
||||||
|
|----------------------------------|---------:|-------:|
|
||||||
|
| OFFSET, no index (true BEFORE) | 1,266.3 | 1.0× |
|
||||||
|
| OFFSET + index (index alone) | 482.7 | 2.6× |
|
||||||
|
| KEYSET + index (AFTER) | 76.7 | **16.5×** |
|
||||||
|
|
||||||
|
- Full-folder listing cost drops **16.5×**; unlike OFFSET (even indexed),
|
||||||
|
keyset stays O(page) at any depth, so the gap widens with folder size.
|
||||||
|
- Companion fix in the same commit: the Photos timeline cursor
|
||||||
|
(`list_media_files`) wrapped its keyset column in
|
||||||
|
`EXTRACT(EPOCH FROM …)::bigint` plus an `IS NULL OR` disjunction —
|
||||||
|
non-sargable, so page k re-scanned all k·limit rows already scrolled past.
|
||||||
|
It now compares the raw `media_sort_date` against a timestamptz bind
|
||||||
|
(identical row semantics — the cursor is whole seconds) and splits the
|
||||||
|
cursor/no-cursor query shapes, restoring the
|
||||||
|
`idx_files_media_timeline_by_drive` boundary condition the index was built
|
||||||
|
for. Same mechanism as measured above (index-boundary vs per-row filter);
|
||||||
|
the deep-scroll effect mirrors the OFFSET column.
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# Quota path — narrow 2-column read + skip-when-not-requested
|
||||||
|
|
||||||
|
Two independent fixes on the quota resolution that runs on every upload check
|
||||||
|
and every quota-reporting folder PROPFIND:
|
||||||
|
|
||||||
|
1. **Narrow read.** `check_storage_quota` / `get_user_storage_info` called
|
||||||
|
`get_user_by_id`, whose SELECT drags the entire `auth.users` row —
|
||||||
|
including `image`, an avatar data URI of up to 512 KiB — to read two i64s.
|
||||||
|
New `UserPgRepository::get_storage_usage` reads exactly
|
||||||
|
`(storage_used_bytes, storage_quota_bytes)` (same pattern as the existing
|
||||||
|
`get_user_flags`).
|
||||||
|
2. **Skip entirely when not asked.** `resolve_webdav_quota` (2 round-trips:
|
||||||
|
drive row + user row) ran on EVERY folder PROPFIND on both surfaces, even
|
||||||
|
when the client's `<D:prop>` list named no quota property — which is the
|
||||||
|
common shape for sync-client polls. `PropFindRequest::wants_quota()` now
|
||||||
|
gates it: `AllProp`/`PropName` keep quota (the writers emit RFC 4331 props
|
||||||
|
there), explicit prop lists trigger the lookups only if they name
|
||||||
|
`quota-used-bytes` / `quota-available-bytes`. Responses are byte-identical
|
||||||
|
for every request that names quota or asks for allprop.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_quota_path
|
||||||
|
# tunables: BENCH_SECONDS=4 BENCH_CONCURRENCIES=8,64 BENCH_IMAGE_KB=512
|
||||||
|
```
|
||||||
|
|
||||||
|
## Results (4 cores, local PG16, pool=20, 512 KiB avatar on the row)
|
||||||
|
|
||||||
|
| conc | mode | ops/s | p50 µs | p99 µs |
|
||||||
|
|-----:|--------|-------:|---------:|---------:|
|
||||||
|
| 8 | FULL | 2,222 | 3,369.4 | 8,452.2 |
|
||||||
|
| 8 | NARROW | 25,118 | 294.9 | 867.9 |
|
||||||
|
| 64 | FULL | 2,567 | 24,642.3 | 36,164.0 |
|
||||||
|
| 64 | NARROW | 40,195 | 1,468.9 | 3,964.9 |
|
||||||
|
|
||||||
|
- **11–16× throughput, p50 3.4 ms → 0.29 ms** for the user-row half of every
|
||||||
|
quota resolution (the avatar bytes dominated the wire+decode cost).
|
||||||
|
- With `wants_quota()` the common PROPFIND pays **zero** quota queries — the
|
||||||
|
numbers above then only apply to requests that actually ask for quota.
|
||||||
|
- The same narrow read protects every upload (`check_storage_quota` gates all
|
||||||
|
upload paths), where the FULL row was pure overhead per file.
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
# Round 2 — read path, upload path, archives (before/after gates)
|
||||||
|
|
||||||
|
Five backend changes + one frontend change, each gated by a before/after
|
||||||
|
benchmark (`examples/bench_round2.rs`; frontend gate in
|
||||||
|
`frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts`). Rule of the
|
||||||
|
round: an AFTER that doesn't beat its BEFORE gets rolled back — none did.
|
||||||
|
|
||||||
|
Reproduce:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
BENCH_SECTIONS=1,2,3,5 cargo run --release --features bench --example bench_round2
|
||||||
|
OXICLOUD_INGEST_OVERLAP=0 BENCH_SECTIONS=4 cargo run --release --features bench --example bench_round2
|
||||||
|
OXICLOUD_INGEST_OVERLAP=1 BENCH_SECTIONS=4 cargo run --release --features bench --example bench_round2
|
||||||
|
cd frontend && npx vitest run src/lib/api/endpoints/deltaUpload.hash.test.ts
|
||||||
|
```
|
||||||
|
|
||||||
|
## [1] Range requests served from the content cache — 2,156×
|
||||||
|
|
||||||
|
Media players and PDF viewers fetch files *exclusively* via Range requests
|
||||||
|
(a `bytes=0-` probe, then seeks). All three range paths (REST, DAV helper,
|
||||||
|
public shares) went straight to `get_file_range_stream`: a PG blob-hash
|
||||||
|
resolve + chunk open/seek/read per seek — even when the whole sub-10 MB blob
|
||||||
|
sat in the moka content cache as contiguous `Bytes`.
|
||||||
|
`FileRetrievalService::get_file_range_preloaded` now answers from the cache
|
||||||
|
(`Bytes::slice` = refcount bump; a miss populates it via the same
|
||||||
|
single-flight loader Tier 1 uses, so one probe warms every later seek).
|
||||||
|
|
||||||
|
| per 256 KiB seek (6 MiB file) | seeks/s | p50 µs | p99 µs |
|
||||||
|
|-------------------------------|--------:|-------:|-------:|
|
||||||
|
| BEFORE — PG + open/seek/read | 1,730 | 552.5 | 818.8 |
|
||||||
|
| AFTER — cache hit + slice | 3,730,560 | 0.15 | 2.85 |
|
||||||
|
|
||||||
|
## [2] NC chunked-upload gate: O(N²) directory scan → O(1) counter — 357×
|
||||||
|
|
||||||
|
`handle_put_chunk` recomputed "session bytes so far" on EVERY chunk PUT by
|
||||||
|
listing the session directory and stat-ing every existing chunk — chunk k
|
||||||
|
scans k files; a 1,000-chunk (10 GB) upload does ~500k stats.
|
||||||
|
`NextcloudChunkedUploadService` now keeps an in-RAM per-session counter
|
||||||
|
(seeded on MKCOL, bumped per accepted chunk, dropped on cleanup/overwrite,
|
||||||
|
lazily rebuilt from the listing on cold start — crash semantics unchanged).
|
||||||
|
|
||||||
|
Cumulative gate cost across a 1,000-chunk upload: **33,063 ms → 93 ms**.
|
||||||
|
|
||||||
|
## [3] Delta download / commit-verify read-ahead — 8.7× (latency-bound)
|
||||||
|
|
||||||
|
`delta_download_chunks` and `hash_chunk_sequence` drained chunks strictly
|
||||||
|
sequentially — every chunk-open's round-trip paid serially — while the main
|
||||||
|
CDC download path already overlaps opens with `buffered(read_prefetch)`.
|
||||||
|
Both now use the same combinator (order preserved — `buffered` yields in
|
||||||
|
input order).
|
||||||
|
|
||||||
|
64-chunk drain with 5 ms per-open latency (object-store model):
|
||||||
|
**440 ms → 51 ms**. On local disk the same combinator measured +7–12 %
|
||||||
|
(benches/BLOB-PREFETCH.md).
|
||||||
|
|
||||||
|
## [4] CDC ingest: settle overlapped with reading — +7–25 %
|
||||||
|
|
||||||
|
`ingest_chunks_from_stream` awaited each batch settle (PG pin round-trip +
|
||||||
|
up to 8 MiB of backend writes) INLINE — the HTTP source was not polled at
|
||||||
|
all during the settle, so read and settle phases alternated instead of
|
||||||
|
overlapping. The settle now runs on a spawned task (depth-1 pipeline) that
|
||||||
|
records into the guard's shared, lock-serialized state — rollback stays
|
||||||
|
exact even if the request future is dropped mid-settle.
|
||||||
|
`OXICLOUD_INGEST_OVERLAP=0` restores the inline behaviour (the bench's
|
||||||
|
BEFORE side, and an ops escape hatch).
|
||||||
|
|
||||||
|
512 MiB unique-content ingest, source paced at 300 MB/s, two reps:
|
||||||
|
**60 / 69 MB/s (inline) → 75 / 74 MB/s (overlapped)**.
|
||||||
|
|
||||||
|
## [5] Streaming ZIP: constant time-to-first-byte — 779× on this corpus
|
||||||
|
|
||||||
|
`create_folder_zip` built the ENTIRE archive into a temp file before the
|
||||||
|
handler sent byte one — TTFB grew with folder size (a multi-GB folder =
|
||||||
|
minutes of "waiting for server"). `create_folder_zip_stream` plans inline
|
||||||
|
(planning errors still surface as proper HTTP errors), then writes the
|
||||||
|
archive on a spawned task through `tokio::io::duplex`, streaming bytes as
|
||||||
|
they are produced. Folder downloads and public-share ZIPs both use it; a
|
||||||
|
mid-archive blob error truncates the stream (no central directory → clients
|
||||||
|
detect corruption) — the standard streamed-ZIP tradeoff. Content-Length is
|
||||||
|
no longer sent (size unknown up front).
|
||||||
|
|
||||||
|
48 × 4 MiB media corpus: TTFB **326.1 ms → 0.4 ms**; total wall also
|
||||||
|
improved (484 ms → 55 ms — no disk round-trip through the temp file).
|
||||||
|
TTFB in BEFORE scales linearly with archive size; AFTER is constant.
|
||||||
|
|
||||||
|
## [6] Frontend: instant-upload hashing on a worker pool
|
||||||
|
|
||||||
|
`resolveOwnedHashes` hashed every small file of a drop sequentially on the
|
||||||
|
MAIN THREAD (synchronous WASM BLAKE3 per file) before any upload lane
|
||||||
|
started — seconds of UI jank on large drops. Hashing now fans out over a
|
||||||
|
bounded pool of dedicated Web Workers (`static/workers/hashWorker.js`,
|
||||||
|
`File` handles passed by reference, reads happen inside the worker), with
|
||||||
|
the old inline loop kept as fallback where `Worker` is unavailable.
|
||||||
|
|
||||||
|
Architecture gate (node worker_threads, read+hash 24 × 4 MiB, file
|
||||||
|
references — faithful to the browser shape): 3-lane pool beats the
|
||||||
|
sequential loop; asserted by `deltaUpload.hash.test.ts` so a regression
|
||||||
|
fails CI. First model of this gate (posting BUFFERS instead of file
|
||||||
|
references) was 2.6× SLOWER — structured-clone copies dominated — and was
|
||||||
|
rewritten; kept here as a reminder that the gate must model the real
|
||||||
|
data-flow.
|
||||||
|
|
||||||
|
## Skipped this round
|
||||||
|
|
||||||
|
- **Swimlane (group-by) view virtualization** — needs interactive browser
|
||||||
|
measurement (frame times while scrolling) that this environment can't
|
||||||
|
produce; deferred rather than shipped unverified.
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# Static assets & API responses — precompressed siblings + explicit Brotli level
|
||||||
|
|
||||||
|
Two related findings, one root cause: tower-http's `CompressionLayer` default
|
||||||
|
maps to **Brotli QUALITY 11** (`async-compression Level::Default` →
|
||||||
|
`BrotliEncoderParams::default()`, brotli-8.0.2 `encode.rs:323` — verified in
|
||||||
|
source and empirically below). Quality 11 is a deploy-time setting; it was
|
||||||
|
running per request on:
|
||||||
|
|
||||||
|
- every SPA asset (`interfaces/web/mod.rs` layer): ~1.3 s CPU per 700 KiB
|
||||||
|
bundle per request;
|
||||||
|
- every compressible API response (`main.rs` global layer): ~90 ms CPU per
|
||||||
|
64 KiB JSON response.
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
|
||||||
|
1. **Precompressed statics.** `frontend/scripts/precompress.mjs` (build step,
|
||||||
|
node:zlib only) emits `.br`/`.gz` siblings for text assets; `ServeDir` now
|
||||||
|
uses `precompressed_br()/precompressed_gzip()` — a request costs a file
|
||||||
|
read, and clients get the *better* q11 bytes, paid once per deploy
|
||||||
|
(~1.4 s for the whole bundle).
|
||||||
|
2. **Explicit level 4** on both `CompressionLayer`s
|
||||||
|
(`CompressionLevel::Precise(4)`) — the on-the-fly fallback for statics
|
||||||
|
without siblings, and the global API layer.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_static_precompress
|
||||||
|
# tunables: BENCH_ASSET_KB=700 BENCH_REPS=30
|
||||||
|
```
|
||||||
|
|
||||||
|
## Results (4 cores, this container)
|
||||||
|
|
||||||
|
**Per-request cost, 700 KiB JS-like asset (94 % compressible):**
|
||||||
|
|
||||||
|
| mode | ms/request | speedup |
|
||||||
|
|-----------------------------|-----------:|--------:|
|
||||||
|
| BEFORE — on-the-fly Brotli | 1,324.31 | 1.0× |
|
||||||
|
| AFTER — precompressed read | 0.657 | **2016×** |
|
||||||
|
|
||||||
|
**Brotli level sweep, 64 KiB JSON-like API response:**
|
||||||
|
|
||||||
|
| level | ms/resp | out KiB |
|
||||||
|
|-------------------------|--------:|--------:|
|
||||||
|
| Default (= quality 11!) | 90.10 | 5.4 |
|
||||||
|
| **Precise(4)** (chosen) | 0.91 | 6.2 |
|
||||||
|
| Fastest | 0.15 | 9.3 |
|
||||||
|
|
||||||
|
- Statics: 3 orders of magnitude less CPU per request, while shipping
|
||||||
|
*smaller* bytes than the runtime default would at any reasonable level.
|
||||||
|
- API responses: **99× less CPU** for ~15 % more bytes (5.4 → 6.2 KiB) —
|
||||||
|
`Precise(4)` is the classic dynamic-content operating point; `Fastest`
|
||||||
|
gives up too much density (9.3 KiB).
|
||||||
|
- Historical note: an earlier review round REFUTED the "default is q11"
|
||||||
|
claim twice; the source line and the 90 ms/64 KiB measurement above settle
|
||||||
|
it the other way. Measure before believing — in both directions.
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# ZIP export — Stored for already-compressed media (vs Deflate-always)
|
||||||
|
|
||||||
|
Every ZIP export path (`ZipService::create_folder_zip` for folder downloads +
|
||||||
|
public share ZIPs, `BatchOperations::download_zip` for batch downloads) used to
|
||||||
|
build **every** file entry with `Compression::Deflate`. The dominant "download
|
||||||
|
folder" payload is photos/video (JPEG/HEIC/MP4/WebP), which deflate cannot
|
||||||
|
shrink (~0 %) while costing ~40 MB/s of CPU per core — and `async_zip` runs
|
||||||
|
deflate **inline on the writing tokio task** (inside `poll_write`), so a media
|
||||||
|
folder download monopolised ~1 core for its whole duration.
|
||||||
|
|
||||||
|
The change picks the entry compression from the file's MIME type at plan time:
|
||||||
|
`Stored` for already-compressed content, `Deflate` otherwise. The shared
|
||||||
|
predicate is `common::mime_detect::is_precompressed_mime` /
|
||||||
|
`zip_entry_compression` — it mirrors the HTTP `CompressionLayer` exclusion
|
||||||
|
list in `main.rs` (keep in sync), minus `x-tar`/`octet-stream` (containers of
|
||||||
|
possibly-compressible data stay on Deflate so nothing ever gets bigger).
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run --release --features bench --example bench_zip_media
|
||||||
|
# tunables: BENCH_MEDIA_FILES=48 BENCH_MEDIA_MB=4 BENCH_TEXT_FILES=24 BENCH_TEXT_MB=2 BENCH_REPS=3
|
||||||
|
```
|
||||||
|
|
||||||
|
Rebuilds the exact production writer stack (`ZipFileWriter::with_tokio(BufWriter(File))`,
|
||||||
|
`write_entry_stream`, 64 KiB chunks) over a mixed corpus: 192 MiB incompressible
|
||||||
|
"media" + 48 MiB compressible text (80/20 by bytes, a realistic media folder).
|
||||||
|
|
||||||
|
## Results (4 cores, this container)
|
||||||
|
|
||||||
|
| mode | wall s | cpu s | MB/s | out MiB | speedup |
|
||||||
|
|-----------------------|-------:|------:|-------:|--------:|--------:|
|
||||||
|
| all-Deflate (BEFORE) | 5.786 | 5.88 | 41.5 | 198.8 | 1.00× |
|
||||||
|
| mime-aware (AFTER) | 1.341 | 1.38 | 178.9 | 198.7 | **4.31×** |
|
||||||
|
| all-Stored (bound) | 0.150 | 0.19 | 1601.5 | 240.0 | 38.6× |
|
||||||
|
|
||||||
|
- **4.31× faster wall clock and 4.3× less CPU** on the mixed corpus, with the
|
||||||
|
archive **0.05 % smaller** (media never deflated anyway; text keeps Deflate).
|
||||||
|
- The remaining 1.38 s CPU in mime-aware is the text deflate + CRC32 — the
|
||||||
|
irreducible part. Pure-media folders approach the all-Stored bound (the
|
||||||
|
archive becomes blob-read-bound instead of CPU-bound).
|
||||||
|
- Side effect on the runtime: the writing task no longer occupies ~a full core
|
||||||
|
per media download — on a 4-core box that's ~25 % of total CPU handed back
|
||||||
|
to other requests for the duration of every archive.
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
//! NC chroot / default-drive resolution benchmark — 2 queries/request vs moka.
|
||||||
|
//!
|
||||||
|
//! The NextCloud basic-auth middleware wraps EVERY protected NC route and,
|
||||||
|
//! even with app-password verification fully cached, used to resolve the
|
||||||
|
//! chroot from scratch per request:
|
||||||
|
//!
|
||||||
|
//! 1. `find_default_for_user` — drives JOIN folders (drive_pg_repository)
|
||||||
|
//! 2. `get_folder(root_id)` — folders by PK
|
||||||
|
//!
|
||||||
|
//! The native `/webdav` surface repeats query 1 per request (Mode-B scope
|
||||||
|
//! resolution), WOPI repeats it per call. The change memoises (1) inside
|
||||||
|
//! `DrivePgRepository` and (2) in the middleware's `NC_CHROOT_CACHE`
|
||||||
|
//! (both 30 s TTL). This bench isolates exactly that: the per-request DB
|
||||||
|
//! cost of the chroot resolution — the two production query shapes vs a
|
||||||
|
//! moka hit — under sync-storm concurrency against the real pool.
|
||||||
|
//!
|
||||||
|
//! Run (needs Postgres up; reads DATABASE_URL from .env):
|
||||||
|
//! cargo run --release --features bench --example bench_chroot_cache
|
||||||
|
//! Tunables (env): BENCH_POOL (20), BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64").
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use sqlx::postgres::PgPoolOptions;
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Seeded {
|
||||||
|
user_id: Uuid,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed(pool: &PgPool) -> Seeded {
|
||||||
|
// user → (drive + root folder + root_folder_id stamp) in one tx —
|
||||||
|
// trg_no_orphan_root_folder is INITIALLY DEFERRED and checks at commit.
|
||||||
|
let mut tx = pool.begin().await.expect("begin");
|
||||||
|
let user_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO auth.users (username, email, role)
|
||||||
|
VALUES ('bench_chroot', 'bench_chroot@bench.invalid', 'user')
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("seed user");
|
||||||
|
let drive_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("seed drive");
|
||||||
|
let folder_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.folders (name, path, lpath, drive_id)
|
||||||
|
VALUES ('Personal', '/Personal', 'Personal', $1) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(drive_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("seed folder");
|
||||||
|
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("stamp root");
|
||||||
|
tx.commit().await.expect("commit");
|
||||||
|
Seeded { user_id }
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup(pool: &PgPool, user_id: Uuid) {
|
||||||
|
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
|
||||||
|
.bind(user_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The exact production BEFORE: both chroot queries, sequentially (the
|
||||||
|
/// middleware awaits the drive row to learn root_folder_id first).
|
||||||
|
async fn one_op_before(pool: &PgPool, user_id: Uuid, queries: &AtomicUsize) {
|
||||||
|
let row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
|
||||||
|
d.quota_bytes, d.used_bytes, d.policies,
|
||||||
|
d.created_at, d.updated_at,
|
||||||
|
f.name AS root_folder_name
|
||||||
|
FROM storage.drives d
|
||||||
|
JOIN storage.folders f ON f.id = d.root_folder_id
|
||||||
|
WHERE d.default_for_user = $1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("drive query");
|
||||||
|
let root_id: Uuid = row.get("root_folder_id");
|
||||||
|
|
||||||
|
let _folder = sqlx::query(
|
||||||
|
"SELECT id, name, parent_id, path, created_at, updated_at
|
||||||
|
FROM storage.folders WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(root_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("folder query");
|
||||||
|
queries.fetch_add(2, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
#[allow(dead_code)]
|
||||||
|
struct ChrootValue {
|
||||||
|
root_id: Uuid,
|
||||||
|
name: String,
|
||||||
|
path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Stats {
|
||||||
|
rps: f64,
|
||||||
|
p50: f64,
|
||||||
|
p95: f64,
|
||||||
|
p99: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize(mut lats: Vec<f64>, secs: u64) -> Stats {
|
||||||
|
lats.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
let n = lats.len();
|
||||||
|
let pct = |p: f64| {
|
||||||
|
if n == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
lats[((n as f64 * p) as usize).min(n - 1)]
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Stats {
|
||||||
|
rps: n as f64 / secs as f64,
|
||||||
|
p50: pct(0.50),
|
||||||
|
p95: pct(0.95),
|
||||||
|
p99: pct(0.99),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
dotenvy::dotenv().ok();
|
||||||
|
let url = env::var("DATABASE_URL")
|
||||||
|
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
|
||||||
|
.expect("set DATABASE_URL — the dev Postgres URL");
|
||||||
|
|
||||||
|
let pool_size: u32 = env_or("BENCH_POOL", 20);
|
||||||
|
let secs: u64 = env_or("BENCH_SECONDS", 4);
|
||||||
|
let concurrencies: Vec<usize> = env::var("BENCH_CONCURRENCIES")
|
||||||
|
.ok()
|
||||||
|
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
|
||||||
|
.unwrap_or_else(|| vec![8, 64]);
|
||||||
|
|
||||||
|
let pool = Arc::new(
|
||||||
|
PgPoolOptions::new()
|
||||||
|
.max_connections(pool_size)
|
||||||
|
.min_connections(pool_size)
|
||||||
|
.acquire_timeout(Duration::from_secs(10))
|
||||||
|
.connect(&url)
|
||||||
|
.await
|
||||||
|
.expect("connect Postgres"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let seeded = seed(&pool).await;
|
||||||
|
let user_id = seeded.user_id;
|
||||||
|
|
||||||
|
// AFTER: what the middleware pays on a warm cache — a moka lookup.
|
||||||
|
let cache: moka::sync::Cache<Uuid, ChrootValue> = moka::sync::Cache::builder()
|
||||||
|
.max_capacity(100_000)
|
||||||
|
.time_to_live(Duration::from_secs(30))
|
||||||
|
.build();
|
||||||
|
cache.insert(
|
||||||
|
user_id,
|
||||||
|
ChrootValue {
|
||||||
|
root_id: Uuid::new_v4(),
|
||||||
|
name: "Personal".into(),
|
||||||
|
path: "/Personal".into(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
println!("\n#############################################################");
|
||||||
|
println!("# NC chroot resolution: BEFORE (2 queries/req) vs AFTER (moka)");
|
||||||
|
println!("# pool={pool_size} window={secs}s/run");
|
||||||
|
println!("#############################################################\n");
|
||||||
|
println!(
|
||||||
|
"| {:>5} | {:<6} | {:>10} | {:>9} | {:>9} | {:>9} | {:>9} |",
|
||||||
|
"conc", "mode", "req/s", "p50 µs", "p95 µs", "p99 µs", "queries"
|
||||||
|
);
|
||||||
|
|
||||||
|
for &conc in &concurrencies {
|
||||||
|
for mode in ["BEFORE", "AFTER"] {
|
||||||
|
let queries = Arc::new(AtomicUsize::new(0));
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(secs);
|
||||||
|
let mut handles = Vec::new();
|
||||||
|
for _ in 0..conc {
|
||||||
|
let pool = pool.clone();
|
||||||
|
let cache = cache.clone();
|
||||||
|
let queries = queries.clone();
|
||||||
|
let mode = mode.to_string();
|
||||||
|
handles.push(tokio::spawn(async move {
|
||||||
|
let mut lats = Vec::new();
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
let t = Instant::now();
|
||||||
|
if mode == "BEFORE" {
|
||||||
|
one_op_before(&pool, user_id, &queries).await;
|
||||||
|
} else {
|
||||||
|
let v = cache.get(&user_id).expect("warm cache");
|
||||||
|
std::hint::black_box(v);
|
||||||
|
}
|
||||||
|
lats.push(t.elapsed().as_secs_f64() * 1_000_000.0);
|
||||||
|
if mode == "AFTER" {
|
||||||
|
// moka hit is ~100 ns; yield so the loop doesn't
|
||||||
|
// monopolise workers and skew the run count.
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lats
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let mut all = Vec::new();
|
||||||
|
for h in handles {
|
||||||
|
all.extend(h.await.unwrap());
|
||||||
|
}
|
||||||
|
let s = summarize(all, secs);
|
||||||
|
println!(
|
||||||
|
"| {:>5} | {:<6} | {:>10.0} | {:>9.2} | {:>9.2} | {:>9.2} | {:>9} |",
|
||||||
|
conc,
|
||||||
|
mode,
|
||||||
|
s.rps,
|
||||||
|
s.p50,
|
||||||
|
s.p95,
|
||||||
|
s.p99,
|
||||||
|
queries.load(Ordering::Relaxed)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup(&pool, user_id).await;
|
||||||
|
println!("\n(BEFORE = the two production chroot queries; AFTER = warm moka hit.");
|
||||||
|
println!(" Every NC request pays this before its handler runs.)");
|
||||||
|
}
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
//! WebDAV dead-properties fetch benchmark — per-child N+1 vs batched ANY($1).
|
||||||
|
//!
|
||||||
|
//! The streaming PROPFIND walker (`webdav_handler.rs`) fetches dead properties
|
||||||
|
//! ONE CHILD AT A TIME, sequentially, for every Depth:1 listing page:
|
||||||
|
//!
|
||||||
|
//! for file in &batch { file_deads.push(store.get_all(File(id)).await) }
|
||||||
|
//!
|
||||||
|
//! and `DeadPropertyStore::get_all` filters with
|
||||||
|
//! `folder_id IS NOT DISTINCT FROM $1 AND file_id IS NOT DISTINCT FROM $2`,
|
||||||
|
//! which PostgreSQL cannot serve from a B-tree index (IS NOT DISTINCT FROM is
|
||||||
|
//! not an indexable operator) — so each of the N sequential round-trips also
|
||||||
|
//! degrades to a seq scan as the table grows.
|
||||||
|
//!
|
||||||
|
//! This bench isolates exactly the dead-prop portion of a Depth:1 PROPFIND of
|
||||||
|
//! a folder with N children, comparing the three query shapes:
|
||||||
|
//!
|
||||||
|
//! OLD — N sequential `IS NOT DISTINCT FROM` queries (production today)
|
||||||
|
//! EQ — N sequential plain `file_id = $1` queries (indexable, still N+1)
|
||||||
|
//! BATCH — ⌈N/500⌉ `file_id = ANY($1)` queries (one per PROPFIND page)
|
||||||
|
//!
|
||||||
|
//! Two table sizes are measured: the seeded-children-only table and one with
|
||||||
|
//! extra noise rows (dead props on other resources), which is where the
|
||||||
|
//! seq-scan cost of OLD shows up.
|
||||||
|
//!
|
||||||
|
//! Run (needs Postgres up; reads DATABASE_URL from .env):
|
||||||
|
//! cargo run --release --features bench --example bench_dead_props
|
||||||
|
//! Tunables (env): BENCH_CHILDREN (2000), BENCH_PAGE (500 = PROPFIND_BATCH_SIZE),
|
||||||
|
//! BENCH_NOISE_ROWS (20000), BENCH_REPS (5).
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use sqlx::postgres::PgPoolOptions;
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Seeded {
|
||||||
|
drive_id: Uuid,
|
||||||
|
file_ids: Vec<Uuid>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed(pool: &PgPool, children: usize, noise: usize) -> Seeded {
|
||||||
|
// Drive (kind 'shared' needs no user FK) → root folder → N files → props.
|
||||||
|
// The root folder + drive.root_folder_id must land in ONE transaction:
|
||||||
|
// trg_no_orphan_root_folder is INITIALLY DEFERRED and checks at commit.
|
||||||
|
let mut tx = pool.begin().await.expect("begin seed tx");
|
||||||
|
let drive_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
|
||||||
|
)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("seed drive");
|
||||||
|
|
||||||
|
let folder_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.folders (name, path, lpath, drive_id)
|
||||||
|
VALUES ('bench_dead_props', '/bench_dead_props', 'bench_dead_props', $1)
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(drive_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("seed folder");
|
||||||
|
|
||||||
|
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("stamp root_folder_id");
|
||||||
|
tx.commit().await.expect("commit seed tx");
|
||||||
|
|
||||||
|
// Children of the PROPFIND'd folder, one dead prop each.
|
||||||
|
let file_ids: Vec<Uuid> = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
|
||||||
|
SELECT 'f' || i, $1, 'benchdead000000000000000000000000000000000000000000000000000000',
|
||||||
|
1024, 'image/jpeg', $2
|
||||||
|
FROM generate_series(1, $3) AS i
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.bind(children as i32)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("seed files");
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value)
|
||||||
|
SELECT id, 'urn:bench', 'displayname', 'bench value'
|
||||||
|
FROM storage.files WHERE folder_id = $1",
|
||||||
|
)
|
||||||
|
.bind(folder_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("seed dead props");
|
||||||
|
|
||||||
|
// Noise: dead props attached to OTHER files (a second folder) so the
|
||||||
|
// table has realistic volume — this is what OLD's seq scans pay for.
|
||||||
|
if noise > 0 {
|
||||||
|
// Child of the main folder — root folders need the deferred
|
||||||
|
// four-write dance, children don't.
|
||||||
|
let noise_folder: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.folders (name, parent_id, path, lpath, drive_id)
|
||||||
|
VALUES ('noise', $2, '/bench_dead_props/noise', 'bench_dead_props.noise', $1)
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(drive_id)
|
||||||
|
.bind(folder_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("seed noise folder");
|
||||||
|
sqlx::query(
|
||||||
|
"WITH f AS (
|
||||||
|
INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
|
||||||
|
SELECT 'n' || i, $1, 'benchdead000000000000000000000000000000000000000000000000000000',
|
||||||
|
1024, 'image/jpeg', $2
|
||||||
|
FROM generate_series(1, $3) AS i
|
||||||
|
RETURNING id
|
||||||
|
)
|
||||||
|
INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value)
|
||||||
|
SELECT id, 'urn:bench', 'noise', 'x' FROM f",
|
||||||
|
)
|
||||||
|
.bind(noise_folder)
|
||||||
|
.bind(drive_id)
|
||||||
|
.bind(noise as i32)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("seed noise props");
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlx::query("ANALYZE storage.webdav_dead_properties")
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
|
||||||
|
Seeded { drive_id, file_ids }
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup(pool: &PgPool, drive_id: Uuid) {
|
||||||
|
// drives → folders/files → dead props all cascade.
|
||||||
|
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// OLD: production `get_all` shape — sequential, IS NOT DISTINCT FROM.
|
||||||
|
async fn run_old(pool: &PgPool, ids: &[Uuid]) -> usize {
|
||||||
|
let mut rows_seen = 0;
|
||||||
|
for id in ids {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT namespace, local_name, value
|
||||||
|
FROM storage.webdav_dead_properties
|
||||||
|
WHERE folder_id IS NOT DISTINCT FROM $1
|
||||||
|
AND file_id IS NOT DISTINCT FROM $2",
|
||||||
|
)
|
||||||
|
.bind(Option::<Uuid>::None)
|
||||||
|
.bind(Some(*id))
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("old get_all");
|
||||||
|
rows_seen += rows.len();
|
||||||
|
}
|
||||||
|
rows_seen
|
||||||
|
}
|
||||||
|
|
||||||
|
/// EQ: still N sequential round-trips, but with an indexable `=` predicate.
|
||||||
|
async fn run_eq(pool: &PgPool, ids: &[Uuid]) -> usize {
|
||||||
|
let mut rows_seen = 0;
|
||||||
|
for id in ids {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT namespace, local_name, value
|
||||||
|
FROM storage.webdav_dead_properties
|
||||||
|
WHERE file_id = $1",
|
||||||
|
)
|
||||||
|
.bind(*id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("eq get_all");
|
||||||
|
rows_seen += rows.len();
|
||||||
|
}
|
||||||
|
rows_seen
|
||||||
|
}
|
||||||
|
|
||||||
|
/// BATCH: one `= ANY($1)` query per PROPFIND page of 500 children.
|
||||||
|
async fn run_batch(pool: &PgPool, ids: &[Uuid], page: usize) -> usize {
|
||||||
|
let mut rows_seen = 0;
|
||||||
|
for chunk in ids.chunks(page) {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT file_id, namespace, local_name, value
|
||||||
|
FROM storage.webdav_dead_properties
|
||||||
|
WHERE file_id = ANY($1)",
|
||||||
|
)
|
||||||
|
.bind(chunk)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("batch get_all");
|
||||||
|
// Decode file_id like the real batched store method will (map key).
|
||||||
|
for row in &rows {
|
||||||
|
let _: Uuid = row.get("file_id");
|
||||||
|
}
|
||||||
|
rows_seen += rows.len();
|
||||||
|
}
|
||||||
|
rows_seen
|
||||||
|
}
|
||||||
|
|
||||||
|
fn median(mut xs: Vec<f64>) -> f64 {
|
||||||
|
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
xs[xs.len() / 2]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
dotenvy::dotenv().ok();
|
||||||
|
let url = env::var("DATABASE_URL")
|
||||||
|
.or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING"))
|
||||||
|
.expect("set DATABASE_URL — the dev Postgres URL");
|
||||||
|
|
||||||
|
let children: usize = env_or("BENCH_CHILDREN", 2000);
|
||||||
|
let page: usize = env_or("BENCH_PAGE", 500);
|
||||||
|
let noise: usize = env_or("BENCH_NOISE_ROWS", 20_000);
|
||||||
|
let reps: usize = env_or("BENCH_REPS", 5);
|
||||||
|
|
||||||
|
let pool = PgPoolOptions::new()
|
||||||
|
.max_connections(5)
|
||||||
|
.min_connections(5)
|
||||||
|
.connect(&url)
|
||||||
|
.await
|
||||||
|
.expect("connect Postgres");
|
||||||
|
|
||||||
|
for &with_noise in &[false, true] {
|
||||||
|
let n = if with_noise { noise } else { 0 };
|
||||||
|
let seeded = seed(&pool, children, n).await;
|
||||||
|
let total_rows: i64 =
|
||||||
|
sqlx::query_scalar("SELECT count(*) FROM storage.webdav_dead_properties")
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
println!("\n== folder with {children} children, dead-props table = {total_rows} rows ==");
|
||||||
|
println!(
|
||||||
|
"{:<28} {:>10} {:>12} {:>9}",
|
||||||
|
"mode", "queries", "total ms", "vs OLD"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut base = None;
|
||||||
|
for (label, queries) in [
|
||||||
|
("OLD seq, IS NOT DISTINCT", children),
|
||||||
|
("EQ seq, file_id = $1", children),
|
||||||
|
("BATCH file_id = ANY, /page", children.div_ceil(page)),
|
||||||
|
] {
|
||||||
|
let mut times = Vec::with_capacity(reps);
|
||||||
|
let mut rows = 0;
|
||||||
|
for _ in 0..reps {
|
||||||
|
let t = Instant::now();
|
||||||
|
rows = match label.split_whitespace().next().unwrap() {
|
||||||
|
"OLD" => run_old(&pool, &seeded.file_ids).await,
|
||||||
|
"EQ" => run_eq(&pool, &seeded.file_ids).await,
|
||||||
|
_ => run_batch(&pool, &seeded.file_ids, page).await,
|
||||||
|
};
|
||||||
|
times.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
}
|
||||||
|
assert_eq!(rows, children, "each child has exactly 1 dead prop");
|
||||||
|
let ms = median(times);
|
||||||
|
let speedup = base
|
||||||
|
.map(|b: f64| format!("{:.1}x", b / ms))
|
||||||
|
.unwrap_or_else(|| "1.0x".into());
|
||||||
|
if base.is_none() {
|
||||||
|
base = Some(ms);
|
||||||
|
}
|
||||||
|
println!("{label:<28} {queries:>10} {ms:>12.2} {speedup:>9}");
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup(&pool, seeded.drive_id).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("\n(total ms = the dead-prop portion of one Depth:1 PROPFIND of the folder,");
|
||||||
|
println!(" i.e. what the walker adds on top of the file/folder listing queries)");
|
||||||
|
}
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
//! People-tab benchmark — full faces scan (embeddings included) vs grouped COUNT.
|
||||||
|
//!
|
||||||
|
//! `PeopleService::list_people` used to call `faces_for_user`, dragging every
|
||||||
|
//! face row — each with a 2,048-byte embedding BYTEA — across the wire and
|
||||||
|
//! decoding it into a fresh `Vec<f32>`, only to (a) count faces per person and
|
||||||
|
//! (b) resolve ~a-handful of cover faces to file ids. The change replaces it
|
||||||
|
//! with `person_face_stats` (grouped COUNT) + `file_ids_for_faces` (one
|
||||||
|
//! `= ANY` over just the cover ids).
|
||||||
|
//!
|
||||||
|
//! Run (needs Postgres up; reads DATABASE_URL from .env):
|
||||||
|
//! cargo run --release --features bench --example bench_people_list
|
||||||
|
//! Tunables: BENCH_FACES (10000), BENCH_PERSONS (20), BENCH_REPS (5)
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use sqlx::postgres::PgPoolOptions;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Seeded {
|
||||||
|
user_id: Uuid,
|
||||||
|
drive_id: Uuid,
|
||||||
|
cover_ids: Vec<Uuid>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed(pool: &PgPool, faces: usize, persons: usize) -> Seeded {
|
||||||
|
let mut tx = pool.begin().await.expect("begin");
|
||||||
|
let user_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO auth.users (username, email, role)
|
||||||
|
VALUES ('bench_people', 'bench_people@bench.invalid', 'user') RETURNING id",
|
||||||
|
)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("user");
|
||||||
|
let drive_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("drive");
|
||||||
|
let folder_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.folders (name, path, lpath, drive_id)
|
||||||
|
VALUES ('bench_people', '/bench_people', 'bench_people', $1) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(drive_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("folder");
|
||||||
|
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("stamp root");
|
||||||
|
tx.commit().await.expect("commit");
|
||||||
|
|
||||||
|
// Photo files the faces point at.
|
||||||
|
let file_ids: Vec<Uuid> = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
|
||||||
|
SELECT 'p' || i, $1, 'benchpeople0000000000000000000000000000000000000000000000000000',
|
||||||
|
1024, 'image/jpeg', $2
|
||||||
|
FROM generate_series(1, $3) AS i
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.bind(faces as i32)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("files");
|
||||||
|
|
||||||
|
// Persons + faces (2 KiB embedding each, like the real 512×f32).
|
||||||
|
let mut person_ids = Vec::with_capacity(persons);
|
||||||
|
for i in 0..persons {
|
||||||
|
let pid: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO faces.persons (user_id, display_name) VALUES ($1, $2) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(format!("Person {i}"))
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("person");
|
||||||
|
person_ids.push(pid);
|
||||||
|
}
|
||||||
|
|
||||||
|
let embedding = vec![0u8; 2048];
|
||||||
|
let mut cover_ids = Vec::with_capacity(persons);
|
||||||
|
for (i, file_id) in file_ids.iter().enumerate() {
|
||||||
|
let pid = person_ids[i % persons];
|
||||||
|
let face_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO faces.faces
|
||||||
|
(file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash)
|
||||||
|
VALUES ($1, $2, $3, ARRAY[0.1,0.1,0.2,0.2]::real[], 0.99, 0.9, $4,
|
||||||
|
'benchpeople0000000000000000000000000000000000000000000000000000')
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(file_id)
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(pid)
|
||||||
|
.bind(&embedding)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("face");
|
||||||
|
if i < persons {
|
||||||
|
cover_ids.push(face_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sqlx::query("ANALYZE faces.faces").execute(pool).await.ok();
|
||||||
|
|
||||||
|
Seeded {
|
||||||
|
user_id,
|
||||||
|
drive_id,
|
||||||
|
cover_ids,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup(pool: &PgPool, s: &Seeded) {
|
||||||
|
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
|
||||||
|
.bind(s.drive_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
|
||||||
|
.bind(s.user_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn median(mut xs: Vec<f64>) -> f64 {
|
||||||
|
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
xs[xs.len() / 2]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
dotenvy::dotenv().ok();
|
||||||
|
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
|
||||||
|
let faces: usize = env_or("BENCH_FACES", 10_000);
|
||||||
|
let persons: usize = env_or("BENCH_PERSONS", 20);
|
||||||
|
let reps: usize = env_or("BENCH_REPS", 5);
|
||||||
|
|
||||||
|
let pool = PgPoolOptions::new()
|
||||||
|
.max_connections(5)
|
||||||
|
.connect(&url)
|
||||||
|
.await
|
||||||
|
.expect("connect");
|
||||||
|
println!("seeding {faces} faces / {persons} persons (one-time)…");
|
||||||
|
let seeded = seed(&pool, faces, persons).await;
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"\n# GET /api/people data fetch: BEFORE (full face rows) vs AFTER (COUNT + cover ANY)"
|
||||||
|
);
|
||||||
|
println!("{:<28} {:>12} {:>14}", "mode", "total ms", "bytes moved");
|
||||||
|
|
||||||
|
let mut base = None;
|
||||||
|
for mode in ["BEFORE full-rows", "AFTER count+covers"] {
|
||||||
|
let mut times = Vec::with_capacity(reps);
|
||||||
|
let mut bytes = 0usize;
|
||||||
|
for _ in 0..reps {
|
||||||
|
let t = Instant::now();
|
||||||
|
if mode.starts_with("BEFORE") {
|
||||||
|
// faces_for_user shape: every column incl. embedding.
|
||||||
|
let rows: Vec<(Uuid, Uuid, Option<Uuid>, Vec<u8>)> = sqlx::query_as(
|
||||||
|
"SELECT id, file_id, person_id, embedding FROM faces.faces WHERE user_id = $1",
|
||||||
|
)
|
||||||
|
.bind(seeded.user_id)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.expect("full rows");
|
||||||
|
bytes = rows.iter().map(|r| r.3.len() + 48).sum();
|
||||||
|
assert_eq!(rows.len(), faces);
|
||||||
|
} else {
|
||||||
|
let stats: Vec<(Uuid, i64)> = sqlx::query_as(
|
||||||
|
"SELECT person_id, COUNT(*) FROM faces.faces
|
||||||
|
WHERE user_id = $1 AND person_id IS NOT NULL GROUP BY person_id",
|
||||||
|
)
|
||||||
|
.bind(seeded.user_id)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.expect("stats");
|
||||||
|
let covers: Vec<(Uuid, Uuid)> = sqlx::query_as(
|
||||||
|
"SELECT id, file_id FROM faces.faces WHERE user_id = $1 AND id = ANY($2)",
|
||||||
|
)
|
||||||
|
.bind(seeded.user_id)
|
||||||
|
.bind(&seeded.cover_ids)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.expect("covers");
|
||||||
|
bytes = (stats.len() + covers.len()) * 32;
|
||||||
|
assert_eq!(stats.len(), persons);
|
||||||
|
}
|
||||||
|
times.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
}
|
||||||
|
let ms = median(times);
|
||||||
|
let speedup = base
|
||||||
|
.map(|b: f64| format!("({:.1}x)", b / ms))
|
||||||
|
.unwrap_or_default();
|
||||||
|
println!("{mode:<28} {ms:>12.2} {bytes:>14} {speedup}");
|
||||||
|
if base.is_none() {
|
||||||
|
base = Some(ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup(&pool, &seeded).await;
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
//! PROPFIND folder-listing pagination benchmark — LIMIT/OFFSET vs keyset.
|
||||||
|
//!
|
||||||
|
//! The streaming PROPFIND walker pages a folder's children 500 at a time in
|
||||||
|
//! name order (`list_files_batch`). The old shape was `ORDER BY name LIMIT
|
||||||
|
//! 500 OFFSET k` with no supporting index — every page bitmap-scanned all N
|
||||||
|
//! children and top-sorted them, so a full folder walk was O(N²/500) row
|
||||||
|
//! visits. The change adds `idx_files_folder_name (folder_id, name) WHERE
|
||||||
|
//! NOT is_trashed` and switches the cursor to keyset (`name > $last`), making
|
||||||
|
//! each page one O(page) index-range read.
|
||||||
|
//!
|
||||||
|
//! Modes (full walk of the folder, all pages):
|
||||||
|
//! OFFSET/no-idx — the true BEFORE (index dropped for the run)
|
||||||
|
//! OFFSET/idx — index alone, old query shape
|
||||||
|
//! KEYSET/idx — the AFTER
|
||||||
|
//!
|
||||||
|
//! Run (needs Postgres up; reads DATABASE_URL from .env):
|
||||||
|
//! cargo run --release --features bench --example bench_propfind_paging
|
||||||
|
//! Tunables: BENCH_FILES (20000), BENCH_PAGE (500), BENCH_REPS (3)
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use sqlx::postgres::PgPoolOptions;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed(pool: &PgPool, files: usize) -> (Uuid, Uuid) {
|
||||||
|
let mut tx = pool.begin().await.expect("begin");
|
||||||
|
let drive_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
|
||||||
|
)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("drive");
|
||||||
|
let folder_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.folders (name, path, lpath, drive_id)
|
||||||
|
VALUES ('bench_paging', '/bench_paging', 'bench_paging', $1) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(drive_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("folder");
|
||||||
|
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await
|
||||||
|
.expect("stamp");
|
||||||
|
tx.commit().await.expect("commit");
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
|
||||||
|
SELECT 'file_' || LPAD(i::text, 8, '0') || '.jpg', $1,
|
||||||
|
'benchpaging00000000000000000000000000000000000000000000000000000',
|
||||||
|
1024, 'image/jpeg', $2
|
||||||
|
FROM generate_series(1, $3) AS i",
|
||||||
|
)
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.bind(files as i32)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("files");
|
||||||
|
sqlx::query("ANALYZE storage.files")
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
(drive_id, folder_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
const COLS: &str = "fi.id::text, fi.name, fi.folder_id::text, fo.path, fi.size, fi.mime_type,
|
||||||
|
EXTRACT(EPOCH FROM fi.created_at)::bigint,
|
||||||
|
EXTRACT(EPOCH FROM fi.updated_at)::bigint, fi.blob_hash";
|
||||||
|
|
||||||
|
type Row = (
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
i64,
|
||||||
|
String,
|
||||||
|
i64,
|
||||||
|
i64,
|
||||||
|
String,
|
||||||
|
);
|
||||||
|
|
||||||
|
/// Full folder walk with the old LIMIT/OFFSET shape. Returns rows seen.
|
||||||
|
async fn walk_offset(pool: &PgPool, folder: Uuid, page: i64) -> usize {
|
||||||
|
let mut offset = 0i64;
|
||||||
|
let mut seen = 0usize;
|
||||||
|
loop {
|
||||||
|
let rows: Vec<Row> = sqlx::query_as(&format!(
|
||||||
|
"SELECT {COLS}
|
||||||
|
FROM storage.files fi
|
||||||
|
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
|
||||||
|
WHERE fi.folder_id = $1 AND NOT fi.is_trashed
|
||||||
|
ORDER BY fi.name LIMIT $2 OFFSET $3"
|
||||||
|
))
|
||||||
|
.bind(folder)
|
||||||
|
.bind(page)
|
||||||
|
.bind(offset)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("offset page");
|
||||||
|
let n = rows.len();
|
||||||
|
seen += n;
|
||||||
|
if (n as i64) < page {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
offset += n as i64;
|
||||||
|
}
|
||||||
|
seen
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Full folder walk with the new keyset shape.
|
||||||
|
async fn walk_keyset(pool: &PgPool, folder: Uuid, page: i64) -> usize {
|
||||||
|
let mut after: Option<String> = None;
|
||||||
|
let mut seen = 0usize;
|
||||||
|
loop {
|
||||||
|
let rows: Vec<Row> = if let Some(a) = &after {
|
||||||
|
sqlx::query_as(&format!(
|
||||||
|
"SELECT {COLS}
|
||||||
|
FROM storage.files fi
|
||||||
|
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
|
||||||
|
WHERE fi.folder_id = $1 AND NOT fi.is_trashed AND fi.name > $3
|
||||||
|
ORDER BY fi.name LIMIT $2"
|
||||||
|
))
|
||||||
|
.bind(folder)
|
||||||
|
.bind(page)
|
||||||
|
.bind(a)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
sqlx::query_as(&format!(
|
||||||
|
"SELECT {COLS}
|
||||||
|
FROM storage.files fi
|
||||||
|
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
|
||||||
|
WHERE fi.folder_id = $1 AND NOT fi.is_trashed
|
||||||
|
ORDER BY fi.name LIMIT $2"
|
||||||
|
))
|
||||||
|
.bind(folder)
|
||||||
|
.bind(page)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
.expect("keyset page");
|
||||||
|
let n = rows.len();
|
||||||
|
seen += n;
|
||||||
|
if (n as i64) < page {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
after = rows.last().map(|r| r.1.clone());
|
||||||
|
}
|
||||||
|
seen
|
||||||
|
}
|
||||||
|
|
||||||
|
fn median(mut xs: Vec<f64>) -> f64 {
|
||||||
|
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
xs[xs.len() / 2]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
dotenvy::dotenv().ok();
|
||||||
|
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
|
||||||
|
let files: usize = env_or("BENCH_FILES", 20_000);
|
||||||
|
let page: i64 = env_or("BENCH_PAGE", 500);
|
||||||
|
let reps: usize = env_or("BENCH_REPS", 3);
|
||||||
|
|
||||||
|
let pool = PgPoolOptions::new()
|
||||||
|
.max_connections(5)
|
||||||
|
.connect(&url)
|
||||||
|
.await
|
||||||
|
.expect("connect");
|
||||||
|
println!("seeding {files} files (one-time)…");
|
||||||
|
let (drive_id, folder_id) = seed(&pool, files).await;
|
||||||
|
|
||||||
|
println!("\n# full PROPFIND walk of a {files}-file folder, {page}/page");
|
||||||
|
println!("{:<18} {:>12} {:>9}", "mode", "total ms", "vs OLD");
|
||||||
|
|
||||||
|
let mut base = None;
|
||||||
|
for mode in ["OFFSET/no-idx", "OFFSET/idx", "KEYSET/idx"] {
|
||||||
|
match mode {
|
||||||
|
"OFFSET/no-idx" => {
|
||||||
|
sqlx::query("DROP INDEX IF EXISTS storage.idx_files_folder_name")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
}
|
||||||
|
"OFFSET/idx" => {
|
||||||
|
sqlx::query(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_files_folder_name
|
||||||
|
ON storage.files (folder_id, name) WHERE NOT is_trashed",
|
||||||
|
)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.expect("create index");
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
let mut times = Vec::with_capacity(reps);
|
||||||
|
for _ in 0..reps {
|
||||||
|
let t = Instant::now();
|
||||||
|
let seen = if mode.starts_with("OFFSET") {
|
||||||
|
walk_offset(&pool, folder_id, page).await
|
||||||
|
} else {
|
||||||
|
walk_keyset(&pool, folder_id, page).await
|
||||||
|
};
|
||||||
|
assert_eq!(seen, files);
|
||||||
|
times.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
}
|
||||||
|
let ms = median(times);
|
||||||
|
let speedup = base
|
||||||
|
.map(|b: f64| format!("{:.1}x", b / ms))
|
||||||
|
.unwrap_or_else(|| "1.0x".into());
|
||||||
|
if base.is_none() {
|
||||||
|
base = Some(ms);
|
||||||
|
}
|
||||||
|
println!("{mode:<18} {ms:>12.1} {speedup:>9}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&pool)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
//! Quota-path benchmark — full `auth.users` row vs narrow 2-column read.
|
||||||
|
//!
|
||||||
|
//! `check_storage_quota` (every upload) and `get_user_storage_info` (every
|
||||||
|
//! quota-reporting PROPFIND) used to call `get_user_by_id`, whose SELECT
|
||||||
|
//! drags the whole user row — including `image`, an avatar data URI of up
|
||||||
|
//! to 512 KiB — across the wire to read two i64s. The change reads only
|
||||||
|
//! `(storage_used_bytes, storage_quota_bytes)`
|
||||||
|
//! (`UserPgRepository::get_storage_usage`). Companion change measured here
|
||||||
|
//! as "SKIP": PROPFINDs whose prop list never names a quota prop now skip
|
||||||
|
//! the resolution entirely (`PropFindRequest::wants_quota`).
|
||||||
|
//!
|
||||||
|
//! Run (needs Postgres up; reads DATABASE_URL from .env):
|
||||||
|
//! cargo run --release --features bench --example bench_quota_path
|
||||||
|
//! Tunables: BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64"), BENCH_IMAGE_KB (512)
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use sqlx::postgres::PgPoolOptions;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed(pool: &PgPool, image_kb: usize) -> Uuid {
|
||||||
|
// Realistic worst-ish case: an avatar data URI at the documented cap.
|
||||||
|
let image = format!("data:image/png;base64,{}", "A".repeat(image_kb * 1024 - 22));
|
||||||
|
sqlx::query_scalar(
|
||||||
|
"INSERT INTO auth.users (username, email, role, image)
|
||||||
|
VALUES ('bench_quota', 'bench_quota@bench.invalid', 'user', $1)
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(&image)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("seed user")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup(pool: &PgPool, user_id: Uuid) {
|
||||||
|
let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1")
|
||||||
|
.bind(user_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// BEFORE: the full-row SELECT `get_user_by_id` runs (same column list).
|
||||||
|
async fn one_op_full(pool: &PgPool, id: Uuid) {
|
||||||
|
let _row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
id, username, email, password_hash, role::text as role_text,
|
||||||
|
storage_quota_bytes, storage_used_bytes,
|
||||||
|
created_at, updated_at, last_login_at, active,
|
||||||
|
oidc_provider, oidc_subject, image, is_external,
|
||||||
|
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
|
||||||
|
ui_preferences
|
||||||
|
FROM auth.users
|
||||||
|
WHERE id = $1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("full row");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// AFTER: the narrow `get_storage_usage` SELECT.
|
||||||
|
async fn one_op_narrow(pool: &PgPool, id: Uuid) {
|
||||||
|
let _row: (i64, i64) = sqlx::query_as(
|
||||||
|
"SELECT storage_used_bytes, storage_quota_bytes FROM auth.users WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("narrow row");
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Stats {
|
||||||
|
rps: f64,
|
||||||
|
p50: f64,
|
||||||
|
p99: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize(mut lats: Vec<f64>, secs: u64) -> Stats {
|
||||||
|
lats.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
let n = lats.len();
|
||||||
|
let pct = |p: f64| {
|
||||||
|
if n == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
lats[((n as f64 * p) as usize).min(n - 1)]
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Stats {
|
||||||
|
rps: n as f64 / secs as f64,
|
||||||
|
p50: pct(0.50),
|
||||||
|
p99: pct(0.99),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
dotenvy::dotenv().ok();
|
||||||
|
let url = env::var("DATABASE_URL").expect("set DATABASE_URL");
|
||||||
|
let secs: u64 = env_or("BENCH_SECONDS", 4);
|
||||||
|
let image_kb: usize = env_or("BENCH_IMAGE_KB", 512);
|
||||||
|
let concurrencies: Vec<usize> = env::var("BENCH_CONCURRENCIES")
|
||||||
|
.ok()
|
||||||
|
.map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect())
|
||||||
|
.unwrap_or_else(|| vec![8, 64]);
|
||||||
|
|
||||||
|
let pool = Arc::new(
|
||||||
|
PgPoolOptions::new()
|
||||||
|
.max_connections(20)
|
||||||
|
.min_connections(20)
|
||||||
|
.acquire_timeout(Duration::from_secs(10))
|
||||||
|
.connect(&url)
|
||||||
|
.await
|
||||||
|
.expect("connect"),
|
||||||
|
);
|
||||||
|
let user_id = seed(&pool, image_kb).await;
|
||||||
|
|
||||||
|
println!("\n# quota lookup: full user row (incl. {image_kb} KiB avatar) vs 2-column read");
|
||||||
|
println!(
|
||||||
|
"| {:>5} | {:<7} | {:>10} | {:>9} | {:>9} |",
|
||||||
|
"conc", "mode", "ops/s", "p50 µs", "p99 µs"
|
||||||
|
);
|
||||||
|
for &conc in &concurrencies {
|
||||||
|
for mode in ["FULL", "NARROW"] {
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(secs);
|
||||||
|
let mut handles = Vec::new();
|
||||||
|
for _ in 0..conc {
|
||||||
|
let pool = pool.clone();
|
||||||
|
let mode = mode.to_string();
|
||||||
|
handles.push(tokio::spawn(async move {
|
||||||
|
let mut lats = Vec::new();
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
let t = Instant::now();
|
||||||
|
if mode == "FULL" {
|
||||||
|
one_op_full(&pool, user_id).await;
|
||||||
|
} else {
|
||||||
|
one_op_narrow(&pool, user_id).await;
|
||||||
|
}
|
||||||
|
lats.push(t.elapsed().as_secs_f64() * 1e6);
|
||||||
|
}
|
||||||
|
lats
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let mut all = Vec::new();
|
||||||
|
for h in handles {
|
||||||
|
all.extend(h.await.unwrap());
|
||||||
|
}
|
||||||
|
let s = summarize(all, secs);
|
||||||
|
println!(
|
||||||
|
"| {:>5} | {:<7} | {:>10.0} | {:>9.1} | {:>9.1} |",
|
||||||
|
conc, mode, s.rps, s.p50, s.p99
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("\n(SKIP: PROPFINDs not naming quota props now issue NEITHER query — 0 round-trips.)");
|
||||||
|
|
||||||
|
cleanup(&pool, user_id).await;
|
||||||
|
}
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
//! Round-2 benchmark battery — five before/after gates in one binary.
|
||||||
|
//!
|
||||||
|
//! Each section isolates exactly what its change touches; a section whose
|
||||||
|
//! AFTER does not beat its BEFORE is grounds for rolling that change back.
|
||||||
|
//!
|
||||||
|
//! [1] range-cache — per-seek: PG resolve + open/seek/read vs moka hit + Bytes::slice
|
||||||
|
//! [2] nc-chunk-gate — per-PUT session-bytes gate: dir scan+stat vs counter
|
||||||
|
//! [3] delta-prefetch — 64-chunk drain: sequential opens vs buffered(8) (5 ms open latency)
|
||||||
|
//! [4] ingest-overlap — real store_from_stream, paced source: OXICLOUD_INGEST_OVERLAP=0 vs 1
|
||||||
|
//! [5] zip-stream — time-to-first-byte: temp-file build vs duplex streaming
|
||||||
|
//!
|
||||||
|
//! Run (needs Postgres for [1] and [4]; reads DATABASE_URL from .env):
|
||||||
|
//! cargo run --release --features bench --example bench_round2
|
||||||
|
//! Select sections: BENCH_SECTIONS="1,2,3,4,5"
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
|
use futures::{StreamExt, TryStreamExt, stream};
|
||||||
|
use sqlx::postgres::PgPoolOptions;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn median(mut xs: Vec<f64>) -> f64 {
|
||||||
|
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
xs[xs.len() / 2]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pct(sorted: &[f64], p: f64) -> f64 {
|
||||||
|
if sorted.is_empty() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
sorted[((sorted.len() as f64 * p) as usize).min(sorted.len() - 1)]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fill_random(buf: &mut [u8], seed: &mut u64) {
|
||||||
|
for chunk in buf.chunks_mut(8) {
|
||||||
|
*seed ^= *seed << 13;
|
||||||
|
*seed ^= *seed >> 7;
|
||||||
|
*seed ^= *seed << 17;
|
||||||
|
let b = seed.wrapping_mul(0x2545F4914F6CDD1D).to_le_bytes();
|
||||||
|
let n = chunk.len();
|
||||||
|
chunk.copy_from_slice(&b[..n]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── [1] range-cache ─────────────────────────────────────────────────────────
|
||||||
|
async fn section_range_cache(url: &str) {
|
||||||
|
println!("\n== [1] range-cache: per-seek cost, 256 KiB ranges over a 6 MiB media file ==");
|
||||||
|
let pool = PgPoolOptions::new()
|
||||||
|
.max_connections(5)
|
||||||
|
.connect(url)
|
||||||
|
.await
|
||||||
|
.expect("pg");
|
||||||
|
|
||||||
|
// Seed: drive→folder→file row (the BEFORE path resolves blob_hash by id)
|
||||||
|
// plus the blob bytes on disk for the open/seek/read.
|
||||||
|
let mut tx = pool.begin().await.expect("tx");
|
||||||
|
let drive_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id",
|
||||||
|
)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let folder_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.folders (name, path, lpath, drive_id)
|
||||||
|
VALUES ('bench_range', '/bench_range', 'bench_range', $1) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(drive_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tx.commit().await.unwrap();
|
||||||
|
|
||||||
|
let blob_hash = "benchrange000000000000000000000000000000000000000000000000000000";
|
||||||
|
let file_id: Uuid = sqlx::query_scalar(
|
||||||
|
"INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id)
|
||||||
|
VALUES ('video.mp4', $1, $2, 6291456, 'video/mp4', $3) RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(folder_id)
|
||||||
|
.bind(blob_hash)
|
||||||
|
.bind(drive_id)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let mut data = vec![0u8; 6 * 1024 * 1024];
|
||||||
|
let mut seed = 7u64;
|
||||||
|
fill_random(&mut data, &mut seed);
|
||||||
|
let blob_path = dir.path().join("blob");
|
||||||
|
std::fs::write(&blob_path, &data).unwrap();
|
||||||
|
|
||||||
|
// AFTER: warm content cache keyed by hash.
|
||||||
|
let cache: moka::sync::Cache<String, Bytes> = moka::sync::Cache::new(1000);
|
||||||
|
cache.insert(blob_hash.to_string(), Bytes::from(data.clone()));
|
||||||
|
|
||||||
|
let secs = 3u64;
|
||||||
|
let range_len = 256 * 1024usize;
|
||||||
|
for mode in ["BEFORE", "AFTER"] {
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(secs);
|
||||||
|
let mut lats = Vec::new();
|
||||||
|
let mut off = 0usize;
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
let t = Instant::now();
|
||||||
|
if mode == "BEFORE" {
|
||||||
|
// 1. resolve blob hash by file id (the real query shape)
|
||||||
|
let _h: String =
|
||||||
|
sqlx::query_scalar("SELECT blob_hash FROM storage.files WHERE id = $1")
|
||||||
|
.bind(file_id)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
// 2. open + seek + read the range (manifest lookup is already
|
||||||
|
// a moka hit post-round-1, so it's omitted on both sides)
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||||
|
let mut f = tokio::fs::File::open(&blob_path).await.unwrap();
|
||||||
|
f.seek(std::io::SeekFrom::Start(off as u64)).await.unwrap();
|
||||||
|
let mut buf = vec![0u8; range_len];
|
||||||
|
f.read_exact(&mut buf).await.unwrap();
|
||||||
|
std::hint::black_box(&buf);
|
||||||
|
} else {
|
||||||
|
let bytes = cache.get(blob_hash).unwrap();
|
||||||
|
let slice = bytes.slice(off..off + range_len);
|
||||||
|
std::hint::black_box(&slice);
|
||||||
|
}
|
||||||
|
lats.push(t.elapsed().as_secs_f64() * 1e6);
|
||||||
|
off = (off + range_len) % (data.len() - range_len);
|
||||||
|
}
|
||||||
|
lats.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
println!(
|
||||||
|
" {:<7} {:>9.0} seeks/s p50 {:>8.2} µs p99 {:>8.2} µs",
|
||||||
|
mode,
|
||||||
|
lats.len() as f64 / secs as f64,
|
||||||
|
pct(&lats, 0.5),
|
||||||
|
pct(&lats, 0.99),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1")
|
||||||
|
.bind(drive_id)
|
||||||
|
.execute(&pool)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── [2] nc-chunk-gate ───────────────────────────────────────────────────────
|
||||||
|
async fn section_nc_chunk_gate() {
|
||||||
|
println!("\n== [2] nc-chunk-gate: cumulative gate cost across a 1000-chunk upload ==");
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let session = dir.path().join("alice").join("upload-1");
|
||||||
|
tokio::fs::create_dir_all(&session).await.unwrap();
|
||||||
|
|
||||||
|
let chunks: usize = env_or("BENCH_CHUNKS", 1000);
|
||||||
|
// BEFORE: every PUT lists the dir and stats every existing chunk.
|
||||||
|
let t0 = Instant::now();
|
||||||
|
for k in 0..chunks {
|
||||||
|
// gate for chunk k: scan the k existing chunks
|
||||||
|
let mut total = 0u64;
|
||||||
|
let mut rd = tokio::fs::read_dir(&session).await.unwrap();
|
||||||
|
while let Some(e) = rd.next_entry().await.unwrap() {
|
||||||
|
total += e.metadata().await.unwrap().len();
|
||||||
|
}
|
||||||
|
std::hint::black_box(total);
|
||||||
|
// accept the chunk (tiny file; the write cost is identical on both
|
||||||
|
// sides so it cancels out — kept for realistic dirent counts)
|
||||||
|
tokio::fs::write(session.join(format!("{k:05}")), b"x")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let before = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
|
// Reset dir.
|
||||||
|
tokio::fs::remove_dir_all(&session).await.unwrap();
|
||||||
|
tokio::fs::create_dir_all(&session).await.unwrap();
|
||||||
|
|
||||||
|
// AFTER: O(1) counter (moka read + insert per PUT).
|
||||||
|
let counter: moka::sync::Cache<String, u64> = moka::sync::Cache::new(10);
|
||||||
|
counter.insert("s".into(), 0);
|
||||||
|
let t0 = Instant::now();
|
||||||
|
for k in 0..chunks {
|
||||||
|
let total = counter.get("s").unwrap();
|
||||||
|
std::hint::black_box(total);
|
||||||
|
tokio::fs::write(session.join(format!("{k:05}")), b"x")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
counter.insert("s".into(), total + 1);
|
||||||
|
}
|
||||||
|
let after = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
|
println!(
|
||||||
|
" BEFORE dir-scan gate: {before:>9.1} ms total AFTER counter gate: {after:>9.1} ms total ({:.1}x)",
|
||||||
|
before / after
|
||||||
|
);
|
||||||
|
println!(" (gate work alone; chunk-write cost included identically on both sides)");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── [3] delta-prefetch ──────────────────────────────────────────────────────
|
||||||
|
async fn section_delta_prefetch() {
|
||||||
|
println!(
|
||||||
|
"\n== [3] delta-prefetch: 64-chunk drain, 5 ms per-open latency (object-store model) =="
|
||||||
|
);
|
||||||
|
let n_chunks = 64usize;
|
||||||
|
let chunk_kb = 256usize;
|
||||||
|
let mut seed = 11u64;
|
||||||
|
let mut payload = vec![0u8; chunk_kb * 1024];
|
||||||
|
fill_random(&mut payload, &mut seed);
|
||||||
|
let payload = Bytes::from(payload);
|
||||||
|
|
||||||
|
// One "chunk open" = latency + a 4-frame byte stream (the shape the
|
||||||
|
// handler drains). Sequential = old; buffered(8) = new combinator.
|
||||||
|
let open = |p: Bytes| async move {
|
||||||
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||||
|
Ok::<_, std::io::Error>(stream::iter(
|
||||||
|
p.chunks(64 * 1024)
|
||||||
|
.map(|c| Ok::<Bytes, std::io::Error>(Bytes::copy_from_slice(c)))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
))
|
||||||
|
};
|
||||||
|
|
||||||
|
for (label, prefetch) in [("BEFORE sequential", 1usize), ("AFTER buffered(8)", 8)] {
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let mut drained = 0u64;
|
||||||
|
let mut s = stream::iter(vec![payload.clone(); n_chunks])
|
||||||
|
.map(&open)
|
||||||
|
.buffered(prefetch)
|
||||||
|
.try_flatten();
|
||||||
|
while let Some(part) = s.next().await {
|
||||||
|
drained += part.unwrap().len() as u64;
|
||||||
|
}
|
||||||
|
let ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
println!(" {label}: {ms:>8.1} ms for {} MiB", drained / 1024 / 1024);
|
||||||
|
}
|
||||||
|
println!(" (local-disk gain for the same combinator: +7-12% — benches/BLOB-PREFETCH.md)");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── [4] ingest-overlap ──────────────────────────────────────────────────────
|
||||||
|
async fn section_ingest_overlap(url: &str) {
|
||||||
|
println!("\n== [4] ingest-overlap: real store_from_stream, source paced at 300 MB/s ==");
|
||||||
|
println!(
|
||||||
|
" (mode fixed per process by OXICLOUD_INGEST_OVERLAP — run twice; current = {})",
|
||||||
|
std::env::var("OXICLOUD_INGEST_OVERLAP").unwrap_or_else(|_| "1/default".into())
|
||||||
|
);
|
||||||
|
use oxicloud::infrastructure::services::dedup_service::DedupService;
|
||||||
|
use oxicloud::infrastructure::services::local_blob_backend::LocalBlobBackend;
|
||||||
|
|
||||||
|
let pool = Arc::new(
|
||||||
|
PgPoolOptions::new()
|
||||||
|
.max_connections(10)
|
||||||
|
.connect(url)
|
||||||
|
.await
|
||||||
|
.expect("pg"),
|
||||||
|
);
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(LocalBlobBackend::new(dir.path()));
|
||||||
|
use oxicloud::application::ports::blob_storage_ports::BlobStorageBackend as _;
|
||||||
|
backend.initialize().await.expect("init backend");
|
||||||
|
let svc = DedupService::new(backend, pool.clone(), pool.clone());
|
||||||
|
|
||||||
|
let total_mb: usize = env_or("BENCH_INGEST_MB", 512);
|
||||||
|
let pace_mbps: f64 = env_or("BENCH_PACE_MBPS", 300.0);
|
||||||
|
let frame = 256 * 1024usize;
|
||||||
|
let mut seed = std::process::id() as u64 | 0xABCD << 32; // unique content per run — no dedup hits
|
||||||
|
let frames: Vec<Bytes> = (0..total_mb * 1024 * 1024 / frame)
|
||||||
|
.map(|_| {
|
||||||
|
let mut b = vec![0u8; frame];
|
||||||
|
fill_random(&mut b, &mut seed);
|
||||||
|
Bytes::from(b)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let frame_interval = Duration::from_secs_f64(frame as f64 / (pace_mbps * 1e6));
|
||||||
|
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let source = stream::iter(frames.into_iter().map(Ok::<Bytes, std::io::Error>)).then(
|
||||||
|
move |f| async move {
|
||||||
|
tokio::time::sleep(frame_interval).await;
|
||||||
|
f
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let result = svc.store_from_stream(source, None).await.expect("ingest");
|
||||||
|
let secs = t0.elapsed().as_secs_f64();
|
||||||
|
println!(
|
||||||
|
" ingested {} MiB in {:.2} s → {:.0} MB/s (blob {})",
|
||||||
|
total_mb,
|
||||||
|
secs,
|
||||||
|
total_mb as f64 / secs,
|
||||||
|
&result.hash()[..12],
|
||||||
|
);
|
||||||
|
// Cleanup: release the reference so GC can reap the bench blobs.
|
||||||
|
let _ = svc.remove_reference(result.hash()).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── [5] zip-stream ──────────────────────────────────────────────────────────
|
||||||
|
async fn section_zip_stream() {
|
||||||
|
println!("\n== [5] zip-stream: time-to-first-byte, 48 x 4 MiB media corpus ==");
|
||||||
|
use async_zip::base::write::ZipFileWriter;
|
||||||
|
use async_zip::{Compression, ZipEntryBuilder};
|
||||||
|
use futures::io::AsyncWriteExt as _;
|
||||||
|
|
||||||
|
let files: usize = env_or("BENCH_ZIP_FILES", 48);
|
||||||
|
let mb: usize = env_or("BENCH_ZIP_MB", 4);
|
||||||
|
let mut seed = 13u64;
|
||||||
|
let corpus: Vec<Bytes> = (0..files)
|
||||||
|
.map(|_| {
|
||||||
|
let mut b = vec![0u8; mb * 1024 * 1024];
|
||||||
|
fill_random(&mut b, &mut seed);
|
||||||
|
Bytes::from(b)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
async fn write_all_entries<W: tokio::io::AsyncWrite + Unpin>(sink: W, corpus: &[Bytes]) {
|
||||||
|
let buf = tokio::io::BufWriter::with_capacity(256 * 1024, sink);
|
||||||
|
let mut zip = ZipFileWriter::with_tokio(buf);
|
||||||
|
for (i, data) in corpus.iter().enumerate() {
|
||||||
|
let entry = ZipEntryBuilder::new(format!("IMG_{i:04}.jpg").into(), Compression::Stored);
|
||||||
|
let mut w = zip.write_entry_stream(entry).await.unwrap();
|
||||||
|
for c in data.chunks(64 * 1024) {
|
||||||
|
w.write_all(c).await.unwrap();
|
||||||
|
}
|
||||||
|
w.close().await.unwrap();
|
||||||
|
}
|
||||||
|
let mut compat = zip.close().await.unwrap();
|
||||||
|
compat.close().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// BEFORE: build the whole archive into a temp file, then "respond".
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let temp = tempfile::NamedTempFile::new().unwrap();
|
||||||
|
let f = tokio::fs::File::create(temp.path()).await.unwrap();
|
||||||
|
write_all_entries(f, &corpus).await;
|
||||||
|
// first byte = read back the first chunk
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
|
let mut rf = tokio::fs::File::open(temp.path()).await.unwrap();
|
||||||
|
let mut first = vec![0u8; 64 * 1024];
|
||||||
|
rf.read_exact(&mut first).await.unwrap();
|
||||||
|
let ttfb_before = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
let mut rest = Vec::new();
|
||||||
|
rf.read_to_end(&mut rest).await.unwrap();
|
||||||
|
let total_before = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
|
// AFTER: duplex — first byte as soon as the first entry flushes.
|
||||||
|
let t0 = Instant::now();
|
||||||
|
let (writer, reader) = tokio::io::duplex(256 * 1024);
|
||||||
|
let corpus2 = corpus.clone();
|
||||||
|
let jh = tokio::spawn(async move { write_all_entries(writer, &corpus2).await });
|
||||||
|
let mut rs = tokio_util::io::ReaderStream::new(reader);
|
||||||
|
let firstb = rs.next().await.unwrap().unwrap();
|
||||||
|
std::hint::black_box(&firstb);
|
||||||
|
let ttfb_after = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
let mut drained = firstb.len();
|
||||||
|
while let Some(c) = rs.next().await {
|
||||||
|
drained += c.unwrap().len();
|
||||||
|
}
|
||||||
|
jh.await.unwrap();
|
||||||
|
let total_after = t0.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
|
println!(" BEFORE temp-file : TTFB {ttfb_before:>8.1} ms total {total_before:>8.1} ms");
|
||||||
|
println!(
|
||||||
|
" AFTER streaming : TTFB {ttfb_after:>8.1} ms total {total_after:>8.1} ms (TTFB {:.0}x, {} MiB drained)",
|
||||||
|
ttfb_before / ttfb_after.max(0.001),
|
||||||
|
drained / 1024 / 1024
|
||||||
|
);
|
||||||
|
println!(" (TTFB scales with archive size in BEFORE; constant in AFTER)");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
dotenvy::dotenv().ok();
|
||||||
|
let url = env::var("DATABASE_URL").unwrap_or_default();
|
||||||
|
let sections: Vec<u32> = env::var("BENCH_SECTIONS")
|
||||||
|
.unwrap_or_else(|_| "1,2,3,4,5".into())
|
||||||
|
.split(',')
|
||||||
|
.filter_map(|x| x.trim().parse().ok())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let _ = median(vec![0.0]); // keep helper linked even if sections change
|
||||||
|
for s in sections {
|
||||||
|
match s {
|
||||||
|
1 => section_range_cache(&url).await,
|
||||||
|
2 => section_nc_chunk_gate().await,
|
||||||
|
3 => section_delta_prefetch().await,
|
||||||
|
4 => section_ingest_overlap(&url).await,
|
||||||
|
5 => section_zip_stream().await,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
//! Static-asset compression benchmark — on-the-fly Brotli per request vs
|
||||||
|
//! serving a precompressed sibling.
|
||||||
|
//!
|
||||||
|
//! The SPA router compressed every compressible static response on the fly
|
||||||
|
//! (tower-http `CompressionLayer`, backed by `async-compression`'s Brotli at
|
||||||
|
//! `Level::Default`) — the same immutable `/_app/immutable` bundle re-encoded
|
||||||
|
//! on EVERY request. The change teaches `ServeDir` to serve build-time
|
||||||
|
//! `.br`/`.gz` siblings (`precompressed_br()/precompressed_gzip()` +
|
||||||
|
//! `frontend/scripts/precompress.mjs`), so a request costs a file read.
|
||||||
|
//!
|
||||||
|
//! This isolates exactly that per-request delta on a JS-bundle-like payload:
|
||||||
|
//! BEFORE — Brotli-encode the asset with async-compression Level::Default
|
||||||
|
//! (what the layer does per request)
|
||||||
|
//! AFTER — read the precompressed sibling from disk (what ServeDir does)
|
||||||
|
//!
|
||||||
|
//! Run (no Postgres needed):
|
||||||
|
//! cargo run --release --features bench --example bench_static_precompress
|
||||||
|
//! Tunables: BENCH_ASSET_KB (700), BENCH_REPS (30)
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::io::Write as _;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JS-like corpus: repetitive identifiers + literals, compresses like a real
|
||||||
|
/// minified bundle (roughly 3-5×).
|
||||||
|
fn synth_js(len: usize, seed: &mut u64) -> Vec<u8> {
|
||||||
|
const FRAGS: &[&str] = &[
|
||||||
|
"function(e,t,n){var r=this;",
|
||||||
|
"return Object.assign({},",
|
||||||
|
"const a=document.querySelector(",
|
||||||
|
"export default{data(){return{",
|
||||||
|
"await fetch(url,{method:'POST',headers:",
|
||||||
|
".map(function(x){return x.id});",
|
||||||
|
"if(void 0!==e&&null!==t){",
|
||||||
|
"console.error('unhandled',err);",
|
||||||
|
];
|
||||||
|
let mut out = Vec::with_capacity(len);
|
||||||
|
while out.len() < len {
|
||||||
|
*seed ^= *seed << 13;
|
||||||
|
*seed ^= *seed >> 7;
|
||||||
|
*seed ^= *seed << 17;
|
||||||
|
out.extend_from_slice(FRAGS[(*seed as usize) % FRAGS.len()].as_bytes());
|
||||||
|
// sprinkle some varying identifiers so it's not pathological
|
||||||
|
let _ = write!(out, "v{}", *seed % 1000);
|
||||||
|
}
|
||||||
|
out.truncate(len);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn median(mut xs: Vec<f64>) -> f64 {
|
||||||
|
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
xs[xs.len() / 2]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
let asset_kb: usize = env_or("BENCH_ASSET_KB", 700);
|
||||||
|
let reps: usize = env_or("BENCH_REPS", 30);
|
||||||
|
let mut seed = 0xC0FFEEu64;
|
||||||
|
let asset = synth_js(asset_kb * 1024, &mut seed);
|
||||||
|
|
||||||
|
// Precompress once (build-time cost, paid once per deploy).
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let br_path = dir.path().join("bundle.js.br");
|
||||||
|
let t = Instant::now();
|
||||||
|
let precompressed = {
|
||||||
|
use async_compression::tokio::bufread::BrotliEncoder;
|
||||||
|
let mut enc = BrotliEncoder::new(std::io::Cursor::new(asset.clone()));
|
||||||
|
let mut out = Vec::new();
|
||||||
|
enc.read_to_end(&mut out).await.expect("precompress");
|
||||||
|
out
|
||||||
|
};
|
||||||
|
let build_ms = t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
std::fs::write(&br_path, &precompressed).expect("write .br");
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"asset: {} KiB JS-like → {} KiB brotli ({}% smaller); one-time build cost {:.1} ms\n",
|
||||||
|
asset.len() / 1024,
|
||||||
|
precompressed.len() / 1024,
|
||||||
|
100 - precompressed.len() * 100 / asset.len(),
|
||||||
|
build_ms
|
||||||
|
);
|
||||||
|
|
||||||
|
// BEFORE: per-request Brotli at the layer's default level.
|
||||||
|
let mut enc_times = Vec::with_capacity(reps);
|
||||||
|
for _ in 0..reps {
|
||||||
|
let t = Instant::now();
|
||||||
|
use async_compression::tokio::bufread::BrotliEncoder;
|
||||||
|
let mut enc = BrotliEncoder::new(std::io::Cursor::new(asset.clone()));
|
||||||
|
let mut out = Vec::new();
|
||||||
|
enc.read_to_end(&mut out).await.expect("encode");
|
||||||
|
std::hint::black_box(&out);
|
||||||
|
enc_times.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// AFTER: per-request read of the precompressed sibling.
|
||||||
|
let mut read_times = Vec::with_capacity(reps);
|
||||||
|
for _ in 0..reps {
|
||||||
|
let t = Instant::now();
|
||||||
|
let mut f = tokio::fs::File::open(&br_path).await.expect("open");
|
||||||
|
let mut out = Vec::new();
|
||||||
|
f.read_to_end(&mut out).await.expect("read");
|
||||||
|
std::hint::black_box(&out);
|
||||||
|
read_times.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Dynamic-response level sweep ─────────────────────────────────────
|
||||||
|
// The global API CompressionLayer (main.rs) compresses JSON responses
|
||||||
|
// per request. async-compression's Level::Default for Brotli is
|
||||||
|
// QUALITY 11 (brotli-8.0.2 encode.rs:323 via compression-codecs) — a
|
||||||
|
// deploy-grade setting on a per-request path. Sweep levels on a
|
||||||
|
// JSON-like 64 KiB body to pick the runtime quality.
|
||||||
|
let json_body = synth_js(64 * 1024, &mut seed); // JSON compresses like JS
|
||||||
|
println!("\n# per-request Brotli level on a 64 KiB JSON-like API response");
|
||||||
|
println!("{:<22} {:>10} {:>12}", "level", "ms/resp", "out KiB");
|
||||||
|
for (label, level) in [
|
||||||
|
("Default (= q11!)", async_compression::Level::Default),
|
||||||
|
("Precise(4)", async_compression::Level::Precise(4)),
|
||||||
|
("Fastest", async_compression::Level::Fastest),
|
||||||
|
] {
|
||||||
|
let mut times = Vec::with_capacity(reps);
|
||||||
|
let mut out_len = 0;
|
||||||
|
for _ in 0..reps {
|
||||||
|
let t = Instant::now();
|
||||||
|
use async_compression::tokio::bufread::BrotliEncoder;
|
||||||
|
let mut enc =
|
||||||
|
BrotliEncoder::with_quality(std::io::Cursor::new(json_body.clone()), level);
|
||||||
|
let mut out = Vec::new();
|
||||||
|
enc.read_to_end(&mut out).await.expect("encode");
|
||||||
|
out_len = out.len();
|
||||||
|
std::hint::black_box(&out);
|
||||||
|
times.push(t.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
"{:<22} {:>10.2} {:>12.1}",
|
||||||
|
label,
|
||||||
|
median(times),
|
||||||
|
out_len as f64 / 1024.0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let enc = median(enc_times);
|
||||||
|
let read = median(read_times);
|
||||||
|
println!(
|
||||||
|
"{:<34} {:>10} {:>9}",
|
||||||
|
"mode (per request)", "ms", "vs BEFORE"
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"{:<34} {:>10.2} {:>9}",
|
||||||
|
"BEFORE on-the-fly Brotli", enc, "1.0x"
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"{:<34} {:>10.3} {:>8.0}x",
|
||||||
|
"AFTER precompressed read",
|
||||||
|
read,
|
||||||
|
enc / read
|
||||||
|
);
|
||||||
|
println!("\n(BEFORE also holds ~1 tokio task busy for the duration on every request;");
|
||||||
|
println!(" AFTER additionally ships the deploy-time q11 encoding, usually smaller than");
|
||||||
|
println!(" the runtime default level.)");
|
||||||
|
}
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
//! ZIP entry-compression benchmark — `Deflate`-always vs MIME-aware `Stored`.
|
||||||
|
//!
|
||||||
|
//! Isolates the ONE variable the ZIP-export change touches: the per-entry
|
||||||
|
//! `Compression` mode chosen by `ZipService::write_prefetched_file` /
|
||||||
|
//! `BatchOperations::add_file_entry_streamed`. It rebuilds the *exact*
|
||||||
|
//! production writer stack —
|
||||||
|
//!
|
||||||
|
//! `ZipFileWriter::with_tokio(BufWriter(File))` + `write_entry_stream`
|
||||||
|
//! fed in ~64 KiB chunks (the blob-stream chunk size)
|
||||||
|
//!
|
||||||
|
//! — and writes the same corpus once per mode, measuring wall time, process
|
||||||
|
//! CPU time (utime+stime from `/proc/self/stat`), and final archive size.
|
||||||
|
//!
|
||||||
|
//! Corpora:
|
||||||
|
//! • `media` — incompressible bytes (models JPEG/HEIC/MP4/WebP, the
|
||||||
|
//! dominant "download folder" payload). Deflate here is pure CPU burn.
|
||||||
|
//! • `text` — compressible text (models docs/source). Deflate genuinely
|
||||||
|
//! shrinks these; the MIME-aware change keeps deflating them.
|
||||||
|
//! • `mixed` — 80 % media / 20 % text by bytes: `all-Deflate` row is the
|
||||||
|
//! production behaviour BEFORE the change; `mime-aware` row (Stored for
|
||||||
|
//! media, Deflate for text) is AFTER.
|
||||||
|
//!
|
||||||
|
//! Run (no Postgres needed):
|
||||||
|
//! cargo run --release --features bench --example bench_zip_media
|
||||||
|
//! Tunables (env):
|
||||||
|
//! BENCH_MEDIA_FILES (48) BENCH_MEDIA_MB (4) per-file size
|
||||||
|
//! BENCH_TEXT_FILES (24) BENCH_TEXT_MB (2)
|
||||||
|
//! BENCH_REPS (3) median reported
|
||||||
|
|
||||||
|
use std::env;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use async_zip::base::write::ZipFileWriter;
|
||||||
|
use async_zip::{Compression, ZipEntryBuilder};
|
||||||
|
use futures::io::AsyncWriteExt as FuturesWriteExt;
|
||||||
|
use tokio::io::BufWriter;
|
||||||
|
|
||||||
|
const CHUNK: usize = 64 * 1024; // blob-stream chunk size on the real path
|
||||||
|
|
||||||
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process CPU seconds (user + system) from /proc/self/stat — covers all
|
||||||
|
/// threads, so it catches deflate work wherever tokio schedules it.
|
||||||
|
fn cpu_seconds() -> f64 {
|
||||||
|
let stat = std::fs::read_to_string("/proc/self/stat").expect("read /proc/self/stat");
|
||||||
|
// utime and stime are fields 14 and 15 (1-based), after the comm field
|
||||||
|
// which may contain spaces — skip past the closing paren first.
|
||||||
|
let after = &stat[stat.rfind(')').unwrap() + 2..];
|
||||||
|
let fields: Vec<&str> = after.split_whitespace().collect();
|
||||||
|
let utime: u64 = fields[11].parse().unwrap(); // field 14 overall
|
||||||
|
let stime: u64 = fields[12].parse().unwrap(); // field 15 overall
|
||||||
|
(utime + stime) as f64 / 100.0 // USER_HZ = 100 on Linux
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deterministic xorshift64* stream — incompressible "media" bytes.
|
||||||
|
fn fill_random(buf: &mut [u8], seed: &mut u64) {
|
||||||
|
for chunk in buf.chunks_mut(8) {
|
||||||
|
*seed ^= *seed << 13;
|
||||||
|
*seed ^= *seed >> 7;
|
||||||
|
*seed ^= *seed << 17;
|
||||||
|
let bytes = seed.wrapping_mul(0x2545F4914F6CDD1D).to_le_bytes();
|
||||||
|
let n = chunk.len();
|
||||||
|
chunk.copy_from_slice(&bytes[..n]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compressible pseudo-text (~3-4× deflate ratio, like real docs/source).
|
||||||
|
fn fill_text(buf: &mut [u8], seed: &mut u64) {
|
||||||
|
const WORDS: &[&str] = &[
|
||||||
|
"the",
|
||||||
|
"quick",
|
||||||
|
"brown",
|
||||||
|
"fox",
|
||||||
|
"jumps",
|
||||||
|
"over",
|
||||||
|
"lazy",
|
||||||
|
"dog",
|
||||||
|
"folder",
|
||||||
|
"file",
|
||||||
|
"storage",
|
||||||
|
"performance",
|
||||||
|
"benchmark",
|
||||||
|
"archive",
|
||||||
|
"download",
|
||||||
|
"stream",
|
||||||
|
];
|
||||||
|
let mut pos = 0;
|
||||||
|
while pos < buf.len() {
|
||||||
|
*seed ^= *seed << 13;
|
||||||
|
*seed ^= *seed >> 7;
|
||||||
|
*seed ^= *seed << 17;
|
||||||
|
let w = WORDS[(*seed as usize) % WORDS.len()].as_bytes();
|
||||||
|
let n = w.len().min(buf.len() - pos);
|
||||||
|
buf[pos..pos + n].copy_from_slice(&w[..n]);
|
||||||
|
pos += n;
|
||||||
|
if pos < buf.len() {
|
||||||
|
buf[pos] = b' ';
|
||||||
|
pos += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CorpusFile {
|
||||||
|
name: String,
|
||||||
|
data: Vec<u8>,
|
||||||
|
is_media: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RunResult {
|
||||||
|
wall: Duration,
|
||||||
|
cpu: f64,
|
||||||
|
bytes_out: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write the corpus through the exact production writer stack, choosing the
|
||||||
|
/// compression mode per entry with `pick`.
|
||||||
|
async fn write_zip(files: &[CorpusFile], pick: impl Fn(&CorpusFile) -> Compression) -> RunResult {
|
||||||
|
let temp = tempfile::NamedTempFile::new().expect("temp file");
|
||||||
|
let tokio_file = tokio::fs::File::create(temp.path()).await.expect("create");
|
||||||
|
let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
|
||||||
|
let mut zip = ZipFileWriter::with_tokio(buf_writer);
|
||||||
|
|
||||||
|
let cpu0 = cpu_seconds();
|
||||||
|
let t0 = Instant::now();
|
||||||
|
for f in files {
|
||||||
|
let entry = ZipEntryBuilder::new(f.name.clone().into(), pick(f));
|
||||||
|
let mut w = zip.write_entry_stream(entry).await.expect("entry start");
|
||||||
|
for chunk in f.data.chunks(CHUNK) {
|
||||||
|
w.write_all(chunk).await.expect("chunk write");
|
||||||
|
}
|
||||||
|
w.close().await.expect("entry close");
|
||||||
|
}
|
||||||
|
let mut compat = zip.close().await.expect("zip close");
|
||||||
|
compat.close().await.expect("flush");
|
||||||
|
let wall = t0.elapsed();
|
||||||
|
let cpu = cpu_seconds() - cpu0;
|
||||||
|
|
||||||
|
let bytes_out = std::fs::metadata(temp.path()).map(|m| m.len()).unwrap_or(0);
|
||||||
|
RunResult {
|
||||||
|
wall,
|
||||||
|
cpu,
|
||||||
|
bytes_out,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn median(mut xs: Vec<f64>) -> f64 {
|
||||||
|
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||||
|
xs[xs.len() / 2]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main(flavor = "multi_thread")]
|
||||||
|
async fn main() {
|
||||||
|
let media_files: usize = env_or("BENCH_MEDIA_FILES", 48);
|
||||||
|
let media_mb: usize = env_or("BENCH_MEDIA_MB", 4);
|
||||||
|
let text_files: usize = env_or("BENCH_TEXT_FILES", 24);
|
||||||
|
let text_mb: usize = env_or("BENCH_TEXT_MB", 2);
|
||||||
|
let reps: usize = env_or("BENCH_REPS", 3);
|
||||||
|
|
||||||
|
let mut seed = 0x9E3779B97F4A7C15u64;
|
||||||
|
let mut corpus: Vec<CorpusFile> = Vec::new();
|
||||||
|
for i in 0..media_files {
|
||||||
|
let mut data = vec![0u8; media_mb * 1024 * 1024];
|
||||||
|
fill_random(&mut data, &mut seed);
|
||||||
|
corpus.push(CorpusFile {
|
||||||
|
name: format!("photos/IMG_{i:04}.jpg"),
|
||||||
|
data,
|
||||||
|
is_media: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for i in 0..text_files {
|
||||||
|
let mut data = vec![0u8; text_mb * 1024 * 1024];
|
||||||
|
fill_text(&mut data, &mut seed);
|
||||||
|
corpus.push(CorpusFile {
|
||||||
|
name: format!("docs/notes_{i:04}.txt"),
|
||||||
|
data,
|
||||||
|
is_media: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let media_bytes: usize = corpus
|
||||||
|
.iter()
|
||||||
|
.filter(|f| f.is_media)
|
||||||
|
.map(|f| f.data.len())
|
||||||
|
.sum();
|
||||||
|
let text_bytes: usize = corpus
|
||||||
|
.iter()
|
||||||
|
.filter(|f| !f.is_media)
|
||||||
|
.map(|f| f.data.len())
|
||||||
|
.sum();
|
||||||
|
let total_mb = (media_bytes + text_bytes) as f64 / 1048576.0;
|
||||||
|
println!(
|
||||||
|
"corpus: {} media files ({} MiB, incompressible) + {} text files ({} MiB, compressible), {} reps\n",
|
||||||
|
media_files,
|
||||||
|
media_bytes / 1048576,
|
||||||
|
text_files,
|
||||||
|
text_bytes / 1048576,
|
||||||
|
reps
|
||||||
|
);
|
||||||
|
|
||||||
|
// (label, per-entry compression picker)
|
||||||
|
type Picker = Box<dyn Fn(&CorpusFile) -> Compression>;
|
||||||
|
let modes: Vec<(&str, Picker)> = vec![
|
||||||
|
(
|
||||||
|
"all-Deflate (BEFORE)",
|
||||||
|
Box::new(|_: &CorpusFile| Compression::Deflate),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"mime-aware (AFTER) ",
|
||||||
|
Box::new(|f: &CorpusFile| {
|
||||||
|
if f.is_media {
|
||||||
|
Compression::Stored
|
||||||
|
} else {
|
||||||
|
Compression::Deflate
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"all-Stored (bound) ",
|
||||||
|
Box::new(|_: &CorpusFile| Compression::Stored),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"{:<22} {:>9} {:>9} {:>10} {:>11} {:>9}",
|
||||||
|
"mode", "wall s", "cpu s", "MB/s", "out MiB", "ratio"
|
||||||
|
);
|
||||||
|
let mut baseline_wall = None;
|
||||||
|
for (label, pick) in &modes {
|
||||||
|
let mut walls = Vec::new();
|
||||||
|
let mut cpus = Vec::new();
|
||||||
|
let mut out = 0u64;
|
||||||
|
for _ in 0..reps {
|
||||||
|
let r = write_zip(&corpus, pick).await;
|
||||||
|
walls.push(r.wall.as_secs_f64());
|
||||||
|
cpus.push(r.cpu);
|
||||||
|
out = r.bytes_out;
|
||||||
|
}
|
||||||
|
let wall = median(walls);
|
||||||
|
let cpu = median(cpus);
|
||||||
|
let speedup = baseline_wall
|
||||||
|
.map(|b: f64| format!("{:.2}x", b / wall))
|
||||||
|
.unwrap_or_else(|| "1.00x".into());
|
||||||
|
if baseline_wall.is_none() {
|
||||||
|
baseline_wall = Some(wall);
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
"{:<22} {:>9.3} {:>9.2} {:>10.1} {:>11.1} {:>9}",
|
||||||
|
label,
|
||||||
|
wall,
|
||||||
|
cpu,
|
||||||
|
total_mb / wall,
|
||||||
|
out as f64 / 1048576.0,
|
||||||
|
speedup
|
||||||
|
);
|
||||||
|
}
|
||||||
|
println!("\n(archive `out MiB` for mime-aware stays ~= all-Deflate: media doesn't deflate,");
|
||||||
|
println!(" text keeps Deflate — the win is CPU/wall, not size loss)");
|
||||||
|
}
|
||||||
@@ -9,7 +9,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite dev",
|
"dev": "vite dev",
|
||||||
"build": "vite build",
|
"build": "vite build",
|
||||||
"postbuild": "node scripts/emit-askama-common.mjs",
|
"postbuild": "node scripts/emit-askama-common.mjs && node scripts/precompress.mjs ../static-dist",
|
||||||
"preview": "vite preview",
|
"preview": "vite preview",
|
||||||
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json && eslint . && stylelint \"src/**/*.{css,svelte}\" && prettier --check .",
|
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json && eslint . && stylelint \"src/**/*.{css,svelte}\" && prettier --check .",
|
||||||
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
|
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
// Precompress built SPA assets so the Rust web layer can serve them with
|
||||||
|
// `ServeDir::precompressed_br()/precompressed_gzip()` instead of re-running
|
||||||
|
// Brotli over the same immutable bundle on every request (the tower-http
|
||||||
|
// CompressionLayer stays as the on-the-fly fallback for anything without a
|
||||||
|
// sibling). Runs as the `build` script's final step; uses only node:zlib —
|
||||||
|
// no dependencies. See benches/STATIC-PRECOMPRESSED.md for the measured win.
|
||||||
|
import { promises as fs } from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import zlib from 'node:zlib';
|
||||||
|
|
||||||
|
const OUT_DIR = process.argv[2] ?? '../static-dist';
|
||||||
|
// Compressible text assets; media formats are already compressed.
|
||||||
|
const EXTENSIONS = new Set([
|
||||||
|
'.js',
|
||||||
|
'.mjs',
|
||||||
|
'.css',
|
||||||
|
'.html',
|
||||||
|
'.svg',
|
||||||
|
'.json',
|
||||||
|
'.txt',
|
||||||
|
'.xml',
|
||||||
|
'.map',
|
||||||
|
'.webmanifest'
|
||||||
|
]);
|
||||||
|
// Below this size the encoding overhead outweighs the transfer win
|
||||||
|
// (mirrors the server's SizeAbove(256) predicate).
|
||||||
|
const MIN_BYTES = 256;
|
||||||
|
|
||||||
|
async function* walk(dir) {
|
||||||
|
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
||||||
|
const p = path.join(dir, entry.name);
|
||||||
|
if (entry.isDirectory()) yield* walk(p);
|
||||||
|
else yield p;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let files = 0;
|
||||||
|
let inBytes = 0;
|
||||||
|
let brBytes = 0;
|
||||||
|
for await (const file of walk(OUT_DIR)) {
|
||||||
|
if (!EXTENSIONS.has(path.extname(file))) continue;
|
||||||
|
const data = await fs.readFile(file);
|
||||||
|
if (data.length < MIN_BYTES) continue;
|
||||||
|
const br = zlib.brotliCompressSync(data, {
|
||||||
|
params: {
|
||||||
|
[zlib.constants.BROTLI_PARAM_QUALITY]: 11,
|
||||||
|
[zlib.constants.BROTLI_PARAM_SIZE_HINT]: data.length
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const gz = zlib.gzipSync(data, { level: 9 });
|
||||||
|
// Only keep siblings that actually shrink the asset.
|
||||||
|
if (br.length < data.length) await fs.writeFile(`${file}.br`, br);
|
||||||
|
if (gz.length < data.length) await fs.writeFile(`${file}.gz`, gz);
|
||||||
|
files += 1;
|
||||||
|
inBytes += data.length;
|
||||||
|
brBytes += Math.min(br.length, data.length);
|
||||||
|
}
|
||||||
|
console.log(
|
||||||
|
`precompress: ${files} assets, ${(inBytes / 1024).toFixed(0)} KiB → ${(brBytes / 1024).toFixed(0)} KiB brotli (${inBytes ? ((1 - brBytes / inBytes) * 100).toFixed(0) : 0}% smaller)`
|
||||||
|
);
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { Worker } from 'node:worker_threads';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { promises as fs } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Benchmark gate for the worker-pool hashing in `resolveOwnedHashes`.
|
||||||
|
*
|
||||||
|
* The browser change moves per-file BLAKE3 hashing from a sequential
|
||||||
|
* main-thread WASM loop onto a small pool of Web Workers. This test measures
|
||||||
|
* the same architecture on this machine with node's worker_threads and a
|
||||||
|
* CPU-bound digest as the stand-in workload: N buffers hashed sequentially
|
||||||
|
* on one thread vs the same work fanned over a 3-lane pool. If the pool
|
||||||
|
* doesn't beat sequential wall-clock, the frontend change must be rolled
|
||||||
|
* back (it would be pure complexity).
|
||||||
|
*/
|
||||||
|
describe('worker-pool hashing (architecture gate)', () => {
|
||||||
|
it('a 3-lane pool beats sequential main-thread hashing on wall clock', async () => {
|
||||||
|
// Faithful to the browser shape: the main thread hands each worker a
|
||||||
|
// FILE REFERENCE (browser: the File handle; here: its path) and the
|
||||||
|
// worker does read + hash. The old shape reads + hashes every file
|
||||||
|
// on the main thread, serially.
|
||||||
|
const nFiles = 24;
|
||||||
|
const size = 4 * 1024 * 1024;
|
||||||
|
const dir = await fs.mkdtemp(join(tmpdir(), 'hashbench-'));
|
||||||
|
const paths: string[] = [];
|
||||||
|
for (let i = 0; i < nFiles; i++) {
|
||||||
|
const p = join(dir, `f${i}`);
|
||||||
|
const b = Buffer.alloc(size);
|
||||||
|
b.fill(i + 1);
|
||||||
|
await fs.writeFile(p, b);
|
||||||
|
paths.push(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sequential (old): read + hash on the calling thread.
|
||||||
|
const t0 = performance.now();
|
||||||
|
for (const p of paths) {
|
||||||
|
const b = await fs.readFile(p);
|
||||||
|
createHash('sha256').update(b).digest('hex');
|
||||||
|
}
|
||||||
|
const seqMs = performance.now() - t0;
|
||||||
|
|
||||||
|
// 3-lane pool (new): each worker reads + hashes its own files.
|
||||||
|
const lanes = 3;
|
||||||
|
const workerSrc = `
|
||||||
|
const { parentPort } = require('node:worker_threads');
|
||||||
|
const { createHash } = require('node:crypto');
|
||||||
|
const { readFileSync } = require('node:fs');
|
||||||
|
parentPort.on('message', (path) => {
|
||||||
|
const b = readFileSync(path);
|
||||||
|
parentPort.postMessage(createHash('sha256').update(b).digest('hex'));
|
||||||
|
});
|
||||||
|
`;
|
||||||
|
const workers = Array.from({ length: lanes }, () => new Worker(workerSrc, { eval: true }));
|
||||||
|
let next = 0;
|
||||||
|
const t1 = performance.now();
|
||||||
|
await Promise.all(
|
||||||
|
workers.map(
|
||||||
|
(w) =>
|
||||||
|
new Promise<void>((resolve, reject) => {
|
||||||
|
const feed = () => {
|
||||||
|
if (next >= paths.length) {
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const i = next++;
|
||||||
|
w.once('message', () => feed());
|
||||||
|
w.once('error', reject);
|
||||||
|
w.postMessage(paths[i]);
|
||||||
|
};
|
||||||
|
feed();
|
||||||
|
})
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const poolMs = performance.now() - t1;
|
||||||
|
await Promise.all(workers.map((w) => w.terminate()));
|
||||||
|
await fs.rm(dir, { recursive: true, force: true });
|
||||||
|
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.info(
|
||||||
|
`read+hash ${nFiles} x 4 MiB: sequential ${seqMs.toFixed(0)} ms vs 3-lane pool ${poolMs.toFixed(0)} ms (${(seqMs / poolMs).toFixed(1)}x)`
|
||||||
|
);
|
||||||
|
expect(poolMs).toBeLessThan(seqMs);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -176,6 +176,59 @@ export async function instantUploadOwned(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const HASH_WORKER_URL = '/workers/hashWorker.js';
|
||||||
|
/** Parallel hashing lanes — enough to saturate small-file hashing without
|
||||||
|
* starving the upload workers of cores. */
|
||||||
|
const HASH_POOL_SIZE = Math.min(4, Math.max(1, (navigator.hardwareConcurrency ?? 2) - 1));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* BLAKE3-hash `files` on a bounded pool of dedicated workers (main thread
|
||||||
|
* stays free). A file whose worker errors is simply absent from the result —
|
||||||
|
* the caller uploads it the normal way. Falls back to the sequential inline
|
||||||
|
* hasher when `Worker` is unavailable.
|
||||||
|
*/
|
||||||
|
async function hashFilesPooled(files: File[]): Promise<Map<File, string>> {
|
||||||
|
if (typeof Worker === 'undefined') {
|
||||||
|
const out = new Map<File, string>();
|
||||||
|
for (const f of files) out.set(f, await blake3HexOfFile(f));
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
const lanes = Math.min(HASH_POOL_SIZE, files.length);
|
||||||
|
const workers = Array.from(
|
||||||
|
{ length: lanes },
|
||||||
|
() => new Worker(HASH_WORKER_URL, { type: 'module' })
|
||||||
|
);
|
||||||
|
const out = new Map<File, string>();
|
||||||
|
let next = 0;
|
||||||
|
try {
|
||||||
|
await Promise.all(
|
||||||
|
workers.map(
|
||||||
|
(w) =>
|
||||||
|
new Promise<void>((resolve, reject) => {
|
||||||
|
const feed = () => {
|
||||||
|
if (next >= files.length) {
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const i = next++;
|
||||||
|
const file = files[i];
|
||||||
|
w.onmessage = (ev: MessageEvent<{ id: number; hex?: string; error?: string }>) => {
|
||||||
|
if (ev.data.hex) out.set(file, ev.data.hex);
|
||||||
|
feed(); // per-file errors: skip the file, keep the lane
|
||||||
|
};
|
||||||
|
w.onerror = (e) => reject(e);
|
||||||
|
w.postMessage({ id: i, file });
|
||||||
|
};
|
||||||
|
feed();
|
||||||
|
})
|
||||||
|
)
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
for (const w of workers) w.terminate();
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve which of `files` the server already owns, with a SINGLE batch round
|
* Resolve which of `files` the server already owns, with a SINGLE batch round
|
||||||
* trip (the Dropbox-style "have you got these?" probe). Every file below the
|
* trip (the Dropbox-style "have you got these?" probe). Every file below the
|
||||||
@@ -193,7 +246,13 @@ export async function resolveOwnedHashes(files: File[]): Promise<Map<File, strin
|
|||||||
|
|
||||||
const hashByFile = new Map<File, string>();
|
const hashByFile = new Map<File, string>();
|
||||||
try {
|
try {
|
||||||
for (const f of inBand) hashByFile.set(f, await blake3HexOfFile(f));
|
// Hash off the main thread on a small worker pool — the sequential
|
||||||
|
// main-thread WASM loop blocked the UI for the whole batch and
|
||||||
|
// delayed every upload lane behind the full hashing phase (measured
|
||||||
|
// in deltaUpload.hash.test.ts). Falls back to the inline loop when
|
||||||
|
// Workers are unavailable (some test environments).
|
||||||
|
const hashed = await hashFilesPooled(inBand);
|
||||||
|
for (const [f, h] of hashed) hashByFile.set(f, h);
|
||||||
} catch {
|
} catch {
|
||||||
return new Map(); // WASM/hashing unavailable → skip instant uploads
|
return new Map(); // WASM/hashing unavailable → skip instant uploads
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -108,7 +108,19 @@ export async function getFolder(id: string): Promise<FolderItem> {
|
|||||||
*/
|
*/
|
||||||
export async function fetchFolderListing(
|
export async function fetchFolderListing(
|
||||||
folderId: string,
|
folderId: string,
|
||||||
opts: { etag?: string; forceRefresh?: boolean } = {}
|
opts: {
|
||||||
|
etag?: string;
|
||||||
|
forceRefresh?: boolean;
|
||||||
|
/**
|
||||||
|
* Progressive render hook: invoked after EVERY page with the
|
||||||
|
* accumulated listing so far (the arrays are fresh copies — safe to
|
||||||
|
* hand to reactive state). Without it, a 2,000-item folder waited
|
||||||
|
* for all ⌈N/200⌉ sequential round-trips before the first row
|
||||||
|
* painted; with it the view paints after page one (~200 items) and
|
||||||
|
* fills in as the tail pages land.
|
||||||
|
*/
|
||||||
|
onPage?: (partial: FolderListing, done: boolean) => void;
|
||||||
|
} = {}
|
||||||
): Promise<FolderListingResult> {
|
): Promise<FolderListingResult> {
|
||||||
const folders: FolderItem[] = [];
|
const folders: FolderItem[] = [];
|
||||||
const files: FileItem[] = [];
|
const files: FileItem[] = [];
|
||||||
@@ -132,6 +144,10 @@ export async function fetchFolderListing(
|
|||||||
else files.push(it.resource as FileItem);
|
else files.push(it.resource as FileItem);
|
||||||
}
|
}
|
||||||
cursor = page.next_cursor;
|
cursor = page.next_cursor;
|
||||||
|
opts.onPage?.(
|
||||||
|
{ folders: [...folders], files: [...files], favoriteIds: [], sharedIds: [] },
|
||||||
|
!cursor
|
||||||
|
);
|
||||||
} while (cursor);
|
} while (cursor);
|
||||||
|
|
||||||
return { status: 200, listing: { folders, files, favoriteIds: [], sharedIds: [] } };
|
return { status: 200, listing: { folders, files, favoriteIds: [], sharedIds: [] } };
|
||||||
|
|||||||
@@ -308,7 +308,22 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetchFolderListing(folderId, { etag: cached?.etag });
|
const res = await fetchFolderListing(folderId, {
|
||||||
|
etag: cached?.etag,
|
||||||
|
// Paint page one (~200 items) immediately instead of waiting
|
||||||
|
// for every sequential page of a large folder; later pages
|
||||||
|
// extend the view as they land. Skip when a cached copy is
|
||||||
|
// already on screen — replacing it with a partial list would
|
||||||
|
// briefly shrink the view.
|
||||||
|
onPage: cached
|
||||||
|
? undefined
|
||||||
|
: (partial, done) => {
|
||||||
|
if (seq !== loadSeq || done) return; // final state applied below
|
||||||
|
applyListing(partial);
|
||||||
|
loading = false;
|
||||||
|
showSkeleton = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
if (seq !== loadSeq) return; // superseded by a newer navigation
|
if (seq !== loadSeq) return; // superseded by a newer navigation
|
||||||
if (res.status === 200 && res.listing) {
|
if (res.status === 200 && res.listing) {
|
||||||
applyListing(res.listing);
|
applyListing(res.listing);
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
/**
|
||||||
|
* OxiCloud — whole-file BLAKE3 hashing worker.
|
||||||
|
*
|
||||||
|
* Computes the instant-upload ("does the server already own this?") hashes
|
||||||
|
* OFF the main thread. The previous shape hashed every small file of a
|
||||||
|
* batch drop sequentially on the main thread with synchronous WASM calls —
|
||||||
|
* seconds of UI jank for a large drop, all before the first upload lane
|
||||||
|
* even started (see collateral bench in deltaUpload.hash.test.ts).
|
||||||
|
*
|
||||||
|
* Protocol with the spawner (one worker handles many requests):
|
||||||
|
* in : { id: number, file: File }
|
||||||
|
* out : { id: number, hex: string } — success
|
||||||
|
* { id: number, error: string } — this file failed (caller
|
||||||
|
* falls back to plain upload)
|
||||||
|
*/
|
||||||
|
|
||||||
|
const WASM_GLUE_URL = '/vendors/hash-wasm/oxicloud_hash_wasm.js';
|
||||||
|
|
||||||
|
let modPromise = null;
|
||||||
|
function load() {
|
||||||
|
if (!modPromise) {
|
||||||
|
modPromise = import(WASM_GLUE_URL).then(async (mod) => {
|
||||||
|
await mod.default();
|
||||||
|
return mod;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return modPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.onmessage = async (ev) => {
|
||||||
|
const { id, file } = ev.data;
|
||||||
|
try {
|
||||||
|
const mod = await load();
|
||||||
|
const bytes = new Uint8Array(await file.arrayBuffer());
|
||||||
|
const hex = mod.blake3Hex(bytes);
|
||||||
|
self.postMessage({ id, hex });
|
||||||
|
} catch (err) {
|
||||||
|
self.postMessage({ id, error: String(err) });
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
-- ════════════════════════════════════════════════════════════════════════════
|
||||||
|
-- Name-ordered folder listing index for streaming WebDAV PROPFIND
|
||||||
|
-- ════════════════════════════════════════════════════════════════════════════
|
||||||
|
-- `list_files_batch` walks a folder's children in `ORDER BY name` pages of
|
||||||
|
-- 500 (native + NextCloud PROPFIND). The only index on the filter column
|
||||||
|
-- was `idx_files_folder_id (folder_id)`, so EVERY page did a bitmap scan of
|
||||||
|
-- all N children plus a top-(offset+limit) sort — a quadratic full-folder
|
||||||
|
-- walk (the initial schema's `(folder_id, name, user_id)` index that served
|
||||||
|
-- this was dropped by 20260902000000 when user_id went nullable).
|
||||||
|
--
|
||||||
|
-- This composite index restores the ordered access path: combined with the
|
||||||
|
-- keyset cursor (`name > $last` — see `file_blob_read_repository.rs`
|
||||||
|
-- `list_files_batch`), each page is one O(page) index-range read with no
|
||||||
|
-- sort, regardless of folder size or scroll depth. Benchmarked in
|
||||||
|
-- benches/DEAD-PROPS.md's companion doc benches/PROPFIND-PAGING.md.
|
||||||
|
--
|
||||||
|
-- Partial (`NOT is_trashed`) to match the listing predicate and keep the
|
||||||
|
-- index compact; trashed rows are never listed by PROPFIND.
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_files_folder_name
|
||||||
|
ON storage.files (folder_id, name)
|
||||||
|
WHERE NOT is_trashed;
|
||||||
@@ -129,6 +129,30 @@ pub struct PropFindRequest {
|
|||||||
pub prop_find_type: PropFindType,
|
pub prop_find_type: PropFindType,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl PropFindRequest {
|
||||||
|
/// Whether answering this PROPFIND requires resolving the account /
|
||||||
|
/// drive quota at all.
|
||||||
|
///
|
||||||
|
/// `resolve_webdav_quota` costs two DB round-trips per request; sync
|
||||||
|
/// clients poll folders with an explicit `<D:prop>` list that most of
|
||||||
|
/// the time names only etag/length/type props — computing quota there
|
||||||
|
/// is pure waste (the response never mentions it). `AllProp` and
|
||||||
|
/// `PropName` keep quota: the writers emit RFC 4331 props for both.
|
||||||
|
/// Measured in `benches/QUOTA-PATH.md`.
|
||||||
|
pub fn wants_quota(&self) -> bool {
|
||||||
|
match &self.prop_find_type {
|
||||||
|
PropFindType::AllProp | PropFindType::PropName => true,
|
||||||
|
PropFindType::Prop(props) => props.iter().any(|p| {
|
||||||
|
p.namespace == "DAV:"
|
||||||
|
&& matches!(
|
||||||
|
p.name.as_str(),
|
||||||
|
"quota-used-bytes" | "quota-available-bytes"
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// WebDAV property value
|
/// WebDAV property value
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct PropValue {
|
pub struct PropValue {
|
||||||
|
|||||||
@@ -29,6 +29,30 @@ pub trait AuthorizationEngine: Send + Sync + 'static {
|
|||||||
resource: Resource,
|
resource: Resource,
|
||||||
) -> Result<bool, DomainError>;
|
) -> Result<bool, DomainError>;
|
||||||
|
|
||||||
|
/// Batched `check(subject, Read, File(id))` over a result page: returns
|
||||||
|
/// the subset of `file_ids` the subject may read. Semantically identical
|
||||||
|
/// to looping [`Self::check`] (the default does exactly that); the
|
||||||
|
/// `PgAclEngine` override resolves every file's drive in ONE query and
|
||||||
|
/// reuses the per-drive role cache, so verifying a 200-hit search page
|
||||||
|
/// costs 1 SQL round-trip instead of up to 200 sequential ones
|
||||||
|
/// (benches/SEARCH-REBAC.md).
|
||||||
|
async fn check_files_read_batch(
|
||||||
|
&self,
|
||||||
|
subject: Subject,
|
||||||
|
file_ids: &[Uuid],
|
||||||
|
) -> Result<std::collections::HashSet<Uuid>, DomainError> {
|
||||||
|
let mut allowed = std::collections::HashSet::with_capacity(file_ids.len());
|
||||||
|
for id in file_ids {
|
||||||
|
if self
|
||||||
|
.check(subject, Permission::Read, Resource::File(*id))
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
allowed.insert(*id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(allowed)
|
||||||
|
}
|
||||||
|
|
||||||
/// Convenience wrapper around `check`: returns `Ok(())` when allowed and
|
/// Convenience wrapper around `check`: returns `Ok(())` when allowed and
|
||||||
/// `DomainError::not_found` when denied (anti-enumeration — same error as
|
/// `DomainError::not_found` when denied (anti-enumeration — same error as
|
||||||
/// "resource doesn't exist" so attackers can't probe IDs by error shape).
|
/// "resource doesn't exist" so attackers can't probe IDs by error shape).
|
||||||
|
|||||||
@@ -39,6 +39,23 @@ pub trait FaceRepository: Send + Sync + 'static {
|
|||||||
user_id: Uuid,
|
user_id: Uuid,
|
||||||
blob_hash: &str,
|
blob_hash: &str,
|
||||||
) -> Result<Vec<Face>, DomainError>;
|
) -> Result<Vec<Face>, DomainError>;
|
||||||
|
/// `(person_id, face_count)` per non-empty cluster — a grouped COUNT
|
||||||
|
/// instead of dragging every face row (each with a 2 KiB embedding
|
||||||
|
/// BYTEA) across the wire just to count them. See benches/PEOPLE-LIST.md.
|
||||||
|
async fn person_face_stats(&self, user_id: Uuid) -> Result<Vec<(Uuid, i64)>, DomainError>;
|
||||||
|
/// face id → file id for the given faces (cover-photo resolution).
|
||||||
|
async fn file_ids_for_faces(
|
||||||
|
&self,
|
||||||
|
user_id: Uuid,
|
||||||
|
face_ids: &[Uuid],
|
||||||
|
) -> Result<std::collections::HashMap<Uuid, Uuid>, DomainError>;
|
||||||
|
/// Reassign every face of `from` to `into` in one statement (merge).
|
||||||
|
async fn reassign_person_faces(
|
||||||
|
&self,
|
||||||
|
user_id: Uuid,
|
||||||
|
from: Uuid,
|
||||||
|
into: Uuid,
|
||||||
|
) -> Result<u64, DomainError>;
|
||||||
async fn assign_person(
|
async fn assign_person(
|
||||||
&self,
|
&self,
|
||||||
face_id: Uuid,
|
face_id: Uuid,
|
||||||
|
|||||||
@@ -111,6 +111,17 @@ pub enum OptimizedFileContent {
|
|||||||
Stream(Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>),
|
Stream(Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Result of a cache-aware HTTP-Range read
|
||||||
|
/// (`FileRetrievalService::get_file_range_preloaded`). Same split as
|
||||||
|
/// [`OptimizedFileContent`]: handlers map each variant onto a response body.
|
||||||
|
pub enum RangeContent {
|
||||||
|
/// Zero-copy slice out of the RAM content cache (a `Bytes::slice` is a
|
||||||
|
/// refcount bump — no allocation, no I/O, no DB).
|
||||||
|
Bytes(Bytes),
|
||||||
|
/// Streaming range read from the blob store (cache miss / large file).
|
||||||
|
Stream(Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>),
|
||||||
|
}
|
||||||
|
|
||||||
/// Primary port for file retrieval operations
|
/// Primary port for file retrieval operations
|
||||||
pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
||||||
/// Gets a file by its ID (system/internal — no ownership check).
|
/// Gets a file by its ID (system/internal — no ownership check).
|
||||||
@@ -235,13 +246,14 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
|||||||
async fn list_files_batch(
|
async fn list_files_batch(
|
||||||
&self,
|
&self,
|
||||||
folder_id: Option<&str>,
|
folder_id: Option<&str>,
|
||||||
offset: i64,
|
after_name: Option<&str>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<Vec<FileDto>, DomainError> {
|
) -> Result<Vec<FileDto>, DomainError> {
|
||||||
let all = self.list_files(folder_id).await?;
|
let mut all = self.list_files(folder_id).await?;
|
||||||
|
all.sort_by(|a, b| a.name.cmp(&b.name));
|
||||||
Ok(all
|
Ok(all
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.skip(offset as usize)
|
.filter(|f| after_name.is_none_or(|a| f.name.as_str() > a))
|
||||||
.take(limit as usize)
|
.take(limit as usize)
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
@@ -257,10 +269,10 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
|||||||
&self,
|
&self,
|
||||||
folder_id: Option<&str>,
|
folder_id: Option<&str>,
|
||||||
_owner_id: Uuid,
|
_owner_id: Uuid,
|
||||||
offset: i64,
|
after_name: Option<&str>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<Vec<FileDto>, DomainError> {
|
) -> Result<Vec<FileDto>, DomainError> {
|
||||||
self.list_files_batch(folder_id, offset, limit).await
|
self.list_files_batch(folder_id, after_name, limit).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -99,6 +99,28 @@ pub trait ShareStoragePort: Send + Sync + 'static {
|
|||||||
share: &crate::domain::entities::share::Share,
|
share: &crate::domain::entities::share::Share,
|
||||||
) -> Result<crate::domain::entities::share::Share, DomainError>;
|
) -> Result<crate::domain::entities::share::Share, DomainError>;
|
||||||
|
|
||||||
|
/// Atomically bump a link's access counter (public share landing).
|
||||||
|
/// Returns the number of rows updated — 0 means "no live share for
|
||||||
|
/// this token" (missing OR expired).
|
||||||
|
///
|
||||||
|
/// The default is the legacy read-modify-write (kept for test mocks);
|
||||||
|
/// `SharePgRepository` overrides it with a single `UPDATE … SET
|
||||||
|
/// access_count = access_count + 1`, replacing 2 correlated-subquery
|
||||||
|
/// round-trips per anonymous visit with 1 and removing the lost-update
|
||||||
|
/// race between concurrent visitors (benches/SHARE-ACCESS.md).
|
||||||
|
async fn increment_access_count(&self, token: &str) -> Result<u64, DomainError> {
|
||||||
|
let share = match self.find_share_by_token(token).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) if e.kind == crate::common::errors::ErrorKind::NotFound => return Ok(0),
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
if share.is_expired() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
self.update_share(&share.increment_access_count()).await?;
|
||||||
|
Ok(1)
|
||||||
|
}
|
||||||
|
|
||||||
async fn find_shares_by_user(
|
async fn find_shares_by_user(
|
||||||
&self,
|
&self,
|
||||||
user_id: Uuid,
|
user_id: Uuid,
|
||||||
|
|||||||
@@ -107,20 +107,30 @@ pub trait FileReadPort: Send + Sync + 'static {
|
|||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lists files in a folder with LIMIT/OFFSET pagination.
|
/// Lists files in a folder in name order, keyset-paginated.
|
||||||
///
|
///
|
||||||
/// Used by streaming WebDAV PROPFIND to avoid loading all files at once.
|
/// Used by streaming WebDAV PROPFIND to avoid loading all files at once.
|
||||||
|
/// `after_name` is the last name of the previous page (`None` = first
|
||||||
|
/// page); names are unique within a folder (unique index on
|
||||||
|
/// `(drive_id, folder_id, name)`), so `name > after_name` is a total,
|
||||||
|
/// stable cursor. Unlike LIMIT/OFFSET, every page is O(page) — the old
|
||||||
|
/// offset shape re-scanned and re-sorted the whole folder per page
|
||||||
|
/// (benches/PROPFIND-PAGING.md).
|
||||||
|
///
|
||||||
/// Default: falls back to `list_files` (loads all, then slices in memory).
|
/// Default: falls back to `list_files` (loads all, then slices in memory).
|
||||||
async fn list_files_batch(
|
async fn list_files_batch(
|
||||||
&self,
|
&self,
|
||||||
folder_id: Option<&str>,
|
folder_id: Option<&str>,
|
||||||
offset: i64,
|
after_name: Option<&str>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<Vec<File>, DomainError> {
|
) -> Result<Vec<File>, DomainError> {
|
||||||
let all = self.list_files(folder_id).await?;
|
let mut all = self.list_files(folder_id).await?;
|
||||||
let start = (offset as usize).min(all.len());
|
all.sort_by(|a, b| a.name().cmp(b.name()));
|
||||||
let end = (start + limit as usize).min(all.len());
|
Ok(all
|
||||||
Ok(all.into_iter().skip(start).take(end - start).collect())
|
.into_iter()
|
||||||
|
.filter(|f| after_name.is_none_or(|a| f.name() > a))
|
||||||
|
.take(limit as usize)
|
||||||
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Streams every file in the subtree rooted at `folder_id`.
|
/// Streams every file in the subtree rooted at `folder_id`.
|
||||||
|
|||||||
@@ -734,7 +734,13 @@ impl BatchOperationService {
|
|||||||
{
|
{
|
||||||
Ok(file_dto) => {
|
Ok(file_dto) => {
|
||||||
match self
|
match self
|
||||||
.add_file_entry_streamed(&mut zip, file_id, &file_dto.name, user_id)
|
.add_file_entry_streamed(
|
||||||
|
&mut zip,
|
||||||
|
file_id,
|
||||||
|
&file_dto.name,
|
||||||
|
&file_dto.mime_type,
|
||||||
|
Some(user_id),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_) => items_added += 1,
|
Ok(_) => items_added += 1,
|
||||||
@@ -758,7 +764,7 @@ impl BatchOperationService {
|
|||||||
{
|
{
|
||||||
Ok(root_folder) => {
|
Ok(root_folder) => {
|
||||||
match self
|
match self
|
||||||
.add_folder_subtree_to_zip(&mut zip, folder_id, &root_folder, user_id)
|
.add_folder_subtree_to_zip(&mut zip, folder_id, &root_folder)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_) => items_added += 1,
|
Ok(_) => items_added += 1,
|
||||||
@@ -803,24 +809,44 @@ impl BatchOperationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Streams a single file into an async ZIP entry (~64 KB peak RAM per file).
|
/// Streams a single file into an async ZIP entry (~64 KB peak RAM per file).
|
||||||
|
///
|
||||||
|
/// Already-compressed content (per its MIME type) is `Stored` — deflating
|
||||||
|
/// JPEG/MP4/… burns ~a CPU core per download for ~0 % size gain.
|
||||||
|
///
|
||||||
|
/// `caller_id = Some(uid)` enforces the per-file Read check and records
|
||||||
|
/// the access in Recents (explicitly-selected top-level files).
|
||||||
|
/// `None` = the file was enumerated from a folder subtree whose ROOT the
|
||||||
|
/// caller already passed `get_folder_with_perms` for — per-file
|
||||||
|
/// re-authorization and per-file Recent spam (2 writes/file via the
|
||||||
|
/// recent hook) are skipped, mirroring `ZipService::create_folder_zip`
|
||||||
|
/// on the native folder-download path (benches/ZIP-BATCH-AUTHZ.md).
|
||||||
async fn add_file_entry_streamed(
|
async fn add_file_entry_streamed(
|
||||||
&self,
|
&self,
|
||||||
zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
|
zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
|
||||||
file_id: &str,
|
file_id: &str,
|
||||||
entry_name: &str,
|
entry_name: &str,
|
||||||
caller_id: Uuid,
|
mime_type: &str,
|
||||||
|
caller_id: Option<Uuid>,
|
||||||
) -> Result<(), BatchOperationError> {
|
) -> Result<(), BatchOperationError> {
|
||||||
let entry = ZipEntryBuilder::new(entry_name.to_string().into(), Compression::Deflate);
|
let compression = crate::common::mime_detect::zip_entry_compression(mime_type);
|
||||||
|
let entry = ZipEntryBuilder::new(entry_name.to_string().into(), compression);
|
||||||
let mut writer = zip
|
let mut writer = zip
|
||||||
.write_entry_stream(entry)
|
.write_entry_stream(entry)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| BatchOperationError::Internal(format!("zip entry start: {}", e)))?;
|
.map_err(|e| BatchOperationError::Internal(format!("zip entry start: {}", e)))?;
|
||||||
|
|
||||||
let stream = self
|
let stream = match caller_id {
|
||||||
.file_retrieval
|
Some(uid) => self
|
||||||
.get_file_stream_with_perms(file_id, caller_id)
|
.file_retrieval
|
||||||
.await
|
.get_file_stream_with_perms(file_id, uid)
|
||||||
.map_err(BatchOperationError::Domain)?;
|
.await
|
||||||
|
.map_err(BatchOperationError::Domain)?,
|
||||||
|
None => self
|
||||||
|
.file_retrieval
|
||||||
|
.get_file_stream(file_id)
|
||||||
|
.await
|
||||||
|
.map_err(BatchOperationError::Domain)?,
|
||||||
|
};
|
||||||
let mut stream = std::pin::Pin::from(stream);
|
let mut stream = std::pin::Pin::from(stream);
|
||||||
|
|
||||||
while let Some(chunk) = stream.next().await {
|
while let Some(chunk) = stream.next().await {
|
||||||
@@ -849,7 +875,6 @@ impl BatchOperationService {
|
|||||||
zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
|
zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
|
||||||
folder_id: &str,
|
folder_id: &str,
|
||||||
root_folder: &FolderDto,
|
root_folder: &FolderDto,
|
||||||
caller_id: Uuid,
|
|
||||||
) -> Result<(), BatchOperationError> {
|
) -> Result<(), BatchOperationError> {
|
||||||
// Bulk-fetch folder tree (small — one entry per folder)
|
// Bulk-fetch folder tree (small — one entry per folder)
|
||||||
let all_folders = self
|
let all_folders = self
|
||||||
@@ -903,8 +928,10 @@ impl BatchOperationService {
|
|||||||
if let Some(files) = files_by_folder.get(&folder.id) {
|
if let Some(files) = files_by_folder.get(&folder.id) {
|
||||||
for file in files {
|
for file in files {
|
||||||
let file_path = format!("{}{}", zip_dir, file.name);
|
let file_path = format!("{}{}", zip_dir, file.name);
|
||||||
|
// Subtree pre-authorized at the root folder — see
|
||||||
|
// `add_file_entry_streamed` docs for why `None`.
|
||||||
if let Err(e) = self
|
if let Err(e) = self
|
||||||
.add_file_entry_streamed(zip, &file.id, &file_path, caller_id)
|
.add_file_entry_streamed(zip, &file.id, &file_path, &file.mime_type, None)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
info!("Could not add file {} to ZIP: {}", file.name, e);
|
info!("Could not add file {} to ZIP: {}", file.name, e);
|
||||||
|
|||||||
@@ -607,6 +607,12 @@ impl DeltaUploadService {
|
|||||||
Ok(DeltaDownloadOutcome::Ready(ordered))
|
Ok(DeltaDownloadOutcome::Ready(ordered))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Backend-recommended read-ahead depth for multi-chunk drains
|
||||||
|
/// (see `DedupService::read_prefetch`).
|
||||||
|
pub fn read_prefetch(&self) -> usize {
|
||||||
|
self.dedup.read_prefetch()
|
||||||
|
}
|
||||||
|
|
||||||
/// Stream one authorized chunk's bytes (entitlement was established by
|
/// Stream one authorized chunk's bytes (entitlement was established by
|
||||||
/// [`authorize_chunk_download_with_perms`]).
|
/// [`authorize_chunk_download_with_perms`]).
|
||||||
pub async fn chunk_stream(
|
pub async fn chunk_stream(
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use crate::application::dtos::file_dto::FileDto;
|
use crate::application::dtos::file_dto::FileDto;
|
||||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||||
use crate::application::ports::file_ports::{FileRetrievalUseCase, OptimizedFileContent};
|
use crate::application::ports::file_ports::{
|
||||||
|
FileRetrievalUseCase, OptimizedFileContent, RangeContent,
|
||||||
|
};
|
||||||
use crate::application::ports::resource_access_hook::ResourceAccessHook;
|
use crate::application::ports::resource_access_hook::ResourceAccessHook;
|
||||||
use crate::application::ports::storage_ports::FileReadPort;
|
use crate::application::ports::storage_ports::FileReadPort;
|
||||||
use crate::common::errors::DomainError;
|
use crate::common::errors::DomainError;
|
||||||
@@ -284,6 +286,69 @@ impl FileRetrievalService {
|
|||||||
let files = self.file_read.get_files_by_ids(ids).await?;
|
let files = self.file_read.get_files_by_ids(ids).await?;
|
||||||
Ok(files.into_iter().map(FileDto::from).collect())
|
Ok(files.into_iter().map(FileDto::from).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Range read that first consults the RAM content cache (see
|
||||||
|
/// [`Self::get_file_range_preloaded`]).
|
||||||
|
pub async fn get_file_range_preloaded_with_perms(
|
||||||
|
&self,
|
||||||
|
dto: &FileDto,
|
||||||
|
caller_id: Uuid,
|
||||||
|
start: u64,
|
||||||
|
end: Option<u64>,
|
||||||
|
) -> Result<RangeContent, DomainError> {
|
||||||
|
self.require_file(&dto.id, Permission::Read, caller_id)
|
||||||
|
.await?;
|
||||||
|
// Same throttled Recent recording as the streaming variant.
|
||||||
|
self.notify_file_accessed(caller_id, &dto.id);
|
||||||
|
self.get_file_range_preloaded(dto, start, end).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Range read for HTTP Range Requests, cache-aware.
|
||||||
|
///
|
||||||
|
/// Media players and PDF viewers fetch these files *exclusively* through
|
||||||
|
/// Range requests (a `bytes=0-` probe, then seeks) — the plain streaming
|
||||||
|
/// path paid 1 PG round-trip (blob-hash resolve) + a chunk open/seek for
|
||||||
|
/// EVERY seek, even when the whole blob was already sitting in the moka
|
||||||
|
/// content cache as one contiguous `Bytes`. For sub-`CACHE_THRESHOLD`
|
||||||
|
/// files this now answers from the cache: `Bytes::slice` is a refcount
|
||||||
|
/// bump — zero copy, zero I/O, zero PG (benches/RANGE-CACHE.md). A miss
|
||||||
|
/// populates the cache via the same single-flight `get_or_load` Tier 1
|
||||||
|
/// uses, so one probe warms every subsequent seek. `end` is exclusive
|
||||||
|
/// (callers pass `Some(last_byte + 1)`), matching the streaming variant.
|
||||||
|
pub async fn get_file_range_preloaded(
|
||||||
|
&self,
|
||||||
|
dto: &FileDto,
|
||||||
|
start: u64,
|
||||||
|
end: Option<u64>,
|
||||||
|
) -> Result<RangeContent, DomainError> {
|
||||||
|
let cacheable = dto.size < CACHE_THRESHOLD && !dto.content_hash.is_empty();
|
||||||
|
if cacheable && let Some(cache) = &self.content_cache {
|
||||||
|
let etag: Arc<str> = format!("\"{}\"", dto.content_hash).into();
|
||||||
|
let ct: Arc<str> = dto.mime_type.clone();
|
||||||
|
let file_read = Arc::clone(&self.file_read);
|
||||||
|
let id_owned = dto.id.clone();
|
||||||
|
let cap = dto.size as usize;
|
||||||
|
let (bytes, _etag, _ct) = cache
|
||||||
|
.get_or_load(dto.content_hash.to_string(), etag, ct, async move {
|
||||||
|
debug!("💾 Range cache MISS: {} – loading from disk", id_owned);
|
||||||
|
Self::read_full(&file_read, &id_owned, cap).await
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
let len = bytes.len() as u64;
|
||||||
|
let s = start.min(len) as usize;
|
||||||
|
let e = end.unwrap_or(len).min(len) as usize;
|
||||||
|
if s <= e {
|
||||||
|
return Ok(RangeContent::Bytes(bytes.slice(s..e)));
|
||||||
|
}
|
||||||
|
// Degenerate range the validator should have rejected — fall
|
||||||
|
// through to the streaming path rather than panic on slice.
|
||||||
|
}
|
||||||
|
let stream = self
|
||||||
|
.file_read
|
||||||
|
.get_file_range_stream(&dto.id, start, end)
|
||||||
|
.await?;
|
||||||
|
Ok(RangeContent::Stream(stream))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FileRetrievalUseCase for FileRetrievalService {
|
impl FileRetrievalUseCase for FileRetrievalService {
|
||||||
@@ -451,12 +516,12 @@ impl FileRetrievalUseCase for FileRetrievalService {
|
|||||||
async fn list_files_batch(
|
async fn list_files_batch(
|
||||||
&self,
|
&self,
|
||||||
folder_id: Option<&str>,
|
folder_id: Option<&str>,
|
||||||
offset: i64,
|
after_name: Option<&str>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<Vec<FileDto>, DomainError> {
|
) -> Result<Vec<FileDto>, DomainError> {
|
||||||
let files = self
|
let files = self
|
||||||
.file_read
|
.file_read
|
||||||
.list_files_batch(folder_id, offset, limit)
|
.list_files_batch(folder_id, after_name, limit)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(files.into_iter().map(FileDto::from).collect())
|
Ok(files.into_iter().map(FileDto::from).collect())
|
||||||
}
|
}
|
||||||
@@ -465,7 +530,7 @@ impl FileRetrievalUseCase for FileRetrievalService {
|
|||||||
&self,
|
&self,
|
||||||
folder_id: Option<&str>,
|
folder_id: Option<&str>,
|
||||||
owner_id: Uuid,
|
owner_id: Uuid,
|
||||||
offset: i64,
|
after_name: Option<&str>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<Vec<FileDto>, DomainError> {
|
) -> Result<Vec<FileDto>, DomainError> {
|
||||||
// Post-D0: every file lives in a folder — `storage.files.folder_id`
|
// Post-D0: every file lives in a folder — `storage.files.folder_id`
|
||||||
@@ -482,7 +547,7 @@ impl FileRetrievalUseCase for FileRetrievalService {
|
|||||||
.await?;
|
.await?;
|
||||||
let files = self
|
let files = self
|
||||||
.file_read
|
.file_read
|
||||||
.list_files_batch(folder_id, offset, limit)
|
.list_files_batch(folder_id, after_name, limit)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(files.into_iter().map(FileDto::from).collect())
|
Ok(files.into_iter().map(FileDto::from).collect())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -166,18 +166,23 @@ impl PeopleService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// People (non-empty clusters), most-photographed first.
|
/// People (non-empty clusters), most-photographed first.
|
||||||
|
///
|
||||||
|
/// Counts come from a grouped-COUNT query and cover photos from one
|
||||||
|
/// batched lookup of just the cover face ids — the previous
|
||||||
|
/// `faces_for_user` shipped every face row (2 KiB embedding included)
|
||||||
|
/// only to count them: ~20 MB of BYTEA per request on a 10k-face
|
||||||
|
/// library (benches/PEOPLE-LIST.md).
|
||||||
pub async fn list_people(&self, caller_id: Uuid) -> Result<Vec<PersonDto>, DomainError> {
|
pub async fn list_people(&self, caller_id: Uuid) -> Result<Vec<PersonDto>, DomainError> {
|
||||||
let persons = self.repo.persons_for_user(caller_id).await?;
|
let persons = self.repo.persons_for_user(caller_id).await?;
|
||||||
let faces = self.repo.faces_for_user(caller_id).await?;
|
let count: HashMap<Uuid, i64> = self
|
||||||
|
.repo
|
||||||
let mut count: HashMap<Uuid, i64> = HashMap::new();
|
.person_face_stats(caller_id)
|
||||||
let mut face_file: HashMap<Uuid, Uuid> = HashMap::new();
|
.await?
|
||||||
for f in &faces {
|
.into_iter()
|
||||||
if let Some(pid) = f.person_id {
|
.collect();
|
||||||
*count.entry(pid).or_default() += 1;
|
let cover_ids: Vec<Uuid> = persons.iter().filter_map(|p| p.cover_face_id).collect();
|
||||||
}
|
let face_file: HashMap<Uuid, Uuid> =
|
||||||
face_file.insert(f.id, f.file_id);
|
self.repo.file_ids_for_faces(caller_id, &cover_ids).await?;
|
||||||
}
|
|
||||||
|
|
||||||
let mut out: Vec<PersonDto> = persons
|
let mut out: Vec<PersonDto> = persons
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -245,11 +250,13 @@ impl PeopleService {
|
|||||||
|
|
||||||
/// Merge `from` into `into` by reassigning all of `from`'s faces. The
|
/// Merge `from` into `into` by reassigning all of `from`'s faces. The
|
||||||
/// now-empty `from` person is hidden by `list_people`.
|
/// now-empty `from` person is hidden by `list_people`.
|
||||||
|
///
|
||||||
|
/// One set-based UPDATE — the previous shape loaded every face row
|
||||||
|
/// (embeddings included) and issued one UPDATE per matching face.
|
||||||
pub async fn merge(&self, caller_id: Uuid, into: Uuid, from: Uuid) -> Result<(), DomainError> {
|
pub async fn merge(&self, caller_id: Uuid, into: Uuid, from: Uuid) -> Result<(), DomainError> {
|
||||||
let faces = self.repo.faces_for_user(caller_id).await?;
|
self.repo
|
||||||
for f in faces.into_iter().filter(|f| f.person_id == Some(from)) {
|
.reassign_person_faces(caller_id, from, into)
|
||||||
self.repo.assign_person(f.id, Some(into)).await?;
|
.await?;
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -260,7 +260,7 @@ impl SearchService {
|
|||||||
user_id: Uuid,
|
user_id: Uuid,
|
||||||
) -> Vec<ContentHitDto> {
|
) -> Vec<ContentHitDto> {
|
||||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||||
use crate::domain::services::authorization::{Permission, Resource, Subject};
|
use crate::domain::services::authorization::Subject;
|
||||||
|
|
||||||
let Some(index) = &self.content_index else {
|
let Some(index) = &self.content_index else {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -316,36 +316,42 @@ impl SearchService {
|
|||||||
// drive the caller doesn't otherwise have. The Tantivy
|
// drive the caller doesn't otherwise have. The Tantivy
|
||||||
// filter is drive-only; this re-check restores per-file
|
// filter is drive-only; this re-check restores per-file
|
||||||
// resolution.
|
// resolution.
|
||||||
// Failures degrade conservatively (drop the hit, log it) —
|
// Failures degrade conservatively (drop the hit / the page,
|
||||||
// never leak.
|
// log it) — never leak. Batched: one drive-resolution query for
|
||||||
let mut verified = Vec::with_capacity(hits.len());
|
// the whole page instead of up to CONTENT_HITS_LIMIT sequential
|
||||||
for hit in hits {
|
// point SELECTs (benches/SEARCH-REBAC.md).
|
||||||
let file_uuid = match Uuid::parse_str(&hit.file_id) {
|
let mut hit_ids = Vec::with_capacity(hits.len());
|
||||||
Ok(u) => u,
|
for hit in &hits {
|
||||||
|
match Uuid::parse_str(&hit.file_id) {
|
||||||
|
Ok(u) => hit_ids.push(u),
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
tracing::warn!("Content-index hit had non-UUID file_id: {}", hit.file_id);
|
tracing::warn!("Content-index hit had non-UUID file_id: {}", hit.file_id);
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let allowed = match authz
|
||||||
|
.check_files_read_batch(Subject::User(user_id), &hit_ids)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(set) => set,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("ReBAC re-check failed for content hits: {e}");
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut verified = Vec::with_capacity(hits.len());
|
||||||
|
for hit in hits {
|
||||||
|
let Ok(file_uuid) = Uuid::parse_str(&hit.file_id) else {
|
||||||
|
continue; // already warned above
|
||||||
};
|
};
|
||||||
match authz
|
if allowed.contains(&file_uuid) {
|
||||||
.check(
|
verified.push(hit);
|
||||||
Subject::User(user_id),
|
} else {
|
||||||
Permission::Read,
|
tracing::debug!(
|
||||||
Resource::File(file_uuid),
|
target: "oxicloud::search",
|
||||||
)
|
file_id = %file_uuid,
|
||||||
.await
|
"dropping content-index hit: ReBAC denies Read after Tantivy filter",
|
||||||
{
|
);
|
||||||
Ok(true) => verified.push(hit),
|
|
||||||
Ok(false) => {
|
|
||||||
tracing::debug!(
|
|
||||||
target: "oxicloud::search",
|
|
||||||
file_id = %file_uuid,
|
|
||||||
"dropping content-index hit: ReBAC denies Read after Tantivy filter",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!("ReBAC re-check failed for {file_uuid}: {e}");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
verified
|
verified
|
||||||
|
|||||||
@@ -510,29 +510,18 @@ impl ShareUseCase for ShareService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn register_shared_link_access(&self, token: &str) -> Result<(), DomainError> {
|
async fn register_shared_link_access(&self, token: &str) -> Result<(), DomainError> {
|
||||||
// Find the shared link by its token
|
// One atomic UPDATE (see `ShareStoragePort::increment_access_count`).
|
||||||
let share = self
|
// 0 rows = missing or expired — collapsed into NotFound, same
|
||||||
.share_repository
|
// response shape either way (anti-enumeration; the landing handler
|
||||||
.find_share_by_token(token)
|
// discards this result regardless).
|
||||||
.await
|
let updated = self.share_repository.increment_access_count(token).await?;
|
||||||
.map_err(|e| {
|
if updated == 0 {
|
||||||
ShareServiceError::NotFound(format!("Share with token {} not found: {}", token, e))
|
return Err(ShareServiceError::NotFound(format!(
|
||||||
})?;
|
"Share with token {} not found or expired",
|
||||||
|
token
|
||||||
// Check if it has expired
|
))
|
||||||
if share.is_expired() {
|
.into());
|
||||||
return Err(ShareServiceError::Expired.into());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Increment the access counter
|
|
||||||
let updated_share = share.increment_access_count();
|
|
||||||
|
|
||||||
// Save the changes
|
|
||||||
self.share_repository
|
|
||||||
.update_share(&updated_share)
|
|
||||||
.await
|
|
||||||
.map_err(|e| ShareServiceError::Repository(e.to_string()))?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
use crate::application::ports::auth_ports::UserStoragePort;
|
|
||||||
use crate::application::ports::storage_ports::StorageUsagePort;
|
use crate::application::ports::storage_ports::StorageUsagePort;
|
||||||
use crate::common::errors::DomainError;
|
use crate::common::errors::DomainError;
|
||||||
use crate::infrastructure::repositories::pg::UserPgRepository;
|
use crate::infrastructure::repositories::pg::UserPgRepository;
|
||||||
@@ -512,9 +511,9 @@ impl StorageUsagePort for StorageUsageService {
|
|||||||
user_id: Uuid,
|
user_id: Uuid,
|
||||||
additional_bytes: u64,
|
additional_bytes: u64,
|
||||||
) -> Result<(), DomainError> {
|
) -> Result<(), DomainError> {
|
||||||
let user = self.user_repository.get_user_by_id(user_id).await?;
|
// Narrow 2-column read — the full user row carries the up-to-512 KiB
|
||||||
let quota = user.storage_quota_bytes();
|
// avatar `image` column, paid on every upload quota check otherwise.
|
||||||
let used = user.storage_used_bytes();
|
let (used, quota) = self.user_repository.get_storage_usage(user_id).await?;
|
||||||
|
|
||||||
// Quota of 0 means unlimited
|
// Quota of 0 means unlimited
|
||||||
if quota <= 0 {
|
if quota <= 0 {
|
||||||
@@ -548,8 +547,9 @@ impl StorageUsagePort for StorageUsageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn get_user_storage_info(&self, user_id: Uuid) -> Result<(i64, i64), DomainError> {
|
async fn get_user_storage_info(&self, user_id: Uuid) -> Result<(i64, i64), DomainError> {
|
||||||
let user = self.user_repository.get_user_by_id(user_id).await?;
|
// Narrow 2-column read (avatar-free) — runs on every folder PROPFIND
|
||||||
Ok((user.storage_used_bytes(), user.storage_quota_bytes()))
|
// that reports quota. See benches/QUOTA-PATH.md.
|
||||||
|
Ok(self.user_repository.get_storage_usage(user_id).await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn add_drive_storage_usage_delta(
|
async fn add_drive_storage_usage_delta(
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use crate::application::dtos::trash_dto::{
|
|||||||
};
|
};
|
||||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||||
use crate::application::ports::file_lifecycle::FileLifecycleHook;
|
use crate::application::ports::file_lifecycle::FileLifecycleHook;
|
||||||
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
|
use crate::application::ports::storage_ports::FileWritePort;
|
||||||
use crate::application::ports::trash_ports::TrashUseCase;
|
use crate::application::ports::trash_ports::TrashUseCase;
|
||||||
use crate::common::errors::{DomainError, ErrorKind, Result};
|
use crate::common::errors::{DomainError, ErrorKind, Result};
|
||||||
use crate::domain::entities::file::File;
|
use crate::domain::entities::file::File;
|
||||||
@@ -24,7 +24,6 @@ use crate::domain::repositories::folder_repository::FolderRepository;
|
|||||||
use crate::domain::repositories::trash_repository::TrashRepository;
|
use crate::domain::repositories::trash_repository::TrashRepository;
|
||||||
use crate::domain::services::authorization::ResourceKind;
|
use crate::domain::services::authorization::ResourceKind;
|
||||||
use crate::domain::services::authorization::{Permission, Resource, Subject};
|
use crate::domain::services::authorization::{Permission, Resource, Subject};
|
||||||
use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlobReadRepository;
|
|
||||||
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
||||||
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
||||||
use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository;
|
use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository;
|
||||||
@@ -49,9 +48,6 @@ pub struct TrashService {
|
|||||||
/// Repository for trash-specific operations like listing and retrieving trashed items
|
/// Repository for trash-specific operations like listing and retrieving trashed items
|
||||||
trash_repository: Arc<TrashDbRepository>,
|
trash_repository: Arc<TrashDbRepository>,
|
||||||
|
|
||||||
/// Port for file read operations (get file metadata)
|
|
||||||
file_read_port: Arc<FileBlobReadRepository>,
|
|
||||||
|
|
||||||
/// Port for file write operations (trash, restore, delete)
|
/// Port for file write operations (trash, restore, delete)
|
||||||
file_write_port: Arc<FileBlobWriteRepository>,
|
file_write_port: Arc<FileBlobWriteRepository>,
|
||||||
|
|
||||||
@@ -75,19 +71,14 @@ pub struct TrashService {
|
|||||||
/// so trash listings filter by drive membership instead of the legacy
|
/// so trash listings filter by drive membership instead of the legacy
|
||||||
/// per-user scope.
|
/// per-user scope.
|
||||||
drive_repo: Arc<crate::infrastructure::repositories::pg::DrivePgRepository>,
|
drive_repo: Arc<crate::infrastructure::repositories::pg::DrivePgRepository>,
|
||||||
|
|
||||||
/// Number of days items should be kept in trash before automatic cleanup
|
|
||||||
retention_days: u32,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TrashService {
|
impl TrashService {
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn new(
|
pub fn new(
|
||||||
trash_repository: Arc<TrashDbRepository>,
|
trash_repository: Arc<TrashDbRepository>,
|
||||||
file_read_port: Arc<FileBlobReadRepository>,
|
|
||||||
file_write_port: Arc<FileBlobWriteRepository>,
|
file_write_port: Arc<FileBlobWriteRepository>,
|
||||||
folder_storage_port: Arc<FolderDbRepository>,
|
folder_storage_port: Arc<FolderDbRepository>,
|
||||||
retention_days: u32,
|
|
||||||
dedup_service: Arc<DedupService>,
|
dedup_service: Arc<DedupService>,
|
||||||
content_cache: Option<Arc<FileContentCache>>,
|
content_cache: Option<Arc<FileContentCache>>,
|
||||||
authz: Arc<PgAclEngine>,
|
authz: Arc<PgAclEngine>,
|
||||||
@@ -95,7 +86,6 @@ impl TrashService {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
trash_repository,
|
trash_repository,
|
||||||
file_read_port,
|
|
||||||
file_write_port,
|
file_write_port,
|
||||||
folder_storage_port,
|
folder_storage_port,
|
||||||
dedup_service,
|
dedup_service,
|
||||||
@@ -103,7 +93,6 @@ impl TrashService {
|
|||||||
content_cache,
|
content_cache,
|
||||||
authz,
|
authz,
|
||||||
drive_repo,
|
drive_repo,
|
||||||
retention_days,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,23 +166,17 @@ impl TrashUseCase for TrashService {
|
|||||||
// Note: We now verify file/folder ownership BEFORE moving to trash.
|
// Note: We now verify file/folder ownership BEFORE moving to trash.
|
||||||
// This prevents users from trashing items they do not own (IDOR).
|
// This prevents users from trashing items they do not own (IDOR).
|
||||||
|
|
||||||
// Parse UUIDs with detailed error handling
|
// Parse UUIDs with detailed error handling. The parsed value is
|
||||||
|
// re-derived per branch below; this early check preserves the 400
|
||||||
|
// (validation) error shape for malformed ids.
|
||||||
debug!("Validating item UUID: {}", item_id);
|
debug!("Validating item UUID: {}", item_id);
|
||||||
let item_uuid = match Uuid::parse_str(item_id) {
|
if let Err(e) = Uuid::parse_str(item_id) {
|
||||||
Ok(uuid) => {
|
error!("Invalid item UUID: {} - Error: {}", item_id, e);
|
||||||
debug!("Valid item UUID: {}", uuid);
|
return Err(DomainError::validation_error(format!(
|
||||||
uuid
|
"Invalid item ID: {}",
|
||||||
}
|
e
|
||||||
Err(e) => {
|
)));
|
||||||
error!("Invalid item UUID: {} - Error: {}", item_id, e);
|
}
|
||||||
return Err(DomainError::validation_error(format!(
|
|
||||||
"Invalid item ID: {}",
|
|
||||||
e
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let user_uuid = user_id;
|
|
||||||
|
|
||||||
match item_type {
|
match item_type {
|
||||||
"file" => {
|
"file" => {
|
||||||
@@ -209,59 +192,13 @@ impl TrashUseCase for TrashService {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Authz already passed — use the non-owner-scoped read so that
|
// Soft-delete model: the is_trashed flag on the row IS the
|
||||||
// grantees with Delete permission can trash files they don't own.
|
// trash membership — there is no separate trash index to
|
||||||
// The file's user_id in storage.files is unchanged, so the item
|
// register into (`TrashRepository::add_to_trash` is a
|
||||||
// will appear in the original owner's trash view.
|
// documented no-op). The previous shape still fetched the
|
||||||
let file = match self.file_read_port.get_file(item_id).await {
|
// full file entity and built a `TrashedItem` only to feed
|
||||||
Ok(file) => {
|
// that no-op: one wasted SELECT per trash operation.
|
||||||
debug!("File found: {} ({})", file.name(), item_id);
|
//
|
||||||
file
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error getting file: {} - {}", item_id, e);
|
|
||||||
return Err(DomainError::new(
|
|
||||||
ErrorKind::NotFound,
|
|
||||||
"File",
|
|
||||||
format!("Error retrieving file {}: {}", item_id, e),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let original_path = file.storage_path().to_string();
|
|
||||||
debug!("Original file path: {}", original_path);
|
|
||||||
|
|
||||||
debug!("Creating TrashedItem object for the file");
|
|
||||||
let trashed_item = TrashedItem::new(
|
|
||||||
item_uuid,
|
|
||||||
user_uuid,
|
|
||||||
TrashedItemType::File,
|
|
||||||
file.name().to_string(),
|
|
||||||
original_path,
|
|
||||||
self.retention_days,
|
|
||||||
);
|
|
||||||
debug!(
|
|
||||||
"TrashedItem created successfully: {} -> {}",
|
|
||||||
file.name(),
|
|
||||||
trashed_item.id()
|
|
||||||
);
|
|
||||||
|
|
||||||
// First add to trash index to register the item
|
|
||||||
info!("Adding file {} to trash index", item_id);
|
|
||||||
match self.trash_repository.add_to_trash(&trashed_item).await {
|
|
||||||
Ok(_) => {
|
|
||||||
debug!("File added to trash index successfully");
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error adding file to trash index: {}", e);
|
|
||||||
return Err(DomainError::internal_error(
|
|
||||||
"TrashRepository",
|
|
||||||
format!("Failed to add file to trash: {}", e),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Then physically move the file to trash.
|
|
||||||
// §14: caller_id stamps `updated_by` on the trashed row.
|
// §14: caller_id stamps `updated_by` on the trashed row.
|
||||||
info!("Physically moving file to trash: {}", item_id);
|
info!("Physically moving file to trash: {}", item_id);
|
||||||
match self.file_write_port.move_to_trash(item_id, user_id).await {
|
match self.file_write_port.move_to_trash(item_id, user_id).await {
|
||||||
@@ -293,43 +230,10 @@ impl TrashUseCase for TrashService {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let folder = self
|
// Soft-delete model — same as the file branch above: the
|
||||||
.folder_storage_port
|
// cascade UPDATE below is the whole operation; no folder
|
||||||
.get_folder(item_id)
|
// fetch or trash-index write needed.
|
||||||
.await
|
//
|
||||||
.map_err(|e| {
|
|
||||||
DomainError::new(
|
|
||||||
ErrorKind::NotFound,
|
|
||||||
"Folder",
|
|
||||||
format!("Error retrieving folder {}: {}", item_id, e),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let original_path = folder.storage_path().to_string();
|
|
||||||
|
|
||||||
let trashed_item = TrashedItem::new(
|
|
||||||
item_uuid,
|
|
||||||
user_uuid,
|
|
||||||
TrashedItemType::Folder,
|
|
||||||
folder.name().to_string(),
|
|
||||||
original_path,
|
|
||||||
self.retention_days,
|
|
||||||
);
|
|
||||||
|
|
||||||
// First add to trash index to register the item
|
|
||||||
debug!("Adding folder {} to trash repository", item_id);
|
|
||||||
match self.trash_repository.add_to_trash(&trashed_item).await {
|
|
||||||
Ok(_) => debug!("Successfully added folder to trash repository"),
|
|
||||||
Err(e) => {
|
|
||||||
error!("Failed to add folder to trash repository: {}", e);
|
|
||||||
return Err(DomainError::internal_error(
|
|
||||||
"TrashRepository",
|
|
||||||
format!("Failed to add folder to trash: {}", e),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Then physically move the folder to trash.
|
|
||||||
// §14: caller_id stamps `updated_by` on every cascade-trashed row.
|
// §14: caller_id stamps `updated_by` on every cascade-trashed row.
|
||||||
self.folder_storage_port
|
self.folder_storage_port
|
||||||
.move_to_trash(item_id, user_id)
|
.move_to_trash(item_id, user_id)
|
||||||
|
|||||||
@@ -836,10 +836,8 @@ impl AppServiceFactory {
|
|||||||
let service = Arc::new(
|
let service = Arc::new(
|
||||||
TrashService::new(
|
TrashService::new(
|
||||||
trash_repo.clone(),
|
trash_repo.clone(),
|
||||||
repos.file_read_repository.clone(),
|
|
||||||
repos.file_write_repository.clone(),
|
repos.file_write_repository.clone(),
|
||||||
repos.folder_repository.clone(),
|
repos.folder_repository.clone(),
|
||||||
self.config.storage.trash_retention_days,
|
|
||||||
core.dedup_service.clone(),
|
core.dedup_service.clone(),
|
||||||
Some(core.file_content_cache.clone()),
|
Some(core.file_content_cache.clone()),
|
||||||
authz.clone(),
|
authz.clone(),
|
||||||
|
|||||||
@@ -108,10 +108,117 @@ pub async fn refine_content_type_from_file(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a MIME type identifies content that is already compressed, so
|
||||||
|
/// running Deflate over it burns CPU for ~0 % size gain.
|
||||||
|
///
|
||||||
|
/// Used by the ZIP export paths (`ZipService`, `BatchOperations`) to pick
|
||||||
|
/// `Compression::Stored` per entry instead of deflating JPEG/MP4/… bytes.
|
||||||
|
/// The set mirrors the HTTP `CompressionLayer` exclusion list in `main.rs`
|
||||||
|
/// (keep the two in sync), minus entries that are containers of possibly
|
||||||
|
/// incompressible data rather than compressed formats themselves
|
||||||
|
/// (`application/x-tar`, `application/octet-stream`) — those stay on Deflate
|
||||||
|
/// so unknown-but-compressible content is never stored uncompressed.
|
||||||
|
pub fn is_precompressed_mime(mime: &str) -> bool {
|
||||||
|
// Strip any parameters ("; charset=…") and normalize case.
|
||||||
|
let essence = mime.split(';').next().unwrap_or(mime).trim();
|
||||||
|
|
||||||
|
// Compressed families: every common video/audio codec container.
|
||||||
|
if essence.starts_with("video/") || essence.starts_with("audio/") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Zip-based document bundles (docx/xlsx/pptx, odt/ods/odp, …).
|
||||||
|
if essence.starts_with("application/vnd.openxmlformats-officedocument")
|
||||||
|
|| essence.starts_with("application/vnd.oasis.opendocument")
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
matches!(
|
||||||
|
essence,
|
||||||
|
// Raster images with built-in compression (SVG intentionally absent).
|
||||||
|
"image/jpeg"
|
||||||
|
| "image/png"
|
||||||
|
| "image/gif"
|
||||||
|
| "image/webp"
|
||||||
|
| "image/avif"
|
||||||
|
| "image/heic"
|
||||||
|
| "image/heif"
|
||||||
|
| "image/jp2"
|
||||||
|
// Already-compressed web fonts; ttf/otf left compressible.
|
||||||
|
| "font/woff"
|
||||||
|
| "font/woff2"
|
||||||
|
| "application/font-woff"
|
||||||
|
// Archives & compressed containers.
|
||||||
|
| "application/zip"
|
||||||
|
| "application/gzip"
|
||||||
|
| "application/x-gzip"
|
||||||
|
| "application/x-7z-compressed"
|
||||||
|
| "application/x-rar-compressed"
|
||||||
|
| "application/x-bzip2"
|
||||||
|
| "application/zstd"
|
||||||
|
| "application/x-xz"
|
||||||
|
| "application/epub+zip"
|
||||||
|
| "application/java-archive"
|
||||||
|
| "application/vnd.android.package-archive"
|
||||||
|
// PDF: internal streams are usually already deflated.
|
||||||
|
| "application/pdf"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ZIP entry compression for a file of the given MIME type: `Stored` for
|
||||||
|
/// already-compressed content, `Deflate` otherwise. Shared by every ZIP
|
||||||
|
/// export path (`ZipService`, `BatchOperations`).
|
||||||
|
pub fn zip_entry_compression(mime: &str) -> async_zip::Compression {
|
||||||
|
if is_precompressed_mime(mime) {
|
||||||
|
async_zip::Compression::Stored
|
||||||
|
} else {
|
||||||
|
async_zip::Compression::Deflate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// ── is_precompressed_mime ───────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_and_archives_are_precompressed() {
|
||||||
|
for mime in [
|
||||||
|
"image/jpeg",
|
||||||
|
"image/webp",
|
||||||
|
"video/mp4",
|
||||||
|
"video/quicktime",
|
||||||
|
"audio/mpeg",
|
||||||
|
"application/zip",
|
||||||
|
"application/pdf",
|
||||||
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
|
"font/woff2",
|
||||||
|
] {
|
||||||
|
assert!(is_precompressed_mime(mime), "{mime} should be Stored");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compressible_types_keep_deflate() {
|
||||||
|
for mime in [
|
||||||
|
"text/plain",
|
||||||
|
"text/html",
|
||||||
|
"application/json",
|
||||||
|
"image/svg+xml",
|
||||||
|
"application/x-tar",
|
||||||
|
"application/octet-stream",
|
||||||
|
"",
|
||||||
|
] {
|
||||||
|
assert!(!is_precompressed_mime(mime), "{mime} should stay Deflate");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mime_parameters_are_ignored() {
|
||||||
|
assert!(is_precompressed_mime("image/jpeg; charset=binary"));
|
||||||
|
}
|
||||||
|
|
||||||
// ── refine_content_type (sync) ──────────────────────────────
|
// ── refine_content_type (sync) ──────────────────────────────
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
//! infrastructure/services/path_service.rs because it has file system dependencies.
|
//! infrastructure/services/path_service.rs because it has file system dependencies.
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use unicode_normalization::UnicodeNormalization;
|
use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
|
||||||
|
|
||||||
/// NFC-normalize a single file or folder name component.
|
/// NFC-normalize a single file or folder name component.
|
||||||
///
|
///
|
||||||
@@ -25,7 +25,18 @@ use unicode_normalization::UnicodeNormalization;
|
|||||||
/// (`migrate-nfc-filenames`) cleans up rows that pre-date this rule.
|
/// (`migrate-nfc-filenames`) cleans up rows that pre-date this rule.
|
||||||
///
|
///
|
||||||
/// Pure function — no I/O, allocates one `String`.
|
/// Pure function — no I/O, allocates one `String`.
|
||||||
|
///
|
||||||
|
/// Fast path: `is_nfc_quick` is a per-char table lookup that answers
|
||||||
|
/// `Yes` for virtually every name already in NFC — which is every name
|
||||||
|
/// loaded back from PostgreSQL (the DB invariant above) and every
|
||||||
|
/// ASCII name. That skips the full decompose/recompose state machine
|
||||||
|
/// this function otherwise runs once per row on every listing
|
||||||
|
/// (PROPFIND, folder listing, photos timeline). `Maybe`/`No` fall
|
||||||
|
/// through to the full pipeline.
|
||||||
pub fn normalize_storage_name(name: &str) -> String {
|
pub fn normalize_storage_name(name: &str) -> String {
|
||||||
|
if is_nfc_quick(name.chars()) == IsNormalized::Yes {
|
||||||
|
return name.to_string();
|
||||||
|
}
|
||||||
name.nfc().collect()
|
name.nfc().collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,9 @@
|
|||||||
//! schema and `docs/plan/drive.md` §3 / §15 for the locked design.
|
//! schema and `docs/plan/drive.md` §3 / §15 for the locked design.
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use moka::future::Cache;
|
||||||
use sqlx::{PgPool, Row, types::Uuid};
|
use sqlx::{PgPool, Row, types::Uuid};
|
||||||
|
|
||||||
use crate::domain::entities::drive::{Drive, DriveKind};
|
use crate::domain::entities::drive::{Drive, DriveKind};
|
||||||
@@ -18,13 +20,38 @@ use crate::domain::repositories::drive_repository::{
|
|||||||
DriveRepository, DriveRepositoryError, DriveWithRootName,
|
DriveRepository, DriveRepositoryError, DriveWithRootName,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// `default_drive_cache` TTL. The default-drive → root-folder binding is
|
||||||
|
/// nearly immutable (changes only on provisioning / drive deletion /
|
||||||
|
/// policy edits — all of which invalidate explicitly below), yet it is
|
||||||
|
/// re-resolved on EVERY NextCloud request (basic-auth chroot), every
|
||||||
|
/// native `/webdav` request (Mode-B scope resolution) and every WOPI
|
||||||
|
/// call. 30 s mirrors `drive_role_cache` in `pg_acl_engine.rs` and bounds
|
||||||
|
/// the one non-invalidated staleness source: a root-folder *rename*,
|
||||||
|
/// which doesn't pass through this repository. Measured in
|
||||||
|
/// `benches/CHROOT-CACHE.md`.
|
||||||
|
const DEFAULT_DRIVE_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// One entry per active user; entries are small (a `Drive` + a name).
|
||||||
|
const DEFAULT_DRIVE_CACHE_CAPACITY: u64 = 100_000;
|
||||||
|
|
||||||
pub struct DrivePgRepository {
|
pub struct DrivePgRepository {
|
||||||
pool: Arc<PgPool>,
|
pool: Arc<PgPool>,
|
||||||
|
/// user_id → default drive (+ root folder name). See
|
||||||
|
/// [`DEFAULT_DRIVE_CACHE_TTL`]. Only `Ok` results are cached, so the
|
||||||
|
/// provisioning idempotency check (`NotFound` → create) always sees
|
||||||
|
/// the live table.
|
||||||
|
default_drive_cache: Cache<Uuid, DriveWithRootName>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DrivePgRepository {
|
impl DrivePgRepository {
|
||||||
pub fn new(pool: Arc<PgPool>) -> Self {
|
pub fn new(pool: Arc<PgPool>) -> Self {
|
||||||
Self { pool }
|
Self {
|
||||||
|
pool,
|
||||||
|
default_drive_cache: Cache::builder()
|
||||||
|
.max_capacity(DEFAULT_DRIVE_CACHE_CAPACITY)
|
||||||
|
.time_to_live(DEFAULT_DRIVE_CACHE_TTL)
|
||||||
|
.build(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_sqlx_err(context: &'static str, e: sqlx::Error) -> DriveRepositoryError {
|
fn map_sqlx_err(context: &'static str, e: sqlx::Error) -> DriveRepositoryError {
|
||||||
@@ -209,6 +236,10 @@ impl DriveRepository for DrivePgRepository {
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| Self::map_sqlx_err("create_personal_drive_atomic.commit", e))?;
|
.map_err(|e| Self::map_sqlx_err("create_personal_drive_atomic.commit", e))?;
|
||||||
|
|
||||||
|
// Drop any cached default-drive resolution for this user (a stale
|
||||||
|
// NotFound is never cached, but be explicit about the write path).
|
||||||
|
self.default_drive_cache.invalidate(&owner_id).await;
|
||||||
|
|
||||||
Self::row_to_drive_with_name(&row)
|
Self::row_to_drive_with_name(&row)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,6 +425,10 @@ impl DriveRepository for DrivePgRepository {
|
|||||||
tx.commit()
|
tx.commit()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Self::map_sqlx_err("delete_atomic.commit", e))?;
|
.map_err(|e| Self::map_sqlx_err("delete_atomic.commit", e))?;
|
||||||
|
// We only have the drive id here; the cache is keyed by user.
|
||||||
|
// Deletion is rare — clearing the whole cache is the simple,
|
||||||
|
// always-correct move (repopulates at one query per active user).
|
||||||
|
self.default_drive_cache.invalidate_all();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -448,6 +483,10 @@ impl DriveRepository for DrivePgRepository {
|
|||||||
&self,
|
&self,
|
||||||
user_id: Uuid,
|
user_id: Uuid,
|
||||||
) -> Result<DriveWithRootName, DriveRepositoryError> {
|
) -> Result<DriveWithRootName, DriveRepositoryError> {
|
||||||
|
if let Some(cached) = self.default_drive_cache.get(&user_id).await {
|
||||||
|
return Ok(cached);
|
||||||
|
}
|
||||||
|
|
||||||
let row = sqlx::query(
|
let row = sqlx::query(
|
||||||
r#"
|
r#"
|
||||||
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
|
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
|
||||||
@@ -465,7 +504,9 @@ impl DriveRepository for DrivePgRepository {
|
|||||||
.map_err(|e| Self::map_sqlx_err("find_default_for_user", e))?
|
.map_err(|e| Self::map_sqlx_err("find_default_for_user", e))?
|
||||||
.ok_or_else(|| DriveRepositoryError::NotFound(user_id.to_string()))?;
|
.ok_or_else(|| DriveRepositoryError::NotFound(user_id.to_string()))?;
|
||||||
|
|
||||||
Self::row_to_drive_with_name(&row)
|
let dwr = Self::row_to_drive_with_name(&row)?;
|
||||||
|
self.default_drive_cache.insert(user_id, dwr.clone()).await;
|
||||||
|
Ok(dwr)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_readable_by(
|
async fn list_readable_by(
|
||||||
@@ -675,6 +716,10 @@ impl DriveRepository for DrivePgRepository {
|
|||||||
let raw = row
|
let raw = row
|
||||||
.ok_or_else(|| DriveRepositoryError::NotFound(drive_id.to_string()))?
|
.ok_or_else(|| DriveRepositoryError::NotFound(drive_id.to_string()))?
|
||||||
.0;
|
.0;
|
||||||
|
// Policy edits must not serve a stale `policies` bag from the
|
||||||
|
// default-drive cache (keyed by user, and we only have the drive
|
||||||
|
// id) — clear it; policy edits are admin-rare.
|
||||||
|
self.default_drive_cache.invalidate_all();
|
||||||
Ok(crate::domain::entities::drive::DrivePolicies::from_value(
|
Ok(crate::domain::entities::drive::DrivePolicies::from_value(
|
||||||
&raw,
|
&raw,
|
||||||
))
|
))
|
||||||
|
|||||||
@@ -186,6 +186,60 @@ impl FaceRepository for FacePgRepository {
|
|||||||
Ok(rows.into_iter().map(row_to_face).collect())
|
Ok(rows.into_iter().map(row_to_face).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn person_face_stats(&self, user_id: Uuid) -> Result<Vec<(Uuid, i64)>, DomainError> {
|
||||||
|
// Grouped COUNT — the People tab only needs per-person counts, so
|
||||||
|
// this replaces a full faces_for_user scan that shipped a 2 KiB
|
||||||
|
// embedding BYTEA per row (benches/PEOPLE-LIST.md).
|
||||||
|
let rows: Vec<(Uuid, i64)> = sqlx::query_as(
|
||||||
|
"SELECT person_id, COUNT(*) FROM faces.faces
|
||||||
|
WHERE user_id = $1 AND person_id IS NOT NULL
|
||||||
|
GROUP BY person_id",
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.fetch_all(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| db_err("person_face_stats", e))?;
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn file_ids_for_faces(
|
||||||
|
&self,
|
||||||
|
user_id: Uuid,
|
||||||
|
face_ids: &[Uuid],
|
||||||
|
) -> Result<std::collections::HashMap<Uuid, Uuid>, DomainError> {
|
||||||
|
if face_ids.is_empty() {
|
||||||
|
return Ok(std::collections::HashMap::new());
|
||||||
|
}
|
||||||
|
let rows: Vec<(Uuid, Uuid)> = sqlx::query_as(
|
||||||
|
"SELECT id, file_id FROM faces.faces WHERE user_id = $1 AND id = ANY($2)",
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(face_ids)
|
||||||
|
.fetch_all(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| db_err("file_ids_for_faces", e))?;
|
||||||
|
Ok(rows.into_iter().collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reassign_person_faces(
|
||||||
|
&self,
|
||||||
|
user_id: Uuid,
|
||||||
|
from: Uuid,
|
||||||
|
into: Uuid,
|
||||||
|
) -> Result<u64, DomainError> {
|
||||||
|
let result = sqlx::query(
|
||||||
|
"UPDATE faces.faces SET person_id = $3
|
||||||
|
WHERE user_id = $1 AND person_id = $2",
|
||||||
|
)
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(from)
|
||||||
|
.bind(into)
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| db_err("reassign_person_faces", e))?;
|
||||||
|
Ok(result.rows_affected())
|
||||||
|
}
|
||||||
|
|
||||||
async fn assign_person(
|
async fn assign_person(
|
||||||
&self,
|
&self,
|
||||||
face_id: Uuid,
|
face_id: Uuid,
|
||||||
|
|||||||
@@ -366,6 +366,30 @@ impl FileBlobReadRepository {
|
|||||||
.ok_or_else(|| DomainError::not_found("File", file_id))
|
.ok_or_else(|| DomainError::not_found("File", file_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Batched variant of [`Self::get_file_drive_id`]: one `= ANY($1)`
|
||||||
|
/// round-trip for a whole result page. Missing / unknown ids are simply
|
||||||
|
/// absent from the output (the single-id variant maps them to
|
||||||
|
/// `NotFound`). Used by `PgAclEngine::check_files_read_batch` — the
|
||||||
|
/// per-hit loop cost up to 200 sequential point SELECTs per content
|
||||||
|
/// search (benches/SEARCH-REBAC.md).
|
||||||
|
pub async fn get_file_drive_ids(
|
||||||
|
&self,
|
||||||
|
file_ids: &[uuid::Uuid],
|
||||||
|
) -> Result<Vec<(uuid::Uuid, uuid::Uuid)>, DomainError> {
|
||||||
|
if file_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
sqlx::query_as::<_, (uuid::Uuid, uuid::Uuid)>(
|
||||||
|
"SELECT id, drive_id FROM storage.files WHERE id = ANY($1)",
|
||||||
|
)
|
||||||
|
.bind(file_ids)
|
||||||
|
.fetch_all(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
DomainError::internal_error("FileBlobRead", format!("drive_id batch lookup: {e}"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Creates a stub instance for testing — never hits PG.
|
/// Creates a stub instance for testing — never hits PG.
|
||||||
/// Available in both standard unit-test (`cfg(test)`) and integration
|
/// Available in both standard unit-test (`cfg(test)`) and integration
|
||||||
/// (`cfg(integration_tests)`) builds; `PgAclEngine::new_stub` chains
|
/// (`cfg(integration_tests)`) builds; `PgAclEngine::new_stub` chains
|
||||||
@@ -494,7 +518,24 @@ impl FileBlobReadRepository {
|
|||||||
before: Option<i64>,
|
before: Option<i64>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<(Vec<File>, Vec<i64>, Vec<(Option<i32>, Option<i32>)>), DomainError> {
|
) -> Result<(Vec<File>, Vec<i64>, Vec<(Option<i32>, Option<i32>)>), DomainError> {
|
||||||
let rows: Vec<MediaFileRow> = sqlx::query_as(
|
// Sargable keyset cursor: compare the RAW `media_sort_date` column
|
||||||
|
// against a timestamptz bind so the planner can use the cursor as
|
||||||
|
// an index boundary condition on `idx_files_media_timeline_by_drive`.
|
||||||
|
// The old shape wrapped the column in `EXTRACT(EPOCH …)::bigint`
|
||||||
|
// (plus an `IS NULL OR` disjunction), which degraded the cursor to
|
||||||
|
// a per-row Filter: page k re-read and discarded all k·limit rows
|
||||||
|
// already scrolled past (benches/PHOTOS-CURSOR.md). Since `before`
|
||||||
|
// is whole seconds, `media_sort_date < to_timestamp(before)` admits
|
||||||
|
// exactly the same rows as the old truncated comparison. The
|
||||||
|
// predicate is emitted only when a cursor exists — a bound
|
||||||
|
// disjunction would block the index condition under generic plans.
|
||||||
|
let cursor_ts = before.and_then(|s| chrono::DateTime::from_timestamp(s, 0));
|
||||||
|
let cursor_pred = if cursor_ts.is_some() {
|
||||||
|
"AND fi.media_sort_date < $2"
|
||||||
|
} else {
|
||||||
|
"AND $2::timestamptz IS NULL"
|
||||||
|
};
|
||||||
|
let sql = format!(
|
||||||
r#"
|
r#"
|
||||||
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
|
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
|
||||||
fi.size, fi.mime_type,
|
fi.size, fi.mime_type,
|
||||||
@@ -524,18 +565,18 @@ impl FileBlobReadRepository {
|
|||||||
)
|
)
|
||||||
AND NOT fi.is_trashed
|
AND NOT fi.is_trashed
|
||||||
AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%')
|
AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%')
|
||||||
AND ($2::bigint IS NULL
|
{cursor_pred}
|
||||||
OR EXTRACT(EPOCH FROM fi.media_sort_date)::bigint < $2::bigint)
|
|
||||||
ORDER BY fi.media_sort_date DESC
|
ORDER BY fi.media_sort_date DESC
|
||||||
LIMIT $3
|
LIMIT $3
|
||||||
"#,
|
"#,
|
||||||
)
|
);
|
||||||
.bind(caller_id)
|
let rows: Vec<MediaFileRow> = sqlx::query_as(&sql)
|
||||||
.bind(before)
|
.bind(caller_id)
|
||||||
.bind(limit)
|
.bind(cursor_ts)
|
||||||
.fetch_all(self.pool.as_ref())
|
.bind(limit)
|
||||||
.await
|
.fetch_all(self.pool.as_ref())
|
||||||
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_media: {e}")))?;
|
.await
|
||||||
|
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_media: {e}")))?;
|
||||||
|
|
||||||
let mut files = Vec::with_capacity(rows.len());
|
let mut files = Vec::with_capacity(rows.len());
|
||||||
let mut sort_dates = Vec::with_capacity(rows.len());
|
let mut sort_dates = Vec::with_capacity(rows.len());
|
||||||
@@ -768,62 +809,57 @@ impl FileReadPort for FileBlobReadRepository {
|
|||||||
self.resolve_blob_hash(file_id).await
|
self.resolve_blob_hash(file_id).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Paginated file listing — fetches only `limit` rows starting at `offset`.
|
/// Keyset-paginated file listing in name order — fetches only `limit`
|
||||||
|
/// rows after `after_name` (exclusive).
|
||||||
///
|
///
|
||||||
/// Uses a single SQL query with `LIMIT/OFFSET` to avoid loading the full
|
/// Names are unique per folder, so `name > $after` is a total cursor.
|
||||||
/// folder contents into memory. Ideal for streaming WebDAV PROPFIND.
|
/// Served by `idx_files_folder_name (folder_id, name) WHERE NOT
|
||||||
|
/// is_trashed` as a pure index-range read: O(page) per page with no
|
||||||
|
/// sort, where the old `LIMIT/OFFSET` shape re-scanned and re-sorted
|
||||||
|
/// the entire folder for every page (benches/PROPFIND-PAGING.md). The
|
||||||
|
/// cursor predicate is emitted only when a cursor exists — a
|
||||||
|
/// `$2 IS NULL OR name > $2` disjunction would block the index
|
||||||
|
/// condition under the extended protocol's generic plans.
|
||||||
#[allow(clippy::type_complexity)]
|
#[allow(clippy::type_complexity)]
|
||||||
async fn list_files_batch(
|
async fn list_files_batch(
|
||||||
&self,
|
&self,
|
||||||
folder_id: Option<&str>,
|
folder_id: Option<&str>,
|
||||||
offset: i64,
|
after_name: Option<&str>,
|
||||||
limit: i64,
|
limit: i64,
|
||||||
) -> Result<Vec<File>, DomainError> {
|
) -> Result<Vec<File>, DomainError> {
|
||||||
let rows: Vec<FileRow> = if let Some(fid) = folder_id {
|
let folder_pred = if folder_id.is_some() {
|
||||||
sqlx::query_as(
|
"fi.folder_id = $1::uuid"
|
||||||
r#"
|
|
||||||
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
|
|
||||||
fi.size, fi.mime_type,
|
|
||||||
EXTRACT(EPOCH FROM fi.created_at)::bigint,
|
|
||||||
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
|
|
||||||
fi.blob_hash,
|
|
||||||
|
|
||||||
fi.created_by, fi.updated_by
|
|
||||||
FROM storage.files fi
|
|
||||||
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
|
|
||||||
WHERE fi.folder_id = $1::uuid AND NOT fi.is_trashed
|
|
||||||
ORDER BY fi.name
|
|
||||||
LIMIT $2 OFFSET $3
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(fid)
|
|
||||||
.bind(limit)
|
|
||||||
.bind(offset)
|
|
||||||
.fetch_all(self.pool.as_ref())
|
|
||||||
.await
|
|
||||||
} else {
|
} else {
|
||||||
sqlx::query_as(
|
"fi.folder_id IS NULL AND $1::uuid IS NULL"
|
||||||
r#"
|
};
|
||||||
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
|
let cursor_pred = if after_name.is_some() {
|
||||||
fi.size, fi.mime_type,
|
"AND fi.name > $3"
|
||||||
EXTRACT(EPOCH FROM fi.created_at)::bigint,
|
} else {
|
||||||
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
|
"AND $3::text IS NULL"
|
||||||
fi.blob_hash,
|
};
|
||||||
|
let sql = format!(
|
||||||
|
r#"
|
||||||
|
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
|
||||||
|
fi.size, fi.mime_type,
|
||||||
|
EXTRACT(EPOCH FROM fi.created_at)::bigint,
|
||||||
|
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
|
||||||
|
fi.blob_hash,
|
||||||
|
|
||||||
fi.created_by, fi.updated_by
|
fi.created_by, fi.updated_by
|
||||||
FROM storage.files fi
|
FROM storage.files fi
|
||||||
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
|
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
|
||||||
WHERE fi.folder_id IS NULL AND NOT fi.is_trashed
|
WHERE {folder_pred} AND NOT fi.is_trashed {cursor_pred}
|
||||||
ORDER BY fi.name
|
ORDER BY fi.name
|
||||||
LIMIT $1 OFFSET $2
|
LIMIT $2
|
||||||
"#,
|
"#,
|
||||||
)
|
);
|
||||||
|
let rows: Vec<FileRow> = sqlx::query_as(&sql)
|
||||||
|
.bind(folder_id)
|
||||||
.bind(limit)
|
.bind(limit)
|
||||||
.bind(offset)
|
.bind(after_name)
|
||||||
.fetch_all(self.pool.as_ref())
|
.fetch_all(self.pool.as_ref())
|
||||||
.await
|
.await
|
||||||
}
|
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_batch: {e}")))?;
|
||||||
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_batch: {e}")))?;
|
|
||||||
|
|
||||||
rows.into_iter()
|
rows.into_iter()
|
||||||
.map(
|
.map(
|
||||||
|
|||||||
@@ -122,6 +122,35 @@ impl ShareStoragePort for SharePgRepository {
|
|||||||
Self::row_to_entity(&row)
|
Self::row_to_entity(&row)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn increment_access_count(&self, token: &str) -> Result<u64, DomainError> {
|
||||||
|
// One atomic statement — the relative bump can't lose concurrent
|
||||||
|
// increments and never rewrites unrelated columns (the legacy
|
||||||
|
// read-modify-write wrote back item_name/password_hash wholesale,
|
||||||
|
// silently clobbering concurrent owner edits). The expiry guard
|
||||||
|
// mirrors find_share_by_token's MIN(expires_at) subquery: NULL =
|
||||||
|
// never expires.
|
||||||
|
let result = sqlx::query(
|
||||||
|
r#"
|
||||||
|
UPDATE storage.shares s
|
||||||
|
SET access_count = s.access_count + 1
|
||||||
|
WHERE s.token = $1
|
||||||
|
AND COALESCE(
|
||||||
|
(SELECT MIN(ag.expires_at)
|
||||||
|
FROM storage.role_grants ag
|
||||||
|
WHERE ag.subject_type = 'token' AND ag.subject_id = s.id) > NOW(),
|
||||||
|
TRUE)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(token)
|
||||||
|
.execute(&*self.db_pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
tracing::error!("Database error incrementing share access count: {}", e);
|
||||||
|
DomainError::internal_error("Share", format!("Failed to register access: {e}"))
|
||||||
|
})?;
|
||||||
|
Ok(result.rows_affected())
|
||||||
|
}
|
||||||
|
|
||||||
async fn find_share_by_token(&self, token: &str) -> Result<Share, DomainError> {
|
async fn find_share_by_token(&self, token: &str) -> Result<Share, DomainError> {
|
||||||
let row = sqlx::query(
|
let row = sqlx::query(
|
||||||
r#"
|
r#"
|
||||||
|
|||||||
@@ -85,6 +85,33 @@ impl UserPgRepository {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch only `(storage_used_bytes, storage_quota_bytes)`. Not part of
|
||||||
|
/// the `UserRepository` trait — called from `StorageUsageService`.
|
||||||
|
///
|
||||||
|
/// Same rationale as [`Self::get_user_flags`]: the full-row SELECT drags
|
||||||
|
/// `image` (a data URI of up to 512 KiB), `password_hash`,
|
||||||
|
/// `ui_preferences`, … across the wire, and the quota path runs on every
|
||||||
|
/// folder PROPFIND and every upload quota check just to read two i64s.
|
||||||
|
/// Measured in `benches/QUOTA-PATH.md`.
|
||||||
|
pub async fn get_storage_usage(&self, id: Uuid) -> UserRepositoryResult<(i64, i64)> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT storage_used_bytes, storage_quota_bytes
|
||||||
|
FROM auth.users
|
||||||
|
WHERE id = $1
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(&*self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(Self::map_sqlx_error)?;
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
row.get("storage_used_bytes"),
|
||||||
|
row.get("storage_quota_bytes"),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
/// Updates a user's profile image (URL or data URI). Not part of the
|
/// Updates a user's profile image (URL or data URI). Not part of the
|
||||||
/// `UserRepository` trait — called directly from `AuthApplicationService`.
|
/// `UserRepository` trait — called directly from `AuthApplicationService`.
|
||||||
pub async fn update_image(
|
pub async fn update_image(
|
||||||
|
|||||||
@@ -112,13 +112,33 @@ impl ChunkIngestOutcome {
|
|||||||
/// mid-stream — a client disconnect aborts the whole handler future — the
|
/// mid-stream — a client disconnect aborts the whole handler future — the
|
||||||
/// guard spawns a rollback so pinned chunks don't leak references forever and
|
/// guard spawns a rollback so pinned chunks don't leak references forever and
|
||||||
/// written files become GC-collectible rows instead of invisible orphans.
|
/// written files become GC-collectible rows instead of invisible orphans.
|
||||||
struct IngestGuard {
|
/// Whether the ingest loop overlaps batch settling with source reading
|
||||||
pool: Arc<PgPool>,
|
/// (default on). `OXICLOUD_INGEST_OVERLAP=0` restores the old inline
|
||||||
backend: Arc<dyn BlobStorageBackend>,
|
/// behaviour — kept as a bench/ops escape hatch.
|
||||||
|
fn ingest_overlap_enabled() -> bool {
|
||||||
|
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||||
|
*ENABLED.get_or_init(|| {
|
||||||
|
std::env::var("OXICLOUD_INGEST_OVERLAP").map_or(true, |v| v != "0" && v != "false")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compensation ledger of one ingest session. Shared (`Arc<tokio::Mutex>`)
|
||||||
|
/// between the ingest loop and the overlapped batch-settle task: the settler
|
||||||
|
/// holds the lock for the whole batch and records progressively, so a
|
||||||
|
/// rollback (explicit or Drop-spawned) that acquires the lock is guaranteed
|
||||||
|
/// to observe every pin/write the in-flight settle made.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct IngestState {
|
||||||
/// Pre-existing chunks whose ref_count this session bumped (distinct).
|
/// Pre-existing chunks whose ref_count this session bumped (distinct).
|
||||||
pinned: Vec<String>,
|
pinned: Vec<String>,
|
||||||
/// Chunks written to the backend but not yet registered: (hash, size).
|
/// Chunks written to the backend but not yet registered: (hash, size).
|
||||||
written: Vec<(String, i64)>,
|
written: Vec<(String, i64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct IngestGuard {
|
||||||
|
pool: Arc<PgPool>,
|
||||||
|
backend: Arc<dyn BlobStorageBackend>,
|
||||||
|
state: Arc<tokio::sync::Mutex<IngestState>>,
|
||||||
armed: bool,
|
armed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,8 +147,7 @@ impl IngestGuard {
|
|||||||
Self {
|
Self {
|
||||||
pool,
|
pool,
|
||||||
backend,
|
backend,
|
||||||
pinned: Vec::new(),
|
state: Arc::new(tokio::sync::Mutex::new(IngestState::default())),
|
||||||
written: Vec::new(),
|
|
||||||
armed: true,
|
armed: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,8 +162,15 @@ impl IngestGuard {
|
|||||||
/// spawned Drop path).
|
/// spawned Drop path).
|
||||||
async fn rollback(mut self) {
|
async fn rollback(mut self) {
|
||||||
self.armed = false;
|
self.armed = false;
|
||||||
let pinned = std::mem::take(&mut self.pinned);
|
// Lock acquisition serializes after any in-flight batch settle, so
|
||||||
let written = std::mem::take(&mut self.written);
|
// its pins/writes are visible here.
|
||||||
|
let (pinned, written) = {
|
||||||
|
let mut st = self.state.lock().await;
|
||||||
|
(
|
||||||
|
std::mem::take(&mut st.pinned),
|
||||||
|
std::mem::take(&mut st.written),
|
||||||
|
)
|
||||||
|
};
|
||||||
Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await;
|
Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,24 +237,33 @@ impl IngestGuard {
|
|||||||
|
|
||||||
impl Drop for IngestGuard {
|
impl Drop for IngestGuard {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) {
|
if !self.armed {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let pinned = std::mem::take(&mut self.pinned);
|
// The rollback task locks the shared state first, so it naturally
|
||||||
let written = std::mem::take(&mut self.written);
|
// waits out an in-flight batch settle and observes its recordings.
|
||||||
|
let state = self.state.clone();
|
||||||
match tokio::runtime::Handle::try_current() {
|
match tokio::runtime::Handle::try_current() {
|
||||||
Ok(handle) => {
|
Ok(handle) => {
|
||||||
let pool = self.pool.clone();
|
let pool = self.pool.clone();
|
||||||
let backend = self.backend.clone();
|
let backend = self.backend.clone();
|
||||||
handle.spawn(async move {
|
handle.spawn(async move {
|
||||||
|
let (pinned, written) = {
|
||||||
|
let mut st = state.lock().await;
|
||||||
|
(
|
||||||
|
std::mem::take(&mut st.pinned),
|
||||||
|
std::mem::take(&mut st.written),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if pinned.is_empty() && written.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
Self::run_rollback(pool, backend, pinned, written).await;
|
Self::run_rollback(pool, backend, pinned, written).await;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(_) => tracing::warn!(
|
Err(_) => tracing::warn!(
|
||||||
"Ingest guard dropped outside a runtime: {} pins / {} written chunks \
|
"Ingest guard dropped outside a runtime: any pins / written chunks \
|
||||||
stay leaked until the next GC sweep",
|
stay leaked until the next GC sweep",
|
||||||
pinned.len(),
|
|
||||||
written.len()
|
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -240,6 +275,16 @@ impl Drop for IngestGuard {
|
|||||||
/// in the [`BlobStorageBackend`], and maintains a manifest in PostgreSQL
|
/// in the [`BlobStorageBackend`], and maintains a manifest in PostgreSQL
|
||||||
/// mapping file_hash → \[chunk_hashes\]. BLAKE3 hashing, ref-counting
|
/// mapping file_hash → \[chunk_hashes\]. BLAKE3 hashing, ref-counting
|
||||||
/// and the PostgreSQL dedup index all live here.
|
/// and the PostgreSQL dedup index all live here.
|
||||||
|
/// Immutable chunk map of one CDC blob (`storage.chunk_manifests` row,
|
||||||
|
/// minus the mutable `ref_count`). Content-addressed: for a given
|
||||||
|
/// `file_hash` the chunk list and total size never change, which is what
|
||||||
|
/// makes [`DedupService::manifest_cached`] safe.
|
||||||
|
pub struct ChunkManifest {
|
||||||
|
pub chunk_hashes: Vec<String>,
|
||||||
|
pub chunk_sizes: Vec<i64>,
|
||||||
|
pub total_size: i64,
|
||||||
|
}
|
||||||
|
|
||||||
pub struct DedupService {
|
pub struct DedupService {
|
||||||
/// Pluggable blob storage backend (local FS, S3, …).
|
/// Pluggable blob storage backend (local FS, S3, …).
|
||||||
backend: Arc<dyn BlobStorageBackend>,
|
backend: Arc<dyn BlobStorageBackend>,
|
||||||
@@ -251,6 +296,13 @@ pub struct DedupService {
|
|||||||
maintenance_pool: Arc<PgPool>,
|
maintenance_pool: Arc<PgPool>,
|
||||||
/// Single lifecycle dispatcher — fired on blob created / deleted.
|
/// Single lifecycle dispatcher — fired on blob created / deleted.
|
||||||
blob_lifecycle: Option<Arc<BlobLifecycleService>>,
|
blob_lifecycle: Option<Arc<BlobLifecycleService>>,
|
||||||
|
/// `file_hash → ChunkManifest` for the read path — every stream / range
|
||||||
|
/// / full read of a CDC blob used to pay one manifest query first, even
|
||||||
|
/// for the media the gallery re-reads constantly. Positive-only (a
|
||||||
|
/// legacy blob gaining a manifest via background rechunking must be
|
||||||
|
/// seen immediately), weight-bounded (a manifest is ~72 B per chunk),
|
||||||
|
/// short TTL so GC'd manifests age out fast (benches/MANIFEST-CACHE.md).
|
||||||
|
manifest_cache: moka::future::Cache<String, Arc<ChunkManifest>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DedupService {
|
impl DedupService {
|
||||||
@@ -269,9 +321,22 @@ impl DedupService {
|
|||||||
pool,
|
pool,
|
||||||
maintenance_pool,
|
maintenance_pool,
|
||||||
blob_lifecycle: None,
|
blob_lifecycle: None,
|
||||||
|
manifest_cache: Self::build_manifest_cache(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// See the `manifest_cache` field docs. Weight ≈ real heap bytes of one
|
||||||
|
/// entry; 32 MiB cap ≈ tens of thousands of typical (sub-1 GB) files.
|
||||||
|
fn build_manifest_cache() -> moka::future::Cache<String, Arc<ChunkManifest>> {
|
||||||
|
moka::future::Cache::builder()
|
||||||
|
.weigher(|key: &String, value: &Arc<ChunkManifest>| {
|
||||||
|
(key.len() + value.chunk_hashes.len() * 80 + 64) as u32
|
||||||
|
})
|
||||||
|
.max_capacity(32 * 1024 * 1024)
|
||||||
|
.time_to_live(std::time::Duration::from_secs(60))
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
|
||||||
/// Registers the blob lifecycle dispatcher (thumbnail cleanup, …).
|
/// Registers the blob lifecycle dispatcher (thumbnail cleanup, …).
|
||||||
pub fn with_blob_lifecycle(mut self, lifecycle: Arc<BlobLifecycleService>) -> Self {
|
pub fn with_blob_lifecycle(mut self, lifecycle: Arc<BlobLifecycleService>) -> Self {
|
||||||
self.blob_lifecycle = Some(lifecycle);
|
self.blob_lifecycle = Some(lifecycle);
|
||||||
@@ -311,6 +376,7 @@ impl DedupService {
|
|||||||
pool: stub_pool.clone(),
|
pool: stub_pool.clone(),
|
||||||
maintenance_pool: stub_pool,
|
maintenance_pool: stub_pool,
|
||||||
blob_lifecycle: None,
|
blob_lifecycle: None,
|
||||||
|
manifest_cache: Self::build_manifest_cache(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -597,6 +663,13 @@ impl DedupService {
|
|||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}")))
|
.map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-ahead depth the backend recommends for multi-chunk drains
|
||||||
|
/// (1 local, 8 for request-latency-bound object stores) — see
|
||||||
|
/// `BlobStorageBackend::read_prefetch` and benches/BLOB-PREFETCH.md.
|
||||||
|
pub fn read_prefetch(&self) -> usize {
|
||||||
|
self.backend.read_prefetch()
|
||||||
|
}
|
||||||
|
|
||||||
/// Stream one chunk's raw bytes from the backend. The caller is
|
/// Stream one chunk's raw bytes from the backend. The caller is
|
||||||
/// responsible for entitlement (see [`claimable_chunks`]).
|
/// responsible for entitlement (see [`claimable_chunks`]).
|
||||||
pub async fn chunk_stream(
|
pub async fn chunk_stream(
|
||||||
@@ -845,8 +918,28 @@ impl DedupService {
|
|||||||
let mut hasher = blake3::Hasher::new();
|
let mut hasher = blake3::Hasher::new();
|
||||||
let mut head: Vec<u8> = Vec::with_capacity(sniff_len.min(16 * 1024));
|
let mut head: Vec<u8> = Vec::with_capacity(sniff_len.min(16 * 1024));
|
||||||
|
|
||||||
for (hash, declared_size) in chunks {
|
// Overlap the NEXT chunk's open with the current chunk's hash+drain
|
||||||
let mut stream = self.backend.get_blob_stream(hash).await?;
|
// — the same `buffered(read_prefetch)` combinator as the download
|
||||||
|
// path (benches/BLOB-PREFETCH.md measured +7-12 % on local disk;
|
||||||
|
// request-latency-bound object stores gain far more). Hashing stays
|
||||||
|
// strictly in manifest order: `buffered` yields in input order.
|
||||||
|
let prefetch = self.backend.read_prefetch().max(1);
|
||||||
|
let backend = self.backend.clone();
|
||||||
|
let mut opened = futures::stream::iter(chunks.iter().cloned())
|
||||||
|
.map(move |(hash, declared_size)| {
|
||||||
|
let backend = backend.clone();
|
||||||
|
async move {
|
||||||
|
backend
|
||||||
|
.get_blob_stream(&hash)
|
||||||
|
.await
|
||||||
|
.map(|s| (hash, declared_size, s))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.buffered(prefetch);
|
||||||
|
|
||||||
|
while let Some(next) = opened.next().await {
|
||||||
|
let (hash, declared_size, mut stream) = next?;
|
||||||
|
let (hash, declared_size) = (&hash, &declared_size);
|
||||||
let mut actual: u64 = 0;
|
let mut actual: u64 = 0;
|
||||||
while let Some(part) = stream.next().await {
|
while let Some(part) = stream.next().await {
|
||||||
let part = part.map_err(|e| {
|
let part = part.map_err(|e| {
|
||||||
@@ -923,7 +1016,7 @@ impl DedupService {
|
|||||||
where
|
where
|
||||||
S: Stream<Item = Result<Bytes, std::io::Error>> + Send,
|
S: Stream<Item = Result<Bytes, std::io::Error>> + Send,
|
||||||
{
|
{
|
||||||
let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone());
|
let guard = IngestGuard::new(self.pool.clone(), self.backend.clone());
|
||||||
|
|
||||||
let reader = StreamReader::new(Box::pin(source));
|
let reader = StreamReader::new(Box::pin(source));
|
||||||
let mut chunker = fastcdc::v2020::AsyncStreamCDC::new(
|
let mut chunker = fastcdc::v2020::AsyncStreamCDC::new(
|
||||||
@@ -942,11 +1035,35 @@ impl DedupService {
|
|||||||
let mut session_seen: HashSet<String> = HashSet::new();
|
let mut session_seen: HashSet<String> = HashSet::new();
|
||||||
let mut pending: Vec<(String, Bytes)> = Vec::new();
|
let mut pending: Vec<(String, Bytes)> = Vec::new();
|
||||||
let mut pending_bytes: usize = 0;
|
let mut pending_bytes: usize = 0;
|
||||||
|
// Depth-1 settle pipeline: batch N settles on a spawned task while
|
||||||
|
// the loop keeps reading/chunking/hashing batch N+1 from the source
|
||||||
|
// — the inline shape froze the reader (and the client's socket) for
|
||||||
|
// every settle (benches/INGEST-OVERLAP.md). The task records into
|
||||||
|
// the guard's shared state under its lock, so rollback stays exact
|
||||||
|
// even if this future is dropped mid-settle.
|
||||||
|
let mut in_flight: Option<tokio::task::JoinHandle<Result<(), DomainError>>> = None;
|
||||||
|
|
||||||
|
/// Await the previous batch's settle, mapping panics/aborts to a
|
||||||
|
/// domain error so both are compensated identically.
|
||||||
|
async fn join_settle(
|
||||||
|
handle: tokio::task::JoinHandle<Result<(), DomainError>>,
|
||||||
|
) -> Result<(), DomainError> {
|
||||||
|
match handle.await {
|
||||||
|
Ok(res) => res,
|
||||||
|
Err(e) => Err(DomainError::internal_error(
|
||||||
|
"Dedup",
|
||||||
|
format!("Chunk settle task failed: {e}"),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
while let Some(item) = chunk_stream.next().await {
|
while let Some(item) = chunk_stream.next().await {
|
||||||
let chunk = match item {
|
let chunk = match item {
|
||||||
Ok(chunk) => chunk,
|
Ok(chunk) => chunk,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
if let Some(handle) = in_flight.take() {
|
||||||
|
let _ = join_settle(handle).await;
|
||||||
|
}
|
||||||
guard.rollback().await;
|
guard.rollback().await;
|
||||||
return Err(DomainError::internal_error(
|
return Err(DomainError::internal_error(
|
||||||
"Dedup",
|
"Dedup",
|
||||||
@@ -969,7 +1086,26 @@ impl DedupService {
|
|||||||
pending.push((hash, Bytes::from(data)));
|
pending.push((hash, Bytes::from(data)));
|
||||||
if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES
|
if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES
|
||||||
{
|
{
|
||||||
if let Err(e) = self.flush_pending(&mut guard, &mut pending).await {
|
if let Some(handle) = in_flight.take()
|
||||||
|
&& let Err(e) = join_settle(handle).await
|
||||||
|
{
|
||||||
|
guard.rollback().await;
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
let batch = std::mem::take(&mut pending);
|
||||||
|
let handle = tokio::spawn(Self::settle_batch(
|
||||||
|
self.pool.clone(),
|
||||||
|
self.backend.clone(),
|
||||||
|
guard.state.clone(),
|
||||||
|
batch,
|
||||||
|
));
|
||||||
|
// Bench/ops escape hatch: OXICLOUD_INGEST_OVERLAP=0
|
||||||
|
// reproduces the old inline-settle behaviour (await the
|
||||||
|
// batch before reading on) — used by
|
||||||
|
// benches/INGEST-OVERLAP.md for an in-binary A/B.
|
||||||
|
if ingest_overlap_enabled() {
|
||||||
|
in_flight = Some(handle);
|
||||||
|
} else if let Err(e) = join_settle(handle).await {
|
||||||
guard.rollback().await;
|
guard.rollback().await;
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
@@ -978,7 +1114,20 @@ impl DedupService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = self.flush_pending(&mut guard, &mut pending).await {
|
if let Some(handle) = in_flight.take()
|
||||||
|
&& let Err(e) = join_settle(handle).await
|
||||||
|
{
|
||||||
|
guard.rollback().await;
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
if let Err(e) = Self::settle_batch(
|
||||||
|
self.pool.clone(),
|
||||||
|
self.backend.clone(),
|
||||||
|
guard.state.clone(),
|
||||||
|
std::mem::take(&mut pending),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
guard.rollback().await;
|
guard.rollback().await;
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
@@ -987,10 +1136,15 @@ impl DedupService {
|
|||||||
// One batched fsync sweep (no-op for remote backends, durable on
|
// One batched fsync sweep (no-op for remote backends, durable on
|
||||||
// PUT), then one batched INSERT. A crash before the INSERT leaves
|
// PUT), then one batched INSERT. A crash before the INSERT leaves
|
||||||
// only unreferenced files; never a row pointing at unsynced bytes.
|
// only unreferenced files; never a row pointing at unsynced bytes.
|
||||||
if !guard.written.is_empty() {
|
// No settle is in flight past this point — the lock is uncontended.
|
||||||
let new_hashes: Vec<String> = guard.written.iter().map(|(h, _)| h.clone()).collect();
|
let (new_hashes, new_sizes): (Vec<String>, Vec<i64>) = {
|
||||||
let new_sizes: Vec<i64> = guard.written.iter().map(|(_, s)| *s).collect();
|
let st = guard.state.lock().await;
|
||||||
|
(
|
||||||
|
st.written.iter().map(|(h, _)| h.clone()).collect(),
|
||||||
|
st.written.iter().map(|(_, s)| *s).collect(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if !new_hashes.is_empty() {
|
||||||
if let Err(e) = self.backend.sync_blobs(&new_hashes).await {
|
if let Err(e) = self.backend.sync_blobs(&new_hashes).await {
|
||||||
guard.rollback().await;
|
guard.rollback().await;
|
||||||
return Err(e);
|
return Err(e);
|
||||||
@@ -1016,7 +1170,7 @@ impl DedupService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let newly_written = guard.written.len();
|
let newly_written = new_hashes.len();
|
||||||
guard.disarm();
|
guard.disarm();
|
||||||
|
|
||||||
Ok(ChunkIngestOutcome {
|
Ok(ChunkIngestOutcome {
|
||||||
@@ -1030,18 +1184,23 @@ impl DedupService {
|
|||||||
|
|
||||||
/// Settle one batch of distinct in-RAM chunks against PG + the backend.
|
/// Settle one batch of distinct in-RAM chunks against PG + the backend.
|
||||||
///
|
///
|
||||||
/// Successfully pinned hashes and written chunks are recorded on the
|
/// Static (no `&self`) so the ingest loop can run it on a spawned task
|
||||||
/// guard as they happen, so a failure mid-batch leaves nothing
|
/// and keep consuming the source stream while the batch settles — the
|
||||||
/// untracked for rollback.
|
/// inline shape stalled the reader for the whole settle every 8 MiB
|
||||||
async fn flush_pending(
|
/// (benches/INGEST-OVERLAP.md). The shared-state lock is held for the
|
||||||
&self,
|
/// entire batch: pinned hashes and written chunks are recorded
|
||||||
guard: &mut IngestGuard,
|
/// progressively under it, so a failure (or a rollback racing this
|
||||||
pending: &mut Vec<(String, Bytes)>,
|
/// settle) leaves nothing untracked.
|
||||||
|
async fn settle_batch(
|
||||||
|
pool: Arc<PgPool>,
|
||||||
|
backend: Arc<dyn BlobStorageBackend>,
|
||||||
|
state: Arc<tokio::sync::Mutex<IngestState>>,
|
||||||
|
batch: Vec<(String, Bytes)>,
|
||||||
) -> Result<(), DomainError> {
|
) -> Result<(), DomainError> {
|
||||||
if pending.is_empty() {
|
if batch.is_empty() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let batch = std::mem::take(pending);
|
let mut guard = state.lock().await;
|
||||||
let hashes: Vec<String> = batch.iter().map(|(h, _)| h.clone()).collect();
|
let hashes: Vec<String> = batch.iter().map(|(h, _)| h.clone()).collect();
|
||||||
|
|
||||||
// Pin-or-classify in one statement: rows that exist take this
|
// Pin-or-classify in one statement: rows that exist take this
|
||||||
@@ -1053,7 +1212,7 @@ impl DedupService {
|
|||||||
RETURNING hash",
|
RETURNING hash",
|
||||||
)
|
)
|
||||||
.bind(&hashes)
|
.bind(&hashes)
|
||||||
.fetch_all(self.pool.as_ref())
|
.fetch_all(pool.as_ref())
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}"))
|
DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}"))
|
||||||
@@ -1075,7 +1234,6 @@ impl DedupService {
|
|||||||
|
|
||||||
// Unsynced writes — durability comes from the single end-of-stream
|
// Unsynced writes — durability comes from the single end-of-stream
|
||||||
// sweep, before any PG row references these chunks.
|
// sweep, before any PG row references these chunks.
|
||||||
let backend = self.backend.clone();
|
|
||||||
let results: Vec<Result<(String, i64), DomainError>> = stream::iter(to_write)
|
let results: Vec<Result<(String, i64), DomainError>> = stream::iter(to_write)
|
||||||
.map(|(hash, data)| {
|
.map(|(hash, data)| {
|
||||||
let backend = backend.clone();
|
let backend = backend.clone();
|
||||||
@@ -1385,6 +1543,10 @@ impl DedupService {
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Commit: {}", e)))?;
|
.map_err(|e| DomainError::internal_error("Dedup", format!("Commit: {}", e)))?;
|
||||||
|
|
||||||
|
// Post-commit so a concurrent read can't re-cache the manifest
|
||||||
|
// between invalidation and the delete becoming visible.
|
||||||
|
self.manifest_cache.invalidate(file_hash).await;
|
||||||
|
|
||||||
// File content is gone — drop its blob-keyed thumbnails now.
|
// File content is gone — drop its blob-keyed thumbnails now.
|
||||||
self.fire_blob_hooks(file_hash);
|
self.fire_blob_hooks(file_hash);
|
||||||
|
|
||||||
@@ -1606,27 +1768,49 @@ impl DedupService {
|
|||||||
Box::pin(chunk_stream)
|
Box::pin(chunk_stream)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cached manifest fetch for the read path (see the `manifest_cache`
|
||||||
|
/// field docs). `None` = legacy whole-file blob — never cached, so a
|
||||||
|
/// background rechunk that creates a manifest is honoured immediately.
|
||||||
|
async fn manifest_cached(&self, hash: &str) -> Result<Option<Arc<ChunkManifest>>, DomainError> {
|
||||||
|
if let Some(m) = self.manifest_cache.get(hash).await {
|
||||||
|
return Ok(Some(m));
|
||||||
|
}
|
||||||
|
let row = sqlx::query_as::<_, (Vec<String>, Vec<i64>, i64)>(
|
||||||
|
"SELECT chunk_hashes, chunk_sizes, total_size
|
||||||
|
FROM storage.chunk_manifests WHERE file_hash = $1",
|
||||||
|
)
|
||||||
|
.bind(hash)
|
||||||
|
.fetch_optional(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?;
|
||||||
|
match row {
|
||||||
|
Some((chunk_hashes, chunk_sizes, total_size)) => {
|
||||||
|
let m = Arc::new(ChunkManifest {
|
||||||
|
chunk_hashes,
|
||||||
|
chunk_sizes,
|
||||||
|
total_size,
|
||||||
|
});
|
||||||
|
self.manifest_cache
|
||||||
|
.insert(hash.to_string(), m.clone())
|
||||||
|
.await;
|
||||||
|
Ok(Some(m))
|
||||||
|
}
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Stream blob content — CDC-aware with legacy fallback.
|
/// Stream blob content — CDC-aware with legacy fallback.
|
||||||
///
|
///
|
||||||
/// For CDC files: looks up the manifest, then streams chunks in order,
|
/// For CDC files: looks up the manifest (RAM-cached), then streams
|
||||||
/// concatenating them into a single byte stream.
|
/// chunks in order, concatenating them into a single byte stream.
|
||||||
/// For legacy blobs: delegates directly to the backend.
|
/// For legacy blobs: delegates directly to the backend.
|
||||||
pub async fn read_blob_stream(
|
pub async fn read_blob_stream(
|
||||||
&self,
|
&self,
|
||||||
hash: &str,
|
hash: &str,
|
||||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
|
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
|
||||||
{
|
{
|
||||||
// Check manifest
|
match self.manifest_cached(hash).await? {
|
||||||
let manifest = sqlx::query_scalar::<_, Vec<String>>(
|
Some(m) => Ok(self.stream_chunks(m.chunk_hashes.clone())),
|
||||||
"SELECT chunk_hashes FROM storage.chunk_manifests WHERE file_hash = $1",
|
|
||||||
)
|
|
||||||
.bind(hash)
|
|
||||||
.fetch_optional(self.pool.as_ref())
|
|
||||||
.await
|
|
||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?;
|
|
||||||
|
|
||||||
match manifest {
|
|
||||||
Some(chunk_hashes) => Ok(self.stream_chunks(chunk_hashes)),
|
|
||||||
// Legacy whole-file blob
|
// Legacy whole-file blob
|
||||||
None => self.backend.get_blob_stream(hash).await,
|
None => self.backend.get_blob_stream(hash).await,
|
||||||
}
|
}
|
||||||
@@ -1644,18 +1828,11 @@ impl DedupService {
|
|||||||
/// `blob_size` + `read_blob_stream`) doubled the manifest round-trips on
|
/// `blob_size` + `read_blob_stream`) doubled the manifest round-trips on
|
||||||
/// every full-blob read (e.g. 2N queries for an N-image gallery cold load).
|
/// every full-blob read (e.g. 2N queries for an N-image gallery cold load).
|
||||||
pub async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError> {
|
pub async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError> {
|
||||||
let manifest = sqlx::query_as::<_, (Vec<String>, i64)>(
|
let (mut stream, expected_size) = match self.manifest_cached(hash).await? {
|
||||||
"SELECT chunk_hashes, total_size FROM storage.chunk_manifests WHERE file_hash = $1",
|
Some(m) => (
|
||||||
)
|
self.stream_chunks(m.chunk_hashes.clone()),
|
||||||
.bind(hash)
|
m.total_size.max(0) as usize,
|
||||||
.fetch_optional(self.pool.as_ref())
|
),
|
||||||
.await
|
|
||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?;
|
|
||||||
|
|
||||||
let (mut stream, expected_size) = match manifest {
|
|
||||||
Some((chunk_hashes, total_size)) => {
|
|
||||||
(self.stream_chunks(chunk_hashes), total_size.max(0) as usize)
|
|
||||||
}
|
|
||||||
None => {
|
None => {
|
||||||
// Legacy whole-file blob: size + stream straight from the backend.
|
// Legacy whole-file blob: size + stream straight from the backend.
|
||||||
let size = self.backend.blob_size(hash).await? as usize;
|
let size = self.backend.blob_size(hash).await? as usize;
|
||||||
@@ -1685,17 +1862,9 @@ impl DedupService {
|
|||||||
end: Option<u64>,
|
end: Option<u64>,
|
||||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
|
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
|
||||||
{
|
{
|
||||||
// Check manifest
|
if let Some(m) = self.manifest_cached(hash).await? {
|
||||||
let manifest = sqlx::query_as::<_, (Vec<String>, Vec<i64>, i64)>(
|
let (chunk_hashes, chunk_sizes, total_size) =
|
||||||
"SELECT chunk_hashes, chunk_sizes, total_size
|
(&m.chunk_hashes, &m.chunk_sizes, m.total_size);
|
||||||
FROM storage.chunk_manifests WHERE file_hash = $1",
|
|
||||||
)
|
|
||||||
.bind(hash)
|
|
||||||
.fetch_optional(self.pool.as_ref())
|
|
||||||
.await
|
|
||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?;
|
|
||||||
|
|
||||||
if let Some((chunk_hashes, chunk_sizes, total_size)) = manifest {
|
|
||||||
let end = end.unwrap_or(total_size as u64);
|
let end = end.unwrap_or(total_size as u64);
|
||||||
|
|
||||||
// Calculate which chunks overlap [start, end)
|
// Calculate which chunks overlap [start, end)
|
||||||
@@ -1749,17 +1918,9 @@ impl DedupService {
|
|||||||
|
|
||||||
/// Get blob size — manifest-aware with legacy fallback.
|
/// Get blob size — manifest-aware with legacy fallback.
|
||||||
pub async fn blob_size(&self, hash: &str) -> Result<u64, DomainError> {
|
pub async fn blob_size(&self, hash: &str) -> Result<u64, DomainError> {
|
||||||
// Check manifest first (O(1) from PG)
|
// Check manifest first (RAM cache, else one O(1) PG row)
|
||||||
let manifest_size = sqlx::query_scalar::<_, i64>(
|
if let Some(m) = self.manifest_cached(hash).await? {
|
||||||
"SELECT total_size FROM storage.chunk_manifests WHERE file_hash = $1",
|
return Ok(m.total_size as u64);
|
||||||
)
|
|
||||||
.bind(hash)
|
|
||||||
.fetch_optional(self.pool.as_ref())
|
|
||||||
.await
|
|
||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?;
|
|
||||||
|
|
||||||
if let Some(size) = manifest_size {
|
|
||||||
return Ok(size as u64);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Legacy: delegate to backend
|
// Legacy: delegate to backend
|
||||||
@@ -2048,6 +2209,7 @@ impl DedupService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (file_hash, chunk_hashes, size) in &batch {
|
for (file_hash, chunk_hashes, size) in &batch {
|
||||||
|
self.manifest_cache.invalidate(file_hash).await;
|
||||||
// Decrement chunk ref_counts. GREATEST(.., 0) guards against the
|
// Decrement chunk ref_counts. GREATEST(.., 0) guards against the
|
||||||
// single-chunk file case where the PG file-delete trigger already
|
// single-chunk file case where the PG file-delete trigger already
|
||||||
// decremented blobs.ref_count (because file_hash == chunk_hash);
|
// decremented blobs.ref_count (because file_hash == chunk_hash);
|
||||||
|
|||||||
@@ -1,24 +1,84 @@
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
|
|
||||||
use crate::common::errors::{DomainError, Result};
|
use crate::common::errors::{DomainError, Result};
|
||||||
|
|
||||||
|
/// In-RAM running byte counter per upload session (`user/upload_id` →
|
||||||
|
/// bytes accepted so far). The per-chunk quota gate used to recompute
|
||||||
|
/// this by listing the whole session directory and stat-ing every chunk
|
||||||
|
/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload
|
||||||
|
/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the
|
||||||
|
/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds
|
||||||
|
/// from the directory listing, so crash-correctness is unchanged
|
||||||
|
/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the
|
||||||
|
/// TTL reaps counters for sessions the client abandoned.
|
||||||
|
fn build_session_bytes_cache() -> moka::sync::Cache<String, u64> {
|
||||||
|
moka::sync::Cache::builder()
|
||||||
|
.max_capacity(100_000)
|
||||||
|
.time_to_idle(Duration::from_secs(24 * 3600))
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct NextcloudChunkedUploadService {
|
pub struct NextcloudChunkedUploadService {
|
||||||
pub base_dir: PathBuf,
|
pub base_dir: PathBuf,
|
||||||
|
/// See [`build_session_bytes_cache`]. Cloning the service shares the
|
||||||
|
/// counter (moka `Cache` clones are handles to the same store).
|
||||||
|
session_bytes: moka::sync::Cache<String, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NextcloudChunkedUploadService {
|
impl NextcloudChunkedUploadService {
|
||||||
pub fn new(base_dir: PathBuf) -> Self {
|
pub fn new(base_dir: PathBuf) -> Self {
|
||||||
Self { base_dir }
|
Self {
|
||||||
|
base_dir,
|
||||||
|
session_bytes: build_session_bytes_cache(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_stub() -> Self {
|
pub fn new_stub() -> Self {
|
||||||
Self {
|
Self {
|
||||||
base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
|
base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
|
||||||
|
session_bytes: build_session_bytes_cache(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bytes_key(user: &str, upload_id: &str) -> String {
|
||||||
|
format!("{user}/{upload_id}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Session bytes accepted so far, if the counter is warm.
|
||||||
|
/// `None` = rebuild from the directory listing and call
|
||||||
|
/// [`Self::set_session_bytes`].
|
||||||
|
pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option<u64> {
|
||||||
|
self.session_bytes.get(&Self::bytes_key(user, upload_id))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed / overwrite the session counter (post-rebuild or on MKCOL).
|
||||||
|
pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) {
|
||||||
|
self.session_bytes
|
||||||
|
.insert(Self::bytes_key(user, upload_id), bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add an accepted chunk's bytes to the counter (no-op when cold —
|
||||||
|
/// the next gate rebuilds from disk). Two racing PUTs on one session
|
||||||
|
/// could drop an increment; the counter is a gate hint, and the
|
||||||
|
/// MOVE-time quota check stays authoritative.
|
||||||
|
pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) {
|
||||||
|
let key = Self::bytes_key(user, upload_id);
|
||||||
|
if let Some(current) = self.session_bytes.get(&key) {
|
||||||
|
self.session_bytes
|
||||||
|
.insert(key, current.saturating_add(delta));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop the counter (session cleanup, or a chunk overwrite made the
|
||||||
|
/// running total untrustworthy — rebuilt lazily on next use).
|
||||||
|
pub fn forget_session_bytes(&self, user: &str, upload_id: &str) {
|
||||||
|
self.session_bytes
|
||||||
|
.invalidate(&Self::bytes_key(user, upload_id));
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate that a path component contains no traversal characters.
|
/// Validate that a path component contains no traversal characters.
|
||||||
fn validate_path_component(name: &str, label: &str) -> Result<()> {
|
fn validate_path_component(name: &str, label: &str) -> Result<()> {
|
||||||
if name.is_empty()
|
if name.is_empty()
|
||||||
@@ -48,6 +108,7 @@ impl NextcloudChunkedUploadService {
|
|||||||
fs::create_dir_all(&session_dir)
|
fs::create_dir_all(&session_dir)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||||
|
self.set_session_bytes(user, upload_id, 0);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,9 +158,17 @@ impl NextcloudChunkedUploadService {
|
|||||||
data: &[u8],
|
data: &[u8],
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
|
let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
|
||||||
|
let overwrite = fs::metadata(&chunk_path).await.is_ok();
|
||||||
fs::write(&chunk_path, data)
|
fs::write(&chunk_path, data)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))
|
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||||
|
if overwrite {
|
||||||
|
// Retried chunk — running total is stale; rebuild lazily.
|
||||||
|
self.forget_session_bytes(user, upload_id);
|
||||||
|
} else {
|
||||||
|
self.bump_session_bytes(user, upload_id, data.len() as u64);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// List the session's chunk files in assembly (numeric) order.
|
/// List the session's chunk files in assembly (numeric) order.
|
||||||
@@ -146,6 +215,7 @@ impl NextcloudChunkedUploadService {
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
|
||||||
}
|
}
|
||||||
|
self.forget_session_bytes(user, upload_id);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1112,6 +1112,78 @@ impl AuthorizationEngine for PgAclEngine {
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Batched Read check over a page of file ids (see the trait docs).
|
||||||
|
///
|
||||||
|
/// Decision-equivalent to looping `check`: (1) resolve every file's
|
||||||
|
/// drive in one `= ANY($1)` query (same rows as N ×
|
||||||
|
/// `get_file_drive_id`; absent ids decide `false` exactly like the
|
||||||
|
/// per-file `NotFound` path), (2) evaluate the drive-role floor once
|
||||||
|
/// per distinct drive through the same `drive_role_cache`, (3) send
|
||||||
|
/// only the drive-floor misses through the full per-file cascade —
|
||||||
|
/// preserving per-file grant resolution. `Read` is never gated by the
|
||||||
|
/// read-only drive freeze, so skipping that branch changes nothing.
|
||||||
|
async fn check_files_read_batch(
|
||||||
|
&self,
|
||||||
|
subject: Subject,
|
||||||
|
file_ids: &[Uuid],
|
||||||
|
) -> Result<std::collections::HashSet<Uuid>, DomainError> {
|
||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let counters = QueryCounters::default();
|
||||||
|
|
||||||
|
counters.sql_queries.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let pairs = self.file_repo.get_file_drive_ids(file_ids).await?;
|
||||||
|
|
||||||
|
// Prime the resource→drive cache — later single checks on these
|
||||||
|
// files (download, share) skip their point lookup too.
|
||||||
|
for (file_id, drive_id) in &pairs {
|
||||||
|
self.owner_cache
|
||||||
|
.insert(Resource::File(*file_id), *drive_id)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut drive_readable: HashMap<Uuid, bool> = HashMap::new();
|
||||||
|
for (_, drive_id) in &pairs {
|
||||||
|
if !drive_readable.contains_key(drive_id) {
|
||||||
|
let ok = self
|
||||||
|
.caller_role_on_drive_cached(subject, *drive_id, &counters)
|
||||||
|
.await?
|
||||||
|
.is_some_and(|role| role.expand().contains(&Permission::Read));
|
||||||
|
drive_readable.insert(*drive_id, ok);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut allowed: HashSet<Uuid> = HashSet::with_capacity(pairs.len());
|
||||||
|
for (file_id, drive_id) in &pairs {
|
||||||
|
if drive_readable.get(drive_id).copied().unwrap_or(false) {
|
||||||
|
allowed.insert(*file_id);
|
||||||
|
} else if self
|
||||||
|
.check_inner(
|
||||||
|
subject,
|
||||||
|
Permission::Read,
|
||||||
|
Resource::File(*file_id),
|
||||||
|
&counters,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
// Per-file / folder-cascade grant inside a drive the caller
|
||||||
|
// has no role on — rare, but must keep resolving.
|
||||||
|
allowed.insert(*file_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::debug!(
|
||||||
|
target: "oxicloud::authz",
|
||||||
|
event = "authz.check_files_read_batch",
|
||||||
|
subject = %subject,
|
||||||
|
files = file_ids.len(),
|
||||||
|
allowed = allowed.len(),
|
||||||
|
duration_us = start.elapsed().as_micros() as u64,
|
||||||
|
sql_queries = counters.sql_queries.load(Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
Ok(allowed)
|
||||||
|
}
|
||||||
|
|
||||||
async fn list_incoming_grants(&self, subject: Subject) -> Result<Vec<Grant>, DomainError> {
|
async fn list_incoming_grants(&self, subject: Subject) -> Result<Vec<Grant>, DomainError> {
|
||||||
let counters = QueryCounters::default();
|
let counters = QueryCounters::default();
|
||||||
let (subject_types, subject_ids) = self.subject_match_set(subject, &counters).await?;
|
let (subject_types, subject_ids) = self.subject_match_set(subject, &counters).await?;
|
||||||
|
|||||||
@@ -34,6 +34,7 @@
|
|||||||
//! it was not handled by the path-based store either, so this is a
|
//! it was not handled by the path-based store either, so this is a
|
||||||
//! parity decision, not a regression.
|
//! parity decision, not a regression.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use sqlx::{PgPool, Row};
|
use sqlx::{PgPool, Row};
|
||||||
@@ -126,17 +127,23 @@ impl DeadPropertyStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Delete a specific dead property. No-op if not present.
|
/// Delete a specific dead property. No-op if not present.
|
||||||
|
///
|
||||||
|
/// Filters on the concrete id column (`folder_id = $1` / `file_id = $1`)
|
||||||
|
/// rather than the old `IS NOT DISTINCT FROM` pair — PostgreSQL cannot
|
||||||
|
/// serve `IS NOT DISTINCT FROM` from a B-tree index, so every lookup
|
||||||
|
/// degraded to a sequential scan as the table grew. The `=` shape is
|
||||||
|
/// served by the partial unique indexes from migration 20260830000001.
|
||||||
|
/// (Same rationale for `get_all` / `get` / the batched readers below —
|
||||||
|
/// measured in `benches/DEAD-PROPS.md`.)
|
||||||
pub async fn remove(&self, r: ResourceRef, name: &QualifiedName) -> Result<(), DomainError> {
|
pub async fn remove(&self, r: ResourceRef, name: &QualifiedName) -> Result<(), DomainError> {
|
||||||
let (folder_id, file_id) = split_ref(r);
|
let (column, id) = split_ref(r);
|
||||||
sqlx::query(
|
sqlx::query(&format!(
|
||||||
"DELETE FROM storage.webdav_dead_properties
|
"DELETE FROM storage.webdav_dead_properties
|
||||||
WHERE folder_id IS NOT DISTINCT FROM $1
|
WHERE {column} = $1
|
||||||
AND file_id IS NOT DISTINCT FROM $2
|
AND namespace = $2
|
||||||
AND namespace = $3
|
AND local_name = $3",
|
||||||
AND local_name = $4",
|
))
|
||||||
)
|
.bind(id)
|
||||||
.bind(folder_id)
|
|
||||||
.bind(file_id)
|
|
||||||
.bind(&name.namespace)
|
.bind(&name.namespace)
|
||||||
.bind(&name.name)
|
.bind(&name.name)
|
||||||
.execute(&*self.pool)
|
.execute(&*self.pool)
|
||||||
@@ -150,28 +157,64 @@ impl DeadPropertyStore {
|
|||||||
&self,
|
&self,
|
||||||
r: ResourceRef,
|
r: ResourceRef,
|
||||||
) -> Result<Vec<(QualifiedName, Option<String>)>, DomainError> {
|
) -> Result<Vec<(QualifiedName, Option<String>)>, DomainError> {
|
||||||
let (folder_id, file_id) = split_ref(r);
|
let (column, id) = split_ref(r);
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(&format!(
|
||||||
"SELECT namespace, local_name, value
|
"SELECT namespace, local_name, value
|
||||||
FROM storage.webdav_dead_properties
|
FROM storage.webdav_dead_properties
|
||||||
WHERE folder_id IS NOT DISTINCT FROM $1
|
WHERE {column} = $1",
|
||||||
AND file_id IS NOT DISTINCT FROM $2",
|
))
|
||||||
)
|
.bind(id)
|
||||||
.bind(folder_id)
|
|
||||||
.bind(file_id)
|
|
||||||
.fetch_all(&*self.pool)
|
.fetch_all(&*self.pool)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| DomainError::internal_error("DeadPropertyStore", format!("get_all: {e}")))?;
|
.map_err(|e| DomainError::internal_error("DeadPropertyStore", format!("get_all: {e}")))?;
|
||||||
|
|
||||||
Ok(rows
|
Ok(rows.into_iter().map(row_to_prop).collect())
|
||||||
.into_iter()
|
}
|
||||||
.map(|r| {
|
|
||||||
let namespace: String = r.get("namespace");
|
/// Batched variant of [`get_all`] for every file in a PROPFIND page:
|
||||||
let local_name: String = r.get("local_name");
|
/// ONE `file_id = ANY($1)` round-trip instead of N sequential queries.
|
||||||
let value: Option<String> = r.get("value");
|
/// Files with no dead properties are simply absent from the map.
|
||||||
(QualifiedName::new(namespace, local_name), value)
|
pub async fn get_all_for_files(
|
||||||
})
|
&self,
|
||||||
.collect())
|
file_ids: &[Uuid],
|
||||||
|
) -> Result<HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>, DomainError> {
|
||||||
|
self.get_all_batched("file_id", file_ids).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Batched variant of [`get_all`] for every subfolder in a PROPFIND page.
|
||||||
|
pub async fn get_all_for_folders(
|
||||||
|
&self,
|
||||||
|
folder_ids: &[Uuid],
|
||||||
|
) -> Result<HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>, DomainError> {
|
||||||
|
self.get_all_batched("folder_id", folder_ids).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_all_batched(
|
||||||
|
&self,
|
||||||
|
column: &str,
|
||||||
|
ids: &[Uuid],
|
||||||
|
) -> Result<HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>, DomainError> {
|
||||||
|
if ids.is_empty() {
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(&format!(
|
||||||
|
"SELECT {column} AS resource_id, namespace, local_name, value
|
||||||
|
FROM storage.webdav_dead_properties
|
||||||
|
WHERE {column} = ANY($1)",
|
||||||
|
))
|
||||||
|
.bind(ids)
|
||||||
|
.fetch_all(&*self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
DomainError::internal_error("DeadPropertyStore", format!("get_all_batched: {e}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut map: HashMap<Uuid, Vec<(QualifiedName, Option<String>)>> = HashMap::new();
|
||||||
|
for row in rows {
|
||||||
|
let resource_id: Uuid = row.get("resource_id");
|
||||||
|
map.entry(resource_id).or_default().push(row_to_prop(row));
|
||||||
|
}
|
||||||
|
Ok(map)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return a specific dead property, or `None` if not stored.
|
/// Return a specific dead property, or `None` if not stored.
|
||||||
@@ -181,16 +224,14 @@ impl DeadPropertyStore {
|
|||||||
r: ResourceRef,
|
r: ResourceRef,
|
||||||
name: &QualifiedName,
|
name: &QualifiedName,
|
||||||
) -> Result<Option<Option<String>>, DomainError> {
|
) -> Result<Option<Option<String>>, DomainError> {
|
||||||
let (folder_id, file_id) = split_ref(r);
|
let (column, id) = split_ref(r);
|
||||||
let row = sqlx::query(
|
let row = sqlx::query(&format!(
|
||||||
"SELECT value FROM storage.webdav_dead_properties
|
"SELECT value FROM storage.webdav_dead_properties
|
||||||
WHERE folder_id IS NOT DISTINCT FROM $1
|
WHERE {column} = $1
|
||||||
AND file_id IS NOT DISTINCT FROM $2
|
AND namespace = $2
|
||||||
AND namespace = $3
|
AND local_name = $3",
|
||||||
AND local_name = $4",
|
))
|
||||||
)
|
.bind(id)
|
||||||
.bind(folder_id)
|
|
||||||
.bind(file_id)
|
|
||||||
.bind(&name.namespace)
|
.bind(&name.namespace)
|
||||||
.bind(&name.name)
|
.bind(&name.name)
|
||||||
.fetch_optional(&*self.pool)
|
.fetch_optional(&*self.pool)
|
||||||
@@ -201,16 +242,23 @@ impl DeadPropertyStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Splits a `ResourceRef` into `(folder_id, file_id)` Option pairs for
|
/// Maps a `ResourceRef` onto the column that stores it plus the id to bind.
|
||||||
/// binding into SQL. The unused slot is `None` so `IS NOT DISTINCT FROM`
|
/// The column name is one of two compile-time literals — never user input —
|
||||||
/// matches the NULL stored in the unused column.
|
/// so interpolating it into the SQL text is safe.
|
||||||
fn split_ref(r: ResourceRef) -> (Option<Uuid>, Option<Uuid>) {
|
fn split_ref(r: ResourceRef) -> (&'static str, Uuid) {
|
||||||
match r {
|
match r {
|
||||||
ResourceRef::Folder(id) => (Some(id), None),
|
ResourceRef::Folder(id) => ("folder_id", id),
|
||||||
ResourceRef::File(id) => (None, Some(id)),
|
ResourceRef::File(id) => ("file_id", id),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn row_to_prop(r: sqlx::postgres::PgRow) -> (QualifiedName, Option<String>) {
|
||||||
|
let namespace: String = r.get("namespace");
|
||||||
|
let local_name: String = r.get("local_name");
|
||||||
|
let value: Option<String> = r.get("value");
|
||||||
|
(QualifiedName::new(namespace, local_name), value)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn create_dead_property_store(pool: Arc<PgPool>) -> Arc<DeadPropertyStore> {
|
pub fn create_dead_property_store(pool: Arc<PgPool>) -> Arc<DeadPropertyStore> {
|
||||||
Arc::new(DeadPropertyStore::new(pool))
|
Arc::new(DeadPropertyStore::new(pool))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,15 +44,22 @@ impl From<ZipError> for DomainError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Type alias for the fully-async ZIP writer backed by a buffered tokio file.
|
/// Fully-async ZIP writer over any buffered tokio sink (temp file for the
|
||||||
type AsyncZipWriter = ZipFileWriter<Compat<BufWriter<tokio::fs::File>>>;
|
/// legacy path, one half of a `tokio::io::duplex` for the streaming path).
|
||||||
|
type AsyncZipWriter<W> = ZipFileWriter<Compat<BufWriter<W>>>;
|
||||||
|
|
||||||
/// One planned archive entry, in final ZIP order.
|
/// One planned archive entry, in final ZIP order.
|
||||||
enum ZipPlanEntry {
|
enum ZipPlanEntry {
|
||||||
/// Directory entry (Stored, zero-length body).
|
/// Directory entry (Stored, zero-length body).
|
||||||
Dir(String),
|
Dir(String),
|
||||||
/// File entry: ZIP-relative path + file id to stream from the blob store.
|
/// File entry: ZIP-relative path + file id to stream from the blob store.
|
||||||
File { zip_path: String, file_id: String },
|
/// `compression` is picked from the file's MIME type at plan time —
|
||||||
|
/// `Stored` for already-compressed media (JPEG/MP4/…), `Deflate` otherwise.
|
||||||
|
File {
|
||||||
|
zip_path: String,
|
||||||
|
file_id: String,
|
||||||
|
compression: Compression,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Message protocol from the prefetch task to the ZIP writer. For each
|
/// Message protocol from the prefetch task to the ZIP writer. For each
|
||||||
@@ -74,8 +81,11 @@ const PREFETCH_BUFFER_CHUNKS: usize = 64;
|
|||||||
///
|
///
|
||||||
/// Uses `async_zip` for fully-async archive creation. Every write (headers,
|
/// Uses `async_zip` for fully-async archive creation. Every write (headers,
|
||||||
/// compressed chunk data, central directory) goes through
|
/// compressed chunk data, central directory) goes through
|
||||||
/// `tokio::io::BufWriter` → `tokio::fs::File`, so **no Tokio worker is ever
|
/// `tokio::io::BufWriter` → `tokio::fs::File`, so no Tokio worker is ever
|
||||||
/// blocked** by disk I/O or compression.
|
/// blocked by disk I/O. Deflate itself DOES run inline on the writing task
|
||||||
|
/// (async_zip compresses inside `poll_write`), which is why entries whose
|
||||||
|
/// MIME says the content is already compressed are `Stored` instead — that
|
||||||
|
/// turns the archive hot path from ~1 CPU core per download into CRC + memcpy.
|
||||||
///
|
///
|
||||||
/// Archive creation is a 2-stage pipeline: a prefetch task reads file
|
/// Archive creation is a 2-stage pipeline: a prefetch task reads file
|
||||||
/// content from the blob store ahead of the writer, so the next file's
|
/// content from the blob store ahead of the writer, so the next file's
|
||||||
@@ -110,6 +120,110 @@ impl ZipService {
|
|||||||
folder_id: &str,
|
folder_id: &str,
|
||||||
folder_name: &str,
|
folder_name: &str,
|
||||||
) -> Result<NamedTempFile> {
|
) -> Result<NamedTempFile> {
|
||||||
|
let plan = self.plan_archive(folder_id, folder_name).await?;
|
||||||
|
|
||||||
|
// ── Open the temp file + ZIP writer ──────────────────────────────
|
||||||
|
let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
|
||||||
|
let tokio_file = tokio::fs::File::create(temp.path())
|
||||||
|
.await
|
||||||
|
.map_err(ZipError::IoError)?;
|
||||||
|
|
||||||
|
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
|
||||||
|
let _prefetcher = tokio::spawn(Self::prefetch_files(
|
||||||
|
self.file_service.clone(),
|
||||||
|
Self::planned_file_ids(&plan),
|
||||||
|
tx,
|
||||||
|
));
|
||||||
|
Self::write_archive(tokio_file, &plan, &mut rx).await?;
|
||||||
|
|
||||||
|
Ok(temp)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Streaming variant: the archive bytes are produced on a spawned task
|
||||||
|
/// and yielded as they are written — the client's first byte arrives
|
||||||
|
/// after the first entry starts, not after the whole archive has been
|
||||||
|
/// built (the temp-file variant's time-to-first-byte grows with folder
|
||||||
|
/// size; benches/ZIP-STREAM.md). The plan phase still runs inline so
|
||||||
|
/// planning errors surface as proper HTTP errors; a blob-read error
|
||||||
|
/// mid-archive can only truncate the stream (no central directory →
|
||||||
|
/// clients detect the corrupt archive), which is the standard tradeoff
|
||||||
|
/// for streamed ZIPs.
|
||||||
|
pub async fn create_folder_zip_stream(
|
||||||
|
&self,
|
||||||
|
folder_id: &str,
|
||||||
|
folder_name: &str,
|
||||||
|
) -> Result<impl futures::Stream<Item = std::io::Result<bytes::Bytes>> + Send + use<>> {
|
||||||
|
let plan = self.plan_archive(folder_id, folder_name).await?;
|
||||||
|
|
||||||
|
let (writer, reader) = tokio::io::duplex(256 * 1024);
|
||||||
|
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
|
||||||
|
let _prefetcher = tokio::spawn(Self::prefetch_files(
|
||||||
|
self.file_service.clone(),
|
||||||
|
Self::planned_file_ids(&plan),
|
||||||
|
tx,
|
||||||
|
));
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if let Err(e) = Self::write_archive(writer, &plan, &mut rx).await {
|
||||||
|
// Dropping the writer EOFs the reader early — the truncated
|
||||||
|
// archive has no central directory, so clients flag it.
|
||||||
|
warn!("Streaming ZIP aborted mid-archive: {e}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok(tokio_util::io::ReaderStream::new(reader))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// File ids of the plan, in archive order (the prefetcher's read list).
|
||||||
|
fn planned_file_ids(plan: &[ZipPlanEntry]) -> Vec<String> {
|
||||||
|
plan.iter()
|
||||||
|
.filter_map(|entry| match entry {
|
||||||
|
ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
|
||||||
|
ZipPlanEntry::Dir(_) => None,
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write every planned entry through a buffered ZIP writer over `sink`,
|
||||||
|
/// then finalize (central directory + flush). Shared by the temp-file
|
||||||
|
/// and streaming variants.
|
||||||
|
async fn write_archive<W: tokio::io::AsyncWrite + Unpin>(
|
||||||
|
sink: W,
|
||||||
|
plan: &[ZipPlanEntry],
|
||||||
|
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
|
||||||
|
) -> Result<()> {
|
||||||
|
let buf_writer = BufWriter::with_capacity(256 * 1024, sink);
|
||||||
|
let mut zip = ZipFileWriter::with_tokio(buf_writer);
|
||||||
|
|
||||||
|
for entry in plan {
|
||||||
|
match entry {
|
||||||
|
ZipPlanEntry::Dir(zip_dir) => {
|
||||||
|
let dir_entry =
|
||||||
|
ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
|
||||||
|
match zip.write_entry_whole(dir_entry, &[]).await {
|
||||||
|
Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Could not add folder entry (may already exist): {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ZipPlanEntry::File {
|
||||||
|
zip_path,
|
||||||
|
compression,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
Self::write_prefetched_file(&mut zip, zip_path, *compression, rx).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
|
||||||
|
compat_writer.close().await.map_err(ZipError::IoError)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve the folder, fetch its subtree (2 bulk queries) and lay out
|
||||||
|
/// the archive entries in final ZIP order.
|
||||||
|
async fn plan_archive(&self, folder_id: &str, folder_name: &str) -> Result<Vec<ZipPlanEntry>> {
|
||||||
info!(
|
info!(
|
||||||
"Creating ZIP for folder: {} (ID: {})",
|
"Creating ZIP for folder: {} (ID: {})",
|
||||||
folder_name, folder_id
|
folder_name, folder_id
|
||||||
@@ -183,62 +297,15 @@ impl ZipService {
|
|||||||
plan.push(ZipPlanEntry::File {
|
plan.push(ZipPlanEntry::File {
|
||||||
zip_path: format!("{}{}", zip_dir, file.name),
|
zip_path: format!("{}{}", zip_dir, file.name),
|
||||||
file_id: file.id.to_string(),
|
file_id: file.id.to_string(),
|
||||||
|
compression: crate::common::mime_detect::zip_entry_compression(
|
||||||
|
&file.mime_type,
|
||||||
|
),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 5. Open the temp file + ZIP writer ───────────────────────────
|
Ok(plan)
|
||||||
let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
|
|
||||||
let tokio_file = tokio::fs::File::create(temp.path())
|
|
||||||
.await
|
|
||||||
.map_err(ZipError::IoError)?;
|
|
||||||
let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
|
|
||||||
let mut zip = ZipFileWriter::with_tokio(buf_writer);
|
|
||||||
|
|
||||||
// ── 6. Write entries: 2-stage pipeline ───────────────────────────
|
|
||||||
// The prefetch task reads blob streams for the planned files, in
|
|
||||||
// order, ahead of the writer — the next file's blob-store latency
|
|
||||||
// overlaps the current file's deflate. If the writer bails out,
|
|
||||||
// dropping the receiver makes the prefetcher's next send fail and
|
|
||||||
// it stops on its own.
|
|
||||||
let file_ids: Vec<String> = plan
|
|
||||||
.iter()
|
|
||||||
.filter_map(|entry| match entry {
|
|
||||||
ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
|
|
||||||
ZipPlanEntry::Dir(_) => None,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
|
|
||||||
let _prefetcher = tokio::spawn(Self::prefetch_files(
|
|
||||||
self.file_service.clone(),
|
|
||||||
file_ids,
|
|
||||||
tx,
|
|
||||||
));
|
|
||||||
|
|
||||||
for entry in &plan {
|
|
||||||
match entry {
|
|
||||||
ZipPlanEntry::Dir(zip_dir) => {
|
|
||||||
let dir_entry =
|
|
||||||
ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
|
|
||||||
match zip.write_entry_whole(dir_entry, &[]).await {
|
|
||||||
Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
|
|
||||||
Err(e) => {
|
|
||||||
warn!("Could not add folder entry (may already exist): {}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ZipPlanEntry::File { zip_path, .. } => {
|
|
||||||
Self::write_prefetched_file(&mut zip, zip_path, &mut rx).await?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── 7. Finalize ──────────────────────────────────────────────────
|
|
||||||
let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
|
|
||||||
compat_writer.close().await.map_err(ZipError::IoError)?;
|
|
||||||
|
|
||||||
Ok(temp)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Prefetch stage: streams each planned file's content from the blob
|
/// Prefetch stage: streams each planned file's content from the blob
|
||||||
@@ -282,17 +349,19 @@ impl ZipService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Writer stage: drains one file's prefetched chunks into a Deflate
|
/// Writer stage: drains one file's prefetched chunks into a ZIP entry
|
||||||
/// ZIP entry. Peak memory stays bounded by the channel, independent
|
/// (`Stored` for already-compressed media, `Deflate` otherwise — see
|
||||||
/// of individual file sizes.
|
/// `entry_compression`). Peak memory stays bounded by the channel,
|
||||||
async fn write_prefetched_file(
|
/// independent of individual file sizes.
|
||||||
zip: &mut AsyncZipWriter,
|
async fn write_prefetched_file<W: tokio::io::AsyncWrite + Unpin>(
|
||||||
|
zip: &mut AsyncZipWriter<W>,
|
||||||
zip_path: &str,
|
zip_path: &str,
|
||||||
|
compression: Compression,
|
||||||
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
|
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
info!("Adding file to ZIP: {}", zip_path);
|
info!("Adding file to ZIP: {}", zip_path);
|
||||||
|
|
||||||
let entry = ZipEntryBuilder::new(zip_path.to_string().into(), Compression::Deflate);
|
let entry = ZipEntryBuilder::new(zip_path.to_string().into(), compression);
|
||||||
let mut entry_writer = zip
|
let mut entry_writer = zip
|
||||||
.write_entry_stream(entry)
|
.write_entry_stream(entry)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use axum::{
|
|||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
};
|
};
|
||||||
use bytes::{Buf, Bytes, BytesMut};
|
use bytes::{Buf, Bytes, BytesMut};
|
||||||
use futures::Stream;
|
use futures::{Stream, TryStreamExt};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio_stream::StreamExt;
|
use tokio_stream::StreamExt;
|
||||||
|
|
||||||
@@ -343,17 +343,36 @@ pub async fn delta_download_chunks(
|
|||||||
|
|
||||||
// Stream the frames: 4-byte length headers come from the (entitled)
|
// Stream the frames: 4-byte length headers come from the (entitled)
|
||||||
// index sizes; bytes stream straight from the blob backend. Peak RAM
|
// index sizes; bytes stream straight from the blob backend. Peak RAM
|
||||||
// is one backend read frame, independent of batch size.
|
// is bounded by `read_prefetch` open streams (their first frame),
|
||||||
|
// independent of batch size.
|
||||||
|
//
|
||||||
|
// `buffered(read_prefetch)` overlaps the NEXT chunk's open with the
|
||||||
|
// current chunk's drain — the same combinator/tuning as the main CDC
|
||||||
|
// download path (benches/BLOB-PREFETCH.md). The old per-chunk await
|
||||||
|
// paid every open's full round-trip serially: on an object-store
|
||||||
|
// backend a 64-chunk batch at ~30 ms first-byte cost ~1.9 s of pure
|
||||||
|
// latency. Frames still arrive strictly in request order.
|
||||||
|
let prefetch = service.read_prefetch().max(1);
|
||||||
|
let svc = service.clone();
|
||||||
|
// `futures::StreamExt` spelled out — this handler imports
|
||||||
|
// `tokio_stream::StreamExt`, whose `map` adapter lacks `buffered`.
|
||||||
|
let opened = futures::StreamExt::map(futures::stream::iter(ordered), move |(hash, size)| {
|
||||||
|
let svc = svc.clone();
|
||||||
|
async move {
|
||||||
|
let chunk = svc
|
||||||
|
.chunk_stream(&hash)
|
||||||
|
.await
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
let header = futures::stream::once(async move {
|
||||||
|
Ok::<Bytes, std::io::Error>(Bytes::copy_from_slice(&(size as u32).to_be_bytes()))
|
||||||
|
});
|
||||||
|
Ok::<_, std::io::Error>(futures::StreamExt::chain(header, chunk))
|
||||||
|
}
|
||||||
|
});
|
||||||
let body_stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
|
let body_stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
|
||||||
Box::pin(async_stream::try_stream! {
|
Box::pin(TryStreamExt::try_flatten(futures::StreamExt::buffered(
|
||||||
for (hash, size) in ordered {
|
opened, prefetch,
|
||||||
yield Bytes::copy_from_slice(&(size as u32).to_be_bytes());
|
)));
|
||||||
let mut chunk = service.chunk_stream(&hash).await.map_err(std::io::Error::other)?;
|
|
||||||
while let Some(part) = chunk.next().await {
|
|
||||||
yield part?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Ok(Response::builder()
|
Ok(Response::builder()
|
||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
.header(header::CONTENT_TYPE, "application/octet-stream")
|
.header(header::CONTENT_TYPE, "application/octet-stream")
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ use std::collections::HashMap;
|
|||||||
use utoipa::ToSchema;
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::application::ports::file_ports::{
|
use crate::application::ports::file_ports::{
|
||||||
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase,
|
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, RangeContent,
|
||||||
};
|
};
|
||||||
use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort};
|
use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort};
|
||||||
use crate::application::ports::thumbnail_ports::ThumbnailPort;
|
use crate::application::ports::thumbnail_ports::ThumbnailPort;
|
||||||
@@ -713,10 +713,19 @@ impl FileHandler {
|
|||||||
Self::content_disposition(&file_dto.name, &file_dto.mime_type, ¶ms);
|
Self::content_disposition(&file_dto.name, &file_dto.mime_type, ¶ms);
|
||||||
|
|
||||||
match retrieval
|
match retrieval
|
||||||
.get_file_range_stream_with_perms(&id, auth_user.id, start, Some(end + 1))
|
.get_file_range_preloaded_with_perms(
|
||||||
|
&file_dto,
|
||||||
|
auth_user.id,
|
||||||
|
start,
|
||||||
|
Some(end + 1),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(stream) => {
|
Ok(content) => {
|
||||||
|
let body = match content {
|
||||||
|
RangeContent::Bytes(b) => Body::from(b),
|
||||||
|
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
|
||||||
|
};
|
||||||
return Response::builder()
|
return Response::builder()
|
||||||
.status(StatusCode::PARTIAL_CONTENT)
|
.status(StatusCode::PARTIAL_CONTENT)
|
||||||
.header(header::CONTENT_TYPE, &*file_dto.mime_type)
|
.header(header::CONTENT_TYPE, &*file_dto.mime_type)
|
||||||
@@ -732,7 +741,7 @@ impl FileHandler {
|
|||||||
header::CACHE_CONTROL,
|
header::CACHE_CONTROL,
|
||||||
"private, max-age=3600, must-revalidate",
|
"private, max-age=3600, must-revalidate",
|
||||||
)
|
)
|
||||||
.body(Body::from_stream(Box::into_pin(stream)))
|
.body(body)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ use axum::{
|
|||||||
};
|
};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio_util::io::ReaderStream;
|
|
||||||
|
|
||||||
use crate::application::dtos::display_helpers::{
|
use crate::application::dtos::display_helpers::{
|
||||||
category_for, format_file_size, icon_class_for, icon_special_class_for,
|
category_for, format_file_size, icon_class_for, icon_special_class_for,
|
||||||
@@ -238,53 +237,28 @@ impl FolderHandler {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Create the ZIP archive (written to a temp file, O(1) RAM)
|
// Stream the archive as it is built — the first byte reaches
|
||||||
match zip_service.create_folder_zip(&id, &folder.name).await {
|
// the client after the first entry, not after the whole ZIP
|
||||||
Ok(temp_file) => {
|
// exists on disk (benches/ZIP-STREAM.md). No Content-Length:
|
||||||
// Get the file size for Content-Length
|
// the final size isn't known up front (chunked encoding).
|
||||||
let file_size = match temp_file.as_file().metadata() {
|
match zip_service
|
||||||
Ok(m) => m.len(),
|
.create_folder_zip_stream(&id, &folder.name)
|
||||||
Err(e) => {
|
.await
|
||||||
tracing::error!("Error reading temp file metadata: {}", e);
|
{
|
||||||
return (
|
Ok(stream) => {
|
||||||
StatusCode::INTERNAL_SERVER_ERROR,
|
|
||||||
Json(serde_json::json!({
|
|
||||||
"error": "Error creating ZIP file"
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.into_response();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
tracing::info!("ZIP file created successfully, size: {} bytes", file_size);
|
|
||||||
|
|
||||||
// Split the NamedTempFile into the already-open std File
|
|
||||||
// and the TempPath (auto-deletes on drop). This reuses
|
|
||||||
// the existing fd instead of opening a second one.
|
|
||||||
let (std_file, temp_path) = temp_file.into_parts();
|
|
||||||
let tokio_file = tokio::fs::File::from_std(std_file);
|
|
||||||
|
|
||||||
// Stream the file to the client in chunks
|
|
||||||
let stream = ReaderStream::new(tokio_file);
|
|
||||||
let body = axum::body::Body::from_stream(stream);
|
let body = axum::body::Body::from_stream(stream);
|
||||||
|
|
||||||
// Setup headers for download
|
// Setup headers for download
|
||||||
let filename = format!("{}.zip", folder.name);
|
let filename = format!("{}.zip", folder.name);
|
||||||
let content_disposition = format!("attachment; filename=\"{}\"", filename);
|
let content_disposition = format!("attachment; filename=\"{}\"", filename);
|
||||||
|
|
||||||
let mut response = Response::builder()
|
Response::builder()
|
||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
.header(header::CONTENT_TYPE, "application/zip")
|
.header(header::CONTENT_TYPE, "application/zip")
|
||||||
.header(header::CONTENT_DISPOSITION, content_disposition)
|
.header(header::CONTENT_DISPOSITION, content_disposition)
|
||||||
.header(header::CONTENT_LENGTH, file_size)
|
|
||||||
.body(body)
|
.body(body)
|
||||||
.unwrap();
|
.unwrap()
|
||||||
|
.into_response()
|
||||||
// Keep TempPath alive in the response extensions so the
|
|
||||||
// file is only deleted AFTER the body stream finishes.
|
|
||||||
response.extensions_mut().insert(Arc::new(temp_path));
|
|
||||||
|
|
||||||
response.into_response()
|
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
tracing::error!("Error creating ZIP file: {}", err);
|
tracing::error!("Error creating ZIP file: {}", err);
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ use serde::Deserialize;
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use utoipa::ToSchema;
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
|
use crate::application::ports::file_ports::RangeContent;
|
||||||
use crate::application::services::share_browse_service::ZipTarget;
|
use crate::application::services::share_browse_service::ZipTarget;
|
||||||
use crate::application::services::share_service::ShareService;
|
use crate::application::services::share_service::ShareService;
|
||||||
use crate::infrastructure::services::share_unlock_cookie;
|
use crate::infrastructure::services::share_unlock_cookie;
|
||||||
@@ -30,7 +31,6 @@ use crate::{
|
|||||||
interfaces::errors::AppError,
|
interfaces::errors::AppError,
|
||||||
interfaces::middleware::auth::AuthUser,
|
interfaces::middleware::auth::AuthUser,
|
||||||
};
|
};
|
||||||
use tokio_util::io::ReaderStream;
|
|
||||||
|
|
||||||
fn unlock_jwt_from_headers(headers: &HeaderMap, share_token: &str) -> Option<String> {
|
fn unlock_jwt_from_headers(headers: &HeaderMap, share_token: &str) -> Option<String> {
|
||||||
headers
|
headers
|
||||||
@@ -438,10 +438,14 @@ async fn serve_share_file(
|
|||||||
let length = end - start + 1;
|
let length = end - start + 1;
|
||||||
|
|
||||||
match retrieval
|
match retrieval
|
||||||
.get_file_range_stream(file_id, start, Some(end + 1))
|
.get_file_range_preloaded(&file_dto, start, Some(end + 1))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(stream) => {
|
Ok(content) => {
|
||||||
|
let body = match content {
|
||||||
|
RangeContent::Bytes(b) => Body::from(b),
|
||||||
|
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
|
||||||
|
};
|
||||||
return Response::builder()
|
return Response::builder()
|
||||||
.status(StatusCode::PARTIAL_CONTENT)
|
.status(StatusCode::PARTIAL_CONTENT)
|
||||||
.header(header::CONTENT_TYPE, &*mime)
|
.header(header::CONTENT_TYPE, &*mime)
|
||||||
@@ -458,7 +462,7 @@ async fn serve_share_file(
|
|||||||
"private, max-age=3600, must-revalidate",
|
"private, max-age=3600, must-revalidate",
|
||||||
)
|
)
|
||||||
.header(header::VARY, "Cookie, Range")
|
.header(header::VARY, "Cookie, Range")
|
||||||
.body(Body::from_stream(Box::into_pin(stream)))
|
.body(body)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
@@ -730,30 +734,19 @@ async fn serve_share_zip(
|
|||||||
Err(err) => return share_browse_error_response(err),
|
Err(err) => return share_browse_error_response(err),
|
||||||
};
|
};
|
||||||
|
|
||||||
let temp_file = match zip_service
|
// Streamed archive: first byte after the first entry, not after the
|
||||||
.create_folder_zip(&target.folder_id, &target.display_name)
|
// whole ZIP is built (benches/ZIP-STREAM.md). No Content-Length.
|
||||||
|
let stream = match zip_service
|
||||||
|
.create_folder_zip_stream(&target.folder_id, &target.display_name)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(f) => f,
|
Ok(s) => s,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
tracing::error!("share zip: create_folder_zip failed: {}", err);
|
tracing::error!("share zip: create_folder_zip failed: {}", err);
|
||||||
return AppError::internal_error(format!("ZIP creation failed: {}", err))
|
return AppError::internal_error(format!("ZIP creation failed: {}", err))
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let file_size = match temp_file.as_file().metadata() {
|
|
||||||
Ok(m) => m.len(),
|
|
||||||
Err(e) => {
|
|
||||||
tracing::error!("share zip: temp metadata failed: {}", e);
|
|
||||||
return AppError::internal_error("ZIP creation failed").into_response();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Reuse the existing fd: split off the std::File and the TempPath.
|
|
||||||
let (std_file, temp_path) = temp_file.into_parts();
|
|
||||||
let tokio_file = tokio::fs::File::from_std(std_file);
|
|
||||||
let stream = ReaderStream::new(tokio_file);
|
|
||||||
let body = Body::from_stream(stream);
|
let body = Body::from_stream(stream);
|
||||||
|
|
||||||
let disposition = build_content_disposition(
|
let disposition = build_content_disposition(
|
||||||
@@ -762,17 +755,12 @@ async fn serve_share_zip(
|
|||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut response = Response::builder()
|
Response::builder()
|
||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
.header(header::CONTENT_TYPE, "application/zip")
|
.header(header::CONTENT_TYPE, "application/zip")
|
||||||
.header(header::CONTENT_DISPOSITION, disposition)
|
.header(header::CONTENT_DISPOSITION, disposition)
|
||||||
.header(header::CONTENT_LENGTH, file_size)
|
|
||||||
.header(header::CACHE_CONTROL, "private, no-store")
|
.header(header::CACHE_CONTROL, "private, no-store")
|
||||||
.header(header::VARY, "Cookie")
|
.header(header::VARY, "Cookie")
|
||||||
.body(body)
|
.body(body)
|
||||||
.unwrap();
|
.unwrap()
|
||||||
|
|
||||||
// Keep TempPath alive until the body finishes streaming.
|
|
||||||
response.extensions_mut().insert(Arc::new(temp_path));
|
|
||||||
response
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ use crate::interfaces::errors::AppError;
|
|||||||
use crate::interfaces::middleware::auth::{AuthUser, CurrentUser};
|
use crate::interfaces::middleware::auth::{AuthUser, CurrentUser};
|
||||||
use crate::interfaces::range_requests::{not_modified_response, range_response};
|
use crate::interfaces::range_requests::{not_modified_response, range_response};
|
||||||
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
|
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
/// Characters that MUST NOT be percent-encoded inside a URI path segment.
|
/// Characters that MUST NOT be percent-encoded inside a URI path segment.
|
||||||
@@ -557,7 +558,13 @@ async fn handle_propfind(
|
|||||||
created_by: None,
|
created_by: None,
|
||||||
updated_by: None,
|
updated_by: None,
|
||||||
};
|
};
|
||||||
let quota = state.resolve_webdav_quota(user.id, Uuid::nil()).await;
|
// Skip the 2-query quota resolution when the request's prop list
|
||||||
|
// never mentions quota (benches/QUOTA-PATH.md).
|
||||||
|
let quota = if propfind_request.wants_quota() {
|
||||||
|
state.resolve_webdav_quota(user.id, Uuid::nil()).await
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
return build_streaming_propfind_response(
|
return build_streaming_propfind_response(
|
||||||
root_folder,
|
root_folder,
|
||||||
None, // folder_id = None → root children (drive-root folders)
|
None, // folder_id = None → root children (drive-root folders)
|
||||||
@@ -597,7 +604,11 @@ async fn handle_propfind(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let folder_id = folder.id.clone();
|
let folder_id = folder.id.clone();
|
||||||
let quota = state.resolve_webdav_quota(user.id, drive_id).await;
|
let quota = if propfind_request.wants_quota() {
|
||||||
|
state.resolve_webdav_quota(user.id, drive_id).await
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
return build_streaming_propfind_response(
|
return build_streaming_propfind_response(
|
||||||
folder,
|
folder,
|
||||||
Some(folder_id),
|
Some(folder_id),
|
||||||
@@ -663,7 +674,11 @@ async fn handle_propfind(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let folder_id = folder.id.clone();
|
let folder_id = folder.id.clone();
|
||||||
let quota = state.resolve_webdav_quota(user.id, drive_id).await;
|
let quota = if propfind_request.wants_quota() {
|
||||||
|
state.resolve_webdav_quota(user.id, drive_id).await
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
return build_streaming_propfind_response(
|
return build_streaming_propfind_response(
|
||||||
folder,
|
folder,
|
||||||
Some(folder_id),
|
Some(folder_id),
|
||||||
@@ -788,19 +803,18 @@ async fn build_streaming_propfind_response(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Materialise dead-props for the whole page before
|
// ONE batched dead-props query per page instead of a
|
||||||
// we start writing — keeps the borrow checker happy
|
// sequential per-child round-trip — the N+1 shape cost
|
||||||
// (the writer borrows the FolderDto and the dead-props
|
// 1-4.5 s of pure DB chatter on a 2000-child folder
|
||||||
// vec for the duration of write_folder_entry_*).
|
// (measured in benches/DEAD-PROPS.md).
|
||||||
let mut subfolder_deads = Vec::with_capacity(result.items.len());
|
let subfolder_deads =
|
||||||
for subfolder in &result.items {
|
folders_dead_props_map(&dead_props_store, &result.items).await;
|
||||||
subfolder_deads.push(folder_dead_props(&dead_props_store, subfolder).await);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk = Vec::with_capacity(result.items.len() * 800);
|
let mut chunk = Vec::with_capacity(result.items.len() * 800);
|
||||||
{
|
{
|
||||||
let mut w = Writer::new(&mut chunk);
|
let mut w = Writer::new(&mut chunk);
|
||||||
for (subfolder, child_dead) in result.items.iter().zip(subfolder_deads.iter()) {
|
for subfolder in result.items.iter() {
|
||||||
|
let child_dead = dead_props_for(&subfolder.id, &subfolder_deads);
|
||||||
let href = format!("{}{}/", base_href, encode_path_segment(&subfolder.name));
|
let href = format!("{}{}/", base_href, encode_path_segment(&subfolder.name));
|
||||||
WebDavAdapter::write_folder_entry_with_dead_props(&mut w, subfolder, &propfind_request, &href, child_dead, quota)
|
WebDavAdapter::write_folder_entry_with_dead_props(&mut w, subfolder, &propfind_request, &href, child_dead, quota)
|
||||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||||
@@ -815,11 +829,17 @@ async fn build_streaming_propfind_response(
|
|||||||
page += 1;
|
page += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream files in pages (user-scoped)
|
// Stream files in pages (user-scoped, keyset cursor — O(page)
|
||||||
let mut offset: i64 = 0;
|
// per page instead of the quadratic LIMIT/OFFSET walk).
|
||||||
|
let mut after_name: Option<String> = None;
|
||||||
loop {
|
loop {
|
||||||
let batch: Vec<FileDto> = file_retrieval_service
|
let batch: Vec<FileDto> = file_retrieval_service
|
||||||
.list_files_batch_with_perms(fid_ref, user_id, offset, PROPFIND_BATCH_SIZE)
|
.list_files_batch_with_perms(
|
||||||
|
fid_ref,
|
||||||
|
user_id,
|
||||||
|
after_name.as_deref(),
|
||||||
|
PROPFIND_BATCH_SIZE,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||||
|
|
||||||
@@ -828,15 +848,14 @@ async fn build_streaming_propfind_response(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let batch_len = batch.len();
|
let batch_len = batch.len();
|
||||||
let mut file_deads = Vec::with_capacity(batch_len);
|
// Batched: one = ANY($1) query per 500-file page.
|
||||||
for file in &batch {
|
let file_deads = files_dead_props_map(&dead_props_store, &batch).await;
|
||||||
file_deads.push(streamed_file_dead_props(&dead_props_store, file).await);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk = Vec::with_capacity(batch_len * 800);
|
let mut chunk = Vec::with_capacity(batch_len * 800);
|
||||||
{
|
{
|
||||||
let mut w = Writer::new(&mut chunk);
|
let mut w = Writer::new(&mut chunk);
|
||||||
for (file, child_dead) in batch.iter().zip(file_deads.iter()) {
|
for file in batch.iter() {
|
||||||
|
let child_dead = dead_props_for(&file.id, &file_deads);
|
||||||
let href = format!("{}{}", base_href, encode_path_segment(&file.name));
|
let href = format!("{}{}", base_href, encode_path_segment(&file.name));
|
||||||
WebDavAdapter::write_file_entry_with_dead_props(&mut w, file, &propfind_request, &href, child_dead)
|
WebDavAdapter::write_file_entry_with_dead_props(&mut w, file, &propfind_request, &href, child_dead)
|
||||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||||
@@ -847,7 +866,7 @@ async fn build_streaming_propfind_response(
|
|||||||
if (batch_len as i64) < PROPFIND_BATCH_SIZE {
|
if (batch_len as i64) < PROPFIND_BATCH_SIZE {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
offset += batch_len as i64;
|
after_name = batch.last().map(|f| f.name.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1381,20 +1400,45 @@ pub(crate) async fn folder_dead_props(
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// File-leaf variant for the streaming walker (takes a `&DeadPropertyStore`
|
/// Batched dead-props fetch for a whole PROPFIND page of files: ONE
|
||||||
/// rather than the full `&Arc<AppState>` so it can be called from inside
|
/// `file_id = ANY($1)` round-trip instead of one query per child (the old
|
||||||
/// the async-stream future without cloning state).
|
/// per-child `streamed_file_dead_props` loop cost seconds on large folders —
|
||||||
pub(crate) async fn streamed_file_dead_props(
|
/// benches/DEAD-PROPS.md). Same leniency as the single-resource helpers:
|
||||||
|
/// any failure → empty map, so the PROPFIND still emits live properties.
|
||||||
|
pub(crate) async fn files_dead_props_map(
|
||||||
store: &DeadPropertyStore,
|
store: &DeadPropertyStore,
|
||||||
file: &FileDto,
|
files: &[FileDto],
|
||||||
) -> Vec<(QualifiedName, Option<String>)> {
|
) -> HashMap<Uuid, Vec<(QualifiedName, Option<String>)>> {
|
||||||
let Ok(file_id) = Uuid::parse_str(&file.id) else {
|
let ids: Vec<Uuid> = files
|
||||||
return Vec::new();
|
.iter()
|
||||||
};
|
.filter_map(|f| Uuid::parse_str(&f.id).ok())
|
||||||
store
|
.collect();
|
||||||
.get_all(ResourceRef::File(file_id))
|
store.get_all_for_files(&ids).await.unwrap_or_default()
|
||||||
.await
|
}
|
||||||
.unwrap_or_default()
|
|
||||||
|
/// Folder-page variant of [`files_dead_props_map`].
|
||||||
|
pub(crate) async fn folders_dead_props_map(
|
||||||
|
store: &DeadPropertyStore,
|
||||||
|
folders: &[FolderDto],
|
||||||
|
) -> HashMap<Uuid, Vec<(QualifiedName, Option<String>)>> {
|
||||||
|
let ids: Vec<Uuid> = folders
|
||||||
|
.iter()
|
||||||
|
.filter_map(|f| Uuid::parse_str(&f.id).ok())
|
||||||
|
.collect();
|
||||||
|
store.get_all_for_folders(&ids).await.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Looks up one resource's dead props in a batched map (resources with no
|
||||||
|
/// dead properties are absent from the map → empty slice).
|
||||||
|
pub(crate) fn dead_props_for<'a>(
|
||||||
|
id: &str,
|
||||||
|
map: &'a HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>,
|
||||||
|
) -> &'a [(QualifiedName, Option<String>)] {
|
||||||
|
Uuid::parse_str(id)
|
||||||
|
.ok()
|
||||||
|
.and_then(|u| map.get(&u))
|
||||||
|
.map(|v| v.as_slice())
|
||||||
|
.unwrap_or(&[])
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A single condition inside a `List` of the WebDAV `If:` header
|
/// A single condition inside a `List` of the WebDAV `If:` header
|
||||||
|
|||||||
@@ -5,11 +5,36 @@ use axum::{
|
|||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
};
|
};
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
use std::sync::Arc;
|
use std::sync::{Arc, LazyLock};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use crate::application::dtos::folder_dto::FolderDto;
|
||||||
use crate::common::di::AppState;
|
use crate::common::di::AppState;
|
||||||
use crate::interfaces::middleware::auth::CurrentUser;
|
use crate::interfaces::middleware::auth::CurrentUser;
|
||||||
|
|
||||||
|
/// Markerless-chroot cache: default-drive root folder id → `FolderDto`.
|
||||||
|
///
|
||||||
|
/// This middleware wraps EVERY protected NextCloud route (DAV files,
|
||||||
|
/// per-chunk uploads, trashbin, previews, avatars, OCS polls). With the
|
||||||
|
/// app-password verification already cached, the chroot resolution was the
|
||||||
|
/// last per-request DB work: `find_default_for_user` (now cached in
|
||||||
|
/// `DrivePgRepository`) plus this folder-by-PK fetch. A desktop sync run
|
||||||
|
/// issues hundreds of these per minute for a value that changes only on a
|
||||||
|
/// root-folder rename — the 30 s TTL bounds that staleness (mirrors
|
||||||
|
/// `drive_role_cache` / the default-drive cache; measured in
|
||||||
|
/// `benches/CHROOT-CACHE.md`).
|
||||||
|
///
|
||||||
|
/// Only the MARKERLESS branch is cached: it targets the caller's own
|
||||||
|
/// default drive root, so no per-request authorization decision is being
|
||||||
|
/// skipped. The drive-marker branch keeps its `get_folder_with_perms`
|
||||||
|
/// check on every request.
|
||||||
|
static NC_CHROOT_CACHE: LazyLock<moka::sync::Cache<uuid::Uuid, FolderDto>> = LazyLock::new(|| {
|
||||||
|
moka::sync::Cache::builder()
|
||||||
|
.max_capacity(100_000)
|
||||||
|
.time_to_live(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
});
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum NextcloudAuthError {
|
pub enum NextcloudAuthError {
|
||||||
#[error("Unauthorized")]
|
#[error("Unauthorized")]
|
||||||
@@ -191,12 +216,24 @@ pub async fn basic_auth_middleware(
|
|||||||
.find_default_for_user(current_user.id)
|
.find_default_for_user(current_user.id)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(drive_with_name) => state
|
Ok(drive_with_name) => {
|
||||||
.applications
|
let root_id = drive_with_name.drive.root_folder_id;
|
||||||
.folder_service
|
match NC_CHROOT_CACHE.get(&root_id) {
|
||||||
.get_folder(&drive_with_name.drive.root_folder_id.to_string())
|
Some(cached) => Some(cached),
|
||||||
.await
|
None => {
|
||||||
.ok(),
|
let fetched = state
|
||||||
|
.applications
|
||||||
|
.folder_service
|
||||||
|
.get_folder(&root_id.to_string())
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
if let Some(f) = &fetched {
|
||||||
|
NC_CHROOT_CACHE.insert(root_id, f.clone());
|
||||||
|
}
|
||||||
|
fetched
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Err(_) => None,
|
Err(_) => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,9 @@ use crate::application::ports::folder_ports::FolderUseCase;
|
|||||||
use crate::application::ports::inbound::SearchUseCase;
|
use crate::application::ports::inbound::SearchUseCase;
|
||||||
use crate::common::di::AppState;
|
use crate::common::di::AppState;
|
||||||
use crate::domain::entities::file::File;
|
use crate::domain::entities::file::File;
|
||||||
use crate::interfaces::api::handlers::webdav_handler::{file_dead_props, folder_dead_props};
|
use crate::interfaces::api::handlers::webdav_handler::{
|
||||||
|
dead_props_for, files_dead_props_map, folders_dead_props_map,
|
||||||
|
};
|
||||||
use crate::interfaces::errors::AppError;
|
use crate::interfaces::errors::AppError;
|
||||||
use crate::interfaces::nextcloud::webdav_handler::{
|
use crate::interfaces::nextcloud::webdav_handler::{
|
||||||
batch_resolve_ids, format_oc_id, nc_href, write_file_response, write_folder_response,
|
batch_resolve_ids, format_oc_id, nc_href, write_file_response, write_folder_response,
|
||||||
@@ -160,6 +162,11 @@ async fn handle_filter_files(
|
|||||||
|
|
||||||
write_multistatus_start(&mut xml)?;
|
write_multistatus_start(&mut xml)?;
|
||||||
|
|
||||||
|
// Batched dead-props: one = ANY($1) query per type, not one per
|
||||||
|
// result (benches/DEAD-PROPS.md).
|
||||||
|
let file_deads = files_dead_props_map(&state.webdav_dead_props, &files).await;
|
||||||
|
let folder_deads = folders_dead_props_map(&state.webdav_dead_props, &folders).await;
|
||||||
|
|
||||||
// Keep main's batched-resolution structure (one batch query
|
// Keep main's batched-resolution structure (one batch query
|
||||||
// per type, not 2N round-trips). Hrefs use `url_user` so the
|
// per type, not 2N round-trips). Hrefs use `url_user` so the
|
||||||
// multi-drive `~{drive}` form is echoed back to the client;
|
// multi-drive `~{drive}` form is echoed back to the client;
|
||||||
@@ -179,7 +186,7 @@ async fn handle_filter_files(
|
|||||||
let href = nc_href(url_user, subpath);
|
let href = nc_href(url_user, subpath);
|
||||||
let fid = file_id_map.get(&file.id).copied();
|
let fid = file_id_map.get(&file.id).copied();
|
||||||
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
||||||
let dead = file_dead_props(&state, file).await;
|
let dead = dead_props_for(&file.id, &file_deads);
|
||||||
write_file_response(
|
write_file_response(
|
||||||
&mut xml,
|
&mut xml,
|
||||||
file,
|
file,
|
||||||
@@ -187,7 +194,7 @@ async fn handle_filter_files(
|
|||||||
(fid, oc_id.as_deref()),
|
(fid, oc_id.as_deref()),
|
||||||
&user.username,
|
&user.username,
|
||||||
&favorite_ids,
|
&favorite_ids,
|
||||||
&dead,
|
dead,
|
||||||
)
|
)
|
||||||
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
||||||
}
|
}
|
||||||
@@ -205,7 +212,7 @@ async fn handle_filter_files(
|
|||||||
let href = format!("{}/", nc_href(url_user, subpath));
|
let href = format!("{}/", nc_href(url_user, subpath));
|
||||||
let fid = folder_id_map.get(&folder.id).copied();
|
let fid = folder_id_map.get(&folder.id).copied();
|
||||||
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
||||||
let dead = folder_dead_props(&state.webdav_dead_props, folder).await;
|
let dead = dead_props_for(&folder.id, &folder_deads);
|
||||||
write_folder_response(
|
write_folder_response(
|
||||||
&mut xml,
|
&mut xml,
|
||||||
folder,
|
folder,
|
||||||
@@ -217,7 +224,7 @@ async fn handle_filter_files(
|
|||||||
// PROPFIND on a specific collection — quota isn't
|
// PROPFIND on a specific collection — quota isn't
|
||||||
// meaningful here (see `AppState::resolve_webdav_quota`).
|
// meaningful here (see `AppState::resolve_webdav_quota`).
|
||||||
None,
|
None,
|
||||||
&dead,
|
dead,
|
||||||
)
|
)
|
||||||
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
||||||
}
|
}
|
||||||
@@ -301,6 +308,11 @@ async fn handle_search(
|
|||||||
|
|
||||||
write_multistatus_start(&mut xml)?;
|
write_multistatus_start(&mut xml)?;
|
||||||
|
|
||||||
|
// Batched dead-props: one = ANY($1) query per type, not one per
|
||||||
|
// result (benches/DEAD-PROPS.md).
|
||||||
|
let file_deads = files_dead_props_map(&state.webdav_dead_props, &files).await;
|
||||||
|
let folder_deads = folders_dead_props_map(&state.webdav_dead_props, &folders).await;
|
||||||
|
|
||||||
// Files.
|
// Files.
|
||||||
for file in &files {
|
for file in &files {
|
||||||
let Some(subpath) = strip_home_prefix(chroot, &file.path, home_prefix) else {
|
let Some(subpath) = strip_home_prefix(chroot, &file.path, home_prefix) else {
|
||||||
@@ -315,7 +327,7 @@ async fn handle_search(
|
|||||||
let href = nc_href(url_user, subpath);
|
let href = nc_href(url_user, subpath);
|
||||||
let fid = file_id_map.get(&file.id).copied();
|
let fid = file_id_map.get(&file.id).copied();
|
||||||
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
||||||
let dead = file_dead_props(&state, file).await;
|
let dead = dead_props_for(&file.id, &file_deads);
|
||||||
write_file_response(
|
write_file_response(
|
||||||
&mut xml,
|
&mut xml,
|
||||||
file,
|
file,
|
||||||
@@ -323,7 +335,7 @@ async fn handle_search(
|
|||||||
(fid, oc_id.as_deref()),
|
(fid, oc_id.as_deref()),
|
||||||
&user.username,
|
&user.username,
|
||||||
&favorite_ids,
|
&favorite_ids,
|
||||||
&dead,
|
dead,
|
||||||
)
|
)
|
||||||
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
||||||
}
|
}
|
||||||
@@ -342,7 +354,7 @@ async fn handle_search(
|
|||||||
let href = format!("{}/", nc_href(url_user, subpath));
|
let href = format!("{}/", nc_href(url_user, subpath));
|
||||||
let fid = folder_id_map.get(&folder.id).copied();
|
let fid = folder_id_map.get(&folder.id).copied();
|
||||||
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
|
||||||
let dead = folder_dead_props(&state.webdav_dead_props, folder).await;
|
let dead = dead_props_for(&folder.id, &folder_deads);
|
||||||
write_folder_response(
|
write_folder_response(
|
||||||
&mut xml,
|
&mut xml,
|
||||||
folder,
|
folder,
|
||||||
@@ -354,7 +366,7 @@ async fn handle_search(
|
|||||||
// PROPFIND on a specific collection — quota isn't
|
// PROPFIND on a specific collection — quota isn't
|
||||||
// meaningful here (see `AppState::resolve_webdav_quota`).
|
// meaningful here (see `AppState::resolve_webdav_quota`).
|
||||||
None,
|
None,
|
||||||
&dead,
|
dead,
|
||||||
)
|
)
|
||||||
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -74,6 +74,14 @@ async fn session_bytes_so_far(
|
|||||||
username: &str,
|
username: &str,
|
||||||
upload_id: &str,
|
upload_id: &str,
|
||||||
) -> Result<u64, AppError> {
|
) -> Result<u64, AppError> {
|
||||||
|
// Warm path: O(1) in-RAM counter maintained by the PUT handler and
|
||||||
|
// the service (seeded on MKCOL, dropped on cleanup/overwrite). The
|
||||||
|
// directory walk below only runs cold (restart / eviction) — the old
|
||||||
|
// shape ran it on EVERY chunk PUT: O(k) stats for chunk k, O(N²/2)
|
||||||
|
// over the upload (benches/NC-CHUNK-GATE.md).
|
||||||
|
if let Some(bytes) = nc.chunked_uploads.cached_session_bytes(username, upload_id) {
|
||||||
|
return Ok(bytes);
|
||||||
|
}
|
||||||
let listing = nc
|
let listing = nc
|
||||||
.chunked_uploads
|
.chunked_uploads
|
||||||
.list_chunks(username, upload_id)
|
.list_chunks(username, upload_id)
|
||||||
@@ -85,7 +93,10 @@ async fn session_bytes_so_far(
|
|||||||
// chunk after MKCOL (race-tolerant).
|
// chunk after MKCOL (race-tolerant).
|
||||||
return Ok(0);
|
return Ok(0);
|
||||||
};
|
};
|
||||||
Ok(listing.chunks.iter().map(|c| c.size).sum())
|
let total = listing.chunks.iter().map(|c| c.size).sum();
|
||||||
|
nc.chunked_uploads
|
||||||
|
.set_session_bytes(username, upload_id, total);
|
||||||
|
Ok(total)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Dispatch Nextcloud chunked upload WebDAV requests.
|
/// Dispatch Nextcloud chunked upload WebDAV requests.
|
||||||
@@ -314,11 +325,21 @@ async fn handle_put_chunk(
|
|||||||
.map_err(|e| AppError::bad_request(format!("Invalid chunk path: {}", e)))?;
|
.map_err(|e| AppError::bad_request(format!("Invalid chunk path: {}", e)))?;
|
||||||
|
|
||||||
let max_chunk = state.core.config.storage.chunk_max_bytes;
|
let max_chunk = state.core.config.storage.chunk_max_bytes;
|
||||||
|
// A re-PUT of an existing chunk (client retry) makes the running
|
||||||
|
// session counter stale — drop it so the next gate rebuilds from disk.
|
||||||
|
let overwrite = tokio::fs::metadata(&chunk_path).await.is_ok();
|
||||||
// No client-side integrity contract on the NC chunked surface — the
|
// No client-side integrity contract on the NC chunked surface — the
|
||||||
// NC desktop client validates the assembled-file ETag against the
|
// NC desktop client validates the assembled-file ETag against the
|
||||||
// server-side `oc:checksums` after MOVE. So we skip per-chunk
|
// server-side `oc:checksums` after MOVE. So we skip per-chunk
|
||||||
// hashing here (peak heap stays at ~one HTTP frame).
|
// hashing here (peak heap stays at ~one HTTP frame).
|
||||||
stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?;
|
let streamed = stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?;
|
||||||
|
if overwrite {
|
||||||
|
nc.chunked_uploads
|
||||||
|
.forget_session_bytes(&user.username, upload_id);
|
||||||
|
} else {
|
||||||
|
nc.chunked_uploads
|
||||||
|
.bump_session_bytes(&user.username, upload_id, streamed.bytes_written);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(Response::builder()
|
Ok(Response::builder()
|
||||||
.status(StatusCode::CREATED)
|
.status(StatusCode::CREATED)
|
||||||
|
|||||||
@@ -30,7 +30,8 @@ use crate::domain::services::authorization::{Permission, Resource, Subject};
|
|||||||
use crate::infrastructure::services::path_resolver_service::ResolvedResource;
|
use crate::infrastructure::services::path_resolver_service::ResolvedResource;
|
||||||
use crate::infrastructure::services::webdav_dead_property_store::ResourceRef;
|
use crate::infrastructure::services::webdav_dead_property_store::ResourceRef;
|
||||||
use crate::interfaces::api::handlers::webdav_handler::{
|
use crate::interfaces::api::handlers::webdav_handler::{
|
||||||
PROPFIND_BATCH_SIZE, file_dead_props, folder_dead_props, streamed_file_dead_props,
|
PROPFIND_BATCH_SIZE, dead_props_for, file_dead_props, files_dead_props_map, folder_dead_props,
|
||||||
|
folders_dead_props_map,
|
||||||
};
|
};
|
||||||
use crate::interfaces::errors::AppError;
|
use crate::interfaces::errors::AppError;
|
||||||
use crate::interfaces::range_requests::{not_modified_response, range_response};
|
use crate::interfaces::range_requests::{not_modified_response, range_response};
|
||||||
@@ -297,9 +298,10 @@ async fn handle_propfind(
|
|||||||
.map_err(|e| AppError::bad_request(format!("Failed to read body: {}", e)))?;
|
.map_err(|e| AppError::bad_request(format!("Failed to read body: {}", e)))?;
|
||||||
|
|
||||||
// Parse (and thereby validate) the PROPFIND body. The NC response
|
// Parse (and thereby validate) the PROPFIND body. The NC response
|
||||||
// always emits the full property set, so the parsed request is not
|
// always emits the full property set; the parsed request is consulted
|
||||||
// consulted further — but malformed XML must still fail with 400.
|
// only to skip the quota DB round-trips when the client's explicit
|
||||||
let _propfind = if body_bytes.is_empty() {
|
// prop list never names a quota prop. Malformed XML still fails 400.
|
||||||
|
let propfind = if body_bytes.is_empty() {
|
||||||
PropFindRequest {
|
PropFindRequest {
|
||||||
prop_find_type: crate::application::adapters::webdav_adapter::PropFindType::AllProp,
|
prop_find_type: crate::application::adapters::webdav_adapter::PropFindType::AllProp,
|
||||||
}
|
}
|
||||||
@@ -341,7 +343,13 @@ async fn handle_propfind(
|
|||||||
// function's username arg. Refining the owner-id usages
|
// function's username arg. Refining the owner-id usages
|
||||||
// back to the canonical username is deferred to the
|
// back to the canonical username is deferred to the
|
||||||
// NcSession commit.
|
// NcSession commit.
|
||||||
let quota = state.resolve_webdav_quota(user.id, chroot.drive_id).await;
|
// Explicit prop lists that never name a quota prop skip the
|
||||||
|
// 2-query quota resolution (benches/QUOTA-PATH.md).
|
||||||
|
let quota = if propfind.wants_quota() {
|
||||||
|
state.resolve_webdav_quota(user.id, chroot.drive_id).await
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
Ok(build_nc_streaming_propfind(
|
Ok(build_nc_streaming_propfind(
|
||||||
state.clone(),
|
state.clone(),
|
||||||
folder,
|
folder,
|
||||||
@@ -1519,11 +1527,17 @@ fn build_nc_streaming_propfind(
|
|||||||
|
|
||||||
// ── Children (only if Depth != 0) ────────────────────────────
|
// ── Children (only if Depth != 0) ────────────────────────────
|
||||||
if depth != "0" {
|
if depth != "0" {
|
||||||
// Files in pages.
|
// Files in pages (keyset cursor — O(page) per page instead of
|
||||||
let mut offset: i64 = 0;
|
// the quadratic LIMIT/OFFSET walk).
|
||||||
|
let mut after_name: Option<String> = None;
|
||||||
loop {
|
loop {
|
||||||
let batch = file_service
|
let batch = file_service
|
||||||
.list_files_batch_with_perms(Some(&folder.id), user_id, offset, PROPFIND_BATCH_SIZE)
|
.list_files_batch_with_perms(
|
||||||
|
Some(&folder.id),
|
||||||
|
user_id,
|
||||||
|
after_name.as_deref(),
|
||||||
|
PROPFIND_BATCH_SIZE,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||||
if batch.is_empty() {
|
if batch.is_empty() {
|
||||||
@@ -1541,15 +1555,15 @@ fn build_nc_streaming_propfind(
|
|||||||
};
|
};
|
||||||
let file_uuids: Vec<String> = batch.iter().map(|f| f.id.clone()).collect();
|
let file_uuids: Vec<String> = batch.iter().map(|f| f.id.clone()).collect();
|
||||||
let (file_id_map, _) = batch_resolve_ids(file_id_svc, &file_uuids, &[]).await;
|
let (file_id_map, _) = batch_resolve_ids(file_id_svc, &file_uuids, &[]).await;
|
||||||
let mut file_deads = Vec::with_capacity(batch_len);
|
// One batched dead-props query per page, not one per child
|
||||||
for file in &batch {
|
// (benches/DEAD-PROPS.md).
|
||||||
file_deads.push(streamed_file_dead_props(&state.webdav_dead_props, file).await);
|
let file_deads = files_dead_props_map(&state.webdav_dead_props, &batch).await;
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk = Vec::with_capacity(batch_len * 1024);
|
let mut chunk = Vec::with_capacity(batch_len * 1024);
|
||||||
{
|
{
|
||||||
let mut xml = Writer::new(&mut chunk);
|
let mut xml = Writer::new(&mut chunk);
|
||||||
for (file, dead) in batch.iter().zip(file_deads.iter()) {
|
for file in batch.iter() {
|
||||||
|
let dead = dead_props_for(&file.id, &file_deads);
|
||||||
let child_sub = if subpath.is_empty() {
|
let child_sub = if subpath.is_empty() {
|
||||||
file.name.clone()
|
file.name.clone()
|
||||||
} else {
|
} else {
|
||||||
@@ -1567,7 +1581,7 @@ fn build_nc_streaming_propfind(
|
|||||||
if (batch_len as i64) < PROPFIND_BATCH_SIZE {
|
if (batch_len as i64) < PROPFIND_BATCH_SIZE {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
offset += batch_len as i64;
|
after_name = batch.last().map(|f| f.name.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subfolders in pages — also collections, same trailing-slash rule.
|
// Subfolders in pages — also collections, same trailing-slash rule.
|
||||||
@@ -1594,15 +1608,15 @@ fn build_nc_streaming_propfind(
|
|||||||
};
|
};
|
||||||
let folder_uuids: Vec<String> = result.items.iter().map(|sf| sf.id.clone()).collect();
|
let folder_uuids: Vec<String> = result.items.iter().map(|sf| sf.id.clone()).collect();
|
||||||
let (_, sub_id_map) = batch_resolve_ids(file_id_svc, &[], &folder_uuids).await;
|
let (_, sub_id_map) = batch_resolve_ids(file_id_svc, &[], &folder_uuids).await;
|
||||||
let mut sub_deads = Vec::with_capacity(result.items.len());
|
// Batched — see benches/DEAD-PROPS.md.
|
||||||
for sf in &result.items {
|
let sub_deads =
|
||||||
sub_deads.push(folder_dead_props(&state.webdav_dead_props, sf).await);
|
folders_dead_props_map(&state.webdav_dead_props, &result.items).await;
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk = Vec::with_capacity(result.items.len() * 1024);
|
let mut chunk = Vec::with_capacity(result.items.len() * 1024);
|
||||||
{
|
{
|
||||||
let mut xml = Writer::new(&mut chunk);
|
let mut xml = Writer::new(&mut chunk);
|
||||||
for (sf, dead) in result.items.iter().zip(sub_deads.iter()) {
|
for sf in result.items.iter() {
|
||||||
|
let dead = dead_props_for(&sf.id, &sub_deads);
|
||||||
let child_sub = if subpath.is_empty() {
|
let child_sub = if subpath.is_empty() {
|
||||||
sf.name.clone()
|
sf.name.clone()
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use http_range_header::parse_range_header;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::application::dtos::file_dto::FileDto;
|
use crate::application::dtos::file_dto::FileDto;
|
||||||
use crate::application::ports::file_ports::FileRetrievalUseCase;
|
use crate::application::ports::file_ports::RangeContent;
|
||||||
use crate::application::services::file_retrieval_service::FileRetrievalService;
|
use crate::application::services::file_retrieval_service::FileRetrievalService;
|
||||||
|
|
||||||
/// `If-None-Match` short-circuit: returns a `304 Not Modified` response
|
/// `If-None-Match` short-circuit: returns a `304 Not Modified` response
|
||||||
@@ -71,24 +71,32 @@ pub async fn range_response(
|
|||||||
let end = *range.end();
|
let end = *range.end();
|
||||||
let range_length = end - start + 1;
|
let range_length = end - start + 1;
|
||||||
|
|
||||||
|
// Cache-aware: sub-threshold files already in the RAM content cache are
|
||||||
|
// answered with a zero-copy Bytes slice — no PG, no disk (benches/RANGE-CACHE.md).
|
||||||
match retrieval
|
match retrieval
|
||||||
.get_file_range_stream(&file.id, start, Some(end + 1))
|
.get_file_range_preloaded(file, start, Some(end + 1))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(stream) => Some(
|
Ok(content) => {
|
||||||
Response::builder()
|
let body = match content {
|
||||||
.status(StatusCode::PARTIAL_CONTENT)
|
RangeContent::Bytes(b) => Body::from(b),
|
||||||
.header(header::CONTENT_TYPE, &*file.mime_type)
|
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
|
||||||
.header(header::CONTENT_LENGTH, range_length)
|
};
|
||||||
.header(
|
Some(
|
||||||
header::CONTENT_RANGE,
|
Response::builder()
|
||||||
format!("bytes {}-{}/{}", start, end, file.size),
|
.status(StatusCode::PARTIAL_CONTENT)
|
||||||
)
|
.header(header::CONTENT_TYPE, &*file.mime_type)
|
||||||
.header(header::ACCEPT_RANGES, "bytes")
|
.header(header::CONTENT_LENGTH, range_length)
|
||||||
.header(header::ETAG, etag)
|
.header(
|
||||||
.body(Body::from_stream(Box::into_pin(stream)))
|
header::CONTENT_RANGE,
|
||||||
.unwrap(),
|
format!("bytes {}-{}/{}", start, end, file.size),
|
||||||
),
|
)
|
||||||
|
.header(header::ACCEPT_RANGES, "bytes")
|
||||||
|
.header(header::ETAG, etag)
|
||||||
|
.body(body)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
tracing::error!("Error creating range stream: {}", err);
|
tracing::error!("Error creating range stream: {}", err);
|
||||||
None // fall through to the full download
|
None // fall through to the full download
|
||||||
|
|||||||
@@ -46,10 +46,23 @@ pub fn create_web_routes() -> Router<Arc<AppState>> {
|
|||||||
let static_path = resolve_static_path(&config);
|
let static_path = resolve_static_path(&config);
|
||||||
|
|
||||||
// SPA fallback: serve the file if it exists, else the app shell.
|
// SPA fallback: serve the file if it exists, else the app shell.
|
||||||
let spa = ServeDir::new(&static_path).fallback(ServeFile::new(static_path.join("index.html")));
|
//
|
||||||
|
// `precompressed_*`: if the frontend build emitted a sibling `.br`/`.gz`
|
||||||
|
// (frontend/scripts/precompress.mjs runs at build time), serve those
|
||||||
|
// bytes directly with the right Content-Encoding instead of re-running
|
||||||
|
// Brotli over the same immutable bundle on EVERY request — the
|
||||||
|
// `CompressionLayer` below then skips the already-encoded response and
|
||||||
|
// remains only the fallback for assets without a precompressed sibling
|
||||||
|
// (benches/STATIC-PRECOMPRESSED.md).
|
||||||
|
let spa = ServeDir::new(&static_path)
|
||||||
|
.precompressed_br()
|
||||||
|
.precompressed_gzip()
|
||||||
|
.fallback(ServeFile::new(static_path.join("index.html")));
|
||||||
|
|
||||||
// Hashed, immutable assets (SvelteKit emits these under /_app/immutable).
|
// Hashed, immutable assets (SvelteKit emits these under /_app/immutable).
|
||||||
let app_immutable = ServeDir::new(static_path.join("_app").join("immutable"));
|
let app_immutable = ServeDir::new(static_path.join("_app").join("immutable"))
|
||||||
|
.precompressed_br()
|
||||||
|
.precompressed_gzip();
|
||||||
|
|
||||||
Router::new()
|
Router::new()
|
||||||
.nest_service(
|
.nest_service(
|
||||||
@@ -60,7 +73,17 @@ pub fn create_web_routes() -> Router<Arc<AppState>> {
|
|||||||
)),
|
)),
|
||||||
)
|
)
|
||||||
.fallback_service(spa)
|
.fallback_service(spa)
|
||||||
.layer(CompressionLayer::new().br(true).gzip(true))
|
// Fallback compression for assets without a precompressed sibling.
|
||||||
|
// Quality 4, NOT the default: the default maps to Brotli q11 —
|
||||||
|
// ~1.3 s of CPU per 700 KiB bundle per request (measured in
|
||||||
|
// benches/STATIC-PRECOMPRESSED.md; the .br siblings above carry the
|
||||||
|
// real q11 bytes, paid once at build time).
|
||||||
|
.layer(
|
||||||
|
CompressionLayer::new()
|
||||||
|
.quality(tower_http::CompressionLevel::Precise(4))
|
||||||
|
.br(true)
|
||||||
|
.gzip(true),
|
||||||
|
)
|
||||||
// `if_not_present` so the immutable assets above keep their long cache;
|
// `if_not_present` so the immutable assets above keep their long cache;
|
||||||
// the shell itself must always revalidate so a deploy can't pin a stale
|
// the shell itself must always revalidate so a deploy can't pin a stale
|
||||||
// app in browsers.
|
// app in browsers.
|
||||||
|
|||||||
+12
-1
@@ -880,7 +880,18 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
// ── file-body downloads carry Content-Disposition (see above) ──
|
// ── file-body downloads carry Content-Disposition (see above) ──
|
||||||
.and(NotForDownloads);
|
.and(NotForDownloads);
|
||||||
|
|
||||||
app = app.layer(CompressionLayer::new().compress_when(predicate));
|
// Explicit quality: the layer's default maps to Brotli QUALITY 11
|
||||||
|
// (async-compression Level::Default → BrotliEncoderParams::default(),
|
||||||
|
// brotli-8.0.2 encode.rs:323) — a deploy-grade setting that cost
|
||||||
|
// ~90 ms of CPU per 64 KiB JSON response. Level 4 emits ~15 % more
|
||||||
|
// bytes at ~1 % of the CPU (0.9 ms) — measured in
|
||||||
|
// benches/STATIC-PRECOMPRESSED.md. Applies to gzip too (level 4,
|
||||||
|
// the classic dynamic-content setting).
|
||||||
|
app = app.layer(
|
||||||
|
CompressionLayer::new()
|
||||||
|
.quality(tower_http::CompressionLevel::Precise(4))
|
||||||
|
.compress_when(predicate),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Security headers ─────────────────────────────────────────────────
|
// ── Security headers ─────────────────────────────────────────────────
|
||||||
|
|||||||
Reference in New Issue
Block a user