From e3f04d58aa9aa759a236cb57164a59725e6720f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 13:06:33 +0000 Subject: [PATCH 1/6] Stream uploads directly into the CDC chunk store (no spool, single write) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every upload surface previously wrote each byte to disk twice: the HTTP body was spooled to a temp file (or assembled from chunk parts), then mmap-re-read for FastCDC analysis, and finally the new chunks were written to the blob backend. CDC could not start until the last byte arrived, so large uploads paid receive + reread + rewrite latency. The dedup engine now chunks, hashes and settles the stream WHILE it arrives (fastcdc AsyncStreamCDC + incremental BLAKE3): - Each batch of distinct chunks is pinned-or-classified by ONE `UPDATE … RETURNING` (no check-then-bump TOCTOU; pinned chunks can't be reclaimed mid-upload), and only chunks the store doesn't have are written — a full dedup hit performs zero content writes. - Durability before visibility is preserved: one batched fsync sweep, then one batched INSERT, then the manifest. Identical concurrent uploads are resolved at the manifest INSERT via ON CONFLICT (the loser releases its references and becomes a dedup hit). - A drop guard rolls back pins and surfaces written-but-unregistered chunks to GC if the request future is cancelled mid-stream. - MIME sniffing now peeks the first bytes in-flight; client-requested MD5/SHA-256 checksums are computed by a stream tee — the post-upload re-read of the assembled file is gone. All surfaces converge on the new interfaces::upload_ingest helper: REST multipart, WebDAV PUT, NextCloud PUT, WOPI PutFile, the dedup endpoint, and both chunked-upload completions (which now stream their ordered parts straight into the store instead of writing an assembled file — chunk parts persist until finalize, so completion is genuinely retryable). The legacy blob re-chunk migration streams from the backend with no spool file either. Legacy removed: store_from_file + mmap CDC analysers + temp-path plumbing through every port (pre_computed_hash, save_file_from_temp, update_file_content_from_temp), upload_spool + assembled-file assembly in both chunked services, create_file/update_file byte-slice variants (no callers), common::temp, the OXICLOUD_UPLOAD_TMPDIR config, and the memmap2 dependency. Verified end-to-end against PostgreSQL 16: 8 MB upload (26 chunks), identical re-upload (dedup hit, zero writes), 3-byte edit re-upload (26 chunks, 1 written), byte-identical downloads, Range across chunk boundaries, concurrent identical-upload race (manifest ref 2), and trash-empty reclaiming exactly the unshared chunk while the shared 25 survive for the edited file. The empty/sub-8KB multipart path found a post-EOF re-poll panic in the MIME peek (fixed with fuse + regression test). https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- Cargo.lock | 6 +- Cargo.toml | 3 +- docs/config/env.md | 3 +- docs/config/storage-fine-tuning.md | 299 ++-- example.env | 18 +- src/application/ports/chunked_upload_ports.rs | 24 +- src/application/ports/dedup_ports.rs | 12 - src/application/ports/file_ports.rs | 82 +- src/application/ports/storage_ports.rs | 30 +- .../services/file_upload_service.rs | 214 +-- .../services/idor_protection_test.rs | 13 +- .../services/trash_service_test.rs | 11 +- src/common/config.rs | 23 +- src/common/di.rs | 3 +- src/common/mime_detect.rs | 109 +- src/common/mod.rs | 1 - src/common/stubs.rs | 67 +- src/common/temp.rs | 26 - src/domain/repositories/file_repository.rs | 8 +- .../pg/file_blob_write_repository.rs | 77 +- .../services/chunked_upload_service.rs | 168 +-- src/infrastructure/services/dedup_service.rs | 1276 +++++++++-------- .../services/local_blob_backend.rs | 13 +- .../nextcloud_chunked_upload_service.rs | 166 +-- .../api/handlers/chunked_upload_handler.rs | 218 ++- src/interfaces/api/handlers/dedup_handler.rs | 161 +-- src/interfaces/api/handlers/file_handler.rs | 138 +- src/interfaces/api/handlers/webdav_handler.rs | 49 +- src/interfaces/api/handlers/wopi_handler.rs | 81 +- src/interfaces/mod.rs | 2 +- src/interfaces/nextcloud/uploads_handler.rs | 77 +- src/interfaces/nextcloud/webdav_handler.rs | 49 +- src/interfaces/upload_ingest.rs | 588 ++++++++ src/interfaces/upload_spool.rs | 289 ---- 34 files changed, 1864 insertions(+), 2440 deletions(-) delete mode 100644 src/common/temp.rs create mode 100644 src/interfaces/upload_ingest.rs delete mode 100644 src/interfaces/upload_spool.rs diff --git a/Cargo.lock b/Cargo.lock index 00cfd3d6..05fbf7b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1906,6 +1906,11 @@ name = "fastcdc" version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" +dependencies = [ + "async-stream", + "tokio", + "tokio-stream", +] [[package]] name = "fastrand" @@ -3824,7 +3829,6 @@ dependencies = [ "lightningcss", "lru", "md-5 0.11.0", - "memmap2", "mimalloc", "mime_guess", "mockall", diff --git a/Cargo.toml b/Cargo.toml index 42fa985e..29f5ce19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,8 +68,7 @@ azure_storage = { version = "0.21", default-features = false, features = ["enabl azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } aes-gcm = "0.10.3" lru = "0.16.4" -fastcdc = "4.0.0" -memmap2 = "0.9.10" +fastcdc = { version = "4.0.0", features = ["tokio"] } lettre = { version = "0.11.18", default-features = false, features = ["smtp-transport", "tokio1-rustls-tls", "rustls-native-certs", "builder"] } idna = "1.1" smol_str = { version = "0.3.2", features = ["serde"] } diff --git a/docs/config/env.md b/docs/config/env.md index f131ccd4..87fcfd7f 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -14,8 +14,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_MAX_UPLOAD_SIZE` | `10737418240` | Whole-file size ceiling, in bytes (10 GB on 64-bit, 1 GB on 32-bit). Applies to BOTH direct PUTs (per-request body) and chunked uploads (declared `total_size`, checked upfront at session creation). | | `OXICLOUD_DIRECT_PUT_MAX_BYTES` | `1073741824` | Per-request cap for non-chunked PUT bodies, in bytes (1 GiB). Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol (resumable on failure). See [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_CHUNK_MAX_BYTES` | `104857600` | Maximum size of a single chunked-upload PUT in bytes (100 MB). Per-chunk cap, independent of `OXICLOUD_MAX_UPLOAD_SIZE` (whole-file cap). See [Storage Fine Tuning](./storage-fine-tuning.md). | -| `OXICLOUD_UPLOAD_TMPDIR` | (OS temp dir) | Spool directory for non-chunked PUT bodies (`/api/files/upload`, WebDAV PUT). Point at a real-disk path on the same FS as `.blobs/` to avoid tmpfs OOMKill and make blob promotion an atomic rename. See [Storage Fine Tuning](./storage-fine-tuning.md). | -| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Same-FS / NVMe placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | +| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Direct (non-chunked) uploads stream straight into the blob store and need no spool directory. Placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | ## Database diff --git a/docs/config/storage-fine-tuning.md b/docs/config/storage-fine-tuning.md index ea51d3e1..dfd59849 100644 --- a/docs/config/storage-fine-tuning.md +++ b/docs/config/storage-fine-tuning.md @@ -1,62 +1,56 @@ # Storage Fine Tuning -This page is for sysadmins who want to tune **where** OxiCloud spools -upload bodies and **why** the placement matters for throughput and -memory. The defaults work; the gains from a tuned layout are +This page is for sysadmins who want to tune **where** OxiCloud places +upload data on disk and **why** the placement matters for throughput +and memory. The defaults work; the gains from a tuned layout are significant on busy instances or constrained containers. ## The upload lifecycle in 30 seconds -Every upload moves through two stages: - ``` - ┌─── direct (single-PUT) upload ────────┐ -client ─┤ ├──► OxiCloud accepts the - └─── multi-chunk upload │ bytes into a SPOOL on - (`/api/uploads` / │ local disk. - `/dav/uploads/...`) │ - │ Direct upload → OXICLOUD_UPLOAD_TMPDIR - │ Chunked upload → OXICLOUD_CHUNK_DIR - │ - ▼ + ┌─── direct (single-PUT / multipart) ───┐ +client ─┤ │ Streamed DIRECTLY into the + │ ├──► content-addressable store: + │ │ CDC chunking + BLAKE3 + + │ │ dedup checks happen while + │ │ the bytes arrive. No spool + │ │ file, no re-read; chunks + │ │ the store already has are + │ │ never written at all. + │ │ + └─── multi-chunk upload ────────────────┤ Chunk PARTS accumulate on + (`/api/uploads` / │ disk under OXICLOUD_CHUNK_DIR + `/dav/uploads/...`) │ until /complete, which + │ streams them (in order) + ▼ through the same CDC path. ┌─────────────────────────┐ - │ Once the upload is │ - │ complete (and verified │ - │ if a checksum was │ - │ supplied), OxiCloud │ - │ MOVES the assembled │ - │ blob into the configured│ - │ STORAGE BACKEND: │ - │ │ - │ • local FS (.blobs/) │ - │ • S3-compatible │ - │ • Azure Blob │ + │ STORAGE BACKEND │ + │ • local FS (.blobs/) │ + │ • S3-compatible │ + │ • Azure Blob │ └─────────────────────────┘ ``` Two practical consequences: -- **The spool/chunk directories see write-heavy churn** during uploads — - fast disk (NVMe) and sufficient free space matter more here than on - the final storage backend. -- **The promotion from spool → storage is a `rename(2)` whenever - source and destination share a filesystem** (i.e. when the backend - is `local` and the spool dir is on the same FS as `.blobs/`). On - remote backends (S3, Azure) the promotion is always a network - upload from the local spool; placement of the spool still matters - for intake throughput but the "same FS" rule doesn't apply. +- **Direct uploads no longer use a spool directory.** Each uploaded + byte is written at most once — straight into the blob backend as a + CDC chunk. Re-uploads and edited files write only the chunks the + store doesn't already have. +- **The chunk-session directory sees write-heavy churn** during + multi-chunk uploads — fast disk (NVMe) and sufficient free space + matter more here than on the final storage backend. ## Upload size caps — what each one bounds Three independent caps control how large an upload OxiCloud will -accept. Pick them with disk and tmpfs sizing in mind: the spool/chunk -directories must be able to hold the worst case (cap × concurrent -uploads). +accept. Pick them with disk sizing in mind: the chunk directory must +be able to hold the worst case (cap × concurrent sessions). | Variable | Default | What it caps | When it fires | |---|---|---|---| | `OXICLOUD_MAX_UPLOAD_SIZE` | 10 GB | **Whole-file ceiling.** Applies to both direct PUT (per-body) and chunked uploads (declared `total_size`). The absolute upper bound on any single file in OxiCloud. | Chunked: at `POST /api/uploads` against the JSON-declared `total_size`, before any chunk is uploaded. Direct PUT: indirectly via `OXICLOUD_DIRECT_PUT_MAX_BYTES`, which is expected to be ≤ `OXICLOUD_MAX_UPLOAD_SIZE`. | -| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `POST /api/files/upload`, `PUT /webdav/...`, and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | +| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `PUT /webdav/...` and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | | `OXICLOUD_CHUNK_MAX_BYTES` | 100 MB | **Per-chunk body** in a chunked-upload session (`PATCH /api/uploads/{id}` or `PUT /remote.php/dav/uploads/.../chunk`). Independent of the whole-file cap — a 5 GB file in 100 MB chunks is 50 PATCHes each bounded by this. | During chunk-body streaming. Excess → 413. | ### Recommendation: prefer chunked uploads for large files @@ -64,244 +58,117 @@ uploads). The defaults (`OXICLOUD_DIRECT_PUT_MAX_BYTES` = 1 GiB, well below `OXICLOUD_MAX_UPLOAD_SIZE` = 10 GB) are deliberately asymmetric. Files between those two caps can only succeed via the chunked -protocol. Three reasons to keep them that way: +protocol. The reason is **resilience**: a direct PUT at 95 % of 5 GB +that drops loses everything (the partially ingested chunks are +reclaimed by GC, but the client must restart from byte 0). The same +drop on a chunked upload loses one ~5 MB chunk; the client retries +that chunk and continues. NextCloud desktop and the OxiCloud web UI +already switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). -- **Resilience.** A direct PUT at 95 % of 5 GB that drops loses - everything. The same drop on a chunked upload loses one ~5 MB - chunk; the client retries that chunk and continues. -- **Memory + disk pressure.** Direct PUT spools the full body to - disk per request. Ten concurrent 5 GB direct PUTs use up to 50 GB - of transient spool disk. Chunked spreads each upload across many - small PATCHes; per-request resource use stays bounded by - `OXICLOUD_CHUNK_MAX_BYTES`. -- **Convention.** NextCloud desktop and the OxiCloud web UI already - switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). +### Disk sizing -### Why caps matter for tmpfs sizing +OxiCloud streams bodies frame-by-frame, so **RAM** is bounded +(~10 MB per in-flight upload for the CDC ingest buffers) regardless +of the caps. **Disk space** scales with the caps: -OxiCloud streams bodies frame-by-frame, so **RAM** is bounded to one -HTTP frame (~64 KB) per request regardless of the caps. **Disk space**, -however, scales with the caps: +- **Direct PUT / multipart**: no transient spool. Bytes land directly + in the blob backend as deduplicated chunks; worst-case extra disk + per upload is the file's own (deduplicated) size — the same space + the stored file occupies afterwards. +- **Chunked upload**: each in-flight session accumulates its chunk + parts under `OXICLOUD_CHUNK_DIR` until `/complete` streams them + into the blob store and the session is cleaned up. Worst case disk + per session = **file_size** (the parts); total = + `OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. -- **Direct PUT**: each in-flight upload spools the full body to disk - under `OXICLOUD_UPLOAD_TMPDIR` until promotion. Worst case disk = - `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs`. -- **Chunked upload**: each in-flight session accumulates chunks - under `OXICLOUD_CHUNK_DIR`, then assembles them into a single temp - file before promotion. Worst case disk per session = **2 × - file_size** (chunks + assembled file); total disk = - `2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. +| Settings | Chunked worst case (5 sessions) | Safe on 4 GB volume? | +|---|---|---| +| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB | 50 GB | ❌ overflows | +| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB | 2.5 GB | ✅ fits | -The chunked formula uses `OXICLOUD_MAX_UPLOAD_SIZE` because that's -what bounds the declared `total_size` at session creation. The -direct-PUT formula uses the smaller `OXICLOUD_DIRECT_PUT_MAX_BYTES` -since that's what bounds each direct PUT body. +### Don't put `OXICLOUD_CHUNK_DIR` on tmpfs -### Sizing examples - -A 4 GB tmpfs serving a small team (5 concurrent direct PUTs OR 5 -concurrent chunked sessions): - -| Settings | Direct-PUT worst case | Chunked worst case | Safe on 4 GB tmpfs? | -|---|---|---|---| -| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=100 MB | 5 GiB (5 × 1 GiB) | 100 GB (5 × 2 × 10 GB) | ❌ chunked overflows | -| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=100 MB, `OXICLOUD_CHUNK_MAX_BYTES`=20 MB | 500 MB | 5 GB | ⚠ direct PUT fits, chunked still overflows | -| `OXICLOUD_MAX_UPLOAD_SIZE`=300 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=50 MB, `OXICLOUD_CHUNK_MAX_BYTES`=10 MB | 250 MB | 3 GB | ✅ both fit | - -A real-disk volume (cheap, large): - -| Settings | Direct-PUT worst case | Chunked worst case | Comment | -|---|---|---|---| -| Defaults (see row above) | 5 GiB | 100 GB | Fine on a 200+ GB volume; almost any real-disk setup | -| `OXICLOUD_MAX_UPLOAD_SIZE`=100 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=5 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=500 MB | 25 GiB | 1 TB | Plausible for video archives; needs a dedicated upload volume | - -### Choosing tmpfs vs real disk - -| Constraint | Choice | -|---|---| -| `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs + 2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions ≤ free RAM × 0.5` | tmpfs OK (fast, atomic with `.blobs/` if also tmpfs) | -| Worst case exceeds half free RAM | **real disk** — same filesystem as `.blobs/` ideal | -| Container with cgroup memory limit | **real disk** — tmpfs spool counts against the cgroup limit and triggers OOMKill | -| Multi-GB uploads expected | **real disk** — even small concurrency on tmpfs runs out of space | -| Small-file workload only (≤ 50 MB), high concurrency | tmpfs gives a noticeable intake speedup | - -The defaults (`OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, -`OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, -`OXICLOUD_CHUNK_MAX_BYTES`=100 MB) assume **real disk**. Don't run -the defaults against tmpfs unless you've sized it for the worst case. +In many container setups the OS temp dir is **tmpfs** — RAM-backed +storage that counts against the cgroup memory limit. A few concurrent +multi-GB chunked sessions on tmpfs will wake the OOMKiller long +before the uploads finish. Point `OXICLOUD_CHUNK_DIR` at a real-disk +directory in containers. ## TL;DR | Variable | Default | Purpose | |---|---|---| | `OXICLOUD_STORAGE_PATH` | `./storage` | Where `.blobs/` lives (the canonical content store) | -| `OXICLOUD_UPLOAD_TMPDIR` | OS temp dir | Where non-chunked PUT bodies are spooled | | `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Where chunked-upload sessions accumulate | The two rules that matter most: -1. **Put all three on the same filesystem.** Blob promotion is an - atomic `rename(2)` when source and destination share an FS — cheap - and crash-safe. Across filesystems it becomes a full `read + write + - unlink`, multiplying the IO and widening the durability window. -2. **Don't leave the spool dir on tmpfs** (the default in many - containers). Spool bodies count against the cgroup memory limit +1. **Keep `OXICLOUD_CHUNK_DIR` off tmpfs** (the default in many + containers) — chunk parts count against the cgroup memory limit and can trigger OOMKill on multi-GB uploads. +2. **NVMe for the chunk dir pays off** on deployments with heavy + large-file traffic: each chunk PUT writes a file and the progress + bitmap, and `/complete` reads them all back in order. -## Where each upload surface spools - -OxiCloud has several entry points that accept request bodies. They -land in different places by default: +## Where each upload surface writes | Surface | Default destination | Configurable via | |---|---|---| | REST chunked PUT (`PATCH /api/uploads/{id}`) | `{STORAGE_PATH}/.uploads/{upload_id}/chunk_NNNNNN` | `OXICLOUD_CHUNK_DIR` | -| REST chunked assemble (during `/complete`) | `{STORAGE_PATH}/.uploads/{upload_id}/assembled` | `OXICLOUD_CHUNK_DIR` | | NextCloud chunked PUT (`PUT /dav/uploads/.../chunk`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/{chunk_name}` | `OXICLOUD_CHUNK_DIR` | -| NextCloud chunked assemble (during `MOVE`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/.assembled` | `OXICLOUD_CHUNK_DIR` | -| Native WebDAV PUT (`PUT /webdav/{path}`) | OS temp dir (`/tmp`) | `OXICLOUD_UPLOAD_TMPDIR` | -| NextCloud single-file PUT (`PUT /dav/files/.../{path}`) | OS temp dir | `OXICLOUD_UPLOAD_TMPDIR` | -| REST multipart upload (`POST /api/files/upload`) | `{STORAGE_PATH}/.dedup_temp/upload-{uuid}` | `OXICLOUD_STORAGE_PATH` (subdir is hard-wired) | -| Final blob storage (after fsync + rename) | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | +| Direct PUT / multipart / WOPI / chunked `/complete` | straight into the blob backend (CDC chunks) | `OXICLOUD_STORAGE_PATH` (local backend) | +| Final blob storage | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | -## Why placement matters - -### 1. Same filesystem ⇒ promotion is a rename - -OxiCloud uses **content-addressable storage**: the final blob path is -derived from the file's BLAKE3 hash, which can only be known after the -last byte arrives. So every upload writes to a temp location first, -then **promotes** the temp file to `.blobs/{ab}/{abc…}.blob` by way of -a `rename(2)` call. - -- **Same FS:** `rename` is atomic, O(1), no data copy. Total upload - cost = body bytes received + one rename syscall. Crash-safe — the - blob either exists at the final path or doesn't. -- **Cross-FS:** the kernel can't `rename(2)` across filesystems. The - blob backend falls back to `fs::copy + fs::remove_file` (visible in - `local_blob_backend.rs` as the EXDEV handler). Total cost = body - bytes received + one full file copy. Doubles the IO bandwidth used - per upload and widens the durability window. - -### 2. Spool off tmpfs - -`tempfile::NamedTempFile::new()` (used when `OXICLOUD_UPLOAD_TMPDIR` -is unset) honors `$TMPDIR`, which in many container setups points at -**tmpfs** — RAM-backed storage. A 2 GB upload spool then consumes 2 GB -of memory until the rename promotes it to disk. - -In a Kubernetes pod with a 4 GB memory limit, the OOMKiller wakes up -long before the upload finishes. With `OXICLOUD_UPLOAD_TMPDIR` pointed -at a real-disk directory, the spool's memory footprint stays at ~one -HTTP frame regardless of file size. - -### 3. NVMe for the hot path - -The chunked-upload session directory sees a LOT of small writes — -each chunk PUT writes a file, the progress bitmap is rewritten after -each PUT, the assemble step reads them all back in order. Pointing -`OXICLOUD_CHUNK_DIR` at an NVMe device is a substantial win on -deployments that handle large file uploads, even if the final blob -storage is on slower disk. - -The same applies to `OXICLOUD_UPLOAD_TMPDIR` (single-file PUTs). - -A common high-throughput layout: - -- **NVMe** (small, fast): `OXICLOUD_CHUNK_DIR`, `OXICLOUD_UPLOAD_TMPDIR` -- **HDD or NAS** (large, cheap): `OXICLOUD_STORAGE_PATH`/`.blobs/` - -Trade-off: the rename optimization (rule 1) DOESN'T apply across -filesystems. If you split the hot path off the blob filesystem, every -upload pays a full file copy on promotion. You have to choose -between **fast intake** and **zero-copy promotion**. - -| Goal | Layout | Cost per upload | -|---|---|---| -| Fastest possible intake | NVMe chunk dir + HDD blobs | 1× write to NVMe + 1× read NVMe + 1× write to HDD (copy) | -| Lowest IO + crash safety | NVMe everything OR HDD everything | 1× write to disk + 1 rename (~0 cost) | -| Default (do nothing) | Everything under `STORAGE_PATH` on whatever FS that is | Depends on `STORAGE_PATH` placement | - -For most deployments **"same FS everywhere"** wins. The NVMe-split is -useful when intake latency dominates the user experience and you can -afford the doubled IO. +The local blob backend stages each chunk write under +`{STORAGE_PATH}/.dedup_temp/` and promotes it with an atomic +`rename(2)` — both directories live under `OXICLOUD_STORAGE_PATH`, +so same-filesystem placement (and therefore atomic promotion) is +automatic and not separately configurable. ## Recommended layouts ### Single-disk box (most common) -Defaults are fine. Optionally set `OXICLOUD_UPLOAD_TMPDIR` to keep -the PUT spool off `/tmp`: +Defaults are fine: ```bash OXICLOUD_STORAGE_PATH=/var/lib/oxicloud -OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/.spool # OXICLOUD_CHUNK_DIR unset → /var/lib/oxicloud/.uploads ``` -All three on the same filesystem → rename promotion → atomic and fast. - ### Container with constrained memory -Critical: make sure neither spool sits on tmpfs. +Critical: make sure the chunk dir doesn't sit on tmpfs. ```bash OXICLOUD_STORAGE_PATH=/data -OXICLOUD_UPLOAD_TMPDIR=/data/.spool OXICLOUD_CHUNK_DIR=/data/.uploads ``` -If you can't mount a writable `/data`, at minimum bind-mount a real -volume at the spool dirs. - ### Split-disk (NVMe intake + HDD blobs) ```bash OXICLOUD_STORAGE_PATH=/mnt/hdd/oxicloud # .blobs/ + .dedup_temp/ -OXICLOUD_UPLOAD_TMPDIR=/mnt/nvme/oxi-spool OXICLOUD_CHUNK_DIR=/mnt/nvme/oxi-chunks ``` -Faster intake; pays a copy on promotion. Worth it when uploads are -many small files (NVMe IOPS dominates) or when intake latency directly -hits user-visible UX. +Chunk parts land on NVMe (fast PUTs, fast `/complete` read-back); +the deduplicated chunks are written once to the HDD-backed blob +store as `/complete` streams through them. -## Sharing the spool and chunk directories +## Sharing the chunk directory -Pointing `OXICLOUD_UPLOAD_TMPDIR` and `OXICLOUD_CHUNK_DIR` at the -**same directory** is supported by design. Each writer tags its -output so the surfaces never interfere with each other: +The REST and NC chunked surfaces can share `OXICLOUD_CHUNK_DIR` by +design. Each writer tags its output so they never interfere: | Writer | On-disk name pattern | |---|---| -| PUT spool (single-file uploads) | `.tmpXXXXXXXX` — files (not directories), random suffix | | REST chunked sessions | `oxi-chunk-{uuid}/` — directories with a well-known prefix | | NC chunked subtree | `nextcloud/{user}/{uuid}/` — under its own root subdir | The 24-hour orphan-session cleanup loop filters strictly on the `oxi-chunk-` prefix, so it can NEVER delete a non-OxiCloud directory -that happens to live alongside chunked sessions. The PUT spool's -`.tmpXXXX` files are files (not directories) and the NC subtree's -`nextcloud/` root has its own name — both are invisible to the -cleanup loop. - -**Recommendation:** for new deployments, use separate directories -anyway (the defaults `.spool/` and `.uploads/` already do this) — -it makes disk-usage attribution clearer and keeps IOPS isolated when -both are busy. Shared directories are safe to use when disk layout -forces it. - -## What's NOT yet configurable - -- **REST multipart upload directory** (`POST /api/files/upload`) is - hard-wired to `{STORAGE_PATH}/.dedup_temp/`. It can't be moved - separately. Same-FS placement is automatic. -- **WOPI PutFile spool** (Office editor saves) uses the bare OS temp - dir without honoring `OXICLOUD_UPLOAD_TMPDIR`. This is a known - inconsistency and on the hardening backlog. -- **Per-user / per-drive spool directories** — all users share the - same `OXICLOUD_CHUNK_DIR` root today. Multi-tenant isolation - through separate spool dirs isn't supported. +that happens to live alongside chunked sessions. ## Quick verification diff --git a/example.env b/example.env index c1dffd63..2084135a 100644 --- a/example.env +++ b/example.env @@ -55,22 +55,14 @@ OXICLOUD_SERVER_HOST=127.0.0.1 #OXICLOUD_CHUNK_MAX_BYTES=104857600 # ── Upload spool directories ───────────────────────────────────────── -# Where in-flight uploads land BEFORE being promoted into final blob -# storage. Same-filesystem placement (with `.blobs/`) makes the -# promotion an atomic rename(2); NVMe placement speeds up intake. +# Direct (non-chunked) uploads stream straight into the blob store — +# no spool directory. Only chunked-upload sessions accumulate on disk. # See docs/config/storage-fine-tuning.md for layout examples. -# Directory for non-chunked PUT spool tempfiles. Default: OS temp dir -# ($TMPDIR / /tmp), often tmpfs (RAM) in containers — writing a large -# upload there fills page-cache that counts against the cgroup memory -# limit and can OOMKill the process. Point this at a real-disk path -# (same FS as the storage backend is ideal). -#OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/tmp - # Root directory for chunked-upload sessions (REST + NextCloud chunked -# share this root). Default: {STORAGE_PATH}/.uploads. Pointing this at -# NVMe accelerates the chunk-write + assembly loop; pointing it at the -# same FS as `.blobs/` makes blob promotion atomic. +# share this root). Default: {STORAGE_PATH}/.uploads. Avoid tmpfs in +# containers (parts count against the cgroup memory limit); NVMe +# placement accelerates chunk PUTs and the /complete streaming pass. #OXICLOUD_CHUNK_DIR=/var/lib/oxicloud/.uploads # How often (seconds) the background sweep reconciles each user's cached diff --git a/src/application/ports/chunked_upload_ports.rs b/src/application/ports/chunked_upload_ports.rs index 5d7d5427..098dd276 100644 --- a/src/application/ports/chunked_upload_ports.rs +++ b/src/application/ports/chunked_upload_ports.rs @@ -98,6 +98,21 @@ pub struct UploadStatusResponseDto { pub is_complete: bool, } +/// A completed upload session, ready to be streamed into the blob store. +/// +/// `chunk_paths` lists every chunk file in assembly order — the caller +/// concatenates them as one byte stream (CDC chunking + hashing happen in +/// that single pass). The files stay on disk until `finalize_upload`, so a +/// failed completion (e.g. checksum mismatch) remains retryable. +#[derive(Debug, Clone)] +pub struct CompletedUploadParts { + pub chunk_paths: Vec, + pub filename: String, + pub folder_id: Option, + pub content_type: String, + pub total_size: u64, +} + /// Port for chunked/resumable upload operations. /// /// Implementations manage upload sessions, chunk storage, reassembly, @@ -137,16 +152,15 @@ pub trait ChunkedUploadPort: Send + Sync + 'static { user_id: Uuid, ) -> Result; - /// Assemble all chunks into the final file. + /// Validate completion and hand back the ordered chunk parts. /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. - /// The hash is computed during assembly (hash-on-write), eliminating a - /// second sequential read of the assembled file. + /// No assembled file is written — the caller streams the parts straight + /// into the content-addressable store. async fn complete_upload( &self, upload_id: &str, user_id: Uuid, - ) -> Result<(PathBuf, String, Option, String, u64, String), DomainError>; + ) -> Result; /// Finalize upload: clean up the session and temporary files. async fn finalize_upload(&self, upload_id: &str, user_id: Uuid) -> Result<(), DomainError>; diff --git a/src/application/ports/dedup_ports.rs b/src/application/ports/dedup_ports.rs index 95f219ea..23ddc962 100644 --- a/src/application/ports/dedup_ports.rs +++ b/src/application/ports/dedup_ports.rs @@ -80,18 +80,6 @@ pub struct DedupStatsDto { /// duplicate storage automatically. Multiple file references can /// point to the same physical blob. pub trait DedupPort: Send + Sync + 'static { - /// Store content with deduplication (streaming from file). - /// - /// If `pre_computed_hash` is provided (e.g. hash-on-write from the handler), - /// the file will NOT be re-read to calculate the hash — saving one full - /// sequential read of the file. - async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result; - /// Check if a blob with the given hash exists. async fn blob_exists(&self, hash: &str) -> bool; diff --git a/src/application/ports/file_ports.rs b/src/application/ports/file_ports.rs index b40e0a35..19d9e926 100644 --- a/src/application/ports/file_ports.rs +++ b/src/application/ports/file_ports.rs @@ -1,6 +1,5 @@ use bytes::Bytes; use futures::Stream; -use std::path::Path; use std::pin::Pin; use std::sync::Arc; use uuid::Uuid; @@ -17,74 +16,51 @@ use crate::domain::services::authorization::Permission; // Upload port // ───────────────────────────────────────────────────── +/// A blob already stored in the content-addressable chunk store, carrying +/// exactly ONE reference that the receiving method takes ownership of. +/// +/// Produced by the upload-ingest layer (`interfaces::upload_ingest`), which +/// streams the request body straight into the CDC dedup store. Methods that +/// accept a `StoredBlob` either attach the reference to a file row or +/// release it on failure — callers never need to compensate themselves. +#[derive(Debug, Clone)] +pub struct StoredBlob { + /// BLAKE3 of the full content (manifest / blob key). + pub hash: String, + /// Content size in bytes. + pub size: u64, + /// `false` when the content already existed (dedup hit) — forwarded to + /// lifecycle hooks so e.g. thumbnails aren't regenerated for known blobs. + pub is_new_blob: bool, +} + /// Primary port for file upload operations. /// -/// **All upload paths converge on streaming-to-disk** — no method accepts -/// `Vec` for content. Even `create_file` / `update_file` (WebDAV -/// helpers that receive `&[u8]`) spool to a temp file internally so that -/// peak RAM stays at ~256 KB regardless of file size. -/// -/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming` -/// - Chunked uploads: chunks already on disk → `upload_file_from_path` -/// - WebDAV PUT (new): handler streams to temp file → `update_file_streaming` -/// - WebDAV PUT (small/compat): `create_file` / `update_file` spool internally +/// **All upload paths converge on streaming-into-the-chunk-store** — content +/// never passes through this port; it is ingested by the interface layer +/// (CDC chunking + hashing while the body arrives, no spool file) and only +/// the resulting [`StoredBlob`] reference travels through here. pub trait FileUploadUseCase: Send + Sync + 'static { - /// Upload from a temp file already on disk (true streaming, ~256 KB RAM). + /// Register a new file row pointing at an already-ingested blob. /// - /// When `pre_computed_hash` is `Some`, the blob store skips the hash - /// re-read — the handler already computed it during the multipart spool. + /// Takes ownership of the blob's reference (released on failure). async fn upload_file_streaming( &self, name: String, folder_id: Option, content_type: String, - temp_path: &Path, - size: u64, - pre_computed_hash: Option, + blob: StoredBlob, ) -> Result; - /// Upload from a file already assembled on disk (chunked uploads). + /// Replace the content of the file at `path` with an already-ingested + /// blob, or create the file when it doesn't exist (WebDAV/WOPI PUT). /// - /// Same as `upload_file_streaming` but with a separate name for clarity. - async fn upload_file_from_path( - &self, - name: String, - folder_id: Option, - content_type: String, - file_path: &Path, - pre_computed_hash: Option, - ) -> Result; - - /// Creates a new file at the specified path (for WebDAV) - async fn create_file( - &self, - parent_path: &str, - filename: &str, - content: &[u8], - content_type: &str, - ) -> Result; - - /// Updates the content of an existing file (for WebDAV) - async fn update_file( - &self, - path: &str, - content: &[u8], - content_type: &str, - modified_at: Option, - ) -> Result; - - /// Streaming update — spools body to a temp file with incremental hash, - /// then atomically replaces the file content via dedup store. - /// - /// Peak RAM: ~256 KB regardless of file size. - /// Used by WebDAV PUT for large files. + /// Takes ownership of the blob's reference (released on failure). async fn update_file_streaming( &self, path: &str, - temp_path: &Path, - size: u64, + blob: StoredBlob, content_type: &str, - pre_computed_hash: Option, modified_at: Option, ) -> Result; } diff --git a/src/application/ports/storage_ports.rs b/src/application/ports/storage_ports.rs index 213cb759..33307c35 100644 --- a/src/application/ports/storage_ports.rs +++ b/src/application/ports/storage_ports.rs @@ -251,21 +251,21 @@ pub struct CopyFolderTreeResult { /// Secondary port for file **writing**. /// -/// Covers: upload (buffered + streaming), move, delete, update, -/// and deferred registration for the write-behind cache. +/// Covers: upload registration, move, delete, update, and deferred +/// registration for the write-behind cache. pub trait FileWritePort: Send + Sync + 'static { - /// Streaming upload — saves a file from a temp file already on disk. + /// Register a file row pointing at a blob already stored in the + /// content-addressable chunk store. /// - /// When `pre_computed_hash` is provided, the dedup service skips the - /// hash re-read — zero extra I/O beyond the initial spool. - async fn save_file_from_temp( + /// Takes ownership of one blob reference: on any failure the reference + /// is released before the error is returned. + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result; /// Moves a file to another folder. @@ -281,22 +281,20 @@ pub trait FileWritePort: Send + Sync + 'static { /// Deletes a file. async fn delete_file(&self, id: &str) -> Result<(), DomainError>; - /// Streaming update — replaces file content from a temp file on disk. + /// Atomically swap a file's content to a blob already stored in the + /// content-addressable chunk store. /// - /// When `pre_computed_hash` is provided, the dedup service skips the - /// hash re-read — zero extra I/O beyond the initial spool. - /// Peak RAM: ~256 KB regardless of file size. + /// Takes ownership of one blob reference (released on failure); the + /// previous content's reference is dropped after the swap. /// /// Returns `(new_blob_hash, updated_at_epoch)` — everything a caller /// needs to rebuild the fresh entity/ETag from a `File` it already /// holds, without re-reading the row it just updated. - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, file_id: &str, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - content_type: Option, - pre_computed_hash: Option, modified_at: Option, ) -> Result<(String, i64), DomainError>; diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 92ec9215..49f00a95 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -1,15 +1,13 @@ -use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::application::dtos::file_dto::FileDto; use crate::application::ports::file_lifecycle::FileLifecycleHook; -use crate::application::ports::file_ports::FileUploadUseCase; +use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; use crate::application::services::storage_usage_service::StorageUsageService; use crate::common::errors::DomainError; use crate::infrastructure::repositories::pg::FileBlobReadRepository; use crate::infrastructure::repositories::pg::FileBlobWriteRepository; -use crate::infrastructure::services::dedup_service::DedupService; use crate::infrastructure::services::file_content_cache::FileContentCache; use tracing::{debug, info, warn}; @@ -34,20 +32,16 @@ fn extract_username_from_path(path: &str) -> Option { /// Service for file upload operations. /// -/// **Every upload path converges on streaming-to-disk** — there is no -/// `Vec` buffer path. -/// -/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming` -/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path` -/// - **WebDAV PUT (large)**: handler streams body to temp file → `update_file_streaming` -/// - **WebDAV PUT (small / compat)**: `create_file` / `update_file` spool `&[u8]` -/// to a temp file internally, then call the streaming path. -/// -/// Peak RAM usage during any upload: ~256 KB (streaming hash) regardless of file size. +/// Content never passes through this service: the interface layer streams +/// the request body straight into the CDC chunk store (no spool file, no +/// full-body buffering) and hands over a [`StoredBlob`] reference. This +/// service registers the metadata row, keeps caches coherent and fires +/// lifecycle hooks. Blob-reference ownership is consumed by the write +/// port, which releases it on failure — callers never compensate. pub struct FileUploadService { - /// Write port — handles save, streaming, deferred registration + /// Write port — registers file rows against ingested blobs. file_write: Arc, - /// Read port — needed for WebDAV create_file / update_file + /// Read port — needed for WebDAV/WOPI update-by-path. file_read: Option>, /// Optional storage usage tracking storage_usage_service: Option>, @@ -55,9 +49,6 @@ pub struct FileUploadService { content_cache: Option>, /// Single lifecycle dispatcher — fires on_file_created / on_file_updated. file_lifecycle_hook: Option>, - /// Directory for spool temp files (`&[u8]` upload variants). When `Some`, - /// keeps spools off tmpfs/RAM so they don't count against the cgroup limit. - upload_temp_dir: Option, } impl FileUploadService { @@ -69,7 +60,6 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, - upload_temp_dir: None, } } @@ -84,16 +74,9 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, - upload_temp_dir: None, } } - /// Configures the spool directory for the `&[u8]` upload variants. - pub fn with_upload_temp_dir(mut self, dir: Option) -> Self { - self.upload_temp_dir = dir; - self - } - /// Configures the content cache for invalidation on file updates. pub fn with_content_cache(mut self, cache: Arc) -> Self { self.content_cache = Some(cache); @@ -117,11 +100,6 @@ impl FileUploadService { // ── private helpers ────────────────────────────────────────── - /// Create a spool temp file, honoring the configured upload temp dir. - fn new_temp(&self) -> std::io::Result { - crate::common::temp::new_spool_temp_file(self.upload_temp_dir.as_deref()) - } - /// Optionally update storage usage after a successful upload. fn maybe_update_storage_usage(&self, file: &FileDto) { if let Some(storage_service) = &self.storage_usage_service { @@ -146,172 +124,37 @@ impl FileUploadService { } impl FileUploadUseCase for FileUploadService { - /// Streaming upload from a temp file on disk. - /// - /// Peak RAM: ~256 KB (hash calculation) regardless of file size. - /// The temp file is consumed (moved/deleted) by the blob store. + /// Register a new file row pointing at an already-ingested blob. async fn upload_file_streaming( &self, name: String, folder_id: Option, content_type: String, - temp_path: &Path, - size: u64, - pre_computed_hash: Option, + blob: StoredBlob, ) -> Result { - let (file, is_new_blob) = self + let file = self .file_write - .save_file_from_temp_with_dedup( - name.clone(), - folder_id, - content_type, - temp_path, - size, - pre_computed_hash, - ) + .save_file_with_blob(name.clone(), folder_id, content_type, &blob.hash, blob.size) .await?; let dto = FileDto::from(file); info!( "📡 STREAMING UPLOAD: {} ({} bytes, ID: {})", - name, size, dto.id + name, blob.size, dto.id ); self.maybe_update_storage_usage(&dto); if let Some(hook) = &self.file_lifecycle_hook { - hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, is_new_blob); + hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, blob.is_new_blob); } Ok(dto) } - /// Upload from a file already on disk (chunked uploads). - async fn upload_file_from_path( - &self, - name: String, - folder_id: Option, - content_type: String, - file_path: &Path, - pre_computed_hash: Option, - ) -> Result { - let size = tokio::fs::metadata(file_path) - .await - .map_err(|e| { - DomainError::internal_error( - "FileUpload", - format!("Failed to read file metadata: {}", e), - ) - })? - .len(); - - self.upload_file_streaming( - name, - folder_id, - content_type, - file_path, - size, - pre_computed_hash, - ) - .await - } - - /// Creates a file at a specific path (for WebDAV PUT on new resource). - /// - /// Spools the in-memory `&[u8]` to a temp file with hash-on-write, - /// then delegates to the streaming path. Peak RAM: the caller's - /// buffer + ~256 KB for the hasher. - async fn create_file( - &self, - parent_path: &str, - filename: &str, - content: &[u8], - content_type: &str, - ) -> Result { - // Look up the folder ID by folder path - let parent_id = if !parent_path.is_empty() { - if let Some(file_read) = &self.file_read { - file_read.get_folder_id_by_path(parent_path).await.ok() - } else { - None - } - } else { - None - }; - - // Spool to temp file + hash - let temp = self - .new_temp() - .map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?; - tokio::fs::write(temp.path(), content) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = DedupService::hash_file(temp.path()) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("hash: {e}")))?; - - let (file, is_new_blob) = self - .file_write - .save_file_from_temp_with_dedup( - filename.to_string(), - parent_id, - content_type.to_string(), - temp.path(), - content.len() as u64, - Some(hash), - ) - .await?; - let dto = FileDto::from(file); - self.maybe_update_storage_usage(&dto); - if let Some(hook) = &self.file_lifecycle_hook { - hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, is_new_blob); - } - Ok(dto) - } - - /// Updates an existing file's content, or creates it if not found (for WebDAV PUT). - /// - /// Spools the in-memory `&[u8]` to a temp file with hash-on-write, - /// then delegates to the streaming update/create path. - async fn update_file( - &self, - path: &str, - content: &[u8], - content_type: &str, - modified_at: Option, - ) -> Result { - // Spool to temp file + hash - let temp = self - .new_temp() - .map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?; - tokio::fs::write(temp.path(), content) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = DedupService::hash_file(temp.path()) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("hash: {e}")))?; - - self.update_file_streaming( - path, - temp.path(), - content.len() as u64, - content_type, - Some(hash), - modified_at, - ) - .await - } - - /// Streaming update — replaces file content from a temp file on disk. - /// - /// Uses `update_file_content_from_temp` which passes the pre-computed hash - /// to dedup, avoiding a second full read of the file. - /// For new files (not found at `path`), falls back to `upload_file_streaming`. - /// - /// Peak RAM: ~256 KB regardless of file size. + /// Swap the content of the file at `path` to an already-ingested blob, + /// creating the file when it doesn't exist (WebDAV/NextCloud/WOPI PUT). async fn update_file_streaming( &self, path: &str, - temp_path: &Path, - size: u64, + blob: StoredBlob, content_type: &str, - pre_computed_hash: Option, modified_at: Option, ) -> Result { // Try to find the existing file first @@ -321,14 +164,7 @@ impl FileUploadUseCase for FileUploadService { let file_id = file.id().to_string(); let (new_hash, updated_at) = self .file_write - .update_file_content_from_temp( - &file_id, - temp_path, - size, - Some(content_type.to_string()), - pre_computed_hash, - modified_at, - ) + .update_file_content_with_blob(&file_id, &blob.hash, blob.size, modified_at) .await?; // Invalidate content cache — file content has changed. if let Some(cc) = &self.content_cache { @@ -343,7 +179,7 @@ impl FileUploadUseCase for FileUploadService { parts.id, parts.name, parts.storage_path, - size, + blob.size, parts.mime_type, parts.folder_id, parts.created_at, @@ -381,15 +217,15 @@ impl FileUploadUseCase for FileUploadService { None }; - let (created, is_new_blob) = self + let is_new_blob = blob.is_new_blob; + let created = self .file_write - .save_file_from_temp_with_dedup( + .save_file_with_blob( filename.to_string(), parent_id, content_type.to_string(), - temp_path, - size, - pre_computed_hash, + &blob.hash, + blob.size, ) .await?; let dto = FileDto::from(created); diff --git a/src/application/services/idor_protection_test.rs b/src/application/services/idor_protection_test.rs index 8682ae28..33d1dcf7 100644 --- a/src/application/services/idor_protection_test.rs +++ b/src/application/services/idor_protection_test.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use futures::Stream; use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::pin::Pin; use std::sync::Mutex; use uuid::Uuid; @@ -169,14 +169,13 @@ impl MockFileWritePort { } impl FileWritePort for MockFileWritePort { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> Result { unimplemented!() } @@ -205,13 +204,11 @@ impl FileWritePort for MockFileWritePort { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result<(String, i64), DomainError> { Ok((String::new(), 0)) diff --git a/src/application/services/trash_service_test.rs b/src/application/services/trash_service_test.rs index 951abf39..675bb300 100644 --- a/src/application/services/trash_service_test.rs +++ b/src/application/services/trash_service_test.rs @@ -554,14 +554,13 @@ impl FileReadPort for MockFileRepository { } impl FileWritePort for MockFileRepository { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &std::path::Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> std::result::Result { unimplemented!() } @@ -586,13 +585,11 @@ impl FileWritePort for MockFileRepository { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &std::path::Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> std::result::Result<(String, i64), DomainError> { Ok((String::new(), 0)) diff --git a/src/common/config.rs b/src/common/config.rs index ffc89a93..6a6217e6 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -228,12 +228,6 @@ pub struct StorageConfig { /// returns 413 with a "use chunked upload" hint when a direct PUT /// exceeds this cap. Env: `OXICLOUD_DIRECT_PUT_MAX_BYTES`. pub direct_put_max_bytes: usize, - /// Directory for upload spool temp files. When `Some`, large uploads are - /// spooled here instead of the OS default temp dir (often tmpfs/RAM in - /// containers, where the spool's page-cache counts against the cgroup - /// memory limit and can trigger OOMKill on large files). Env: - /// `OXICLOUD_UPLOAD_TMPDIR`. - pub upload_temp_dir: Option, /// Root directory for chunked-upload sessions. When `Some`, chunks land /// under `{chunk_dir}/{upload_id}/` (REST) and /// `{chunk_dir}/nextcloud/{user}/{upload_id}/` (NC). When `None`, falls @@ -401,7 +395,6 @@ impl Default for StorageConfig { max_upload_size: MAX_UPLOAD_SIZE, chunk_max_bytes: 100 * 1024 * 1024, // 100 MB — sane upper bound for a single chunked-upload PUT direct_put_max_bytes: 1024 * 1024 * 1024, // 1 GiB — pushes larger uploads onto the chunked protocol - upload_temp_dir: None, chunk_dir: None, usage_reconcile_secs: 600, // 10 minutes tree_etag_flush_ms: 500, @@ -1281,18 +1274,10 @@ impl AppConfig { config.storage.direct_put_max_bytes = val; } - // Upload spool directory — keep large upload temp files off tmpfs/RAM - // (otherwise their page-cache counts against the cgroup memory limit). - if let Ok(dir) = env::var("OXICLOUD_UPLOAD_TMPDIR") - && !dir.trim().is_empty() - { - config.storage.upload_temp_dir = Some(PathBuf::from(dir.trim())); - } - // Chunked-upload session root — separate from the PUT spool because - // chunked sessions accumulate disk on long uploads (multi-chunk - // resumable transfers) while PUT spool is short-lived. Sysadmins - // commonly want one of them on fast/local storage (NVMe) and the - // other on bulk storage; this knob lets that be expressed. + // Chunked-upload session root — chunked sessions accumulate disk on + // long uploads (multi-chunk resumable transfers); sysadmins commonly + // want them on fast/local storage (NVMe). This knob lets that be + // expressed. if let Ok(dir) = env::var("OXICLOUD_CHUNK_DIR") && !dir.trim().is_empty() { diff --git a/src/common/di.rs b/src/common/di.rs index 08b483f0..809611f1 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -452,8 +452,7 @@ impl AppServiceFactory { repos.file_read_repository.clone(), ) .with_content_cache(core.file_content_cache.clone()) - .with_file_lifecycle_hook(core.file_lifecycle.clone()) - .with_upload_temp_dir(self.config.storage.upload_temp_dir.clone()), + .with_file_lifecycle_hook(core.file_lifecycle.clone()), ); let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( diff --git a/src/common/mime_detect.rs b/src/common/mime_detect.rs index 8cecd2c9..d2fcf118 100644 --- a/src/common/mime_detect.rs +++ b/src/common/mime_detect.rs @@ -8,21 +8,25 @@ //! //! Performance: < 1µs for the `infer` check (reads only header bytes, no allocation). -use std::path::Path; -use tokio::io::AsyncReadExt; - -/// Maximum bytes to read for magic-byte detection. -const MAGIC_BYTES_LEN: usize = 8192; +/// Maximum bytes needed for magic-byte detection. Upload ingestion peeks +/// this many bytes off the stream before forwarding them unchanged. +pub const MAGIC_BYTES_LEN: usize = 8192; /// Extract the filename component from a `/`-separated path. pub fn filename_from_path(path: &str) -> &str { path.rsplit('/').next().unwrap_or(path) } +/// Whether a claimed Content-Type is too generic to trust — these trigger +/// magic-byte detection on the upload path. +pub fn is_generic_mime(claimed: &str) -> bool { + claimed.is_empty() || claimed == "application/octet-stream" || claimed == "binary/octet-stream" +} + /// Refine a claimed MIME type using magic bytes and filename extension. /// /// This is a synchronous function — the caller should already have the first -/// bytes of the file available (or call the async wrapper below). +/// bytes of the content available (upload ingestion peeks them in-flight). /// /// # Arguments /// * `buf` — first bytes of the file (at least 8192 for best results) @@ -30,10 +34,7 @@ pub fn filename_from_path(path: &str) -> &str { /// * `claimed` — the Content-Type sent by the client pub fn refine_content_type(buf: &[u8], filename: &str, claimed: &str) -> String { // If the client sent a specific type (not generic), trust it - if !claimed.is_empty() - && claimed != "application/octet-stream" - && claimed != "binary/octet-stream" - { + if !is_generic_mime(claimed) { return claimed.to_string(); } @@ -52,49 +53,9 @@ pub fn refine_content_type(buf: &[u8], filename: &str, claimed: &str) -> String claimed.to_string() } -/// Async helper: reads the first bytes of a file on disk and refines the MIME type. -/// -/// Designed for the upload path where the file has been spooled to a temp path. -pub async fn refine_content_type_from_file( - temp_path: &Path, - filename: &str, - claimed: &str, -) -> String { - // Fast path: if the client gave us a specific type, trust it - if !claimed.is_empty() - && claimed != "application/octet-stream" - && claimed != "binary/octet-stream" - { - return claimed.to_string(); - } - - // Read only the first bytes needed for magic detection (not the whole file). - match tokio::fs::File::open(temp_path).await { - Ok(mut file) => { - let mut buf = vec![0u8; MAGIC_BYTES_LEN]; - let n = file.read(&mut buf).await.unwrap_or(0); - refine_content_type(&buf[..n], filename, claimed) - } - Err(e) => { - tracing::warn!( - "MIME detection: failed to read {} for magic bytes: {}", - temp_path.display(), - e - ); - // Fall back to extension - let guess = mime_guess::from_path(filename); - if let Some(mime) = guess.first() { - return mime.to_string(); - } - claimed.to_string() - } - } -} - #[cfg(test)] mod tests { use super::*; - use std::io::Write; // ── refine_content_type (sync) ────────────────────────────── @@ -145,47 +106,15 @@ mod tests { assert_eq!(result, "image/png"); } - // ── refine_content_type_from_file (async) ─────────────────── + // ── is_generic_mime ───────────────────────────────────────── - #[tokio::test] - async fn from_file_detects_png() { - let mut tmp = tempfile::NamedTempFile::new().unwrap(); - let png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"; - tmp.write_all(png).unwrap(); - tmp.flush().unwrap(); - - let result = - refine_content_type_from_file(tmp.path(), "photo", "application/octet-stream").await; - assert_eq!(result, "image/png"); - } - - #[tokio::test] - async fn from_file_falls_back_to_extension() { - let mut tmp = tempfile::NamedTempFile::new().unwrap(); - tmp.write_all(b"not magic").unwrap(); - tmp.flush().unwrap(); - - let result = - refine_content_type_from_file(tmp.path(), "doc.css", "application/octet-stream").await; - assert_eq!(result, "text/css"); - } - - #[tokio::test] - async fn from_file_trusts_specific_claimed() { - let result = - refine_content_type_from_file(Path::new("/nonexistent"), "file", "image/webp").await; - assert_eq!(result, "image/webp"); - } - - #[tokio::test] - async fn from_file_missing_file_falls_back_to_extension() { - let result = refine_content_type_from_file( - Path::new("/nonexistent/file"), - "photo.jpg", - "application/octet-stream", - ) - .await; - assert_eq!(result, "image/jpeg"); + #[test] + fn generic_mime_detection() { + assert!(is_generic_mime("")); + assert!(is_generic_mime("application/octet-stream")); + assert!(is_generic_mime("binary/octet-stream")); + assert!(!is_generic_mime("image/png")); + assert!(!is_generic_mime("text/plain")); } // ── filename_from_path ────────────────────────────────────── diff --git a/src/common/mod.rs b/src/common/mod.rs index ddb9e2c6..b343f797 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -4,4 +4,3 @@ pub mod errors; pub mod locale; pub mod mime_detect; pub mod stubs; -pub mod temp; diff --git a/src/common/stubs.rs b/src/common/stubs.rs index c2c3957e..e857085a 100644 --- a/src/common/stubs.rs +++ b/src/common/stubs.rs @@ -24,6 +24,7 @@ use crate::application::dtos::search_dto::{ }; use crate::application::ports::file_ports::{ FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, OptimizedFileContent, + StoredBlob, }; use crate::application::ports::folder_ports::FolderUseCase; @@ -149,14 +150,13 @@ impl FileReadPort for StubFileReadPort { pub struct StubFileWritePort; impl FileWritePort for StubFileWritePort { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> Result { Ok(File::default()) } @@ -185,13 +185,11 @@ impl FileWritePort for StubFileWritePort { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result<(String, i64), DomainError> { Ok((String::new(), 0)) @@ -478,40 +476,7 @@ impl FileUploadUseCase for StubFileUploadUseCase { _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, - _size: u64, - _pre_computed_hash: Option, - ) -> Result { - Ok(FileDto::default()) - } - - async fn upload_file_from_path( - &self, - _name: String, - _folder_id: Option, - _content_type: String, - _file_path: &Path, - _pre_computed_hash: Option, - ) -> Result { - Ok(FileDto::default()) - } - - async fn create_file( - &self, - _parent_path: &str, - _filename: &str, - _content: &[u8], - _content_type: &str, - ) -> Result { - Ok(FileDto::default()) - } - - async fn update_file( - &self, - _path: &str, - _content: &[u8], - _content_type: &str, - _modified_at: Option, + _blob: StoredBlob, ) -> Result { Ok(FileDto::default()) } @@ -519,10 +484,8 @@ impl FileUploadUseCase for StubFileUploadUseCase { async fn update_file_streaming( &self, _path: &str, - _temp_path: &Path, - _size: u64, + _blob: StoredBlob, _content_type: &str, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result { Ok(FileDto::default()) @@ -756,25 +719,11 @@ impl SearchUseCase for StubSearchUseCase { // DedupPort // --------------------------------------------------------------------------- -use crate::application::ports::dedup_ports::{ - BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, -}; +use crate::application::ports::dedup_ports::{BlobMetadataDto, DedupPort, DedupStatsDto}; pub struct StubDedupPort; impl DedupPort for StubDedupPort { - async fn store_from_file( - &self, - _source_path: &Path, - _content_type: Option, - _pre_computed_hash: Option, - ) -> Result { - Err(DomainError::internal_error( - "DedupService", - "DedupService not initialized", - )) - } - async fn blob_exists(&self, _hash: &str) -> bool { false } diff --git a/src/common/temp.rs b/src/common/temp.rs deleted file mode 100644 index b83729df..00000000 --- a/src/common/temp.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Shared helper for creating upload spool temp files. -//! -//! Upload paths spool the request body to a temp file before deduplication. -//! By default `tempfile` uses the OS temp dir (`std::env::temp_dir()`, i.e. -//! `$TMPDIR` / `/tmp`), which in many container setups is **tmpfs (RAM)**. -//! Writing a multi-hundred-MB upload there fills page-cache that counts -//! against the cgroup memory limit and can OOMKill the process. Pointing the -//! spool at a real-disk directory (`OXICLOUD_UPLOAD_TMPDIR`) keeps the upload -//! footprint proportional to the streaming buffer, not the file size. - -use std::path::Path; -use tempfile::NamedTempFile; - -/// Create a [`NamedTempFile`], honoring an optional configured spool directory. -/// -/// When `dir` is `Some`, the temp file is created there (the directory is -/// created if missing); otherwise the OS default temp dir is used. -pub fn new_spool_temp_file(dir: Option<&Path>) -> std::io::Result { - match dir { - Some(d) => { - std::fs::create_dir_all(d)?; - NamedTempFile::new_in(d) - } - None => NamedTempFile::new(), - } -} diff --git a/src/domain/repositories/file_repository.rs b/src/domain/repositories/file_repository.rs index 4fba9aae..d9878f2e 100644 --- a/src/domain/repositories/file_repository.rs +++ b/src/domain/repositories/file_repository.rs @@ -71,15 +71,15 @@ pub trait FileWriteRepository: Send + Sync + 'static { content: Vec, ) -> Result; - /// Streaming upload — saves a file from a temp file already on disk. - async fn save_file_from_temp( + /// Registers a file row pointing at a blob already stored in the + /// content-addressable chunk store (one blob reference is consumed). + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result; /// Moves a file to another folder. diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs index 185509bf..c83208fe 100644 --- a/src/infrastructure/repositories/pg/file_blob_write_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -220,26 +220,30 @@ impl FileBlobWriteRepository { Ok((new_hash.to_string(), updated_at)) } - /// Like [`FileWritePort::save_file_from_temp`] but also returns whether the - /// blob was genuinely new (`true`) or a dedup hit (`false`). - /// Used by [`FileUploadService`] to pass `is_new_blob` to lifecycle hooks. - pub async fn save_file_from_temp_with_dedup( + /// Register a file row pointing at a blob already stored in the chunk + /// store (the upload-ingest layer streamed the content in). Consumes the + /// caller's blob reference: any failure releases it before returning. + async fn save_file_with_blob_impl( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, - ) -> Result<(File, bool), DomainError> { - let user_id = self.resolve_user_id(folder_id.as_deref()).await?; - - let dedup_result = self - .dedup - .store_from_file(temp_path, Some(content_type.clone()), pre_computed_hash) - .await?; - let is_new_blob = !dedup_result.was_deduplicated(); - let blob_hash = dedup_result.hash().to_string(); + ) -> Result { + let user_id = match self.resolve_user_id(folder_id.as_deref()).await { + Ok(user_id) => user_id, + Err(e) => { + if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await { + tracing::error!( + "Blob orphaned after owner resolution failure — hash: {}, err: {}", + &blob_hash[..12], + rollback_err + ); + } + return Err(e); + } + }; // Deadlock victims (40P01) retry before the compensation below runs — // a successful retry must keep the blob reference alive. The final @@ -259,7 +263,7 @@ impl FileBlobWriteRepository { .bind(&name) .bind(&folder_id) .bind(user_id) - .bind(&blob_hash) + .bind(blob_hash) .bind(size as i64) .bind(&content_type) .bind(category_order_for(&name, &content_type)) @@ -269,7 +273,7 @@ impl FileBlobWriteRepository { { Ok(row) => row, Err(e) => { - if let Err(rollback_err) = self.dedup.remove_reference(&blob_hash).await { + if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await { tracing::error!( "Blob orphaned after failed INSERT — hash: {}, err: {}", &blob_hash[..12], @@ -309,32 +313,23 @@ impl FileBlobWriteRepository { row.1, row.2, Some(user_id), - blob_hash, + blob_hash.to_string(), )?; - Ok((file, is_new_blob)) + Ok(file) } } impl FileWritePort for FileBlobWriteRepository { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result { - self.save_file_from_temp_with_dedup( - name, - folder_id, - content_type, - temp_path, - size, - pre_computed_hash, - ) - .await - .map(|(file, _)| file) + self.save_file_with_blob_impl(name, folder_id, content_type, blob_hash, size) + .await } async fn move_file( @@ -532,24 +527,18 @@ impl FileWritePort for FileBlobWriteRepository { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, file_id: &str, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - content_type: Option, - pre_computed_hash: Option, modified_at: Option, ) -> Result<(String, i64), DomainError> { - // Streaming: pass pre-computed hash so dedup skips re-reading the file. - let dedup_result = self - .dedup - .store_from_file(temp_path, content_type, pre_computed_hash) - .await?; - let new_hash = dedup_result.hash().to_string(); - + // The content was already ingested into the chunk store by the + // upload-ingest layer; swap_blob_hash consumes its reference and + // releases it on failure. let swapped = self - .swap_blob_hash(file_id, &new_hash, size as i64, modified_at) + .swap_blob_hash(file_id, blob_hash, size as i64, modified_at) .await?; // The file now maps to a different blob — drop the read-side cache // entry so streaming downloads cannot serve the previous content diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index 39aaa7d8..ffc9b8e5 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -6,14 +6,14 @@ //! (updated atomically on each chunk) are stored alongside the chunk files. //! On boot the service scans `temp_base_dir` and recovers any active sessions. //! - Parallel chunk transfers (up to 6 concurrent) -//! - Automatic reassembly with hash-on-write (BLAKE3) +//! - Completion streams the ordered parts straight into the CDC blob store //! - Expiration cleanup (24 h) //! //! Protocol: //! 1. POST /api/uploads → Create upload session, get upload_id //! 2. PATCH /api/uploads/:id → Upload chunks (parallel OK) //! 3. HEAD /api/uploads/:id → Check progress -//! 4. POST /api/uploads/:id/complete → Finalize and assemble +//! 4. POST /api/uploads/:id/complete → Stream parts into the blob store use chrono::{DateTime, Utc}; use dashmap::DashMap; @@ -28,7 +28,8 @@ use tokio::io::AsyncWriteExt; use uuid::Uuid; use crate::application::ports::chunked_upload_ports::{ - ChunkUploadResponseDto, ChunkedUploadPort, CreateUploadResponseDto, UploadStatusResponseDto, + ChunkUploadResponseDto, ChunkedUploadPort, CompletedUploadParts, CreateUploadResponseDto, + UploadStatusResponseDto, }; use crate::domain::errors::{DomainError, ErrorKind}; @@ -603,7 +604,7 @@ impl ChunkedUploadService { /// /// Used by the streaming REST PUT path: the handler calls /// `prepare_chunk` → streams body to disk via - /// `interfaces::upload_spool::stream_body_to_path` → calls + /// `interfaces::upload_ingest::stream_body_to_path` → calls /// `commit_chunk` to finalise. This lets the body bypass the /// in-memory `Bytes` allocation entirely (peak heap ~one HTTP /// frame instead of "chunk size"). @@ -936,24 +937,23 @@ impl ChunkedUploadService { }) } - /// Assemble chunks into final file and return the path + pre-computed BLAKE3 hash. + /// Validate completion and return the chunk parts in assembly order. /// - /// **Hash-on-Write**: BLAKE3 is computed while copying chunks into the - /// assembled file, eliminating the second sequential read that dedup_service - /// would otherwise need. - /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. + /// No assembled file is written and nothing is hashed here — the caller + /// streams the parts straight into the CDC chunk store, which computes + /// BLAKE3 and dedup-checks each CDC chunk in that single read pass. The + /// part files stay on disk until `finalize_upload`, so a completion + /// that fails downstream (e.g. client checksum mismatch) is retryable. async fn complete_upload_inner( &self, upload_id: &str, user_id: &str, - ) -> Result<(PathBuf, String, Option, String, u64, String), String> { - // Verify ownership before assembly + ) -> Result { + // Verify ownership before completion self.verify_session_owner(upload_id, user_id)?; // Get session and validate completion. - // Clone the session data and drop the DashMap ref immediately - // so the shard is not held during the expensive assembly step. + // Clone the session data and drop the DashMap ref immediately. let session = { let entry = self .sessions @@ -971,118 +971,29 @@ impl ChunkedUploadService { entry.clone() }; - // Assemble file with hash-on-write. - // - // The entire loop is offloaded to spawn_blocking because BLAKE3 - // hashing is CPU-bound and would otherwise block a Tokio worker, - // starving all other connections. - // Synchronous I/O is used inside the blocking thread — it avoids - // the async reactor overhead and is actually faster for this - // sequential workload. - let assembled_path = session.temp_dir.join("assembled"); - let chunks_meta: Vec<(usize, PathBuf)> = session - .chunks + // Chunk indices are unique (upload_chunk rejects duplicates) but may + // have arrived out of order — sort to recover the assembly order. + let mut indices: Vec = session.chunks.iter().map(|c| c.index).collect(); + indices.sort_unstable(); + let chunk_paths: Vec = indices .iter() - .map(|c| { - ( - c.index, - session.temp_dir.join(format!("chunk_{:06}", c.index)), - ) - }) + .map(|index| session.temp_dir.join(format!("chunk_{:06}", index))) .collect(); - let total_size = session.total_size; - - let hash = tokio::task::spawn_blocking(move || -> Result { - use std::io::{BufWriter as StdBufWriter, Read, Write}; - - let raw_output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&assembled_path) - .map_err(|e| format!("Failed to create assembled file: {e}"))?; - - // Pre-allocate assembled file to reduce fragmentation - let _ = raw_output.set_len(total_size); - - // 512 KB I/O buffers — 8× fewer syscalls than 64 KB - let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = blake3::Hasher::new(); - - // For files >10 MB, use multithreaded BLAKE3 hashing (all cores) - const RAYON_THRESHOLD: u64 = 10 * 1024 * 1024; - let use_rayon = total_size > RAYON_THRESHOLD; - - // Single 512 KB read buffer reused across all chunks (avoids N allocations) - let mut buf = vec![0u8; 524_288]; - for (index, chunk_path) in &chunks_meta { - let mut chunk_file = std::fs::File::open(chunk_path) - .map_err(|e| format!("Failed to open chunk {index}: {e}"))?; - loop { - let n = chunk_file - .read(&mut buf) - .map_err(|e| format!("Failed to read chunk {index}: {e}"))?; - if n == 0 { - break; - } - if use_rayon { - hasher.update_rayon(&buf[..n]); - } else { - hasher.update(&buf[..n]); - } - output.write_all(&buf[..n]).map_err(|e| { - format!("Failed to write chunk {index} to assembled file: {e}") - })?; - } - } - - output - .flush() - .map_err(|e| format!("Failed to flush assembled file: {e}"))?; - // ── Durability boundary ──────────────────────────────────── - // `flush` drains BufWriter's userspace buffer but leaves the - // bytes in the kernel page cache. Without `sync_all`, a - // power loss between this `complete_upload` returning 2xx - // and the OS writeback timer firing (~5 s default) loses - // the merged blob — and PG's metadata row references a hash - // that no longer exists on disk. Reclaim the BufWriter's - // inner File via `into_inner` so we can `sync_all` it; the - // BufWriter would otherwise drop without flushing on the - // inner handle. - let raw_output = output - .into_inner() - .map_err(|e| format!("into_inner on BufWriter failed: {e}"))?; - raw_output - .sync_all() - .map_err(|e| format!("Failed to fsync assembled file: {e}"))?; - - // Clean up chunk files (keep assembled) — already on a blocking thread - for (_index, chunk_path) in &chunks_meta { - let _ = std::fs::remove_file(chunk_path); - } - - Ok(hasher.finalize().to_hex().to_string()) - }) - .await - .map_err(|e| format!("Assembly task panicked: {e}"))??; - - let assembled_path = session.temp_dir.join("assembled"); tracing::info!( - "✅ Assembled chunked upload: {} ({} bytes from {} chunks)", + "✅ Chunked upload complete: {} ({} bytes in {} chunks, streamed to CAS)", session.filename, session.total_size, - session.chunks.len() + chunk_paths.len() ); - Ok(( - assembled_path, - session.filename.clone(), - session.folder_id.clone(), - session.content_type.clone(), - session.total_size, - hash, - )) + Ok(CompletedUploadParts { + chunk_paths, + filename: session.filename.clone(), + folder_id: session.folder_id.clone(), + content_type: session.content_type.clone(), + total_size: session.total_size, + }) } /// Finalize upload: remove session from RAM, then clean disk OUTSIDE lock. @@ -1187,7 +1098,7 @@ impl ChunkedUploadPort for ChunkedUploadService { &self, upload_id: &str, user_id: Uuid, - ) -> Result<(PathBuf, String, Option, String, u64, String), DomainError> { + ) -> Result { self.complete_upload_inner(upload_id, &user_id.to_string()) .await .map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e)) @@ -1458,18 +1369,21 @@ mod tests { assert_eq!(status.completed_chunks, 2); assert!(status.pending_chunks.is_empty()); - // 4. Complete (assemble) - let (path, filename, _folder, _ct, size, hash) = service + // 4. Complete — returns the ordered chunk parts, no assembled file + let parts = service .complete_upload_inner(&id, "test-user") .await .expect("complete"); - assert_eq!(filename, "test.txt"); - assert_eq!(size, 1024); - assert!(!hash.is_empty()); - assert!(path.exists()); + assert_eq!(parts.filename, "test.txt"); + assert_eq!(parts.total_size, 1024); + assert_eq!(parts.chunk_paths.len(), 2); - // 5. Verify assembled content - let content = fs::read(&path).await.expect("read assembled"); + // 5. Verify the parts concatenate to the original content in order + let mut content = Vec::new(); + for path in &parts.chunk_paths { + assert!(path.exists(), "chunk part must remain until finalize"); + content.extend_from_slice(&fs::read(path).await.expect("read part")); + } assert_eq!(&content[..512], &[b'A'; 512]); assert_eq!(&content[512..], &[b'B'; 512]); diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 2d06f6ec..8f7ff957 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -18,33 +18,41 @@ //! blobs in `storage.blobs`) are served transparently — when no manifest //! row exists for a hash, the service falls back to direct blob reads. //! -//! **Write-first strategy** (store_from_file): -//! 1. CDC-analyse the file (mmap → FastCDC boundaries + per-chunk BLAKE3). -//! 2. Batch-check which chunk hashes already exist in PG (dedup skip). -//! 3. Bump ref_count for existing chunks (no disk I/O). -//! 4. Read + write only *new* chunks to the blob backend (idempotent, -//! no per-chunk fsync). -//! 5. One batched fsync sweep makes the new chunks durable, then ONE -//! batched INSERT registers them — durability before visibility. -//! 6. Single manifest INSERT (~few ms total). -//! 7. PG connection is never held during disk I/O. +//! **Single-pass streaming ingest** (store_from_stream): +//! 1. FastCDC boundaries, per-chunk BLAKE3 and the whole-file BLAKE3 are +//! all computed WHILE the bytes arrive — no spool file, no mmap +//! re-read. Peak RAM stays bounded (current chunk + one small batch). +//! 2. Per batch of distinct chunks, ONE `UPDATE … RETURNING` bumps +//! ref_count on already-known chunks (pinning them against concurrent +//! reclaim for the rest of the upload) and atomically classifies the +//! rest as new — no check-then-bump TOCTOU window. +//! 3. Only *new* chunks are written to the blob backend (unsynced, +//! bounded concurrency). Bytes the store already knows never touch +//! disk — a full dedup hit performs zero content writes. +//! 4. At end of stream ONE batched fsync sweep makes the new chunks +//! durable, then ONE batched INSERT registers them — durability +//! before visibility. +//! 5. Single manifest INSERT (~few ms). An identical concurrent upload +//! is resolved via ON CONFLICT: the loser releases its chunk +//! references and turns into a dedup hit. +//! 6. PG connections are never held during disk I/O. //! //! Benefits: +//! - Each uploaded byte hits the disk at most ONCE (dedup hits: zero) //! - Sub-file dedup: edited files share unchanged chunks //! - ACID durability — crash-safe, zero orphaned index entries -//! - PG connections never blocked by disk I/O (write-first) //! - 60-80% storage reduction for versioned / edited files -//! - Faster uploads when chunks already exist use bytes::Bytes; use futures::stream::{self, StreamExt}; use futures::{Stream, TryStreamExt}; use sqlx::PgPool; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use tokio::fs; +use tokio_util::io::StreamReader; use crate::application::ports::blob_lifecycle::BlobLifecycleHook; use crate::application::ports::blob_storage_ports::BlobStorageBackend; @@ -65,11 +73,163 @@ const CDC_MAX_CHUNK: usize = 1_048_576; // ── CDC helper types ───────────────────────────────────────────────────────── -/// Metadata for a single CDC chunk (offset + length + BLAKE3 hash). -struct ChunkMeta { - hash: String, - offset: usize, - length: usize, +/// Everything a streaming chunk ingest learned about its byte stream. +/// +/// Produced by [`DedupService::ingest_chunks_from_stream`]. On success the +/// ingest session holds exactly ONE `storage.blobs.ref_count` reference per +/// *distinct* chunk hash; the caller must either attach those references to +/// a manifest or hand them back via `release_chunk_refs`. +struct ChunkIngestOutcome { + /// BLAKE3 of the complete byte stream (the future manifest key). + file_hash: String, + /// Total bytes consumed from the stream. + total_size: u64, + /// Per-occurrence chunk hashes, in file order (the manifest layout). + chunk_hashes: Vec, + /// Per-occurrence chunk sizes, in file order. + chunk_sizes: Vec, + /// How many distinct chunks were actually written to the backend. + newly_written: usize, +} + +impl ChunkIngestOutcome { + /// Distinct chunk hashes — the set this ingest holds one reference on each. + fn distinct_hashes(&self) -> Vec { + let mut seen = HashSet::new(); + self.chunk_hashes + .iter() + .filter(|h| seen.insert(h.as_str())) + .cloned() + .collect() + } +} + +/// Compensation guard for an in-flight ingest session. +/// +/// Tracks the two side effects a session accumulates before its chunks are +/// fully registered: ref_count pins taken on pre-existing chunks and freshly +/// written (still unregistered) chunk files. If the session future is dropped +/// mid-stream — a client disconnect aborts the whole handler future — the +/// guard spawns a rollback so pinned chunks don't leak references forever and +/// written files become GC-collectible rows instead of invisible orphans. +struct IngestGuard { + pool: Arc, + backend: Arc, + /// Pre-existing chunks whose ref_count this session bumped (distinct). + pinned: Vec, + /// Chunks written to the backend but not yet registered: (hash, size). + written: Vec<(String, i64)>, + armed: bool, +} + +impl IngestGuard { + fn new(pool: Arc, backend: Arc) -> Self { + Self { + pool, + backend, + pinned: Vec::new(), + written: Vec::new(), + armed: true, + } + } + + /// The session's chunks are fully registered — references now belong to + /// the caller, nothing to compensate. + fn disarm(mut self) { + self.armed = false; + } + + /// Deterministic rollback for handled errors (awaited inline, unlike the + /// spawned Drop path). + async fn rollback(mut self) { + self.armed = false; + let pinned = std::mem::take(&mut self.pinned); + let written = std::mem::take(&mut self.written); + Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await; + } + + /// Release pins and surface written-but-unregistered chunk files to GC. + /// + /// Best-effort: every step logs instead of failing — the worst outcome of + /// a failed rollback is a bounded ref_count over-count (storage leak), + /// never data loss. + async fn run_rollback( + pool: Arc, + backend: Arc, + pinned: Vec, + written: Vec<(String, i64)>, + ) { + if !pinned.is_empty() + && let Err(e) = sqlx::query( + "UPDATE storage.blobs SET ref_count = GREATEST(ref_count - 1, 0) + WHERE hash = ANY($1)", + ) + .bind(&pinned) + .execute(pool.as_ref()) + .await + { + tracing::warn!( + "Ingest rollback: failed to release {} chunk pins: {e}", + pinned.len() + ); + } + + if written.is_empty() { + return; + } + // Durability first, then visibility at ref_count 0 so the existing GC + // sweep can reclaim the bytes — a backend file with no PG row would be + // invisible to it. ON CONFLICT DO NOTHING keeps a concurrent + // uploader's row (and its references) intact. + let hashes: Vec = written.iter().map(|(h, _)| h.clone()).collect(); + let sizes: Vec = written.iter().map(|(_, s)| *s).collect(); + if let Err(e) = backend.sync_blobs(&hashes).await { + tracing::warn!( + "Ingest rollback: sync of {} chunks failed: {e}", + hashes.len() + ); + } + if let Err(e) = sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) + SELECT h, s, 0 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) + ON CONFLICT (hash) DO NOTHING", + ) + .bind(&hashes) + .bind(&sizes) + .execute(pool.as_ref()) + .await + { + tracing::warn!( + "Ingest rollback: failed to register {} orphan chunks for GC: {e}", + hashes.len() + ); + } + } +} + +impl Drop for IngestGuard { + fn drop(&mut self) { + if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) { + return; + } + let pinned = std::mem::take(&mut self.pinned); + let written = std::mem::take(&mut self.written); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + let pool = self.pool.clone(); + let backend = self.backend.clone(); + handle.spawn(async move { + Self::run_rollback(pool, backend, pinned, written).await; + }); + } + Err(_) => tracing::warn!( + "Ingest guard dropped outside a runtime: {} pins / {} written chunks \ + stay leaked until the next GC sweep", + pinned.len(), + written.len() + ), + } + } } /// Content-Addressable Storage Service with CDC (PostgreSQL-backed) @@ -82,7 +242,7 @@ pub struct DedupService { /// Pluggable blob storage backend (local FS, S3, …). backend: Arc, /// PostgreSQL connection pool (dedup index in `storage.blobs`) — primary, - /// used by request-path operations (store_from_file, etc.). + /// used by request-path operations (store_from_stream, etc.). pool: Arc, /// Isolated maintenance pool for long-running operations /// (verify_integrity, garbage_collect) that must never starve the primary. @@ -192,86 +352,12 @@ impl DedupService { .unwrap_or_else(|| PathBuf::from(format!("remote://{}", hash))) } - // ── CDC analysis ─────────────────────────────────────────── - - /// Single-pass CDC: compute whole-file BLAKE3 hash + chunk boundaries + per-chunk hashes. - /// - /// Memory-maps the file and runs FastCDC boundary detection - /// concurrently with BLAKE3 hashing — all in one pass. - async fn cdc_hash_and_chunk_file(path: &Path) -> std::io::Result<(String, Vec)> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path)?; - let file_size = file.metadata()?.len(); - - if file_size == 0 { - return Ok((blake3::hash(b"").to_hex().to_string(), vec![])); - } - - // SAFETY: file is opened read-only; no concurrent writers expected - // (source is a temp upload file owned exclusively by this request). - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let chunker = - fastcdc::v2020::FastCDC::new(&mmap, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK); - - let mut file_hasher = blake3::Hasher::new(); - let mut chunks = Vec::new(); - - for chunk in chunker { - let data = &mmap[chunk.offset..chunk.offset + chunk.length]; - file_hasher.update(data); - chunks.push(ChunkMeta { - hash: blake3::hash(data).to_hex().to_string(), - offset: chunk.offset, - length: chunk.length, - }); - } - - Ok((file_hasher.finalize().to_hex().to_string(), chunks)) - }) - .await - .expect("cdc_hash_and_chunk_file: spawn_blocking panicked") - } - - /// CDC analysis without file-hash computation (when hash is pre-computed). - async fn cdc_chunk_file(path: &Path) -> std::io::Result> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path)?; - let file_size = file.metadata()?.len(); - - if file_size == 0 { - return Ok(vec![]); - } - - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let chunker = - fastcdc::v2020::FastCDC::new(&mmap, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK); - - let chunks: Vec = chunker - .map(|chunk| { - let data = &mmap[chunk.offset..chunk.offset + chunk.length]; - ChunkMeta { - hash: blake3::hash(data).to_hex().to_string(), - offset: chunk.offset, - length: chunk.length, - } - }) - .collect(); - - Ok(chunks) - }) - .await - .expect("cdc_chunk_file: spawn_blocking panicked") - } - // ── Hash helpers ───────────────────────────────────────────── /// Calculate BLAKE3 hash of a file (~5× faster than SHA-256). /// - /// Uses memory-mapped I/O with rayon parallelism. Kept for callers - /// that only need the hash (e.g. upload handlers pre-computing the hash - /// before calling `store_from_file`). + /// Uses memory-mapped I/O with rayon parallelism. Used by + /// `verify_integrity` to re-hash local blob files. pub async fn hash_file(path: &Path) -> std::io::Result { let path = path.to_path_buf(); tokio::task::spawn_blocking(move || { @@ -283,329 +369,238 @@ impl DedupService { .expect("hash_file: spawn_blocking task panicked") } - // ── Core store operations ──────────────────────────────────── - - /// Store content with CDC deduplication (from file). - /// - /// **Fast path**: if `pre_computed_hash` is `Some`, the manifest / - /// legacy-blob index is checked *before* running CDC — returning - /// instantly on a full-file dedup hit. - /// - /// **New-file path**: CDC-analyses the file (single mmap pass), - /// stores unique chunks via the blob backend, then inserts the - /// manifest in PostgreSQL. - pub async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result { - // ── Fast path: pre-computed hash → check before CDC ────── - if let Some(ref hash) = pre_computed_hash - && let Some(result) = self.try_dedup_hit(hash, source_path).await? - { - return Ok(result); - } - - // ── CDC analysis ───────────────────────────────────────── - let (file_hash, chunks) = if let Some(hash) = pre_computed_hash { - let chunks = Self::cdc_chunk_file(source_path) - .await - .map_err(DomainError::from)?; - (hash, chunks) - } else { - let (hash, chunks) = Self::cdc_hash_and_chunk_file(source_path) - .await - .map_err(DomainError::from)?; - // Check dedup with newly computed hash - if let Some(result) = self.try_dedup_hit(&hash, source_path).await? { - return Ok(result); - } - (hash, chunks) - }; - - let file_size = fs::metadata(source_path) - .await - .map_err(DomainError::from)? - .len(); - - // ── Store chunks (write-first — no PG connection held) ─── - let (chunk_hashes, chunk_sizes) = self.store_chunks(source_path, &chunks).await?; - - // ── Insert manifest ────────────────────────────────────── - sqlx::query( - "INSERT INTO storage.chunk_manifests - (file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, content_type, ref_count) - VALUES ($1, $2, $3, $4, $5, $6, 1)", - ) - .bind(&file_hash) - .bind(&chunk_hashes) - .bind(chunk_sizes.iter().map(|s| *s as i64).collect::>()) - .bind(file_size as i64) - .bind(chunk_hashes.len() as i32) - .bind(&content_type) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to insert manifest: {}", e)) - })?; - - // ── Clean up source file ───────────────────────────────── - let _ = fs::remove_file(source_path).await; - - tracing::info!( - "NEW BLOB (CDC): {} ({} bytes, {} chunks)", - &file_hash[..12], - file_size, - chunk_hashes.len() - ); - - self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()); - - Ok(DedupResultDto::NewBlob { - hash: file_hash, - size: file_size, - }) - } - - /// Check manifest or legacy blob for a dedup hit. - /// - /// Returns `Some(ExistingBlob)` if the exact file was already stored. - /// Bumps the appropriate ref_count and removes the source file. - async fn try_dedup_hit( - &self, - hash: &str, - source_path: &Path, - ) -> Result, DomainError> { - // ── CDC manifest hit ───────────────────────────────────── - let manifest = sqlx::query_as::<_, (i64,)>( - "SELECT total_size FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to check manifest: {}", e)) - })?; - - if let Some((total_size,)) = manifest { - sqlx::query( - "UPDATE storage.chunk_manifests SET ref_count = ref_count + 1 WHERE file_hash = $1", - ) - .bind(hash) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to bump manifest ref_count: {}", e), - ) - })?; - - let _ = fs::remove_file(source_path).await; - - tracing::info!( - "DEDUP HIT (manifest): {} ({} bytes saved)", - &hash[..12], - total_size - ); - return Ok(Some(DedupResultDto::ExistingBlob { - hash: hash.to_owned(), - size: total_size as u64, - saved_bytes: total_size as u64, - })); - } - - // ── Legacy whole-file blob hit ─────────────────────────── - let legacy = sqlx::query_as::<_, (i64,)>("SELECT size FROM storage.blobs WHERE hash = $1") - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to check legacy blob: {}", e)) - })?; - - if let Some((size,)) = legacy { - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1") - .bind(hash) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to bump legacy ref_count: {}", e), - ) - })?; - - let _ = fs::remove_file(source_path).await; - tracing::info!( - "DEDUP HIT (legacy blob): {} ({} bytes saved)", - &hash[..12], - size - ); - return Ok(Some(DedupResultDto::ExistingBlob { - hash: hash.to_owned(), - size: size as u64, - saved_bytes: size as u64, - })); - } - - Ok(None) - } + // ── Core store operations (streaming CDC) ─────────────────── /// Maximum concurrent chunk uploads to the blob backend. const CHUNK_UPLOAD_CONCURRENCY: usize = 8; + /// Flush the pending distinct-chunk batch after this many chunks… + const FLUSH_MAX_CHUNKS: usize = 32; + /// …or after this many buffered bytes, whichever comes first. Together + /// with the ≤ 1 MiB chunk in flight this bounds peak RAM per upload to + /// ~9 MiB regardless of file size. + const FLUSH_MAX_BYTES: usize = 8 * 1024 * 1024; - /// Store CDC chunks via the blob backend + upsert in PG. + /// Store content with CDC deduplication, straight from a byte stream — + /// the single write path for every upload surface (REST multipart, + /// WebDAV PUT, NextCloud PUT, chunked-upload assembly, WOPI PutFile). /// - /// Phase 0: Batch-queries PG to discover which chunk hashes already - /// exist in `storage.blobs`. - /// Phase 1: Bumps `ref_count` for chunks that already exist (one - /// batched UPDATE, no disk I/O — the biggest saving for versioned - /// files where most chunks are unchanged). - /// Phase 2: Reads + writes only *new* chunks, with up to - /// [`CHUNK_UPLOAD_CONCURRENCY`] writes in flight and **no per-chunk - /// fsync**. - /// Phase 3: One batched `sync_blobs` sweep makes every new chunk - /// durable (no-op for remote backends, which are durable on PUT). - /// Phase 4: ONE batched INSERT registers the new chunks in PG. The - /// sweep runs first so a crash can never leave a `storage.blobs` row - /// pointing at bytes that were still in the page cache. + /// One pass over the incoming bytes: FastCDC boundary detection, + /// per-chunk BLAKE3, the whole-file BLAKE3, dedup lookups and blob + /// writes all happen while the stream is still arriving. There is no + /// spool file and no re-read — each uploaded byte touches the disk at + /// most once, and not at all when the store already has its chunk. /// - /// `ref_count` is incremented once per *distinct* chunk (one reference per - /// manifest), staying symmetric with `remove_manifest_reference` so a file - /// that repeats a chunk cannot over-count and leak the blob forever. - async fn store_chunks( + /// Identical-content races (two clients uploading the same file + /// concurrently) are resolved at the manifest INSERT via ON CONFLICT: + /// the loser releases its chunk references and returns `ExistingBlob`. + pub async fn store_from_stream( &self, - source_path: &Path, - chunks: &[ChunkMeta], - ) -> Result<(Vec, Vec), DomainError> { - let pool = &self.pool; - let backend = &self.backend; + source: S, + content_type: Option, + ) -> Result + where + S: Stream> + Send, + { + let outcome = self.ingest_chunks_from_stream(source).await?; + let distinct = outcome.distinct_hashes(); + let total_size = outcome.total_size; + let file_hash = outcome.file_hash.clone(); - // ── Phase 0: de-duplicate chunk hashes, then batch-check existence ── - // A single file can legitimately repeat the same chunk many times - // (zero-filled regions in disk/VM images, repeated document structures, - // concatenated archives). `ref_count` is tracked per *distinct* chunk — - // one reference per manifest — to stay symmetric with - // `remove_manifest_reference`, which decrements via - // `WHERE hash = ANY(chunk_hashes)` (matching each row once). Counting - // per-occurrence here would over-increment and leak the blob forever. - // Keep the first occurrence of each hash so new chunks know where to - // read their bytes. - let mut seen = std::collections::HashSet::new(); - let unique_chunks: Vec<&ChunkMeta> = chunks - .iter() - .filter(|c| seen.insert(c.hash.as_str())) - .collect(); - let unique_hashes: Vec = unique_chunks.iter().map(|c| c.hash.clone()).collect(); + // A bounded retry covers the rare interleaving where the manifest + // that beat our INSERT is deleted again before our ref bump lands. + for _ in 0..3 { + let inserted = sqlx::query( + "INSERT INTO storage.chunk_manifests + (file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, content_type, ref_count) + VALUES ($1, $2, $3, $4, $5, $6, 1) + ON CONFLICT (file_hash) DO NOTHING", + ) + .bind(&file_hash) + .bind(&outcome.chunk_hashes) + .bind( + outcome + .chunk_sizes + .iter() + .map(|s| *s as i64) + .collect::>(), + ) + .bind(total_size as i64) + .bind(outcome.chunk_hashes.len() as i32) + .bind(&content_type) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to insert manifest: {}", e)) + })? + .rows_affected(); - let existing_hashes: std::collections::HashSet = - sqlx::query_scalar::<_, String>("SELECT hash FROM storage.blobs WHERE hash = ANY($1)") - .bind(&unique_hashes) - .fetch_all(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( + if inserted > 0 { + tracing::info!( + "NEW BLOB (CDC stream): {} ({} bytes, {} chunks, {} written)", + &file_hash[..12], + total_size, + outcome.chunk_hashes.len(), + outcome.newly_written, + ); + self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()); + return Ok(DedupResultDto::NewBlob { + hash: file_hash, + size: total_size, + }); + } + + // The manifest already exists — either this exact content was + // stored before or an identical concurrent upload just won the + // race. Bump ITS ref_count first and only then hand back this + // session's chunk references; the reverse order could leave the + // caller's file row without any manifest reference behind it. + if let Some(existing_size) = self.bump_manifest_if_exists(&file_hash).await? { + self.release_chunk_refs(self.pool.as_ref(), &distinct).await; + tracing::info!( + "DEDUP HIT (manifest): {} ({} bytes saved)", + &file_hash[..12], + existing_size, + ); + return Ok(DedupResultDto::ExistingBlob { + hash: file_hash, + size: existing_size as u64, + saved_bytes: existing_size as u64, + }); + } + } + + self.release_chunk_refs(self.pool.as_ref(), &distinct).await; + Err(DomainError::internal_error( + "Dedup", + format!("Manifest insert/bump kept racing for {file_hash}"), + )) + } + + /// Bump a manifest's ref_count if it exists; returns its total_size. + /// Single statement — no window between the existence check and the bump. + async fn bump_manifest_if_exists(&self, file_hash: &str) -> Result, DomainError> { + sqlx::query_scalar::<_, i64>( + "UPDATE storage.chunk_manifests SET ref_count = ref_count + 1 + WHERE file_hash = $1 + RETURNING total_size", + ) + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to bump manifest ref_count: {e}")) + }) + } + + /// Stream → chunk store, WITHOUT creating a manifest. + /// + /// Splits the stream with FastCDC while computing per-chunk and + /// whole-stream BLAKE3 hashes, then settles each batch of distinct + /// chunks against PG: + /// + /// 1. ONE `UPDATE … RETURNING` per batch pins every already-known chunk + /// (`ref_count + 1` — protecting it from a concurrent last-reference + /// delete for the rest of the upload) and atomically classifies the + /// remaining hashes as new. No check-then-bump TOCTOU window. + /// 2. New chunks are written to the backend unsynced with bounded + /// concurrency; chunks the store already has are dropped from RAM + /// without any disk I/O. + /// 3. At end of stream, ONE `sync_blobs` sweep makes the new chunks + /// durable, then ONE batched INSERT registers them (`ON CONFLICT` + /// bumps instead — a concurrent identical upload may have registered + /// the same brand-new chunk first). Durability before visibility. + /// + /// `ref_count` is taken once per *distinct* chunk — symmetric with + /// `remove_manifest_reference`, which decrements via + /// `WHERE hash = ANY(chunk_hashes)` (each row once). A repeated chunk + /// (zero-filled regions, concatenated archives) must not over-count or + /// the blob leaks forever. + /// + /// If the returned references are not attached to a manifest, the caller + /// must hand them back via `release_chunk_refs`. If this future is + /// dropped mid-stream (client disconnect), the internal guard rolls the + /// session back in a spawned task. + async fn ingest_chunks_from_stream( + &self, + source: S, + ) -> Result + where + S: Stream> + Send, + { + let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone()); + + let reader = StreamReader::new(Box::pin(source)); + let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( + reader, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ); + let chunk_stream = chunker.as_stream(); + futures::pin_mut!(chunk_stream); + + let mut file_hasher = blake3::Hasher::new(); + let mut total_size: u64 = 0; + let mut chunk_hashes: Vec = Vec::new(); + let mut chunk_sizes: Vec = Vec::new(); + let mut session_seen: HashSet = HashSet::new(); + let mut pending: Vec<(String, Bytes)> = Vec::new(); + let mut pending_bytes: usize = 0; + + while let Some(item) = chunk_stream.next().await { + let chunk = match item { + Ok(chunk) => chunk, + Err(e) => { + guard.rollback().await; + return Err(DomainError::internal_error( "Dedup", - format!("Failed to check existing chunks: {}", e), - ) - })? - .into_iter() - .collect(); - - // ── Phase 1: bump ref_count for every existing chunk in ONE query ── - // (was one UPDATE per occurrence — now a single batched round-trip). - let existing: Vec = unique_hashes - .iter() - .filter(|h| existing_hashes.contains(*h)) - .cloned() - .collect(); - if !existing.is_empty() { - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = ANY($1)") - .bind(&existing) - .execute(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to bump ref_count: {}", e)) - })?; - } - - // ── Phase 2: upload each NEW chunk once, concurrently ────────────── - // Read each new chunk by positioned I/O immediately before its upload - // instead of materializing every chunk's *data* up front. Peak heap for - // file content stays bounded to ~CHUNK_UPLOAD_CONCURRENCY × CDC_MAX_CHUNK - // (≈ 8 MiB) — proportional to the chunk size, never the file size. - // - // Owned metadata (hash + offset + length, no file data) so the stream - // below does not borrow `chunks` across an `.await` (which would make - // this future non-`Send` and break the upload handlers). - let new_ops: Vec<(String, u64, usize)> = unique_chunks - .iter() - .filter(|c| !existing_hashes.contains(&c.hash)) - .map(|c| (c.hash.clone(), c.offset as u64, c.length)) - .collect(); - - let source = Arc::new(std::fs::File::open(source_path).map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to open source file: {}", e)) - })?); - - // Writes are *unsynced*: no per-chunk fsync. Durability comes from - // the single batched sweep below, BEFORE any PG row references the - // new chunks — so a crash can never leave storage.blobs claiming a - // chunk whose bytes didn't reach the platter. - let results: Vec> = stream::iter(new_ops) - .map(|(hash, offset, length)| { - let source = source.clone(); - let backend = backend.clone(); - async move { - // Positioned read of just this chunk (≤ CDC_MAX_CHUNK) off - // the async runtime, then upload. - let bytes = tokio::task::spawn_blocking(move || { - use std::os::unix::fs::FileExt; - let mut buf = vec![0u8; length]; - source.read_exact_at(&mut buf, offset)?; - Ok::, std::io::Error>(buf) - }) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Read task failed: {}", e)) - })? - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to read chunk: {}", e)) - })?; - - backend - .put_blob_from_bytes_unsynced(&hash, Bytes::from(bytes)) - .await?; - Ok((hash, length as i64)) + format!("Upload stream failed: {e}"), + )); } - }) - .buffer_unordered(Self::CHUNK_UPLOAD_CONCURRENCY) - .collect() - .await; + }; - let mut new_rows: Vec<(String, i64)> = Vec::with_capacity(results.len()); - for result in results { - new_rows.push(result?); + let data = chunk.data; + total_size += data.len() as u64; + // Per-chunk hashing is ≤ 1 MiB of BLAKE3 (< 1 ms) — cheaper than + // a spawn_blocking round-trip per chunk. + file_hasher.update(&data); + let hash = blake3::hash(&data).to_hex().to_string(); + chunk_sizes.push(data.len() as u64); + chunk_hashes.push(hash.clone()); + + if session_seen.insert(hash.clone()) { + pending_bytes += data.len(); + pending.push((hash, Bytes::from(data))); + if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES + { + if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + guard.rollback().await; + return Err(e); + } + pending_bytes = 0; + } + } } - if !new_rows.is_empty() { - // ── Phase 3: durability barrier — one batched fsync sweep ────── - // (was 2 fsyncs per chunk: ~8 200 for a 1 GB upload; now one - // parallel sweep over the new files + ≤256 prefix dirs). - // Remote backends are durable on PUT — sync_blobs is a no-op. - let new_hashes: Vec = new_rows.iter().map(|(h, _)| h.clone()).collect(); - backend.sync_blobs(&new_hashes).await?; + if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + guard.rollback().await; + return Err(e); + } - // ── Phase 4: register all new chunks in ONE batched INSERT ───── - // (was one round-trip per chunk). `new_rows` is built from - // `unique_chunks`, so no hash repeats within the batch — safe for - // ON CONFLICT DO UPDATE, which covers a concurrent uploader - // inserting the same brand-new chunk between the existence check - // in Phase 0 and this INSERT. - let new_sizes: Vec = new_rows.iter().map(|(_, s)| *s).collect(); - sqlx::query( + // ── Durability before visibility for the new chunks ────── + // One batched fsync sweep (no-op for remote backends, durable on + // PUT), then one batched INSERT. A crash before the INSERT leaves + // only unreferenced files; never a row pointing at unsynced bytes. + if !guard.written.is_empty() { + let new_hashes: Vec = guard.written.iter().map(|(h, _)| h.clone()).collect(); + let new_sizes: Vec = guard.written.iter().map(|(_, s)| *s).collect(); + + if let Err(e) = self.backend.sync_blobs(&new_hashes).await { + guard.rollback().await; + return Err(e); + } + + let registered = sqlx::query( "INSERT INTO storage.blobs (hash, size, ref_count) SELECT h, s, 1 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) ON CONFLICT (hash) DO UPDATE @@ -613,19 +608,102 @@ impl DedupService { ) .bind(&new_hashes) .bind(&new_sizes) - .execute(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to upsert chunks: {}", e)) - })?; + .execute(self.pool.as_ref()) + .await; + + if let Err(e) = registered { + guard.rollback().await; + return Err(DomainError::internal_error( + "Dedup", + format!("Failed to register chunks: {e}"), + )); + } } - // chunk_hashes/chunk_sizes keep the full per-occurrence CDC sequence — - // the manifest needs every chunk, in order, to reassemble the file. - let chunk_hashes: Vec = chunks.iter().map(|c| c.hash.clone()).collect(); - let chunk_sizes: Vec = chunks.iter().map(|c| c.length as u64).collect(); + let newly_written = guard.written.len(); + guard.disarm(); - Ok((chunk_hashes, chunk_sizes)) + Ok(ChunkIngestOutcome { + file_hash: file_hasher.finalize().to_hex().to_string(), + total_size, + chunk_hashes, + chunk_sizes, + newly_written, + }) + } + + /// Settle one batch of distinct in-RAM chunks against PG + the backend. + /// + /// Successfully pinned hashes and written chunks are recorded on the + /// guard as they happen, so a failure mid-batch leaves nothing + /// untracked for rollback. + async fn flush_pending( + &self, + guard: &mut IngestGuard, + pending: &mut Vec<(String, Bytes)>, + ) -> Result<(), DomainError> { + if pending.is_empty() { + return Ok(()); + } + let batch = std::mem::take(pending); + let hashes: Vec = batch.iter().map(|(h, _)| h.clone()).collect(); + + // Pin-or-classify in one statement: rows that exist take this + // session's reference NOW; hashes not returned don't exist and are + // ours to write. + let pinned: HashSet = sqlx::query_scalar::<_, String>( + "UPDATE storage.blobs SET ref_count = ref_count + 1 + WHERE hash = ANY($1) + RETURNING hash", + ) + .bind(&hashes) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}")) + })? + .into_iter() + .collect(); + + let mut to_write: Vec<(String, Bytes)> = Vec::with_capacity(batch.len()); + for (hash, data) in batch { + if pinned.contains(&hash) { + guard.pinned.push(hash); + } else { + to_write.push((hash, data)); + } + } + if to_write.is_empty() { + return Ok(()); + } + + // Unsynced writes — durability comes from the single end-of-stream + // sweep, before any PG row references these chunks. + let backend = self.backend.clone(); + let results: Vec> = stream::iter(to_write) + .map(|(hash, data)| { + let backend = backend.clone(); + async move { + let len = data.len() as i64; + backend.put_blob_from_bytes_unsynced(&hash, data).await?; + Ok((hash, len)) + } + }) + .buffer_unordered(Self::CHUNK_UPLOAD_CONCURRENCY) + .collect() + .await; + + let mut first_err: Option = None; + for result in results { + match result { + Ok(row) => guard.written.push(row), + Err(e) => first_err = first_err.or(Some(e)), + } + } + match first_err { + Some(e) => Err(e), + None => Ok(()), + } } // ── Reference counting ─────────────────────────────────────── @@ -879,7 +957,7 @@ impl DedupService { DomainError::internal_error("Dedup", format!("Failed to begin transaction: {}", e)) })?; - // Lock the row exclusively — prevents concurrent store_from_file from + // Lock the row exclusively — prevents a concurrent ingest from // incrementing ref_count while we might be deleting let row = sqlx::query_as::<_, (i32, i64)>( "SELECT ref_count, size FROM storage.blobs WHERE hash = $1 FOR UPDATE", @@ -917,7 +995,7 @@ impl DedupService { })?; // Delete blob from backend AFTER committing PG — the row is gone, - // so no concurrent store_from_file can resurrect a reference. + // so no concurrent ingest can resurrect a reference. if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete blob file {}: {}", hash, e); } @@ -1532,15 +1610,16 @@ impl DedupService { // report `legacy re-chunk: nothing to do`). // // Per-hash algorithm: - // 1. Spool the blob to a temp file via the normal read path (this - // decrypts it when encryption is on), verifying BLAKE3 == hash. - // 2. CDC-chunk the spool + store chunks (`store_chunks` bumps each - // distinct chunk once — the manifest's reference). - // 3. One short accounting TX with the blob row locked: + // 1. Stream the blob through the normal read path (this decrypts it + // when encryption is on) straight into the chunk-ingest engine — + // no spool file — verifying BLAKE3 == hash before keeping the + // chunks (each distinct chunk bumped once — the manifest's + // reference). + // 2. One short accounting TX with the blob row locked: // manifest INSERT with ref_count = N (current file rows referencing // the hash), blob ref_count -= N (those references now live on the // manifest), DELETE the blob row only if it hits exactly 0. - // 4. Physically delete the whole-file blob only when its row was + // 3. Physically delete the whole-file blob only when its row was // removed. Single-chunk files (chunk hash == file hash) keep the // physical blob — it IS the chunk; only the bookkeeping moves. // @@ -1548,7 +1627,7 @@ impl DedupService { // and the legacy dedup-hit path. A racing identical upload can land one // legacy reference after our commit; the blob row then survives (> 0) // and that file stays readable through the legacy fallback — a bounded - // space leak, never data loss. A crash between step 2 and 3 leaks one + // space leak, never data loss. A crash between step 1 and 2 leaks one // +1 on that file's chunk refs (re-run re-bumps); also a bounded leak, // never data loss. @@ -1688,22 +1767,12 @@ impl DedupService { hash: &str, content_type: Option, ) -> Result { - // ── 1. Spool + verify (decrypts via the normal read path) ── - // The spooled, hash-verified plaintext is the source of truth for - // sizes — `storage.blobs.size` is legacy metadata we don't trust + // ── 1. Stream + verify (decrypts via the normal read path) ── + // The chunk store is fed directly from the blob read stream — no + // spool file. Sizes come from the CDC pass over the hash-verified + // plaintext; `storage.blobs.size` is legacy metadata we don't trust // for the manifest's Range arithmetic. - // - // The path carries a per-attempt UUID: two processes sharing a temp - // dir and racing on the same hash must never truncate or delete each - // other's in-flight spool. - let spool = std::env::temp_dir().join(format!( - "oxicloud-rechunk-{}-{}.tmp", - &hash[..hash.len().min(16)], - uuid::Uuid::new_v4() - )); - let result = self.spool_and_chunk(hash, &spool).await; - let _ = fs::remove_file(&spool).await; - let (chunk_hashes, chunk_sizes) = result?; + let (chunk_hashes, chunk_sizes) = self.ingest_legacy_blob(hash).await?; let total_size: u64 = chunk_sizes.iter().sum(); // ── 2. Accounting TX: move the file references onto the manifest ── @@ -1756,10 +1825,11 @@ impl DedupService { if inserted == 0 { // A manifest appeared concurrently — only possible if the same - // content was re-uploaded and fully stored during our spool. + // content was re-uploaded and fully stored while we streamed. // Their bookkeeping is already correct; drop ours. tx.rollback().await.ok(); - self.release_chunk_refs(&chunk_hashes).await; + self.release_chunk_refs(self.maintenance_pool.as_ref(), &chunk_hashes) + .await; return Ok(0); } @@ -1828,63 +1898,36 @@ impl DedupService { Ok(freed) } - /// Spool a legacy blob to `spool`, verify its BLAKE3 matches `hash`, - /// CDC-chunk it and store the chunks. Returns (chunk_hashes, chunk_sizes). - async fn spool_and_chunk( - &self, - hash: &str, - spool: &Path, - ) -> Result<(Vec, Vec), DomainError> { - use tokio::io::AsyncWriteExt; - - let mut stream = self.read_blob_stream(hash).await?; - let file = fs::File::create(spool) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk spool: {e}")))?; - let mut writer = tokio::io::BufWriter::with_capacity(512 * 1024, file); - let mut hasher = blake3::Hasher::new(); - - while let Some(chunk) = stream.next().await { - let chunk = chunk - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk read: {e}")))?; - hasher.update(&chunk); - writer - .write_all(&chunk) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk write: {e}")))?; - } - writer - .flush() - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk flush: {e}")))?; - - let actual = hasher.finalize().to_hex().to_string(); - if actual != hash { + /// Re-chunk one legacy whole-file blob straight from the backend read + /// stream (no spool file), verifying that the streamed content still + /// matches its recorded BLAKE3 before the chunks are kept. + /// + /// On mismatch the freshly taken chunk references are released — the + /// written chunk bytes become unreferenced rows the GC sweeps — and an + /// error is returned; the legacy blob itself stays untouched. + async fn ingest_legacy_blob(&self, hash: &str) -> Result<(Vec, Vec), DomainError> { + let stream = self.read_blob_stream(hash).await?; + let outcome = self.ingest_chunks_from_stream(stream).await?; + if outcome.file_hash != hash { + let distinct = outcome.distinct_hashes(); + self.release_chunk_refs(self.maintenance_pool.as_ref(), &distinct) + .await; return Err(DomainError::internal_error( "Dedup", - format!("Blob content does not match its hash (expected {hash}, got {actual})"), + format!( + "Blob content does not match its hash (expected {hash}, got {})", + outcome.file_hash + ), )); } - - // Empty blobs can't be mmap'd by the CDC analyser; they become an - // empty manifest (the chunked read path streams zero chunks). - let spooled_len = fs::metadata(spool) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk stat: {e}")))? - .len(); - if spooled_len == 0 { - return Ok((Vec::new(), Vec::new())); - } - - let chunks = Self::cdc_chunk_file(spool) - .await - .map_err(DomainError::from)?; - self.store_chunks(spool, &chunks).await + Ok((outcome.chunk_hashes, outcome.chunk_sizes)) } - /// Best-effort compensation: drop the per-manifest chunk references - /// taken by `store_chunks` when the manifest insert was abandoned. - async fn release_chunk_refs(&self, chunk_hashes: &[String]) { + /// Best-effort compensation: drop one reference per *distinct* chunk + /// hash (clamped at 0). Used whenever an ingest session's references end + /// up not being attached to a manifest — dedup hit, lost insert race, or + /// content-verification failure. + async fn release_chunk_refs(&self, pool: &PgPool, chunk_hashes: &[String]) { if chunk_hashes.is_empty() { return; } @@ -1893,10 +1936,10 @@ impl DedupService { WHERE hash = ANY($1)", ) .bind(chunk_hashes) - .execute(self.maintenance_pool.as_ref()) + .execute(pool) .await { - tracing::warn!("Legacy re-chunk: failed to release chunk refs: {e}"); + tracing::warn!("Dedup: failed to release chunk refs: {e}"); } } } @@ -1915,16 +1958,6 @@ pub struct LegacyRechunkReport { // ─── Port implementation ───────────────────────────────────────────────────── impl DedupPort for DedupService { - async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result { - self.store_from_file(source_path, content_type, pre_computed_hash) - .await - } - async fn blob_exists(&self, hash: &str) -> bool { self.blob_exists(hash).await } @@ -2002,20 +2035,95 @@ mod tests { file } + /// One chunk as seen by the streaming analyser. + struct TestChunk { + hash: String, + offset: usize, + length: usize, + } + + /// Run the exact same streaming chunker the ingest engine uses + /// (`AsyncStreamCDC` + the production CDC parameters) over an in-memory + /// buffer, feeding it in `frame`-sized pieces to exercise the refill + /// logic the same way HTTP body frames do. + /// + /// Returns the whole-stream BLAKE3 plus per-chunk metadata. + async fn stream_cdc(data: &[u8], frame: usize) -> (String, Vec) { + let frames: Vec> = data + .chunks(frame.max(1)) + .map(|c| Ok(Bytes::copy_from_slice(c))) + .collect(); + let reader = StreamReader::new(Box::pin(stream::iter(frames))); + let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( + reader, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ); + let chunk_stream = chunker.as_stream(); + futures::pin_mut!(chunk_stream); + + let mut file_hasher = blake3::Hasher::new(); + let mut chunks = Vec::new(); + while let Some(item) = chunk_stream.next().await { + let chunk = item.expect("in-memory stream cannot fail"); + file_hasher.update(&chunk.data); + chunks.push(TestChunk { + hash: blake3::hash(&chunk.data).to_hex().to_string(), + offset: chunk.offset as usize, + length: chunk.length, + }); + } + (file_hasher.finalize().to_hex().to_string(), chunks) + } + + const TEST_FRAME: usize = 64 * 1024; // typical HTTP body frame size + + // ── Stream chunking ≡ slice chunking ───────────────────────── + // + // The whole dedup index hinges on this invariant: the boundaries (and + // therefore the chunk hashes) produced by the streaming chunker must be + // identical to FastCDC over the full in-memory slice, regardless of how + // the bytes were framed on the wire. Pre-streaming blobs were chunked + // via mmap + slice FastCDC — their chunks must keep deduplicating + // against newly streamed uploads. + + #[tokio::test] + async fn test_stream_chunking_matches_slice_chunking() { + let data: Vec = (0..4 * 1024 * 1024) + .map(|i| ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1)) as u8) + .collect(); + + let slice_chunks: Vec<(usize, usize)> = + fastcdc::v2020::FastCDC::new(&data, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK) + .map(|c| (c.offset, c.length)) + .collect(); + + for frame in [7usize, 4096, TEST_FRAME, data.len()] { + let (_, streamed) = stream_cdc(&data, frame).await; + assert_eq!( + streamed.len(), + slice_chunks.len(), + "chunk count must not depend on framing (frame={frame})" + ); + for (s, (offset, length)) in streamed.iter().zip(slice_chunks.iter()) { + assert_eq!((s.offset, s.length), (*offset, *length), "frame={frame}"); + let expected = blake3::hash(&data[*offset..*offset + *length]) + .to_hex() + .to_string(); + assert_eq!(s.hash, expected, "frame={frame}"); + } + } + } + // ── Determinism ────────────────────────────────────────────── #[tokio::test] async fn test_cdc_deterministic_same_content() { let data = vec![42u8; 512 * 1024]; // 512 KB of 0x2A - let f1 = write_temp_file(&data).await; - let f2 = write_temp_file(&data).await; - let (hash1, chunks1) = DedupService::cdc_hash_and_chunk_file(f1.path()) - .await - .unwrap(); - let (hash2, chunks2) = DedupService::cdc_hash_and_chunk_file(f2.path()) - .await - .unwrap(); + let (hash1, chunks1) = stream_cdc(&data, TEST_FRAME).await; + let (hash2, chunks2) = stream_cdc(&data, 4096).await; assert_eq!(hash1, hash2, "same content must produce same file hash"); assert_eq!( @@ -2030,16 +2138,13 @@ mod tests { } } - // ── Empty file ─────────────────────────────────────────────── + // ── Empty stream ───────────────────────────────────────────── #[tokio::test] - async fn test_cdc_empty_file() { - let f = write_temp_file(b"").await; - let (hash, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + async fn test_cdc_empty_stream() { + let (hash, chunks) = stream_cdc(b"", TEST_FRAME).await; - assert!(chunks.is_empty(), "empty file must produce zero chunks"); + assert!(chunks.is_empty(), "empty stream must produce zero chunks"); assert_eq!(hash, blake3::hash(b"").to_hex().to_string()); } @@ -2048,10 +2153,7 @@ mod tests { #[tokio::test] async fn test_cdc_small_file_single_chunk() { let data = b"Hello, OxiCloud CDC dedup!"; - let f = write_temp_file(data).await; - let (hash, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (hash, chunks) = stream_cdc(data, TEST_FRAME).await; assert_eq!(chunks.len(), 1, "tiny file must be a single chunk"); assert_eq!(chunks[0].offset, 0); @@ -2067,11 +2169,8 @@ mod tests { let data: Vec = (0..4 * 1024 * 1024) .map(|i| ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1)) as u8) .collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; assert!(chunks.len() > 1, "4 MB should produce multiple chunks"); @@ -2104,78 +2203,22 @@ mod tests { let data: Vec = (0..1024 * 1024).map(|i| (i % 251) as u8).collect(); let f = write_temp_file(&data).await; - let (cdc_hash, _) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (cdc_hash, _) = stream_cdc(&data, TEST_FRAME).await; let standalone_hash = DedupService::hash_file(f.path()).await.unwrap(); assert_eq!( cdc_hash, standalone_hash, - "CDC file hash must match standalone hash_file()" + "streamed file hash must match standalone hash_file()" ); } - // ── Chunk hashes are correct BLAKE3 of chunk data ──────────── - - #[tokio::test] - async fn test_cdc_chunk_hashes_are_correct() { - let data: Vec = (0..2 * 1024 * 1024) - .map(|i| ((i as u64).wrapping_mul(2862933555777941757).wrapping_add(3)) as u8) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - - for chunk in &chunks { - let chunk_data = &data[chunk.offset..chunk.offset + chunk.length]; - let expected_hash = blake3::hash(chunk_data).to_hex().to_string(); - assert_eq!( - chunk.hash, expected_hash, - "chunk at offset {} has wrong hash", - chunk.offset - ); - } - } - - // ── Reassembly matches original ────────────────────────────── - - #[tokio::test] - async fn test_cdc_reassembly_matches_original() { - let data: Vec = (0..3 * 1024 * 1024) - .map(|i| ((i as u64).wrapping_mul(1103515245).wrapping_add(12345)) as u8) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - - // Reassemble from chunks - let mut reassembled = Vec::with_capacity(data.len()); - for chunk in &chunks { - reassembled.extend_from_slice(&data[chunk.offset..chunk.offset + chunk.length]); - } - - assert_eq!( - reassembled.len(), - data.len(), - "reassembled length must match" - ); - assert_eq!(reassembled, data, "reassembled content must match original"); - } - - // ── Chunks cover entire file (no gaps, no overlaps) ────────── + // ── Reassembly: chunks are contiguous and cover the file ───── #[tokio::test] async fn test_cdc_chunks_are_contiguous() { let data: Vec = (0..2 * 1024 * 1024).map(|i| (i % 199) as u8).collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; let mut expected_offset = 0usize; for (i, chunk) in chunks.iter().enumerate() { @@ -2205,15 +2248,8 @@ mod tests { *b = b.wrapping_add(1); } - let f_base = write_temp_file(&base).await; - let f_mod = write_temp_file(&modified).await; - - let (hash_base, chunks_base) = DedupService::cdc_hash_and_chunk_file(f_base.path()) - .await - .unwrap(); - let (hash_mod, chunks_mod) = DedupService::cdc_hash_and_chunk_file(f_mod.path()) - .await - .unwrap(); + let (hash_base, chunks_base) = stream_cdc(&base, TEST_FRAME).await; + let (hash_mod, chunks_mod) = stream_cdc(&modified, TEST_FRAME).await; // File hashes must differ assert_ne!( @@ -2241,28 +2277,6 @@ mod tests { ); } - // ── cdc_chunk_file matches cdc_hash_and_chunk_file ─────────── - - #[tokio::test] - async fn test_cdc_chunk_file_matches_full() { - let data: Vec = (0..1024 * 1024) - .map(|i| (i as u8).wrapping_mul(7)) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks_full) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - let chunks_only = DedupService::cdc_chunk_file(f.path()).await.unwrap(); - - assert_eq!(chunks_full.len(), chunks_only.len()); - for (a, b) in chunks_full.iter().zip(chunks_only.iter()) { - assert_eq!(a.hash, b.hash); - assert_eq!(a.offset, b.offset); - assert_eq!(a.length, b.length); - } - } - // ── Large file produces expected chunk count ────────────────── #[tokio::test] @@ -2271,11 +2285,8 @@ mod tests { let data: Vec = (0..8 * 1024 * 1024) .map(|i| ((i as u64).wrapping_mul(2862933555777941757).wrapping_add(3)) as u8) .collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; // With 256KB avg, expect 20-60 chunks for 8MB assert!( @@ -2306,15 +2317,8 @@ mod tests { let mut with_prefix = prefix; with_prefix.extend_from_slice(&base); - let f_base = write_temp_file(&base).await; - let f_prefix = write_temp_file(&with_prefix).await; - - let (_, chunks_base) = DedupService::cdc_hash_and_chunk_file(f_base.path()) - .await - .unwrap(); - let (_, chunks_prefix) = DedupService::cdc_hash_and_chunk_file(f_prefix.path()) - .await - .unwrap(); + let (_, chunks_base) = stream_cdc(&base, TEST_FRAME).await; + let (_, chunks_prefix) = stream_cdc(&with_prefix, TEST_FRAME).await; let base_set: HashSet<&str> = chunks_base.iter().map(|c| c.hash.as_str()).collect(); let prefix_set: HashSet<&str> = chunks_prefix.iter().map(|c| c.hash.as_str()).collect(); @@ -2331,6 +2335,20 @@ mod tests { chunks_prefix.len() ); } + + // ── ChunkIngestOutcome helpers ─────────────────────────────── + + #[test] + fn test_distinct_hashes_deduplicates_preserving_order() { + let outcome = ChunkIngestOutcome { + file_hash: String::new(), + total_size: 0, + chunk_hashes: vec!["a".into(), "b".into(), "a".into(), "c".into(), "b".into()], + chunk_sizes: vec![1, 2, 1, 3, 2], + newly_written: 0, + }; + assert_eq!(outcome.distinct_hashes(), vec!["a", "b", "c"]); + } } // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/infrastructure/services/local_blob_backend.rs b/src/infrastructure/services/local_blob_backend.rs index 632701cb..8ef8cf3b 100644 --- a/src/infrastructure/services/local_blob_backend.rs +++ b/src/infrastructure/services/local_blob_backend.rs @@ -244,12 +244,13 @@ impl BlobStorageBackend for LocalBlobBackend { // Atomic rename (same filesystem). Falls back to copy+delete for // cross-device moves (EXDEV errno 18). // - // Durability boundary: the caller is responsible for - // having sync_all'd the source file before invoking this - // function — both `interfaces::upload_spool::spool_body_to_temp` - // and `chunked_upload_service::complete_upload_inner` - // (the two production producers of `source_path`) now do - // so. We fsync the parent of `blob_path` AFTER the rename + // Durability boundary: the caller is responsible for having + // sync_all'd the source file before invoking this function. + // (The streaming upload path writes chunks via + // `put_blob_from_bytes_unsynced` + a batched `sync_blobs` + // sweep instead; this move-based entry point remains for + // whole-file producers such as migration tooling and tests.) + // We fsync the parent of `blob_path` AFTER the rename // so the dirent change itself becomes durable; without // that, a power loss can resurrect the old (unrenamed) // name even when the file contents survive. diff --git a/src/infrastructure/services/nextcloud_chunked_upload_service.rs b/src/infrastructure/services/nextcloud_chunked_upload_service.rs index 34e6cb52..fff15b5e 100644 --- a/src/infrastructure/services/nextcloud_chunked_upload_service.rs +++ b/src/infrastructure/services/nextcloud_chunked_upload_service.rs @@ -73,7 +73,7 @@ impl NextcloudChunkedUploadService { /// Store a chunk in the session directory. Buffers `data` in memory — /// use [`safe_chunk_path`](Self::safe_chunk_path) + the - /// `interfaces/upload_spool::stream_body_to_path` helper to stream the + /// `interfaces/upload_ingest::stream_body_to_path` helper to stream the /// HTTP body directly to disk and avoid materialising the whole chunk /// in RAM. pub async fn store_chunk( @@ -93,24 +93,14 @@ impl NextcloudChunkedUploadService { Ok(()) } - /// Assemble all chunks in numeric order into a temp file, computing - /// the BLAKE3 of the concatenated stream **during** the same read/ - /// write pass (hash-on-write). + /// List the session's chunk files in assembly (numeric) order. /// - /// Returns `(temp_path, total_size, blake3_hex)`. The caller passes - /// the hash to the upload service as `pre_computed_hash` so the - /// downstream dedup layer never has to re-read the assembled file - /// to compute it — saving one full file-sized read pass per upload. - /// - /// The read/hash/write loop runs inside `spawn_blocking` because - /// BLAKE3 is CPU-bound and would otherwise starve the Tokio worker - /// running other connections; synchronous I/O is used inside the - /// blocking thread because the workload is sequential and the - /// async reactor overhead would only slow it down. For files larger - /// than ~10 MB BLAKE3's Rayon mode parallelises across cores — - /// mirrors what `ChunkedUploadService::complete_upload_inner` does - /// for the REST chunked path. - pub async fn assemble(&self, user: &str, upload_id: &str) -> Result<(PathBuf, u64, String)> { + /// The caller streams these directly into the CDC chunk store + /// (`interfaces::upload_ingest::stream_from_files`) — chunking, BLAKE3 + /// hashing and dedup checks happen in that single read pass, so no + /// assembled temp file is ever written. The chunk parts stay on disk + /// until [`cleanup`](Self::cleanup), keeping completion retryable. + pub async fn ordered_chunk_paths(&self, user: &str, upload_id: &str) -> Result> { let session_dir = self.safe_session_dir(user, upload_id)?; let mut entries: Vec = Vec::new(); @@ -124,8 +114,11 @@ impl NextcloudChunkedUploadService { .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))? { let name = entry.file_name().to_string_lossy().to_string(); - if name == ".file" { - continue; // Skip the assembly marker. + // `.file` is the NC-protocol assembly marker; `.assembled` is + // the staging file older releases wrote — sessions in flight + // across an upgrade may still contain one. + if name == ".file" || name == ".assembled" { + continue; } entries.push(name); } @@ -133,74 +126,7 @@ impl NextcloudChunkedUploadService { // Sort chunks numerically (Nextcloud sends them as "00001", "00002", ...). entries.sort(); - let temp_path = session_dir.join(".assembled"); - let chunk_paths: Vec = entries.iter().map(|n| session_dir.join(n)).collect(); - let assembled_for_blocking = temp_path.clone(); - - // Read/hash/write loop runs synchronously on the blocking pool. - // BLAKE3 is computed in the same pass that copies bytes from chunk - // files into the assembled file — no second read after the fact. - let (total_size, hash) = - tokio::task::spawn_blocking(move || -> std::io::Result<(u64, String)> { - use std::io::{BufWriter as StdBufWriter, Read, Write}; - - let raw_output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&assembled_for_blocking)?; - - // 512 KB write buffer — 8× fewer syscalls than 64 KB. - let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = blake3::Hasher::new(); - let mut buf = vec![0u8; 524_288]; - let mut total: u64 = 0; - - // Files >10 MB benefit from BLAKE3's multi-threaded mode. - // The threshold matches the REST chunked path's heuristic. - const RAYON_THRESHOLD_PER_FRAME: usize = 128 * 1024; - - for chunk_path in &chunk_paths { - let mut chunk_file = std::fs::File::open(chunk_path)?; - loop { - let n = chunk_file.read(&mut buf)?; - if n == 0 { - break; - } - if n >= RAYON_THRESHOLD_PER_FRAME { - hasher.update_rayon(&buf[..n]); - } else { - hasher.update(&buf[..n]); - } - output.write_all(&buf[..n])?; - total += n as u64; - } - } - - output.flush()?; - // ── Durability boundary ───────────────────────────────── - // sync_all is the actual fsync; without it, a power loss - // before the kernel writeback timer (~5 s) loses - // acknowledged data. Pull the inner File out of the - // BufWriter so we can sync the underlying handle — - // dropping the BufWriter wouldn't trigger fsync. macOS - // caveat: fsync there flushes to the disk controller - // only; true durability needs F_FULLFSYNC, not exposed - // by std. - let raw_output = output - .into_inner() - .map_err(|e| std::io::Error::other(format!("into_inner: {e}")))?; - raw_output.sync_all()?; - - Ok((total, hasher.finalize().to_hex().to_string())) - }) - .await - .map_err(|e| { - DomainError::internal_error("ChunkedUpload", format!("assemble task: {e}")) - })? - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - - Ok((temp_path, total_size, hash)) + Ok(entries.iter().map(|n| session_dir.join(n)).collect()) } /// Delete the upload session directory. @@ -260,9 +186,10 @@ impl NextcloudChunkedUploadService { let name = entry.file_name().to_string_lossy().to_string(); // Filter internal markers — `.file` is the NC-protocol // assembly trigger target (it never reaches the disk - // because MOVE redirects it), `.assembled` is our own - // staging file from `assemble()`. Surfacing either to - // the client would confuse its chunk-count check. + // because MOVE redirects it), `.assembled` is the staging + // file pre-streaming releases wrote (kept for sessions in + // flight across an upgrade). Surfacing either to the + // client would confuse its chunk-count check. if name == ".file" || name == ".assembled" { continue; } @@ -331,8 +258,18 @@ mod tests { assert!(!svc.session_exists("alice", "upload-999").await); } + /// Concatenate the session's chunk files in assembly order — mirrors + /// what `upload_ingest::stream_from_files` feeds the CDC store. + async fn concat_chunks(svc: &NextcloudChunkedUploadService, user: &str, id: &str) -> Vec { + let mut out = Vec::new(); + for path in svc.ordered_chunk_paths(user, id).await.unwrap() { + out.extend_from_slice(&fs::read(&path).await.unwrap()); + } + out + } + #[tokio::test] - async fn test_store_and_assemble_chunks() { + async fn test_store_and_order_chunks() { let (svc, _dir) = test_service(); svc.create_session("alice", "upload-002").await.unwrap(); @@ -343,20 +280,14 @@ mod tests { .await .unwrap(); - let (temp_path, size, hash) = svc.assemble("alice", "upload-002").await.unwrap(); - let assembled = fs::read(&temp_path).await.unwrap(); - assert_eq!(assembled, b"Hello, World!"); - assert_eq!(size, 13); - // BLAKE3("Hello, World!") — proves hash-on-write happens during - // the assemble pass, not via a re-read. assert_eq!( - hash, - "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + concat_chunks(&svc, "alice", "upload-002").await, + b"Hello, World!" ); } #[tokio::test] - async fn test_assemble_chunks_in_sorted_order() { + async fn test_chunk_paths_sorted_regardless_of_upload_order() { let (svc, _dir) = test_service(); svc.create_session("alice", "upload-003").await.unwrap(); @@ -371,16 +302,29 @@ mod tests { .await .unwrap(); - let (temp_path, size, hash) = svc.assemble("alice", "upload-003").await.unwrap(); - let assembled = fs::read(&temp_path).await.unwrap(); - assert_eq!(assembled, b"ABC"); - assert_eq!(size, 3); - // BLAKE3("ABC") — confirms sort happened (chunks were stored in - // order 3,1,2 but the hash matches "ABC", not "CAB" or "BAC"). - assert_eq!( - hash, - "d1717274597cf0289694f75d96d444b992a096f1afd8e7bbfa6ebb1d360fedfc" - ); + // Chunks were stored in order 3,1,2 but must concatenate as "ABC". + assert_eq!(concat_chunks(&svc, "alice", "upload-003").await, b"ABC"); + } + + #[tokio::test] + async fn test_internal_markers_excluded_from_chunk_paths() { + let (svc, _dir) = test_service(); + svc.create_session("alice", "upload-005").await.unwrap(); + + svc.store_chunk("alice", "upload-005", "00001", b"data") + .await + .unwrap(); + // Stale staging file from a pre-streaming release. + svc.store_chunk("alice", "upload-005", ".assembled", b"old") + .await + .unwrap(); + + let paths = svc + .ordered_chunk_paths("alice", "upload-005") + .await + .unwrap(); + assert_eq!(paths.len(), 1, "markers must be filtered out"); + assert!(paths[0].ends_with("00001")); } #[tokio::test] diff --git a/src/interfaces/api/handlers/chunked_upload_handler.rs b/src/interfaces/api/handlers/chunked_upload_handler.rs index f2276e50..0998803b 100644 --- a/src/interfaces/api/handlers/chunked_upload_handler.rs +++ b/src/interfaces/api/handlers/chunked_upload_handler.rs @@ -4,7 +4,7 @@ //! - POST /api/uploads → Create upload session //! - PATCH /api/uploads/:id → Upload a chunk //! - HEAD /api/uploads/:id → Get upload status -//! - POST /api/uploads/:id/complete → Assemble and finalize +//! - POST /api/uploads/:id/complete → Stream parts into the blob store //! - DELETE /api/uploads/:id → Cancel upload use axum::{ @@ -27,7 +27,7 @@ use crate::common::di::AppState; use crate::domain::services::authorization::Permission; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; -use crate::interfaces::upload_spool::stream_body_to_path; +use crate::interfaces::upload_ingest::{self, stream_body_to_path}; /// Request body for creating an upload session #[derive(Debug, Deserialize, ToSchema)] @@ -64,35 +64,32 @@ pub struct CompleteUploadResponse { /// Optional body for `POST /api/uploads/{id}/complete`. /// /// When the client supplies `checksum`, the server compares it against -/// the assembled file's hash BEFORE promoting the blob to storage — -/// failure aborts the upload atomically (no orphaned blob, no DB row). -/// This is the end-to-end integrity check: per-chunk MD5 proves each -/// chunk arrived intact, but only the final hash catches assembly / -/// promotion bugs and mis-ordered chunks. +/// the streamed content's hash BEFORE the file row is created — failure +/// releases the blob reference and returns 400, with the chunk parts +/// kept on disk for a retry. This is the end-to-end integrity check: +/// per-chunk MD5 proves each chunk arrived intact, but only the final +/// hash catches mis-ordered or corrupted assemblies. /// -/// **`blake3` is highly recommended** — it's the algorithm the server -/// already runs over the assembled file during hash-on-write -/// assembly, so verification is a string comparison with zero extra -/// I/O and zero extra CPU. It's also the same algorithm the server -/// uses for blob-storage addressing, so the value the client sends -/// equals the `content_hash` they'd later read back from -/// `GET /api/files/{id}`. `md5` and `sha256` are accepted for -/// compatibility with legacy client tooling but each triggers a -/// second hash pass over the assembled file (~30–100 ms depending -/// on size). +/// **`blake3` is highly recommended** — it's the content-addressing +/// algorithm of the blob store itself, so verification is a string +/// comparison against the hash the store already computed, and the +/// value the client sends equals the `content_hash` they'd later read +/// back from `GET /api/files/{id}`. `md5` and `sha256` are accepted +/// for legacy client tooling; they are computed by an in-flight tee +/// during the same streaming pass — no extra disk read either way. /// /// `Default` keeps the existing wire shape: clients that POST with no /// body get today's behavior (no verification, server just returns /// what it computed). #[derive(Debug, Default, Deserialize, ToSchema)] pub struct CompleteUploadRequest { - /// Lowercase hex digest the client expects the assembled file to + /// Lowercase hex digest the client expects the streamed content to /// hash to. Compared case-insensitively. Omit to skip verification. pub checksum: Option, /// Algorithm name. `blake3` is the recommended choice (default — - /// matches the server's hash-on-write algorithm, zero extra cost). - /// `md5`, `sha256` / `sha-256` are accepted but trigger an extra - /// hash pass. Unknown values return 400. + /// matches the blob store's content-addressing algorithm). `md5`, + /// `sha256` / `sha-256` are accepted and computed in-flight. + /// Unknown values return 400. pub checksumalg: Option, } @@ -307,70 +304,15 @@ impl ChunkedUploadHandler { } } - /// Compute the requested checksum of the assembled file. - /// - /// For `Blake3` the server already has the hash from hash-on-write - /// assembly — we just return it (zero I/O, zero CPU). For `Md5` and - /// `Sha256` we re-read the assembled file on the blocking pool and - /// hash it; the cost (~30–100 ms for typical files) is the trade-off - /// for accepting non-default algorithms. - async fn compute_assembled_hash( - assembled_path: &std::path::Path, - alg: ChecksumAlg, - blake3_already_computed: &str, - ) -> Result { - match alg { - ChecksumAlg::Blake3 => Ok(blake3_already_computed.to_string()), - ChecksumAlg::Md5 | ChecksumAlg::Sha256 => { - let path = assembled_path.to_path_buf(); - tokio::task::spawn_blocking(move || -> Result { - use std::io::Read; - let mut file = std::fs::File::open(&path)?; - let mut buf = vec![0u8; 524_288]; - match alg { - ChecksumAlg::Md5 => { - use md5::Digest as _; - let mut h = md5::Md5::new(); - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - h.update(&buf[..n]); - } - Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) - } - ChecksumAlg::Sha256 => { - use sha2::Digest as _; - let mut h = sha2::Sha256::new(); - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - h.update(&buf[..n]); - } - Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) - } - // Blake3 handled above — this branch is unreachable but - // keeps the match exhaustive without an else-clause. - ChecksumAlg::Blake3 => unreachable!(), - } - }) - .await - .map_err(|e| std::io::Error::other(format!("hash task join failed: {e}")))? - } - } - } - /// POST /api/uploads/:upload_id/complete - Finalize upload /// - /// Assembles all chunks into the final file and creates the file record. - /// When `body.checksum` is supplied, the assembled file's hash is - /// verified before the blob is promoted to storage — mismatch - /// returns 400 and the assembled temp is removed (the session - /// itself is kept so the client can re-issue complete after - /// diagnosing). + /// Streams the uploaded chunk parts, in order, straight into the CDC + /// chunk store and creates the file record — no assembled temp file is + /// ever written. When `body.checksum` is supplied it is verified from + /// the same streaming pass (BLAKE3 comes from the store itself; + /// MD5/SHA-256 are computed by an in-flight tee) — mismatch returns 400 + /// with the blob reference released, and the chunk parts stay on disk + /// so the client can re-issue complete after diagnosing. pub(super) async fn complete_upload_impl( State(state): State>, auth_user: AuthUser, @@ -379,10 +321,11 @@ impl ChunkedUploadHandler { ) -> impl IntoResponse { let chunked_service = &state.core.chunked_upload_service; let upload_service = &state.applications.file_upload_service; + let dedup = &state.core.dedup_service; - // ── Parse the optional algorithm BEFORE assembly so a bad - // `checksumalg` doesn't waste the (potentially expensive) - // hash work on a request we'll reject anyway. + // ── Parse the optional algorithm BEFORE completion so a bad + // `checksumalg` doesn't waste any work on a request we'll + // reject anyway. let alg = match body.checksumalg.as_deref() { Some(name) => match ChecksumAlg::parse(name) { Some(a) => Some(a), @@ -397,37 +340,54 @@ impl ChunkedUploadHandler { }; let expected_checksum = body.checksum.as_deref(); - // Assemble chunks (hash-on-write: BLAKE3 computed during assembly) - let (assembled_path, filename, folder_id, content_type, total_size, hash) = - match chunked_service - .complete_upload(&upload_id, auth_user.id) - .await - { - Ok(result) => result, - Err(e) => { - return AppError::from(e).into_response(); - } - }; + // Validate completion and get the chunk parts in assembly order. + let parts = match chunked_service + .complete_upload(&upload_id, auth_user.id) + .await + { + Ok(result) => result, + Err(e) => { + return AppError::from(e).into_response(); + } + }; + + // MD5/SHA-256 verification taps the stream while it is ingested; + // BLAKE3 needs no tee — the store's own content hash IS BLAKE3. + let alg = expected_checksum.map(|_| alg.unwrap_or(ChecksumAlg::Blake3)); + let tee = match alg { + Some(ChecksumAlg::Md5) | Some(ChecksumAlg::Sha256) => { + Some(upload_ingest::checksum_tee(alg.unwrap())) + } + _ => None, + }; + + // ── Stream the parts into the CDC chunk store ─────────────── + let ingested = match upload_ingest::ingest_stream_to_cas( + upload_ingest::stream_from_files(parts.chunk_paths), + dedup, + &parts.filename, + &parts.content_type, + usize::MAX, + tee.clone(), + ) + .await + { + Ok(ingested) => ingested, + Err(e) => return e.into_response(), + }; // ── End-to-end integrity verification ─────────────────────── - // Only fires when the client supplied an `expected` checksum. - // For BLAKE3 (the documented preferred choice) this is a string - // comparison against the hash assembly already produced. For - // MD5/SHA-256 we re-hash the assembled file on the blocking pool. - if let Some(expected) = expected_checksum { - let alg = alg.unwrap_or(ChecksumAlg::Blake3); - let computed = match Self::compute_assembled_hash(&assembled_path, alg, &hash).await { - Ok(c) => c, - Err(e) => { - let _ = tokio::fs::remove_file(&assembled_path).await; - return AppError::internal_error(format!( - "Failed to compute assembled checksum: {e}" - )) - .into_response(); - } + if let (Some(expected), Some(alg)) = (expected_checksum, alg) { + let computed = match alg { + ChecksumAlg::Blake3 => Some(ingested.hash.clone()), + _ => tee.as_ref().and_then(upload_ingest::finalize_checksum_tee), + }; + let Some(computed) = computed else { + upload_ingest::discard_ingested(dedup, &ingested).await; + return AppError::internal_error("Checksum tee produced no digest").into_response(); }; if !computed.eq_ignore_ascii_case(expected) { - let _ = tokio::fs::remove_file(&assembled_path).await; + upload_ingest::discard_ingested(dedup, &ingested).await; tracing::warn!( target: "audit", event = "chunked_upload.checksum_mismatch", @@ -449,36 +409,28 @@ impl ChunkedUploadHandler { } } - // ── MIME detection (magic bytes + extension fallback) ───── - let content_type = crate::common::mime_detect::refine_content_type_from_file( - &assembled_path, - &filename, - &content_type, - ) - .await; - - // Upload from assembled file on disk — zero extra RAM copies, hash pre-computed + // Register the file row against the ingested blob. + let size = ingested.size; match upload_service - .upload_file_from_path( - filename.clone(), - folder_id.clone(), - content_type, - &assembled_path, - Some(hash), + .upload_file_streaming( + parts.filename.clone(), + parts.folder_id.clone(), + ingested.content_type.clone(), + ingested.stored(), ) .await { Ok(file) => { - // Cleanup session + // Cleanup session (removes the chunk part files) let _ = chunked_service .finalize_upload(&upload_id, auth_user.id) .await; tracing::info!( "✅ CHUNKED UPLOAD COMPLETE: {} (ID: {}, {} bytes)", - filename, + parts.filename, file.id, - total_size + size ); ( @@ -486,14 +438,14 @@ impl ChunkedUploadHandler { Json(CompleteUploadResponse { file_id: file.id, filename: file.name, - size: total_size, + size, path: file.path, }), ) .into_response() } Err(e) => { - tracing::error!("Failed to create file from assembled upload: {:?}", e); + tracing::error!("Failed to create file from chunked upload: {:?}", e); AppError::internal_error(format!("Failed to create file: {}", e)).into_response() } } diff --git a/src/interfaces/api/handlers/dedup_handler.rs b/src/interfaces/api/handlers/dedup_handler.rs index 3defb869..3e2f29db 100644 --- a/src/interfaces/api/handlers/dedup_handler.rs +++ b/src/interfaces/api/handlers/dedup_handler.rs @@ -5,12 +5,11 @@ use axum::{ response::IntoResponse, }; use serde::Serialize; -use tokio::io::AsyncWriteExt; use utoipa::ToSchema; -use crate::application::ports::dedup_ports::DedupResultDto; use crate::common::di::AppState; use crate::interfaces::middleware::auth::AuthUser; +use crate::interfaces::upload_ingest; use std::sync::Arc; /// Global application state for dependency injection @@ -155,10 +154,10 @@ impl DedupHandler { /// Upload content with automatic deduplication (streaming). /// - /// Spools the upload to a temp file while computing the BLAKE3 hash - /// incrementally (hash-on-write). Memory usage is constant (~512 KB) - /// regardless of file size. Then delegates to `store_from_file` with - /// the pre-computed hash so the file is never re-read for hashing. + /// Streams the multipart field straight into the CDC chunk store — + /// chunking, BLAKE3 hashing and dedup checks happen while the bytes + /// arrive (no temp file, no re-read; peak RAM is bounded regardless + /// of file size). /// /// POST /api/dedup/upload pub(super) async fn upload_with_dedup_impl( @@ -177,64 +176,29 @@ impl DedupHandler { .content_type() .unwrap_or("application/octet-stream") .to_string(); + let filename = field.file_name().unwrap_or("unnamed").to_string(); - // ── Spool to temp file + BLAKE3 hash-on-write ──────── - let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp"); - let temp_path = temp_dir.join(format!("dedup-{}", uuid::Uuid::new_v4())); - - let mut total_size: u64 = 0; - let mut hasher = blake3::Hasher::new(); - let mut field = field; - - let spool_result: Result<(), String> = async { - let file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| format!("Failed to create temp file: {}", e))?; - - // 512 KB buffer — reduces write syscalls - let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); - - loop { - match field.chunk().await { - Ok(Some(chunk)) => { - total_size += chunk.len() as u64; - hasher.update(&chunk); - writer - .write_all(&chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; - } - Ok(None) => break, - Err(e) => { - return Err(format!( - "Connection lost during upload (received {} bytes): {}", - total_size, e - )); - } - } + // ── Stream into the CDC chunk store ────────────────── + let source = upload_ingest::multipart_field_stream(field); + let ingested = match upload_ingest::ingest_stream_to_cas( + source, + dedup, + &filename, + &content_type, + usize::MAX, + None, + ) + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::warn!("Dedup upload ingest failed: {}", e.message); + return e.into_response(); } + }; - writer - .flush() - .await - .map_err(|e| format!("Failed to flush temp file: {}", e))?; - Ok(()) - } - .await; - - if let Err(msg) = spool_result { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::warn!("Dedup upload spool failed: {}", msg); - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(format!(r#"{{"error": "{}"}}"#, msg))) - .unwrap() - .into_response(); - } - - if total_size == 0 { - let _ = tokio::fs::remove_file(&temp_path).await; + if ingested.size == 0 { + upload_ingest::discard_ingested(dedup, &ingested).await; return Response::builder() .status(StatusCode::BAD_REQUEST) .header(header::CONTENT_TYPE, "application/json") @@ -243,60 +207,33 @@ impl DedupHandler { .into_response(); } - let hash = hasher.finalize().to_hex().to_string(); + let metadata = dedup.get_blob_metadata(&ingested.hash).await; - // ── Store with deduplication (pre-computed hash) ────── - match dedup - .store_from_file(&temp_path, Some(content_type), Some(hash)) - .await - { - Ok(result) => { - let (is_new, bytes_saved) = match &result { - DedupResultDto::NewBlob { .. } => (true, 0), - DedupResultDto::ExistingBlob { saved_bytes, .. } => { - (false, *saved_bytes) - } - }; + let response = DedupUploadResponse { + is_new: ingested.is_new_blob, + hash: ingested.hash.clone(), + size: ingested.size, + bytes_saved: ingested.bytes_saved, + ref_count: metadata.map(|m| m.ref_count).unwrap_or(1), + }; - let metadata = dedup.get_blob_metadata(result.hash()).await; + tracing::info!( + "🔗 Dedup upload: hash={}, new={}, saved={}", + ingested.hash, + ingested.is_new_blob, + ingested.bytes_saved + ); - let response = DedupUploadResponse { - is_new, - hash: result.hash().to_string(), - size: result.size(), - bytes_saved, - ref_count: metadata.map(|m| m.ref_count).unwrap_or(1), - }; - - tracing::info!( - "🔗 Dedup upload: hash={}, new={}, saved={}", - result.hash(), - is_new, - bytes_saved - ); - - return Response::builder() - .status(if is_new { - StatusCode::CREATED - } else { - StatusCode::OK - }) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(serde_json::to_string(&response).unwrap())) - .unwrap() - .into_response(); - } - Err(e) => { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("Dedup upload failed: {}", e); - return Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(r#"{"error": "Upload failed"}"#)) - .unwrap() - .into_response(); - } - } + return Response::builder() + .status(if ingested.is_new_blob { + StatusCode::CREATED + } else { + StatusCode::OK + }) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&response).unwrap())) + .unwrap() + .into_response(); } } diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index d80cd494..903fd87e 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -21,6 +21,7 @@ use crate::common::di::AppState; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; use crate::interfaces::range_requests::not_modified_response; +use crate::interfaces::upload_ingest; use crate::{application::dtos::file_dto::FileDto, domain::services::authorization::Permission}; use std::sync::Arc; @@ -51,11 +52,12 @@ impl FileHandler { // UPLOAD // ═══════════════════════════════════════════════════════════════════════ - /// Streaming file upload — constant ~64 KB RAM regardless of file size. + /// Streaming file upload — bounded RAM regardless of file size. /// - /// **Hash-on-Write**: BLAKE3 is computed while spooling the multipart - /// body to the temp file. This eliminates the second sequential read - /// that dedup_service would otherwise need, cutting total I/O in half. + /// The multipart body is streamed straight into the CDC chunk store: + /// chunking, hashing and dedup checks happen while the bytes arrive. + /// No spool file, no re-read — chunks the store already has are never + /// written to disk at all. pub async fn upload_file( State(state): State, auth_user: AuthUser, @@ -71,7 +73,7 @@ impl FileHandler { /// [`Self::upload_file_with_thumbnails`]. /// /// Returns `(FileDto, blob_hash)` on success. The blob hash is the - /// BLAKE3 digest computed during the hash-on-write spool and is + /// BLAKE3 digest computed during the streaming ingest and is /// propagated without an extra database round-trip so that callers /// (e.g. thumbnail generation) can resolve the physical blob path /// immediately. @@ -158,120 +160,55 @@ impl FileHandler { } } - // ── Spool multipart field to temp file + hash-on-write ── - // .dedup_temp is created once by DedupService::initialize() at startup - let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp"); - let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4())); - - let mut total_size: u64 = 0; - let mut hasher = blake3::Hasher::new(); - let spool_result: Result<(), String> = async { - let file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| format!("Failed to create temp file: {}", e))?; - - // Pre-allocate if Content-Length is known (reduces fragmentation) - let hint = field - .headers() - .get(axum::http::header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - if let Some(len) = hint { - let _ = file.set_len(len).await; // best-effort - } - - // 512 KB buffer — 8× fewer write syscalls than 64 KB - let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); - let mut field = field; - // IMPORTANT: use explicit match instead of `while let Ok(Some(..))`. - // The old pattern silently swallowed Err (client disconnect) - // and accepted partially received data as a complete upload. - loop { - match field.chunk().await { - Ok(Some(chunk)) => { - total_size += chunk.len() as u64; - hasher.update(&chunk); - tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; - } - Ok(None) => break, // End of field — upload complete - Err(e) => { - return Err(format!( - "Connection lost during upload (received {} bytes): {}", - total_size, e - )); - } - } - } - tokio::io::AsyncWriteExt::flush(&mut writer) - .await - .map_err(|e| format!("Failed to flush temp file: {}", e))?; - Ok(()) - } - .await; - - if let Err(e) = spool_result { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("❌ UPLOAD SPOOL FAILED: {} - {}", filename, e); - return Err(Self::domain_error_response( - crate::common::errors::DomainError::internal_error("FileUpload", e), - )); - } - - // Empty file — use streaming path with the (empty) temp file - if total_size == 0 { - let hash = hasher.finalize().to_hex().to_string(); - let dto = upload_service - .upload_file_streaming( - filename, - folder_id, - content_type, - &temp_path, - 0, - Some(hash.clone()), - ) - .await - .map_err(Self::domain_error_response)?; - return Ok((dto, hash)); - } - - // Finalize hash - let hash = hasher.finalize().to_hex().to_string(); - - // ── MIME detection (magic bytes + extension fallback) ─ - let content_type = crate::common::mime_detect::refine_content_type_from_file( - &temp_path, + // ── Stream the field into the CDC chunk store ──────── + // Chunking (FastCDC) + hashing (BLAKE3) + dedup checks + + // MIME sniffing all happen while the bytes arrive; chunks + // the store already has never touch the disk. Size is + // capped globally by DefaultBodyLimit. + let dedup = &state.core.dedup_service; + let source = upload_ingest::multipart_field_stream(field); + let ingested = match upload_ingest::ingest_stream_to_cas( + source, + dedup, &filename, &content_type, + usize::MAX, + None, ) - .await; + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::error!("❌ UPLOAD INGEST FAILED: {} - {}", filename, e.message); + return Err(e.into_response()); + } + }; - // ── Quota enforcement ──────────────────────────────── + // ── Quota enforcement (exact size now known) ───────── if let Some(storage_svc) = state.storage_usage_service.as_ref() && let Err(err) = storage_svc - .check_storage_quota(auth_user.id, total_size) + .check_storage_quota(auth_user.id, ingested.size) .await { - let _ = tokio::fs::remove_file(&temp_path).await; + upload_ingest::discard_ingested(dedup, &ingested).await; tracing::warn!( "⛔ UPLOAD REJECTED (quota): user={}, file={}, size={}", auth_user.username, filename, - total_size + ingested.size ); return Err(Self::quota_error_response(err)); } - // ── Streaming upload (temp file → blob store, hash pre-computed) ─ + // ── Register the file row against the ingested blob ── + let hash = ingested.hash.clone(); + let size = ingested.size; match upload_service .upload_file_streaming( filename.clone(), folder_id, - content_type, - &temp_path, - total_size, - Some(hash.clone()), + ingested.content_type.clone(), + ingested.stored(), ) .await { @@ -279,13 +216,12 @@ impl FileHandler { tracing::info!( "✅ STREAMING UPLOAD: {} ({} bytes, ID: {})", filename, - total_size, + size, file.id ); return Ok((file, hash)); } Err(err) => { - let _ = tokio::fs::remove_file(&temp_path).await; tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err); return Err(Self::domain_error_response(err)); } diff --git a/src/interfaces/api/handlers/webdav_handler.rs b/src/interfaces/api/handlers/webdav_handler.rs index c0097eb3..4b8c2117 100644 --- a/src/interfaces/api/handlers/webdav_handler.rs +++ b/src/interfaces/api/handlers/webdav_handler.rs @@ -898,10 +898,11 @@ async fn handle_head( /** * Handles PUT requests to create or update files. * - * **Streaming implementation**: the request body is spooled to a temp file - * with incremental BLAKE3 hashing. Peak RAM usage is ~256 KB regardless - * of file size. The temp file is then atomically moved into blob storage - * via `update_file_streaming`. + * **Streaming implementation**: the request body is streamed straight into + * the CDC chunk store (FastCDC + BLAKE3 while the bytes arrive — no spool + * file, no re-read; peak RAM is bounded regardless of file size), then the + * file row is atomically swapped onto the ingested blob via + * `update_file_streaming`. * * @param state The application state containing service dependencies * @param path The requested resource path @@ -913,7 +914,7 @@ async fn handle_put( req: Request, path: String, ) -> Result, AppError> { - use crate::interfaces::upload_spool::spool_body_to_temp; + use crate::interfaces::upload_ingest; let user = extract_user(&req)?; @@ -972,32 +973,31 @@ async fn handle_put( .unwrap_or("application/octet-stream") .to_string(); - // ── Streaming spool: body → temp file + incremental hash ── - // Shared with the NextCloud-compat PUT handler; peak heap ~one frame - // regardless of file size. Honors `upload_temp_dir` to keep the spool - // off tmpfs/RAM. - let spooled = spool_body_to_temp( + // ── Streaming ingest: body → CDC chunk store ────────────── + // Shared with the NextCloud-compat PUT handler; chunking + hashing + + // dedup checks run while the body arrives — no spool file, no re-read. + let filename = crate::common::mime_detect::filename_from_path(&path).to_string(); + let ingested = upload_ingest::ingest_body_to_cas( req.into_body(), + &state.core.dedup_service, + &filename, + &content_type, max_upload, - state.core.config.storage.upload_temp_dir.clone(), ) .await?; - let temp_path = spooled.temp.path().to_path_buf(); - let total_bytes = spooled.size as usize; - let hash = spooled.hash; // ── Quota enforcement ──────────────────────────────────── if let Some(storage_svc) = state.storage_usage_service.as_ref() && let Err(err) = storage_svc - .check_storage_quota(user.id, total_bytes as u64) + .check_storage_quota(user.id, ingested.size) .await { - let _ = tokio::fs::remove_file(&temp_path).await; + upload_ingest::discard_ingested(&state.core.dedup_service, &ingested).await; tracing::warn!( "⛔ WEBDAV PUT REJECTED (quota): user={}, file={}, size={}", user.id, path, - total_bytes + ingested.size ); return Err(AppError::new( StatusCode::INSUFFICIENT_STORAGE, @@ -1006,21 +1006,12 @@ async fn handle_put( )); } - // ── Atomic store: temp file → dedup blob + DB metadata update ── + // ── Atomic store: swap the file row onto the ingested blob ── + let content_type = ingested.content_type.clone(); let result = file_upload_service - .update_file_streaming( - &path, - &temp_path, - total_bytes as u64, - &content_type, - Some(hash), - None, - ) + .update_file_streaming(&path, ingested.stored(), &content_type, None) .await; - // Clean up temp file (may already be moved by dedup, ignore error) - let _ = tokio::fs::remove_file(&temp_path).await; - match result { Ok(_file_dto) => Ok(Response::builder() .status(StatusCode::NO_CONTENT) diff --git a/src/interfaces/api/handlers/wopi_handler.rs b/src/interfaces/api/handlers/wopi_handler.rs index 80ed7b81..93e3320b 100644 --- a/src/interfaces/api/handlers/wopi_handler.rs +++ b/src/interfaces/api/handlers/wopi_handler.rs @@ -165,10 +165,6 @@ async fn put_file( State(state): State, req: Request, ) -> Response { - use http_body_util::BodyStream; - use tokio::io::AsyncWriteExt; - use tokio_stream::StreamExt; - let claims = match state .token_service .validate_token(&token_query.access_token) @@ -218,75 +214,32 @@ async fn put_file( Err(_) => return StatusCode::NOT_FOUND.into_response(), }; - // ── Streaming spool: body → temp file + incremental BLAKE3 ── - let temp_file = match tempfile::NamedTempFile::new() { - Ok(f) => f, - Err(e) => { - tracing::error!("WOPI PutFile: failed to create temp file: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - let temp_path = temp_file.path().to_path_buf(); - - let mut file_out = match tokio::fs::File::create(&temp_path).await { - Ok(f) => f, - Err(e) => { - tracing::error!("WOPI PutFile: failed to open temp file: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - + // ── Streaming ingest: body → CDC chunk store (no temp file) ── let content_type = file.mime_type.clone(); - let mut hasher = blake3::Hasher::new(); - let mut total_bytes: u64 = 0; - let mut stream = BodyStream::new(req.into_body()); - - while let Some(frame_result) = stream.next().await { - let frame = match frame_result { - Ok(f) => f, - Err(e) => { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: body read error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len() as u64; - hasher.update(chunk); - if let Err(e) = file_out.write_all(chunk).await { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: temp write error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } + let ingested = match crate::interfaces::upload_ingest::ingest_body_to_cas( + req.into_body(), + &state.app_state.core.dedup_service, + &file.name, + &content_type, + usize::MAX, + ) + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::error!("WOPI PutFile: ingest failed: {}", e.message); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - } - if let Err(e) = file_out.flush().await { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: flush error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - drop(file_out); + }; - let hash = hasher.finalize().to_hex().to_string(); - - // ── Atomic store: temp file → dedup blob + DB metadata update ── + // ── Atomic store: swap the file row onto the ingested blob ── let result = state .app_state .applications .file_upload_service - .update_file_streaming( - &file.path, - &temp_path, - total_bytes, - &content_type, - Some(hash), - None, - ) + .update_file_streaming(&file.path, ingested.stored(), &content_type, None) .await; - // Clean up temp file (may already be moved by dedup, ignore error) - let _ = tokio::fs::remove_file(&temp_path).await; - match result { Ok(_file_dto) => StatusCode::OK.into_response(), Err(e) => { diff --git a/src/interfaces/mod.rs b/src/interfaces/mod.rs index fef37883..f9c9f41f 100644 --- a/src/interfaces/mod.rs +++ b/src/interfaces/mod.rs @@ -3,7 +3,7 @@ pub mod errors; pub mod middleware; pub mod nextcloud; pub mod range_requests; -pub mod upload_spool; +pub mod upload_ingest; pub mod web; pub use api::create_api_routes; diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index 8bb6ca8f..16dd5b07 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -7,10 +7,12 @@ use std::sync::Arc; use crate::application::ports::file_ports::{FileRetrievalUseCase, FileUploadUseCase}; use crate::common::di::AppState; -use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; +use crate::common::mime_detect::filename_from_path; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; -use crate::interfaces::upload_spool::stream_body_to_path; +use crate::interfaces::upload_ingest::{ + discard_ingested, ingest_stream_to_cas, stream_body_to_path, stream_from_files, +}; /// Dispatch Nextcloud chunked upload WebDAV requests. /// @@ -239,18 +241,17 @@ async fn handle_assemble( let dest_subpath = extract_files_subpath(&destination, &user.username) .ok_or_else(|| AppError::bad_request("Invalid Destination URL"))?; - // Assemble chunks into a temp file with hash-on-write (BLAKE3 computed - // during the same read/write loop that copies chunks into the - // assembled file). The hash is passed downstream as `pre_computed_hash` - // so the dedup layer never re-reads the assembled file just to compute - // it — saves one full file-sized read pass per upload. - let (temp_path, size, blake3_hash) = nc + // Stream the chunk parts, in order, straight into the CDC chunk store — + // no assembled temp file is ever written. Chunking (FastCDC), BLAKE3 + // hashing, dedup checks and MIME sniffing (magic bytes off the first + // part) all happen in that single read pass. The parts stay on disk + // until the session cleanup below, so a failed completion is retryable. + let chunk_paths = nc .chunked_uploads - .assemble(&user.username, upload_id) + .ordered_chunk_paths(&user.username, upload_id) .await - .map_err(|e| AppError::internal_error(format!("Failed to assemble chunks: {}", e)))?; + .map_err(|e| AppError::internal_error(format!("Failed to list chunks: {}", e)))?; - // Write assembled file to storage via the upload service. let upload_service = &state.applications.file_upload_service; let file_service = &state.applications.file_retrieval_service; let folder_service = &state.applications.folder_service; @@ -261,35 +262,31 @@ async fn handle_assemble( dest_subpath.trim_matches('/') ); - // Detect content type via magic bytes + extension fallback. - let filename = filename_from_path(&dest_subpath); - let content_type = - refine_content_type_from_file(&temp_path, filename, "application/octet-stream").await; + let filename = filename_from_path(&dest_subpath).to_string(); + let ingested = ingest_stream_to_cas( + stream_from_files(chunk_paths), + &state.core.dedup_service, + &filename, + "application/octet-stream", + usize::MAX, + None, + ) + .await?; + let content_type = ingested.content_type.clone(); // Check if file exists (update vs create). let existing = file_service.get_file_by_path(&internal_path).await; let etag: Option = if existing.is_ok() { let dto = upload_service - .update_file_streaming( - &internal_path, - &temp_path, - size, - &content_type, - Some(blake3_hash.clone()), - oc_mtime, - ) + .update_file_streaming(&internal_path, ingested.stored(), &content_type, oc_mtime) .await .map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?; Some(dto.etag) } else { - // New-file branch: resolve the parent folder by path and pass the - // assembled file's path directly to `upload_file_from_path` so the - // bytes never get read back into RAM. Previously this branch did - // `tokio::fs::read(&temp_path)` — an extra full file-sized read - // pass AND a peak-RAM allocation equal to the upload size, which - // defeated the streaming model on large NC uploads. + // New-file branch: resolve the parent folder by path and register + // the file row against the already-ingested blob. let (parent_sub, filename) = match dest_subpath.rsplit_once('/') { Some((p, n)) => (p, n), None => ("", dest_subpath.as_str()), @@ -302,18 +299,23 @@ async fn handle_assemble( let parent_internal = parent_internal.trim_end_matches('/'); use crate::application::ports::folder_ports::FolderUseCase; - let parent_folder = folder_service - .get_folder_by_path(parent_internal) - .await - .map_err(|e| AppError::internal_error(format!("Parent folder lookup failed: {}", e)))?; + let parent_folder = match folder_service.get_folder_by_path(parent_internal).await { + Ok(folder) => folder, + Err(e) => { + discard_ingested(&state.core.dedup_service, &ingested).await; + return Err(AppError::internal_error(format!( + "Parent folder lookup failed: {}", + e + ))); + } + }; let dto = upload_service - .upload_file_from_path( + .upload_file_streaming( filename.to_string(), Some(parent_folder.id), content_type.to_string(), - &temp_path, - Some(blake3_hash), + ingested.stored(), ) .await .map_err(|e| AppError::internal_error(format!("Failed to create file: {}", e)))?; @@ -321,9 +323,6 @@ async fn handle_assemble( Some(dto.etag) }; - // Clean up temp file (session cleanup below removes the directory anyway). - let _ = tokio::fs::remove_file(&temp_path).await; - // Cleanup session. let _ = nc.chunked_uploads.cleanup(&user.username, upload_id).await; diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 8999fe04..e74aff48 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -22,12 +22,12 @@ use crate::application::ports::file_ports::{ use crate::application::ports::folder_ports::FolderUseCase; use crate::application::ports::trash_ports::TrashUseCase; use crate::common::di::AppState; -use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; +use crate::common::mime_detect::filename_from_path; use crate::interfaces::api::handlers::webdav_handler::PROPFIND_BATCH_SIZE; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; use crate::interfaces::range_requests::{not_modified_response, range_response}; -use crate::interfaces::upload_spool::spool_body_to_temp; +use crate::interfaces::upload_ingest::ingest_body_to_cas; /// Extension trait to map XML write errors to `String` concisely. trait XmlResultExt { @@ -575,46 +575,35 @@ async fn handle_put( // at 95 % loses everything. let max_upload = state.core.config.storage.direct_put_max_bytes; - // Stream the body to a temp file + incremental hash — never buffer the - // full upload in RAM. The old `body::to_bytes` path loaded the entire - // file (e.g. an 800 MB ISO) into anonymous memory before any dedup logic, - // OOMKilling the process even on dedup hits. Shared with the native - // WebDAV PUT handler; peak heap ~one frame regardless of file size. - let spooled = spool_body_to_temp( - req.into_body(), - max_upload, - state.core.config.storage.upload_temp_dir.clone(), - ) - .await?; - - // Detect real MIME type from the first bytes on disk (no full read). + // Stream the body straight into the CDC chunk store — never buffer the + // full upload in RAM and never spool it to disk. Chunking, hashing, + // dedup checks and MIME sniffing (magic bytes off the first frames) + // all run while the body arrives; chunks the store already has are + // never written at all. Shared with the native WebDAV PUT handler. + // // `filename` is owned so we don't hold a borrow of the `subpath` param // across the await (which would make the handler future non-Send). let filename = filename_from_path(subpath).to_string(); - let content_type = - refine_content_type_from_file(spooled.temp.path(), &filename, &claimed_type).await; + let ingested = ingest_body_to_cas( + req.into_body(), + &state.core.dedup_service, + &filename, + &claimed_type, + max_upload, + ) + .await?; + let content_type = ingested.content_type.clone(); // Distinguish create (201) vs update (204) for the response status. let existed = file_service.get_file_by_path(&internal_path).await.is_ok(); // Single streaming path — handles both update and create internally, - // passing the precomputed hash so the dedup fast path can short-circuit - // without re-reading the file. + // swapping the file row onto the already-ingested blob. let stored = upload_service - .update_file_streaming( - &internal_path, - spooled.temp.path(), - spooled.size, - &content_type, - Some(spooled.hash), - oc_mtime, - ) + .update_file_streaming(&internal_path, ingested.stored(), &content_type, oc_mtime) .await .map_err(|e| AppError::internal_error(format!("Failed to store file: {}", e)))?; - // dedup may have already moved the temp on a new-blob store; ignore error. - let _ = tokio::fs::remove_file(spooled.temp.path()).await; - let status = if existed { StatusCode::NO_CONTENT } else { diff --git a/src/interfaces/upload_ingest.rs b/src/interfaces/upload_ingest.rs new file mode 100644 index 00000000..c1ecf8a8 --- /dev/null +++ b/src/interfaces/upload_ingest.rs @@ -0,0 +1,588 @@ +//! Shared streaming upload ingestion: request body → CDC chunk store. +//! +//! Used by every upload surface (REST multipart, native WebDAV PUT, +//! NextCloud PUT, chunked-upload assembly, WOPI PutFile) so none of them +//! buffers the full body in RAM **or spools it to a temp file**. The bytes +//! flow straight into [`DedupService::store_from_stream`], which chunks +//! (FastCDC), hashes (BLAKE3) and dedup-checks them while they arrive — +//! each uploaded byte touches the disk at most once, and not at all when +//! the store already has its chunk. +//! +//! MIME refinement happens in-flight: when the claimed Content-Type is +//! generic, the first bytes are peeked off the stream for magic-byte +//! detection before being forwarded unchanged. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use axum::body::Body; +use bytes::Bytes; +use futures::stream::{self, Stream, StreamExt, TryStreamExt}; +use http_body_util::BodyStream; +// The `Digest` trait (re-exported by both `md5` and `sha2` from the +// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / +// `finalize` methods. Importing once via `sha2` covers both — +// otherwise every call site would need fully-qualified +// `::…` syntax. +use sha2::Digest as _; +use tokio::io::AsyncWriteExt; +use tokio_util::io::ReaderStream; + +use crate::application::ports::chunked_upload_ports::ChecksumAlg; +use crate::application::ports::file_ports::StoredBlob; +use crate::common::mime_detect::{MAGIC_BYTES_LEN, is_generic_mime, refine_content_type}; +use crate::infrastructure::services::dedup_service::DedupService; +use crate::interfaces::errors::AppError; + +/// Content stored in the chunk store by one upload ingest. +/// +/// The ingest holds ONE blob reference; pass [`IngestedBlob::stored`] to the +/// upload service (which takes ownership of the reference) or hand it back +/// via [`discard_ingested`] when the upload is rejected after the fact. +pub struct IngestedBlob { + /// BLAKE3 of the full content (the blob/manifest key). + pub hash: String, + /// Total bytes ingested. + pub size: u64, + /// Refined content type (claimed type or magic-byte detection). + pub content_type: String, + /// `false` when the exact content already existed (dedup hit). + pub is_new_blob: bool, + /// Bytes that did not need to be transferred to storage (dedup hit). + pub bytes_saved: u64, +} + +impl IngestedBlob { + /// The blob reference to hand to the upload service. + pub fn stored(&self) -> StoredBlob { + StoredBlob { + hash: self.hash.clone(), + size: self.size, + is_new_blob: self.is_new_blob, + } + } +} + +/// Hand back the blob reference taken by a successful ingest when the upload +/// is rejected after the fact (quota exceeded, checksum mismatch, …). +pub async fn discard_ingested(dedup: &DedupService, blob: &IngestedBlob) { + if let Err(e) = dedup.remove_reference(&blob.hash).await { + tracing::warn!( + "Failed to release blob reference of rejected upload {}: {e}", + &blob.hash[..blob.hash.len().min(12)] + ); + } +} + +/// Shared mutable tee for computing a client-requested checksum during the +/// ingest pass (REST chunked uploads) — no post-store re-read needed. +pub type ChecksumTee = Arc>>; + +/// Create a checksum tee for [`ingest_stream_to_cas`]. +pub fn checksum_tee(alg: ChecksumAlg) -> ChecksumTee { + Arc::new(StdMutex::new(Some(IncrementalHasher::new(alg)))) +} + +/// Finalize a checksum tee into its lowercase hex digest. +pub fn finalize_checksum_tee(tee: &ChecksumTee) -> Option { + tee.lock() + .ok() + .and_then(|mut h| h.take()) + .map(IncrementalHasher::finalize_hex) +} + +/// Out-of-band state observed by the stream adapters while the dedup engine +/// consumes the stream — lets the caller map an opaque engine error back to +/// the precise HTTP failure (413 vs 400). +struct IngestFlags { + too_large: AtomicBool, + source_error: StdMutex>, +} + +/// Stream a request body (or any byte stream) into the CDC chunk store. +/// +/// Single pass: size-cap enforcement, optional checksum tee, MIME sniffing +/// (first [`MAGIC_BYTES_LEN`] bytes, only when `claimed_type` is generic) +/// and the CDC chunk/hash/store pipeline all run while the bytes arrive. +/// Peak heap is bounded by the dedup engine (~9 MiB) regardless of size. +/// +/// On error nothing stays referenced — the engine compensates internally. +pub async fn ingest_stream_to_cas( + source: S, + dedup: &Arc, + filename: &str, + claimed_type: &str, + max_bytes: usize, + checksum: Option, +) -> Result +where + S: Stream> + Send, + E: std::fmt::Display, +{ + let flags = Arc::new(IngestFlags { + too_large: AtomicBool::new(false), + source_error: StdMutex::new(None), + }); + + // ── Adapter: cap + checksum tee + error capture ────────────── + let adapter_flags = flags.clone(); + let mut total: usize = 0; + let counted = source.map(move |item| match item { + Ok(bytes) => { + total += bytes.len(); + if total > max_bytes { + adapter_flags.too_large.store(true, Ordering::Relaxed); + return Err(std::io::Error::other("upload exceeds size cap")); + } + if let Some(tee) = &checksum + && let Ok(mut hasher) = tee.lock() + && let Some(hasher) = hasher.as_mut() + { + hasher.update(&bytes); + } + Ok(bytes) + } + Err(e) => { + let message = e.to_string(); + if let Ok(mut slot) = adapter_flags.source_error.lock() { + *slot = Some(message.clone()); + } + Err(std::io::Error::other(message)) + } + }); + // `fuse` is load-bearing: when the source is shorter than the MIME peek + // (< MAGIC_BYTES_LEN), the peek loop drains it to None and the `chain` + // below polls it once more — non-fused sources (e.g. `stream::unfold`, + // as used for multipart fields) panic on a post-None poll. + let mut counted = Box::pin(counted.fuse()); + + // ── In-flight MIME sniff (only when the claimed type is generic) ── + let mut head: Vec> = Vec::new(); + let content_type = if is_generic_mime(claimed_type) { + let mut head_len = 0usize; + while head_len < MAGIC_BYTES_LEN { + match counted.next().await { + Some(Ok(bytes)) => { + head_len += bytes.len(); + head.push(Ok(bytes)); + } + Some(Err(e)) => { + head.push(Err(e)); + break; + } + None => break, + } + } + let mut magic = Vec::with_capacity(head_len.min(MAGIC_BYTES_LEN)); + for item in head.iter().flatten() { + let take = (MAGIC_BYTES_LEN - magic.len()).min(item.len()); + magic.extend_from_slice(&item[..take]); + if magic.len() >= MAGIC_BYTES_LEN { + break; + } + } + refine_content_type(&magic, filename, claimed_type) + } else { + claimed_type.to_string() + }; + + // ── Store: peeked head + remainder, one continuous stream ──── + let full_stream = stream::iter(head).chain(counted); + let result = dedup + .store_from_stream(full_stream, Some(content_type.clone())) + .await; + + match result { + Ok(stored) => { + let is_new_blob = !stored.was_deduplicated(); + let bytes_saved = match &stored { + crate::application::ports::dedup_ports::DedupResultDto::ExistingBlob { + saved_bytes, + .. + } => *saved_bytes, + _ => 0, + }; + Ok(IngestedBlob { + hash: stored.hash().to_string(), + size: stored.size(), + content_type, + is_new_blob, + bytes_saved, + }) + } + Err(e) => { + if flags.too_large.load(Ordering::Relaxed) { + return Err(AppError::payload_too_large(format!( + "Upload body exceeds the direct-PUT cap ({max_bytes} bytes). \ + Use the chunked-upload protocol (REST: `/api/uploads/...`, \ + NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ + Chunked uploads are resumable on transient failure." + ))); + } + let source_error = flags.source_error.lock().ok().and_then(|s| s.clone()); + if let Some(message) = source_error { + return Err(AppError::bad_request(format!( + "Failed to read request body: {message}" + ))); + } + Err(AppError::from(e)) + } + } +} + +/// [`ingest_stream_to_cas`] for an HTTP request body. +pub async fn ingest_body_to_cas( + body: Body, + dedup: &Arc, + filename: &str, + claimed_type: &str, + max_bytes: usize, +) -> Result { + let source = BodyStream::new(body).filter_map(|item| async move { + match item { + Ok(frame) => frame.into_data().ok().map(Ok), + Err(e) => Some(Err(e)), + } + }); + ingest_stream_to_cas(source, dedup, filename, claimed_type, max_bytes, None).await +} + +/// Adapt a multipart field into a byte stream for [`ingest_stream_to_cas`]. +/// +/// Terminates after the first error — multipart fields are not resumable. +pub fn multipart_field_stream( + field: axum::extract::multipart::Field<'_>, +) -> impl Stream> + Send + '_ { + stream::unfold((field, false), |(mut field, done)| async move { + if done { + return None; + } + match field.chunk().await { + Ok(Some(bytes)) => Some((Ok(bytes), (field, false))), + Ok(None) => None, + Err(e) => Some((Err(e), (field, true))), + } + }) +} + +/// Concatenate already-uploaded chunk part files into one byte stream, in +/// the given order — feeds chunked-upload assembly into the CDC store +/// without ever materializing an assembled file on disk. +pub fn stream_from_files( + paths: Vec, +) -> impl Stream> + Send { + stream::iter(paths.into_iter().map(Ok::<_, std::io::Error>)) + .and_then(|path| async move { + tokio::fs::File::open(path) + .await + .map(|file| ReaderStream::with_capacity(file, 64 * 1024)) + }) + .try_flatten() +} + +/// Result of a streamed write to a caller-supplied path. +pub struct StreamedToPath { + /// Total bytes written. + pub bytes_written: u64, + /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` + /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. + pub checksum_hex: Option, + /// Algorithm used to compute `checksum_hex`. Echoed back so the + /// caller can include it in audit logs or response headers. + pub alg: Option, +} + +/// Stream an HTTP request body directly to a known destination file, +/// enforcing `max_bytes` as a hard size limit. +/// +/// Used by the chunked-upload PUT handlers — each chunk has a +/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` +/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the +/// REST surface), so there's no spool/move dance. Peak heap is ~one +/// HTTP frame regardless of chunk size or `max_bytes`. +/// +/// `checksum_alg` is the optional client-requested integrity check +/// (default `md5` per the legacy `Content-MD5` contract; `blake3` +/// available for forward-compat). When `Some`, the hash is computed +/// incrementally during streaming — no extra disk read for verification. +/// +/// On size overflow the partial file is removed before the function +/// returns, so a client retry against the same chunk name starts from +/// a clean slate. On any other I/O error the partial file is also +/// removed and the error surfaces — callers can assume the path is +/// either fully written or absent. +pub async fn stream_body_to_path( + body: Body, + path: &Path, + max_bytes: usize, + checksum_alg: Option, +) -> Result { + let mut file = tokio::fs::File::create(path) + .await + .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; + + let mut total_bytes: usize = 0; + let mut stream = BodyStream::new(body); + let mut hasher = checksum_alg.map(IncrementalHasher::new); + + while let Some(frame_result) = stream.next().await { + let frame = match frame_result { + Ok(f) => f, + Err(e) => { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::bad_request(format!( + "Failed to read request body: {e}" + ))); + } + }; + if let Some(chunk) = frame.data_ref() { + total_bytes += chunk.len(); + if total_bytes > max_bytes { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::payload_too_large(format!( + "Chunk exceeds maximum size of {max_bytes} bytes" + ))); + } + if let Some(h) = hasher.as_mut() { + h.update(chunk); + } + if let Err(e) = file.write_all(chunk).await { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::internal_error(format!( + "Failed to write chunk: {e}" + ))); + } + } + } + file.flush() + .await + .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; + drop(file); + + Ok(StreamedToPath { + bytes_written: total_bytes as u64, + checksum_hex: hasher.map(IncrementalHasher::finalize_hex), + alg: checksum_alg, + }) +} + +/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`] +/// and the [`ChecksumTee`] of chunked-upload completion. +/// Per-frame `update` is sub-millisecond for all three algorithms at the +/// 64 KB frame sizes axum's body stream produces, so we don't need +/// `spawn_blocking` (which the old buffered path used because it hashed +/// the full multi-MB chunk in one shot). +pub enum IncrementalHasher { + Md5(md5::Md5), + Sha256(sha2::Sha256), + // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 + // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the + // outlier keeps the enum size proportional to the common case + // rather than the worst case. + Blake3(Box), +} + +impl IncrementalHasher { + fn new(alg: ChecksumAlg) -> Self { + match alg { + ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), + ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), + ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), + } + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Self::Md5(h) => h.update(bytes), + Self::Sha256(h) => h.update(bytes), + Self::Blake3(h) => { + h.update(bytes); + } + } + } + + fn finalize_hex(self) -> String { + match self { + Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Blake3(h) => h.finalize().to_hex().to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + + #[tokio::test] + async fn stream_body_to_path_caps_oversized() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + // 5 MiB body, 4 MiB cap → must reject. + let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + assert!( + result.is_err(), + "expected PayloadTooLarge, got Ok(bytes_written={})", + result.ok().map(|r| r.bytes_written).unwrap_or(0) + ); + // Partial file must be removed on rejection. + assert!( + !path.exists(), + "rejected chunk file should be removed, but {} still exists", + path.display() + ); + } + + #[tokio::test] + async fn stream_body_to_path_accepts_under_cap() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + let outcome = result.expect("should succeed"); + assert_eq!(outcome.bytes_written, 1024 * 1024); + assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); + assert!(path.exists()); + } + + #[tokio::test] + async fn stream_body_to_path_caps_at_exact_boundary() { + // Edge case: body exactly equal to cap should succeed; cap+1 must fail. + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 100])); + let outcome = stream_body_to_path(body, &path, 100, None) + .await + .expect("100 bytes at 100-byte cap should succeed"); + assert_eq!(outcome.bytes_written, 100); + + let path2 = temp_dir.path().join("chunk2"); + let body = Body::from(Bytes::from(vec![1u8; 101])); + assert!( + stream_body_to_path(body, &path2, 100, None).await.is_err(), + "101 bytes at 100-byte cap must reject" + ); + assert!(!path2.exists()); + } + + #[tokio::test] + async fn ingest_rejects_oversized_before_touching_storage() { + // 1 KiB body against a 100-byte cap: the adapter must abort the + // stream before any flush, so the stub dedup service (which cannot + // reach PG) is never asked to settle a batch. + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from(vec![0u8; 1024]))]); + + let result = ingest_stream_to_cas( + source, + &dedup, + "file.bin", + "application/octet-stream", + 100, + None, + ) + .await; + + let err = result.err().expect("oversized body must be rejected"); + assert_eq!(err.status_code, axum::http::StatusCode::PAYLOAD_TOO_LARGE); + } + + #[tokio::test] + async fn ingest_surfaces_source_errors_as_bad_request() { + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::iter(vec![ + Ok::<_, std::io::Error>(Bytes::from_static(b"partial")), + Err(std::io::Error::other("connection reset by peer")), + ]); + + let result = + ingest_stream_to_cas(source, &dedup, "file.bin", "text/plain", usize::MAX, None).await; + + let err = result.err().expect("source error must surface"); + assert_eq!(err.status_code, axum::http::StatusCode::BAD_REQUEST); + assert!( + err.message.contains("connection reset by peer"), + "original cause must be preserved: {}", + err.message + ); + } + + /// Regression: a source shorter than the MIME peek (< MAGIC_BYTES_LEN) + /// is drained to None during sniffing and then polled once more by the + /// `chain` that re-attaches the peeked head. Non-fused sources — like + /// the `stream::unfold` used for multipart fields — panic on that + /// post-None poll ("Unfold must not be polled after it returned + /// `Poll::Ready(None)`") unless the ingest fuses the stream first. + /// The stub dedup service can't reach PG, so an orderly `Err` (not a + /// panic) proves the stream layer survived. + #[tokio::test] + async fn ingest_short_stream_with_generic_mime_does_not_repoll_source() { + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::unfold(false, |done| async move { + if done { + None + } else { + Some((Ok::<_, std::io::Error>(Bytes::from_static(b"tiny")), true)) + } + }); + + let result = ingest_stream_to_cas( + source, + &dedup, + "tiny.bin", + "application/octet-stream", + usize::MAX, + None, + ) + .await; + + assert!( + result.is_err(), + "stub DB must reject the store — but only AFTER the stream \ + layer survived the post-peek poll" + ); + } + + #[tokio::test] + async fn stream_from_files_concatenates_in_order() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let a = temp_dir.path().join("a"); + let b = temp_dir.path().join("b"); + tokio::fs::write(&a, b"Hello, ").await.unwrap(); + tokio::fs::write(&b, b"World!").await.unwrap(); + + let mut out = Vec::new(); + let s = stream_from_files(vec![a, b]); + futures::pin_mut!(s); + while let Some(chunk) = s.next().await { + out.extend_from_slice(&chunk.expect("read")); + } + assert_eq!(out, b"Hello, World!"); + } + + #[tokio::test] + async fn checksum_tee_roundtrip() { + let tee = checksum_tee(ChecksumAlg::Md5); + if let Ok(mut h) = tee.lock() + && let Some(h) = h.as_mut() + { + h.update(b"hello world"); + } + let hex = finalize_checksum_tee(&tee).expect("digest"); + assert_eq!(hex, "5eb63bbbe01eeed093cb22bb8f5acdc3"); + assert!( + finalize_checksum_tee(&tee).is_none(), + "second finalize returns None" + ); + } +} diff --git a/src/interfaces/upload_spool.rs b/src/interfaces/upload_spool.rs deleted file mode 100644 index 7ac2a56a..00000000 --- a/src/interfaces/upload_spool.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! Shared streaming upload spool: request body → temp file + incremental hash. -//! -//! Used by both the native WebDAV PUT handler and the NextCloud-compat PUT -//! handler so neither buffers the full request body in memory. Peak heap is -//! ~one HTTP frame regardless of file size; the body is written to a temp -//! file (off tmpfs when [`StorageConfig::upload_temp_dir`] is configured) and -//! BLAKE3-hashed on the fly so the dedup layer can short-circuit on a hit. - -use std::path::{Path, PathBuf}; - -use axum::body::Body; -use http_body_util::BodyStream; -// The `Digest` trait (re-exported by both `md5` and `sha2` from the -// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / -// `finalize` methods. Importing once via `sha2` covers both — -// otherwise every call site would need fully-qualified -// `::…` syntax. -use sha2::Digest as _; -use tempfile::NamedTempFile; -use tokio::io::AsyncWriteExt; -use tokio_stream::StreamExt; - -use crate::application::ports::chunked_upload_ports::ChecksumAlg; -use crate::common::temp::new_spool_temp_file; -use crate::interfaces::errors::AppError; - -/// Outcome of spooling a request body to disk. -pub struct SpooledBody { - /// The temp file holding the body. Kept alive by the caller (dropping it - /// removes the file unless the dedup layer already consumed/moved it). - pub temp: NamedTempFile, - /// Hex-encoded BLAKE3 of the full body — matches `DedupService::hash_file`, - /// so passing it as `pre_computed_hash` enables the dedup fast path. - pub hash: String, - /// Total bytes written. - pub size: u64, -} - -/// Stream an HTTP request body to a temp file, computing its BLAKE3 hash -/// incrementally and enforcing `max_upload` as a hard size limit. -/// -/// Peak heap is ~one frame — the body is never fully buffered in RAM. -/// -/// `temp_dir` is taken by value (not `&Path`) so the returned future captures -/// no borrowed lifetime — required for the handler future to stay `Send`. -pub async fn spool_body_to_temp( - body: Body, - max_upload: usize, - temp_dir: Option, -) -> Result { - let temp = new_spool_temp_file(temp_dir.as_deref()) - .map_err(|e| AppError::internal_error(format!("Failed to create temp file: {e}")))?; - let temp_path = temp.path().to_path_buf(); - - let mut file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| AppError::internal_error(format!("Failed to open temp file: {e}")))?; - - let mut hasher = blake3::Hasher::new(); - let mut total_bytes: usize = 0; - let mut stream = BodyStream::new(body); - - while let Some(frame_result) = stream.next().await { - let frame = frame_result - .map_err(|e| AppError::bad_request(format!("Failed to read request body: {e}")))?; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len(); - if total_bytes > max_upload { - // Abort early — stop reading, delete temp file. - drop(file); - let _ = tokio::fs::remove_file(&temp_path).await; - return Err(AppError::payload_too_large(format!( - "Upload body exceeds the direct-PUT cap ({max_upload} bytes). \ - Use the chunked-upload protocol (REST: `/api/uploads/...`, \ - NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ - Chunked uploads are resumable on transient failure." - ))); - } - hasher.update(chunk); - file.write_all(chunk).await.map_err(|e| { - AppError::internal_error(format!("Failed to write to temp file: {e}")) - })?; - } - } - file.flush() - .await - .map_err(|e| AppError::internal_error(format!("Failed to flush temp file: {e}")))?; - drop(file); - - let hash = hasher.finalize().to_hex().to_string(); - Ok(SpooledBody { - temp, - hash, - size: total_bytes as u64, - }) -} - -/// Result of a streamed write to a caller-supplied path. -pub struct StreamedToPath { - /// Total bytes written. - pub bytes_written: u64, - /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` - /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. - pub checksum_hex: Option, - /// Algorithm used to compute `checksum_hex`. Echoed back so the - /// caller can include it in audit logs or response headers. - pub alg: Option, -} - -/// Stream an HTTP request body directly to a known destination file, -/// enforcing `max_bytes` as a hard size limit. -/// -/// Used by the chunked-upload PUT handlers — each chunk has a -/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` -/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the -/// REST surface), so there's no spool/move dance. Peak heap is ~one -/// HTTP frame regardless of chunk size or `max_bytes`. -/// -/// `checksum_alg` is the optional client-requested integrity check -/// (default `md5` per the legacy `Content-MD5` contract; `blake3` -/// available for forward-compat). When `Some`, the hash is computed -/// incrementally during streaming — no extra disk read for verification. -/// -/// On size overflow the partial file is removed before the function -/// returns, so a client retry against the same chunk name starts from -/// a clean slate. On any other I/O error the partial file is also -/// removed and the error surfaces — callers can assume the path is -/// either fully written or absent. -pub async fn stream_body_to_path( - body: Body, - path: &Path, - max_bytes: usize, - checksum_alg: Option, -) -> Result { - let mut file = tokio::fs::File::create(path) - .await - .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; - - let mut total_bytes: usize = 0; - let mut stream = BodyStream::new(body); - let mut hasher = checksum_alg.map(IncrementalHasher::new); - - while let Some(frame_result) = stream.next().await { - let frame = match frame_result { - Ok(f) => f, - Err(e) => { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::bad_request(format!( - "Failed to read request body: {e}" - ))); - } - }; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len(); - if total_bytes > max_bytes { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::payload_too_large(format!( - "Chunk exceeds maximum size of {max_bytes} bytes" - ))); - } - if let Some(h) = hasher.as_mut() { - h.update(chunk); - } - if let Err(e) = file.write_all(chunk).await { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::internal_error(format!( - "Failed to write chunk: {e}" - ))); - } - } - } - file.flush() - .await - .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; - drop(file); - - Ok(StreamedToPath { - bytes_written: total_bytes as u64, - checksum_hex: hasher.map(IncrementalHasher::finalize_hex), - alg: checksum_alg, - }) -} - -/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`]. -/// Per-frame `update` is sub-millisecond for all three algorithms at the -/// 64 KB frame sizes axum's body stream produces, so we don't need -/// `spawn_blocking` (which the old buffered path used because it hashed -/// the full multi-MB chunk in one shot). -enum IncrementalHasher { - Md5(md5::Md5), - Sha256(sha2::Sha256), - // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 - // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the - // outlier keeps the enum size proportional to the common case - // rather than the worst case. - Blake3(Box), -} - -impl IncrementalHasher { - fn new(alg: ChecksumAlg) -> Self { - match alg { - ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), - ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), - ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), - } - } - - fn update(&mut self, bytes: &[u8]) { - match self { - Self::Md5(h) => h.update(bytes), - Self::Sha256(h) => h.update(bytes), - Self::Blake3(h) => { - h.update(bytes); - } - } - } - - fn finalize_hex(self) -> String { - match self { - Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), - Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), - Self::Blake3(h) => h.finalize().to_hex().to_string(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - - #[tokio::test] - async fn stream_body_to_path_caps_oversized() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - // 5 MiB body, 4 MiB cap → must reject. - let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); - let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; - assert!( - result.is_err(), - "expected PayloadTooLarge, got Ok(bytes_written={})", - result.ok().map(|r| r.bytes_written).unwrap_or(0) - ); - // Partial file must be removed on rejection. - assert!( - !path.exists(), - "rejected chunk file should be removed, but {} still exists", - path.display() - ); - } - - #[tokio::test] - async fn stream_body_to_path_accepts_under_cap() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB - let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; - let outcome = result.expect("should succeed"); - assert_eq!(outcome.bytes_written, 1024 * 1024); - assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); - assert!(path.exists()); - } - - #[tokio::test] - async fn stream_body_to_path_caps_at_exact_boundary() { - // Edge case: body exactly equal to cap should succeed; cap+1 must fail. - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - let body = Body::from(Bytes::from(vec![1u8; 100])); - let outcome = stream_body_to_path(body, &path, 100, None) - .await - .expect("100 bytes at 100-byte cap should succeed"); - assert_eq!(outcome.bytes_written, 100); - - let path2 = temp_dir.path().join("chunk2"); - let body = Body::from(Bytes::from(vec![1u8; 101])); - assert!( - stream_body_to_path(body, &path2, 100, None).await.is_err(), - "101 bytes at 100-byte cap must reject" - ); - assert!(!path2.exists()); - } -} From 944c8337870366464565d97c220f20580c954e7b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 13:19:29 +0000 Subject: [PATCH 2/6] Sweep aborted-upload orphans from the periodic trash job; pipeline ZIP reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups to the streaming-upload work: The dedup garbage_collect() pass only ran when a user manually emptied their trash. Zero-reference rows — chunks orphaned by aborted streaming uploads (registered at ref_count 0 by the ingest rollback) and blobs dereferenced by trash expiry itself — could linger indefinitely on instances where nobody empties trash. The periodic TrashCleanupService sweep now ends every run with garbage_collect() (maintenance pool, batched), bounding orphan lifetime to the cleanup interval. Folder-ZIP creation was strictly sequential: open blob stream, deflate, close, repeat — every per-file blob-store round-trip (PG lookup + backend open; a full HTTP round-trip on S3/Azure) added to the wall clock. It now runs as a 2-stage pipeline: a prefetch task streams the planned files' content ahead of the writer through a bounded channel (~4 MiB), so the next file's read latency overlaps the current file's compression. ZIP entries are still written strictly in order, peak RAM stays flat, and a writer error hangs up the channel so the prefetcher stops on its own. Verified end-to-end against PostgreSQL 16: an upload aborted at ~14 MB left exactly 30 ref_count=0 chunk rows which the GC then reclaimed (8.5 MB, rows + physical files); a chunked upload completed with a wrong MD5 returned 400 with the tee-computed digest and the SAME session then completed successfully with the right checksum (parts persist — the old assembly deleted them, so the documented retry never actually worked); a 4-file folder ZIP downloaded and extracted byte-identical. https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- src/common/di.rs | 5 +- .../services/trash_cleanup_service.rs | 45 +++- src/infrastructure/services/zip_service.rs | 204 +++++++++++++----- 3 files changed, 193 insertions(+), 61 deletions(-) diff --git a/src/common/di.rs b/src/common/di.rs index 809611f1..97cf013d 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -557,9 +557,12 @@ impl AppServiceFactory { .with_file_deleted_hook(core.file_lifecycle.clone()), ); - // Initialize cleanup service (bulk-deletes expired items in 2 SQL queries) + // Initialize cleanup service (bulk-deletes expired items in 2 SQL + // queries, then GCs zero-reference blobs — including chunks orphaned + // by aborted streaming uploads). let cleanup_service = TrashCleanupService::new( trash_repo.clone(), + core.dedup_service.clone(), 24, // Run cleanup every 24 hours ); diff --git a/src/infrastructure/services/trash_cleanup_service.rs b/src/infrastructure/services/trash_cleanup_service.rs index 96f62d2d..b4dde9cf 100644 --- a/src/infrastructure/services/trash_cleanup_service.rs +++ b/src/infrastructure/services/trash_cleanup_service.rs @@ -6,21 +6,35 @@ use tracing::{debug, error, info, instrument}; use crate::common::errors::Result; use crate::domain::repositories::trash_repository::TrashRepository; use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository; +use crate::infrastructure::services::dedup_service::DedupService; /// Service for automatic cleanup of expired items in the trash. /// /// Uses `TrashRepository::delete_expired_bulk` to purge all expired items /// in **2 SQL statements inside a single transaction**, instead of the /// previous N+1 pattern that issued 3 queries per expired item. +/// +/// Each sweep ends with a dedup `garbage_collect()` pass: it reclaims the +/// blobs the expiry just dereferenced AND any other zero-reference rows — +/// notably chunks left behind by aborted streaming uploads, whose rollback +/// registers them at ref_count 0 precisely so this sweep can find them. +/// Without it, orphans would only be collected when a user happens to +/// empty their trash by hand. pub struct TrashCleanupService { trash_repository: Arc, + dedup_service: Arc, cleanup_interval_hours: u64, } impl TrashCleanupService { - pub fn new(trash_repository: Arc, cleanup_interval_hours: u64) -> Self { + pub fn new( + trash_repository: Arc, + dedup_service: Arc, + cleanup_interval_hours: u64, + ) -> Self { Self { trash_repository, + dedup_service, cleanup_interval_hours: cleanup_interval_hours.max(1), // Minimum 1 hour } } @@ -29,6 +43,7 @@ impl TrashCleanupService { #[instrument(skip(self))] pub async fn start_cleanup_job(&self) { let trash_repository = self.trash_repository.clone(); + let dedup_service = self.dedup_service.clone(); let interval_hours = self.cleanup_interval_hours; info!( @@ -41,7 +56,7 @@ impl TrashCleanupService { let mut interval = time::interval(interval_duration); // First immediate execution - Self::cleanup_expired_items(trash_repository.clone()) + Self::cleanup_expired_items(trash_repository.clone(), dedup_service.clone()) .await .unwrap_or_else(|e| error!("Error in initial trash cleanup: {:?}", e)); @@ -49,16 +64,24 @@ impl TrashCleanupService { interval.tick().await; debug!("Running scheduled trash cleanup task"); - if let Err(e) = Self::cleanup_expired_items(trash_repository.clone()).await { + if let Err(e) = + Self::cleanup_expired_items(trash_repository.clone(), dedup_service.clone()) + .await + { error!("Error in scheduled trash cleanup: {:?}", e); } } }); } - /// Bulk-delete all expired trash items in a single transaction. - #[instrument(skip(trash_repository))] - async fn cleanup_expired_items(trash_repository: Arc) -> Result<()> { + /// Bulk-delete all expired trash items in a single transaction, then + /// garbage-collect every zero-reference manifest/blob (expired content + /// plus aborted-upload orphans). + #[instrument(skip(trash_repository, dedup_service))] + async fn cleanup_expired_items( + trash_repository: Arc, + dedup_service: Arc, + ) -> Result<()> { debug!("Starting bulk cleanup of expired trash items"); let (files, folders) = trash_repository.delete_expired_bulk().await?; @@ -72,6 +95,16 @@ impl TrashCleanupService { ); } + // Runs on the maintenance pool; batched (500 rows/iteration) with + // yield points, so it never starves request-path queries. + match dedup_service.garbage_collect().await { + Ok((0, _)) => debug!("Trash cleanup GC: nothing to collect"), + Ok((items, bytes)) => { + info!("Trash cleanup GC: reclaimed {items} orphaned blobs ({bytes} bytes)"); + } + Err(e) => error!("Trash cleanup GC failed: {:?}", e), + } + Ok(()) } } diff --git a/src/infrastructure/services/zip_service.rs b/src/infrastructure/services/zip_service.rs index ed7fff13..a05886c4 100644 --- a/src/infrastructure/services/zip_service.rs +++ b/src/infrastructure/services/zip_service.rs @@ -47,12 +47,41 @@ impl From for DomainError { /// Type alias for the fully-async ZIP writer backed by a buffered tokio file. type AsyncZipWriter = ZipFileWriter>>; +/// One planned archive entry, in final ZIP order. +enum ZipPlanEntry { + /// Directory entry (Stored, zero-length body). + Dir(String), + /// File entry: ZIP-relative path + file id to stream from the blob store. + File { zip_path: String, file_id: String }, +} + +/// Message protocol from the prefetch task to the ZIP writer. For each +/// planned file, in order: zero or more `Chunk`s, then exactly one `End`; +/// `Err` aborts the whole archive. +enum Prefetched { + Chunk(bytes::Bytes), + End, + Err(String), +} + +/// Bound on the prefetch channel (messages of ≤ ~64 KB blob-stream chunks): +/// ~4 MiB of read-ahead. Enough to hide the per-file open latency of the +/// blob store (PG lookup + backend round-trip — significant on S3/Azure) +/// behind the deflate of the previous entry, while keeping RAM flat. +const PREFETCH_BUFFER_CHUNKS: usize = 64; + /// Service for creating ZIP files. /// /// Uses `async_zip` for fully-async archive creation. Every write (headers, /// compressed chunk data, central directory) goes through /// `tokio::io::BufWriter` → `tokio::fs::File`, so **no Tokio worker is ever /// blocked** by disk I/O or compression. +/// +/// Archive creation is a 2-stage pipeline: a prefetch task reads file +/// content from the blob store ahead of the writer, so the next file's +/// read latency overlaps the current file's compression instead of adding +/// to it. The ZIP entries themselves are still written strictly in order +/// (the format requires it). pub struct ZipService { file_service: Arc, folder_service: Arc, @@ -144,7 +173,22 @@ impl ZipService { } }; - // ── 4. Open the temp file + ZIP writer ─────────────────────────── + // ── 4. Plan the archive (folders are already sorted by path) ───── + let mut plan: Vec = Vec::new(); + for folder in &all_folders { + let zip_dir = format!("{}/", folder_zip_path(&folder.path)); + plan.push(ZipPlanEntry::Dir(zip_dir.clone())); + if let Some(files) = files_by_folder.get(&folder.id) { + for file in files { + plan.push(ZipPlanEntry::File { + zip_path: format!("{}{}", zip_dir, file.name), + file_id: file.id.to_string(), + }); + } + } + } + + // ── 5. Open the temp file + ZIP writer ─────────────────────────── let temp = NamedTempFile::new().map_err(ZipError::IoError)?; let tokio_file = tokio::fs::File::create(temp.path()) .await @@ -152,79 +196,131 @@ impl ZipService { let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file); let mut zip = ZipFileWriter::with_tokio(buf_writer); - // ── 5. Write entries (folders are already sorted by path) ───────── - for folder in &all_folders { - let zip_dir = format!("{}/", folder_zip_path(&folder.path)); + // ── 6. Write entries: 2-stage pipeline ─────────────────────────── + // The prefetch task reads blob streams for the planned files, in + // order, ahead of the writer — the next file's blob-store latency + // overlaps the current file's deflate. If the writer bails out, + // dropping the receiver makes the prefetcher's next send fail and + // it stops on its own. + let file_ids: Vec = plan + .iter() + .filter_map(|entry| match entry { + ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()), + ZipPlanEntry::Dir(_) => None, + }) + .collect(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(PREFETCH_BUFFER_CHUNKS); + let _prefetcher = tokio::spawn(Self::prefetch_files( + self.file_service.clone(), + file_ids, + tx, + )); - // Directory entry (Stored, zero-length body) - let dir_entry = ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored); - match zip.write_entry_whole(dir_entry, &[]).await { - Ok(()) => debug!("Folder added to ZIP: {}", zip_dir), - Err(e) => { - warn!("Could not add folder entry (may already exist): {}", e); + for entry in &plan { + match entry { + ZipPlanEntry::Dir(zip_dir) => { + let dir_entry = + ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored); + match zip.write_entry_whole(dir_entry, &[]).await { + Ok(()) => debug!("Folder added to ZIP: {}", zip_dir), + Err(e) => { + warn!("Could not add folder entry (may already exist): {}", e); + } + } } - } - - // Files belonging to this folder - if let Some(files) = files_by_folder.get(&folder.id) { - for file in files { - self.add_file_to_zip_streamed(&mut zip, file, &zip_dir) - .await?; + ZipPlanEntry::File { zip_path, .. } => { + Self::write_prefetched_file(&mut zip, zip_path, &mut rx).await?; } } } - // ── 6. Finalize ────────────────────────────────────────────────── + // ── 7. Finalize ────────────────────────────────────────────────── let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?; compat_writer.close().await.map_err(ZipError::IoError)?; Ok(temp) } - /// Streams file content in chunks (~64 KB) into an async ZIP entry, - /// keeping peak memory independent of individual file sizes. - async fn add_file_to_zip_streamed( - &self, + /// Prefetch stage: streams each planned file's content from the blob + /// store, in plan order, into the bounded channel. Stops on the first + /// read error (after forwarding it) or when the writer hangs up. + async fn prefetch_files( + file_service: Arc, + file_ids: Vec, + tx: tokio::sync::mpsc::Sender, + ) { + for file_id in file_ids { + let stream = match file_service.get_file_stream(&file_id).await { + Ok(s) => s, + Err(e) => { + error!("Error opening file stream {}: {}", file_id, e); + let _ = tx + .send(Prefetched::Err(format!( + "Error streaming file {}: {}", + file_id, e + ))) + .await; + return; + } + }; + + let mut stream = std::pin::Pin::from(stream); + while let Some(chunk_result) = stream.next().await { + let message = match chunk_result { + Ok(bytes) => Prefetched::Chunk(bytes), + Err(e) => Prefetched::Err(format!("Error streaming file {}: {}", file_id, e)), + }; + let abort = matches!(message, Prefetched::Err(_)); + if tx.send(message).await.is_err() || abort { + return; // writer gone, or fatal read error forwarded + } + } + + if tx.send(Prefetched::End).await.is_err() { + return; // writer gone + } + } + } + + /// Writer stage: drains one file's prefetched chunks into a Deflate + /// ZIP entry. Peak memory stays bounded by the channel, independent + /// of individual file sizes. + async fn write_prefetched_file( zip: &mut AsyncZipWriter, - file: &FileDto, - folder_path: &str, + zip_path: &str, + rx: &mut tokio::sync::mpsc::Receiver, ) -> Result<()> { - let file_path = format!("{}{}", folder_path, file.name); - info!("Adding file to ZIP: {}", file_path); + info!("Adding file to ZIP: {}", zip_path); - let file_id = file.id.to_string(); - - // Open a streaming entry with Deflate compression - let entry = ZipEntryBuilder::new(file_path.clone().into(), Compression::Deflate); + let entry = ZipEntryBuilder::new(zip_path.to_string().into(), Compression::Deflate); let mut entry_writer = zip .write_entry_stream(entry) .await .map_err(ZipError::AsyncZipError)?; - // Stream file contents in chunks instead of loading all into RAM - let stream = match self.file_service.get_file_stream(&file_id).await { - Ok(s) => s, - Err(e) => { - error!("Error opening file stream {}: {}", file_id, e); - // Close the partially-opened entry before returning - let _ = entry_writer.close().await; - return Err(ZipError::FileReadError(format!( - "Error streaming file {}: {}", - file_id, e - )) - .into()); + loop { + match rx.recv().await { + Some(Prefetched::Chunk(bytes)) => { + entry_writer + .write_all(&bytes) + .await + .map_err(ZipError::IoError)?; + } + Some(Prefetched::End) => break, + Some(Prefetched::Err(message)) => { + // Close the partially-written entry before bailing out. + let _ = entry_writer.close().await; + return Err(ZipError::FileReadError(message).into()); + } + None => { + let _ = entry_writer.close().await; + return Err(ZipError::FileReadError(format!( + "Prefetch stage ended unexpectedly while writing {}", + zip_path + )) + .into()); + } } - }; - - // Pin the stream so StreamExt::next() can be called - let mut stream = std::pin::Pin::from(stream); - - while let Some(chunk_result) = stream.next().await { - let bytes = chunk_result.map_err(ZipError::IoError)?; - entry_writer - .write_all(&bytes) - .await - .map_err(ZipError::IoError)?; } // Finalize the entry (writes data descriptor with CRC + sizes) @@ -233,7 +329,7 @@ impl ZipService { .await .map_err(ZipError::AsyncZipError)?; - debug!("File added to ZIP: {}", file_path); + debug!("File added to ZIP: {}", zip_path); Ok(()) } } From 0fab4ce17d21ecd9a55ea2d05450af7f4cd35c56 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 13:54:32 +0000 Subject: [PATCH 3/6] Instant upload: register already-owned content by hash, zero bytes on the wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 0 of the delta-sync plan. Re-uploading a file the user already has (another device, a restore, a duplicate) used to transfer every byte just for the server to discard them as a dedup hit. The frontend now computes the file's BLAKE3 locally and, on a hit, registers the file with a single ~150-byte metadata call. Server — POST /api/files/by-hash: - All checks live in the application service per the AuthZ rule: Create permission on the target folder via the authorization engine, hash ownership via the existing user-scoped query (a non-owned hash returns 404 — same shape as "no such blob" — and emits an instant_upload.rejected audit event), quota on the logical size. - On success: one ref_count bump + the existing save_file_with_blob row registration (compensation included); is_new_blob=false so lifecycle hooks skip thumbnail regeneration. ~10 ms warm. - The storage-usage service is now built before the application services and injected, instead of only living on AppState. Client — WASM BLAKE3 + worker: - wasm/oxicloud-hash: the exact same blake3 crate the server uses, compiled with WASM SIMD128 (~660 MB/s measured) so browser hashes match server content addresses bit for bit. Built by scripts/build-wasm.sh; the artifacts (45 KB wasm + 8 KB glue) are vendored like pdf.js — no npm dependencies, no wasm toolchain needed for regular builds. - static/js/workers/hashWorker.js streams the File in 8 MiB slices off the main thread (constant RAM at any file size). - features/files/instantUpload.js orchestrates: threshold (8 MiB — below it the round-trips cost more than the bytes), user-scoped /api/dedup/check, by-hash registration, and silent fallback to the normal byte upload on any miss, race or unsupported environment. Wired into both uploadFiles and uploadFolderEntries. - biome.json vendors exclusion fixed to cover nested directories (previous vendors were .mjs and never matched the *.js include). Verified end-to-end against PostgreSQL 16: node-driven WASM hash equals the server's content_hash for a 20 MB file; by-hash returns 201 in ~10 ms warm with a 151-byte request (vs 20,971,873 bytes for the byte upload); the copy downloads byte-identical and the manifest ref_count goes 1→2; a second user probing the same hash gets exists:false and 404 plus the audit line; duplicate name → 409, malformed hash → 400; worker and wasm are served with correct MIME (application/wasm). https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- .gitignore | 1 + biome.json | 2 +- scripts/build-wasm.sh | 39 +++ .../services/file_upload_service.rs | 142 +++++++++- src/common/di.rs | 25 +- src/interfaces/api/handlers/file_handler.rs | 84 ++++++ src/interfaces/api/mod.rs | 1 + src/interfaces/api/routes.rs | 5 +- static/js/core/types.js | 23 ++ static/js/features/files/fileOperations.js | 119 ++++---- static/js/features/files/instantUpload.js | 177 ++++++++++++ .../vendors/hash-wasm/oxicloud_hash_wasm.js | 261 ++++++++++++++++++ .../hash-wasm/oxicloud_hash_wasm_bg.wasm | Bin 0 -> 45242 bytes static/js/workers/hashWorker.js | 79 ++++++ wasm/oxicloud-hash/Cargo.lock | 184 ++++++++++++ wasm/oxicloud-hash/Cargo.toml | 30 ++ wasm/oxicloud-hash/src/lib.rs | 98 +++++++ 17 files changed, 1209 insertions(+), 61 deletions(-) create mode 100755 scripts/build-wasm.sh create mode 100644 static/js/features/files/instantUpload.js create mode 100644 static/js/vendors/hash-wasm/oxicloud_hash_wasm.js create mode 100644 static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm create mode 100644 static/js/workers/hashWorker.js create mode 100644 wasm/oxicloud-hash/Cargo.lock create mode 100644 wasm/oxicloud-hash/Cargo.toml create mode 100644 wasm/oxicloud-hash/src/lib.rs diff --git a/.gitignore b/.gitignore index 0d7b9636..905f8a95 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ tests/e2e/playwright/.auth/ # Test fixtures generated on-the-fly by tests/api/run.sh tests/fixtures/chunk-over-cap-*.bin +wasm/oxicloud-hash/target/ diff --git a/biome.json b/biome.json index 5abc8fe5..9a5abf93 100644 --- a/biome.json +++ b/biome.json @@ -7,7 +7,7 @@ } }, "files": { - "includes": ["static/**/*.js", "static/**/*.css", "static/**/*.json", "!static/js/vendors/"] + "includes": ["static/**/*.js", "static/**/*.css", "static/**/*.json", "!static/js/vendors"] }, "formatter": { "enabled": true, diff --git a/scripts/build-wasm.sh b/scripts/build-wasm.sh new file mode 100755 index 00000000..9a64cff7 --- /dev/null +++ b/scripts/build-wasm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Rebuild the vendored BLAKE3 WASM module (static/js/vendors/hash-wasm/). +# +# The generated artifacts ARE committed — like the other vendored modules +# (pdf.js) — so regular frontend/backend builds never need the wasm +# toolchain. Re-run this script only when wasm/oxicloud-hash/ changes +# (e.g. bumping the blake3 crate, or adding FastCDC for the delta-sync +# client) and commit the regenerated files. +# +# Requirements (one-time): +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --locked +# +# wasm-bindgen-cli's version must match the crate's `wasm-bindgen` +# dependency; cargo prints a clear error when they drift. + +set -euo pipefail +cd "$(dirname "$0")/.." + +CRATE=wasm/oxicloud-hash +OUT=static/js/vendors/hash-wasm + +# SIMD128 is baseline in every evergreen browser (Chrome 91+, Firefox 89+, +# Safari 16.4+) and is worth ~3-4× in hashing throughput. Browsers without +# it fail instantiation; the frontend detects that and falls back to a +# plain byte upload. +RUSTFLAGS="-C target-feature=+simd128" \ + cargo build \ + --manifest-path "$CRATE/Cargo.toml" \ + --target wasm32-unknown-unknown \ + --release + +wasm-bindgen \ + --target web \ + --no-typescript \ + --out-dir "$OUT" \ + "$CRATE/target/wasm32-unknown-unknown/release/oxicloud_hash_wasm.wasm" + +echo "Vendored: $(ls -la "$OUT" | tail -n +2 | awk '{print $9, "("$5" bytes)"}' | xargs)" diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 49f00a95..02ac99a0 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -1,14 +1,19 @@ use std::sync::Arc; +use uuid::Uuid; use crate::application::dtos::file_dto::FileDto; +use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::application::ports::file_lifecycle::FileLifecycleHook; use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; -use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; +use crate::application::ports::storage_ports::{FileReadPort, FileWritePort, StorageUsagePort}; use crate::application::services::storage_usage_service::StorageUsageService; use crate::common::errors::DomainError; +use crate::domain::services::authorization::{Permission, Resource, Subject}; use crate::infrastructure::repositories::pg::FileBlobReadRepository; use crate::infrastructure::repositories::pg::FileBlobWriteRepository; +use crate::infrastructure::services::dedup_service::DedupService; use crate::infrastructure::services::file_content_cache::FileContentCache; +use crate::infrastructure::services::pg_acl_engine::PgAclEngine; use tracing::{debug, info, warn}; /// Helper function to extract username from folder path string. @@ -49,6 +54,18 @@ pub struct FileUploadService { content_cache: Option>, /// Single lifecycle dispatcher — fires on_file_created / on_file_updated. file_lifecycle_hook: Option>, + /// Dependencies of the instant-upload path + /// (`create_file_from_owned_blob_with_perms`); `None` in minimal test + /// wiring. + instant_upload: Option, +} + +/// Everything the instant-upload path needs beyond the upload service's own +/// ports: permission checks, the dedup index, and quota enforcement. +struct InstantUploadDeps { + authz: Arc, + dedup: Arc, + quota: Arc, } impl FileUploadService { @@ -60,6 +77,7 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, + instant_upload: None, } } @@ -74,9 +92,26 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, + instant_upload: None, } } + /// Wires the authorization engine, dedup index and quota service that + /// power the instant-upload path. + pub fn with_instant_upload( + mut self, + authz: Arc, + dedup: Arc, + quota: Arc, + ) -> Self { + self.instant_upload = Some(InstantUploadDeps { + authz, + dedup, + quota, + }); + self + } + /// Configures the content cache for invalidation on file updates. pub fn with_content_cache(mut self, cache: Arc) -> Self { self.content_cache = Some(cache); @@ -98,6 +133,111 @@ impl FileUploadService { self } + // ── Instant upload (zero content bytes) ────────────────────── + + /// Register a new file row pointing at a blob the caller **already + /// owns** — the instant-upload path: the client proved it has the + /// content by hash, so no bytes travel and no chunk is written. Pure + /// metadata: one ref_count bump + one row INSERT. + /// + /// Security model (mirrors `GET /api/dedup/check/{hash}`): + /// - The caller must have `Create` permission on the target folder. + /// - The hash is only claimable when the caller owns at least one + /// non-trashed file referencing it — never a global content oracle. + /// A non-owned hash returns `NotFound` (anti-enumeration: same shape + /// as "no such blob") and emits an `instant_upload.rejected` audit + /// event with the real reason. + /// - Quota is enforced on the logical size, exactly like a byte upload. + pub async fn create_file_from_owned_blob_with_perms( + &self, + caller_id: Uuid, + name: String, + folder_id: String, + hash: &str, + ) -> Result { + let Some(InstantUploadDeps { + authz, + dedup, + quota, + }) = &self.instant_upload + else { + return Err(DomainError::internal_error( + "FileUpload", + "instant upload is not wired (authz/dedup/quota missing)", + )); + }; + + // ── AuthZ: Create on the target folder ─────────────────── + let folder_uuid = Uuid::parse_str(&folder_id) + .map_err(|_| DomainError::not_found("Folder", folder_id.clone()))?; + authz + .require( + Subject::User(caller_id), + Permission::Create, + Resource::Folder(folder_uuid), + ) + .await?; + + // ── Ownership: only blobs the caller can already read ──── + if !dedup + .user_owns_blob_reference(hash, &caller_id.to_string()) + .await + { + tracing::info!( + target: "audit", + event = "instant_upload.rejected", + reason = "hash_not_owned", + caller_id = %caller_id, + blob_hash = %hash, + "👮🏻‍♂️ Instant upload rejected: caller owns no file referencing the claimed hash", + ); + return Err(DomainError::not_found("Blob", hash)); + } + + let Some(metadata) = dedup.get_blob_metadata(hash).await else { + // Lost a race with the last-reference delete — same shape as + // "never existed". + return Err(DomainError::not_found("Blob", hash)); + }; + + // ── Quota on the logical size, before taking any reference ── + quota.check_storage_quota(caller_id, metadata.size).await?; + + // The manifest knows the original content type; fall back to the + // new name's extension when the stored one is generic. + let claimed = metadata.content_type.as_deref().unwrap_or(""); + let content_type = + match crate::common::mime_detect::refine_content_type(&[], &name, claimed) { + ct if ct.is_empty() => "application/octet-stream".to_string(), + ct => ct, + }; + + // Take the reference the row registration will consume (it releases + // it again on any failure). A concurrent GC between the ownership + // check and this bump surfaces as NotFound — the client falls back + // to a normal byte upload. + dedup.add_reference(hash).await?; + + let dto = self + .upload_file_streaming( + name, + Some(folder_id), + content_type, + StoredBlob { + hash: hash.to_string(), + size: metadata.size, + is_new_blob: false, + }, + ) + .await?; + + info!( + "⚡ INSTANT UPLOAD: {} ({} bytes, 0 transferred, ID: {})", + dto.name, metadata.size, dto.id + ); + Ok(dto) + } + // ── private helpers ────────────────────────────────────────── /// Optionally update storage usage after a successful upload. diff --git a/src/common/di.rs b/src/common/di.rs index 97cf013d..49e9bf87 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -439,6 +439,7 @@ impl AppServiceFactory { repos: &RepositoryServices, trash_service: Option>, authz: &Arc, + storage_usage: &Arc, ) -> ApplicationServices { // Main services let folder_service = Arc::new(FolderService::new( @@ -452,7 +453,12 @@ impl AppServiceFactory { repos.file_read_repository.clone(), ) .with_content_cache(core.file_content_cache.clone()) - .with_file_lifecycle_hook(core.file_lifecycle.clone()), + .with_file_lifecycle_hook(core.file_lifecycle.clone()) + .with_instant_upload( + authz.clone(), + core.dedup_service.clone(), + storage_usage.clone(), + ), ); let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( @@ -733,9 +739,19 @@ impl AppServiceFactory { .create_trash_service(&repos, &core, &authorization) .await; + // 3c. Storage usage / quota service (needed by the instant-upload + // path inside the application services, and re-exposed on AppState + // for the handler-side quota checks of the byte-upload paths). + let storage_usage = self.create_storage_usage_service(&repos, &pool, &maintenance_pool); + // 4. Application services (with trash + authz already wired) - let mut apps = - self.create_application_services(&core, &repos, trash_service.clone(), &authorization); + let mut apps = self.create_application_services( + &core, + &repos, + trash_service.clone(), + &authorization, + &storage_usage, + ); // 5. Share service let share_service = self.create_share_service(&repos, &pool, &authorization); @@ -775,8 +791,7 @@ impl AppServiceFactory { recent_service = Some(recent.clone()); apps.recent_service = Some(recent); - storage_usage_service = - Some(self.create_storage_usage_service(&repos, &pool, &maintenance_pool)); + storage_usage_service = Some(storage_usage.clone()); self.start_tree_etag_flush_job(&maintenance_pool); diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index 903fd87e..67c27f57 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -69,6 +69,57 @@ impl FileHandler { } } + /// Instant upload: create a file from a blob the caller already owns. + /// + /// Zero content bytes travel — the client proved possession of the + /// content by hash (it computed BLAKE3 locally and confirmed via + /// `GET /api/dedup/check/{hash}`), so the server only bumps the blob's + /// reference count and registers the metadata row. + /// + /// All authorization (folder Create permission, hash ownership with + /// anti-enumeration, quota) lives in the application service. + pub(super) async fn create_file_by_hash_impl( + State(state): State, + auth_user: AuthUser, + Json(request): Json, + ) -> impl IntoResponse { + // Hash shape check — same contract as /api/dedup/check/{hash}. + if request.hash.len() != 64 || !request.hash.chars().all(|c| c.is_ascii_hexdigit()) { + return AppError::bad_request( + "Invalid hash format. Expected BLAKE3 (64 hex characters)", + ) + .into_response(); + } + // Basename only — same path-traversal guard as the multipart upload. + let filename = request + .name + .rsplit('/') + .next() + .unwrap_or(&request.name) + .rsplit('\\') + .next() + .unwrap_or(&request.name) + .to_string(); + if filename.is_empty() { + return AppError::bad_request("File name must not be empty").into_response(); + } + + match state + .applications + .file_upload_service + .create_file_from_owned_blob_with_perms( + auth_user.id, + filename, + request.folder_id, + &request.hash, + ) + .await + { + Ok(file) => Self::created_json_response(&file).into_response(), + Err(err) => Self::domain_error_response(err).into_response(), + } + } + /// Core upload logic shared by [`Self::upload_file`] and /// [`Self::upload_file_with_thumbnails`]. /// @@ -1039,6 +1090,39 @@ pub async fn upload_file_with_thumbnails( FileHandler::upload_file_with_thumbnails_impl(state, auth_user, multipart).await } +/// Request body for the instant-upload endpoint. +#[derive(Debug, Deserialize, ToSchema)] +pub struct CreateFileByHashRequest { + /// File name to create (path components are stripped). + pub name: String, + /// Target folder ID (the caller needs Create permission on it). + pub folder_id: String, + /// BLAKE3 hash (64 hex chars) of content the caller already owns. + pub hash: String, +} + +#[utoipa::path( + post, + path = "/api/files/by-hash", + request_body = CreateFileByHashRequest, + responses( + (status = 201, description = "File created from an already-owned blob — zero bytes transferred", body = FileDto), + (status = 400, description = "Invalid hash format or empty name"), + (status = 404, description = "No owned blob with this hash (anti-enumeration: same shape as unknown hash)"), + (status = 409, description = "A file with this name already exists in the folder"), + (status = 507, description = "Storage quota exceeded"), + ), + security(("bearerAuth" = [])), + tag = "files" +)] +pub async fn create_file_by_hash( + state: State, + auth_user: AuthUser, + request: Json, +) -> impl IntoResponse { + FileHandler::create_file_by_hash_impl(state, auth_user, request).await +} + #[utoipa::path( get, path = "/api/files/{id}", diff --git a/src/interfaces/api/mod.rs b/src/interfaces/api/mod.rs index e725fc67..51eb8e6d 100644 --- a/src/interfaces/api/mod.rs +++ b/src/interfaces/api/mod.rs @@ -80,6 +80,7 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; // File handlers (free functions — see file_handler.rs for why) handlers::file_handler::list_files_query, handlers::file_handler::upload_file_with_thumbnails, + handlers::file_handler::create_file_by_hash, handlers::file_handler::download_file, handlers::file_handler::get_thumbnail, handlers::file_handler::upload_thumbnail, diff --git a/src/interfaces/api/routes.rs b/src/interfaces/api/routes.rs index f2ac1662..262dfa9c 100644 --- a/src/interfaces/api/routes.rs +++ b/src/interfaces/api/routes.rs @@ -55,8 +55,8 @@ use crate::interfaces::api::handlers::chunked_upload_handler::{ cancel_upload, complete_upload, create_upload, get_upload_status, upload_chunk, }; use crate::interfaces::api::handlers::file_handler::{ - delete_file, download_file, get_file_metadata, get_thumbnail, list_files_query, - move_file_simple, rename_file, upload_file_with_thumbnails, upload_thumbnail, + create_file_by_hash, delete_file, download_file, get_file_metadata, get_thumbnail, + list_files_query, move_file_simple, rename_file, upload_file_with_thumbnails, upload_thumbnail, }; #[allow(deprecated)] use crate::interfaces::api::handlers::folder_handler::{ @@ -229,6 +229,7 @@ pub fn create_api_routes(app_state: &Arc) -> Router> { let basic_file_router = Router::new() .route("/", get(list_files_query)) .route("/upload", post(upload_file_with_thumbnails)) + .route("/by-hash", post(create_file_by_hash)) .route("/{id}", get(download_file)) .route( "/{id}/thumbnail/{size}", diff --git a/static/js/core/types.js b/static/js/core/types.js index 99ba6f02..f0b49ae1 100644 --- a/static/js/core/types.js +++ b/static/js/core/types.js @@ -518,3 +518,26 @@ * @typedef {{kind: 'user', id: string} | {kind: 'group', id: string}} GroupMemberItem */ +// ------------------- Instant upload (dedup) + +/** + * Response from `GET /api/dedup/check/{hash}` — user-scoped: `exists` only + * reflects content the CALLER already owns, never global existence. + * Mirrors `HashCheckResponse` on the server (`dedup_handler.rs`). + * @typedef {Object} HashCheckAnswer + * @property {boolean} exists + * @property {string} hash BLAKE3 echoed back (64 hex chars) + * @property {number} [existing_size] size in bytes, present when `exists` + */ + +/** + * Request body for `POST /api/files/by-hash` (instant upload — registers a + * file from an already-owned blob, zero content bytes on the wire). + * Mirrors `CreateFileByHashRequest` on the server (`file_handler.rs`). + * The 201 response body is a {@link FileItem}. + * @typedef {Object} CreateFileByHash + * @property {string} name file name to create (basename only) + * @property {string} folder_id target folder (caller needs Create on it) + * @property {string} hash BLAKE3 of the owned content (64 hex chars) + */ + diff --git a/static/js/features/files/fileOperations.js b/static/js/features/files/fileOperations.js index 87f7578d..a4bc092d 100644 --- a/static/js/features/files/fileOperations.js +++ b/static/js/features/files/fileOperations.js @@ -12,6 +12,7 @@ import { i18n } from '../../core/i18n.js'; import { notifications } from '../../core/notifications.js'; import { invalidateFolderMeta } from '../../model/filesModel.js'; import { triggerBrowserDownload } from '../../utils/download.js'; +import { tryInstantUpload } from './instantUpload.js'; /** * @typedef {Object} BatchResult @@ -404,20 +405,28 @@ const fileOps = { if (quotaStop) return; const file = readableFiles[idx]; - const formData = new FormData(); - if (targetFolderId) formData.append('folder_id', targetFolderId); - formData.append('file', file); + // ── Instant upload: when the server already has this exact + // content for this user, register it by hash — zero bytes + // on the wire. Any miss/failure falls back to a byte upload. + /** @type {UploadAnswer | null} */ + let result = await tryInstantUpload(file, targetFolderId); + if (result) { + if (batchId) { + try { + notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error'); + } catch (_) {} + } + } else { + const formData = new FormData(); + if (targetFolderId) formData.append('folder_id', targetFolderId); + formData.append('file', file); - console.log(`Uploading file to folder: ${targetFolderId || 'root'}`, { - file: file.name, - size: file.size - }); - - // Scale stall timeout with file size: - // base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit - const sizeGB = file.size / (1024 * 1024 * 1024); - const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000); - const result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout); + // Scale stall timeout with file size: + // base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit + const sizeGB = file.size / (1024 * 1024 * 1024); + const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000); + result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout); + } uploadedCount++; @@ -663,48 +672,54 @@ const fileOps = { const parentPath = parts.slice(0, -1).join('/'); const targetFolderId = folderMap.get(parentPath) || currentFolderId; - // ── FIFO/pipe guard (0-byte files only) ── - // Named pipes (runit supervise/control) report size=0 - // but block on open(). Pre-read only 0-byte files into - // memory; files with size>0 are always regular files and - // go straight to FormData (zero extra memory copy). - /** @type {Blob} */ - let uploadFile = file; // default: use original File - if (file.size === 0) { - try { - const buf = await Promise.race([ - file.arrayBuffer(), - new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000)) - ]); - uploadFile = new Blob([buf], { - type: file.type || 'application/octet-stream' - }); - } catch { - console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`); - uploadedCount++; - successCount++; - if (batchId) { - try { - notifications.fileCompleted(batchId, true); - } catch (_) {} + // ── Instant upload (zero bytes on the wire) ── + // Same fallback contract as uploadFiles: a null result + // means "do the byte upload". The shared accounting + // after this try block handles both outcomes. + const instant = await tryInstantUpload(file, targetFolderId); + if (instant) { + result = instant; + } else { + // ── FIFO/pipe guard (0-byte files only) ── + // Named pipes (runit supervise/control) report size=0 + // but block on open(). Pre-read only 0-byte files into + // memory; files with size>0 are always regular files and + // go straight to FormData (zero extra memory copy). + /** @type {Blob} */ + let uploadFile = file; // default: use original File + if (file.size === 0) { + try { + const buf = await Promise.race([ + file.arrayBuffer(), + new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000)) + ]); + uploadFile = new Blob([buf], { + type: file.type || 'application/octet-stream' + }); + } catch { + console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`); + uploadedCount++; + successCount++; + if (batchId) { + try { + notifications.fileCompleted(batchId, true); + } catch (_) {} + } + return; } - return; } + + const formData = new FormData(); + formData.append('folder_id', targetFolderId); + formData.append('file', uploadFile, file.name); + + const thisTimeout = + file.size === 0 + ? TIMEOUT_MS_ZERO + : Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS); + + result = await this._uploadFileFetch(formData, thisTimeout); } - - const formData = new FormData(); - formData.append('folder_id', targetFolderId); - formData.append('file', uploadFile, file.name); - - const thisTimeout = - file.size === 0 - ? TIMEOUT_MS_ZERO - : Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS); - console.log(`[UPLOAD START] #${idx} ${rel} (${file.size} bytes, timeout=${thisTimeout}ms)`); - - result = await this._uploadFileFetch(formData, thisTimeout); - - console.log(`[UPLOAD END] #${idx} ${rel} ok=${result.ok}${result.errorMsg ? ` err=${result.errorMsg}` : ''}`); } catch (e) { result = { ok: false, diff --git a/static/js/features/files/instantUpload.js b/static/js/features/files/instantUpload.js new file mode 100644 index 00000000..bda924e1 --- /dev/null +++ b/static/js/features/files/instantUpload.js @@ -0,0 +1,177 @@ +/** + * OxiCloud - Instant upload (zero-byte dedup upload) + * + * Before transferring a file's bytes, compute its BLAKE3 locally (in a + * worker, off the main thread) and ask the server whether the caller + * already owns that exact content (`GET /api/dedup/check/{hash}` — the + * check is user-scoped, never a global content oracle). On a hit, a + * single metadata call (`POST /api/files/by-hash`) registers the file + * with ZERO content bytes on the wire. + * + * Performance posture: + * - Hashing runs in a dedicated worker with WASM SIMD128 — the UI thread + * never blocks, RAM stays constant (8 MiB slices). + * - Files below {@link INSTANT_UPLOAD_MIN_SIZE} skip the whole dance: + * two extra round-trips cost more than just uploading them. + * - Any failure (no WASM support, worker error, server miss, races) + * falls back silently to the normal byte upload — instant upload is + * an optimization, never a gate. + */ + +import { getCsrfHeaders } from '../../core/csrf.js'; + +/** + * Files smaller than this upload normally: hashing + two round-trips + * outweigh the transfer. 8 MiB matches the chunked-upload threshold's + * order of magnitude. + */ +export const INSTANT_UPLOAD_MIN_SIZE = 8 * 1024 * 1024; + +// Absolute URL on purpose — works in dev and in the release IIFE bundle +// (same pattern as the pdf.js loader in thumbnail.js). +const HASH_WORKER_URL = '/js/workers/hashWorker.js'; + +/** Hashing budget: 60 s base + 30 s per GB (WASM SIMD does ~0.5-1 GB/s). */ +const HASH_TIMEOUT_BASE_MS = 60000; +const HASH_TIMEOUT_PER_GB_MS = 30000; + +/** + * `false` once the environment proved unable to run the worker/WASM + * (old browser, blocked worker) — later files skip straight to the byte + * upload instead of failing the same way again. `null` = not yet known. + * @type {boolean | null} + */ +let _instantUploadUsable = null; + +/** + * Hash a file in a one-shot worker. Resolves `null` on any failure — + * the caller falls back to a normal upload. + * @param {File} file + * @returns {Promise} + */ +function hashFileInWorker(file) { + return new Promise((resolve) => { + /** @type {Worker} */ + let worker; + try { + worker = new Worker(HASH_WORKER_URL, { type: 'module' }); + } catch (_) { + _instantUploadUsable = false; + resolve(null); + return; + } + + const sizeGB = file.size / (1024 * 1024 * 1024); + const timeoutMs = HASH_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * HASH_TIMEOUT_PER_GB_MS; + + /** @param {string | null} hash */ + const settle = (hash) => { + clearTimeout(timer); + worker.terminate(); + resolve(hash); + }; + const timer = setTimeout(() => settle(null), timeoutMs); + + worker.onmessage = (event) => { + const data = /** @type {{ ok: boolean, hash?: string, error?: string }} */ (event.data); + if (!data.ok) { + // The worker ran but WASM failed (e.g. no SIMD128 support): + // a permanent environment property, don't retry per file. + _instantUploadUsable = false; + } + settle(data.ok && data.hash ? data.hash : null); + }; + worker.onerror = () => { + // Worker script failed to load/parse — permanent. + _instantUploadUsable = false; + settle(null); + }; + + worker.postMessage({ file }); + }); +} + +/** + * Ask the server whether the caller already owns content with this hash. + * @param {string} hash + * @returns {Promise} + */ +async function callerOwnsHash(hash) { + try { + const response = await fetch(`/api/dedup/check/${hash}`, { + headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate' } + }); + if (!response.ok) return false; + const body = /** @type {import('../../core/types.js').HashCheckAnswer} */ (await response.json()); + return body.exists === true; + } catch (_) { + return false; + } +} + +/** + * Try to register `file` as a zero-byte instant upload. + * + * Returns `null` whenever the byte upload should proceed (file too + * small, environment unusable, hash miss, lost race, transient errors). + * Returns an upload-result object compatible with the uploaders' + * `UploadAnswer` shape when the attempt is conclusive — success, quota + * exceeded, or name conflict (a byte upload would fail identically). + * + * @param {File} file + * @param {string | null | undefined} folderId + * @returns {Promise<{ ok: boolean, data?: any, errorMsg?: string, isQuotaError?: boolean } | null>} + */ +export async function tryInstantUpload(file, folderId) { + if (!folderId || file.size < INSTANT_UPLOAD_MIN_SIZE || _instantUploadUsable === false || typeof Worker === 'undefined') { + return null; + } + + const hash = await hashFileInWorker(file); + if (!hash) return null; + + if (!(await callerOwnsHash(hash))) return null; + + try { + const response = await fetch('/api/files/by-hash', { + method: 'POST', + headers: { + ...getCsrfHeaders(), + 'Content-Type': 'application/json', + 'Cache-Control': 'no-cache, no-store, must-revalidate' + }, + body: JSON.stringify( + /** @type {import('../../core/types.js').CreateFileByHash} */ ({ + name: file.name, + folder_id: folderId, + hash + }) + ) + }); + + if (response.status === 201) { + return { ok: true, data: await response.json() }; + } + + /** @type {string} */ + let errorMsg = `Instant upload failed (HTTP ${response.status})`; + try { + const body = await response.json(); + errorMsg = body.message || body.error || errorMsg; + } catch (_) {} + + if (response.status === 507) { + return { ok: false, isQuotaError: true, errorMsg }; + } + if (response.status === 409) { + // Duplicate name in the folder — a byte upload would hit the + // exact same conflict; surface it without transferring. + return { ok: false, errorMsg }; + } + // 404 (ownership race with a delete+GC), 4xx/5xx: fall back to the + // byte upload — the server dedups it on write anyway. + return null; + } catch (_) { + return null; + } +} diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js new file mode 100644 index 00000000..2a341978 --- /dev/null +++ b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js @@ -0,0 +1,261 @@ +/** + * Incremental BLAKE3 hasher. + * + * ```js + * const h = new Blake3Hasher(); + * h.update(chunkBytes); // repeat per slice + * const hex = h.finalizeHex(); + * ``` + */ +export class Blake3Hasher { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + Blake3HasherFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_blake3hasher_free(ptr, 0); + } + /** + * Bytes hashed so far — lets the worker report progress without + * tracking its own counter. + * @returns {number} + */ + count() { + const ret = wasm.blake3hasher_count(this.__wbg_ptr); + return ret; + } + /** + * Finish and return the lowercase hex digest (64 chars). The hasher + * can keep receiving `update` calls afterwards (BLAKE3 finalization + * is non-destructive), but the frontend treats it as terminal. + * @returns {string} + */ + finalizeHex() { + let deferred1_0; + let deferred1_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.blake3hasher_finalizeHex(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred1_0 = r0; + deferred1_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1); + } + } + /** + * Create a fresh hasher. + */ + constructor() { + const ret = wasm.blake3hasher_new(); + this.__wbg_ptr = ret; + Blake3HasherFinalization.register(this, this.__wbg_ptr, this); + return this; + } + /** + * Feed one slice of the file. + * @param {Uint8Array} data + */ + update(data) { + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.blake3hasher_update(this.__wbg_ptr, ptr0, len0); + } +} +if (Symbol.dispose) Blake3Hasher.prototype[Symbol.dispose] = Blake3Hasher.prototype.free; + +/** + * One-shot convenience for small buffers. + * @param {Uint8Array} data + * @returns {string} + */ +export function blake3Hex(data) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.blake3Hex(retptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1); + } +} +function __wbg_get_imports() { + const import0 = { + __proto__: null, + __wbg___wbindgen_throw_bbadd78c1bac3a77: (arg0, arg1) => { + throw new Error(getStringFromWasm0(arg0, arg1)); + } + }; + return { + __proto__: null, + './oxicloud_hash_wasm_bg.js': import0 + }; +} + +const Blake3HasherFinalization = + typeof FinalizationRegistry === 'undefined' + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry((ptr) => wasm.__wbg_blake3hasher_free(ptr, 1)); + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if ( + cachedDataViewMemory0 === null || + cachedDataViewMemory0.buffer.detached === true || + (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer) + ) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +function getStringFromWasm0(ptr, len) { + return decodeText(ptr >>> 0, len); +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function passArray8ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 1, 1) >>> 0; + getUint8ArrayMemory0().set(arg, ptr / 1); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +let WASM_VECTOR_LEN = 0; + +let wasmModule, wasmInstance, wasm; +function __wbg_finalize_init(instance, module) { + wasmInstance = instance; + wasm = instance.exports; + wasmModule = module; + cachedDataViewMemory0 = null; + cachedUint8ArrayMemory0 = null; + return wasm; +} + +async function __wbg_load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + } catch (e) { + const validResponse = module.ok && expectedResponseType(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn( + '`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n', + e + ); + } else { + throw e; + } + } + } + + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + } else { + return instance; + } + } + + function expectedResponseType(type) { + switch (type) { + case 'basic': + case 'cors': + case 'default': + return true; + } + return false; + } +} + +function initSync(module) { + if (wasm !== undefined) return wasm; + + if (module !== undefined) { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({ module } = module); + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead'); + } + } + + const imports = __wbg_get_imports(); + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + const instance = new WebAssembly.Instance(module, imports); + return __wbg_finalize_init(instance, module); +} + +async function __wbg_init(module_or_path) { + if (wasm !== undefined) return wasm; + + if (module_or_path !== undefined) { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({ module_or_path } = module_or_path); + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead'); + } + } + + if (module_or_path === undefined) { + module_or_path = new URL('oxicloud_hash_wasm_bg.wasm', import.meta.url); + } + const imports = __wbg_get_imports(); + + if ( + typeof module_or_path === 'string' || + (typeof Request === 'function' && module_or_path instanceof Request) || + (typeof URL === 'function' && module_or_path instanceof URL) + ) { + module_or_path = fetch(module_or_path); + } + + const { instance, module } = await __wbg_load(await module_or_path, imports); + + return __wbg_finalize_init(instance, module); +} + +export { __wbg_init as default, initSync }; diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm b/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm new file mode 100644 index 0000000000000000000000000000000000000000..7ab00d463942f1b339a82028fd04fccc5e01d2c5 GIT binary patch literal 45242 zcmd_Te{fybb>Dl=xj%sm5Emp!f+8gDdq_#7D3SmO0wm?wI7}p4krhY&qtn;d4hfQ= z2#^9m0Q@0K5#d;|VG~Va zr=C7dpEC8#6RY?6u6@q^fqOwpbk+P(lE67AG-;kJ`r8zkzYLLuIgUDdB}6&$u2w=^vJWnQc$u2t7^B;WN)9LR+*O+ z*xRn`p7QAGGyHl6_|Ze*9s3W=Uzs^|e*WU=<7cOq&K|!Evd2$8zJGS<-s8tFpM3nd ze$ULEetdfF`10As`OC*oo}4;;dgAb@k&{!W4o*!>1cltApYQL>mHMiMa@1Wa^mg}@ zH*G8y3dKU#hGH=+6iS6CA4OeVrAjF(7x`1_iK0SDcg1oLHk4u&?h6kS|6J+(X?6we0fjrhQ8*X+MO=g9re_DbSc+*b;;K@=n~ZWlAsplCf%UxbJ1)ZB-Pm) zm~4EP+o0!s_PoP$q}so6lW-Z>yk%?bT(9aBvpO}`qvtStjyts3>B|H|unf+Y8^05} zA%4ZXZuCsjd-KnY$HW`SLR=8VqAtGsM$h8^JNK#P9|5?>?Q+i1i5Md$3~sw)`_P?t z?T81}$vt%QSUi*r-1uAcnDpFuExhrI@lNa7-R--Q+{JNmSDd6P1H9P<#u?}GcD=U) zb-P{B-tBRrUGM8qXKx1%h4Q&j@>tU9{;1R+2$WX+Y8Xg z;yXOOcVzV5uUGqAkT@^v{TNqzd!Pft!>*XU9>G@9nxh?R9_&DxsB&8o3j7jP5{Lw2 zLb=!Naqy_Ed`|~JPJp%x_50kwI&2ub(eo?6^Y}k#{!xQN?mkCX9oV{6Ko0@-v3R@Z z()Nr?*44dkY#puG<1z6qw@IJ2Cnc{>W4_OA?kqgJfKmEdQHU9|w4tjYkzSHyI&Wr~& zy&89St;2(O-9BhG0bhq5Dj`B!P!>IILY$3|fv`X-b~W(bk(j7=WTG|(?Q%yP&D-eQq2_zt zKDT=v{_Pgvy|}DB>oC%kY@hX9^PH+>oI0YmpK>Pt813Hgu-!JD+T}(ZmfBXnzXP?R z1rNCsGIx41cV=XohrB1Fqsqp z{qC%DYk3G^TePyz9ih(szRvwwopfL|yWb7R(y^lr(|sajx0Ax%?-9H|L(o8sxqa&h zFD-9_I4+3txMph;DFvo-t+XF<K-?5;zsIxvo{a6LTDvn4oe9~WwlrR} zD&?pfbLI^)9k)7b?Z`r*VLVN0nr)bxQy5;~Oh#m-3WwVeOv~FK9+X~;!8bGZvGr*m zWu)#5It6jaTPElW@w*Le6WM|F88PgA(q7cmj6GBM4ko)h z!=kMd!$KfTwf3%WGj$N13CUV^==>1FuZ_@|j$56McC;UVk(y>3re=!!*Ef@q+xHh{ z6B+w%$6t(kmjn9Ji<{87-&@E59P~|x`vZhBH;spG!iZsl4BP|}m-k|s!W-LLO}1|A zZQM5H69}7??OWew>LA+mA_d_+O^41$(F^kinT}hXR)2B8kBNI3>{HWh!}LJ2DVp$F zht&~c!p=sWmbXEy`wJU}GxiziR$uuM{DoO2=r<$TkxsLa{piI_hh+lXOmW)KzO}!w zQM+#JnZg@7twVg>*xNd(`wN@DWLDq7X6hi?^dbdullWr9k9e7m8-lI=BDJG^n8|wn z!Xm)T6b-@F_+mtHW5;nMEpMZ}9$%PEWbEre`z_;(Q9lfW{w@7QY9XJfzpznT#%_UA zrtrr0kL@pPjQzy?g+<6W^%s^b_%resX6zaJZqHw&!|<*Bg~f!Qu)i>$B!7`6E7mpH zy0N#}Ox<6YYJFni*y*iwZBL$FP16%?fDBE zV?Qx}Vcy`T{vsU{|4iZwGxm&qx92a?VffbmBDIiDG`>i4ed#!oDg5ol7Z#I$V*bKL zyqo$9v!g#Fe_=L}vG4Zci*y*iwZBL$gje{@K5eu)N!JThhn1_FCeM^z zwJ~-#(}Kt1%|3szIm=(Pf@q67Ek4d#no_oQzK{8cd4rq!i*!ut@AGO@pMXby}rLtg?JB%oSTX)%aNECw@%FV z`*~`NP@5*;gI+HNGriEOcqix>VQuk+VG+BXex4cw>9Kgw>&0NE7Y1S|1JR}zicILi zw=`a~cCO_E8}as7^5G7T;2jx)dPVBk>}a>Ud)@hnDp0S}TBL!vM+JC^a3y2%U$~t(~U~70G!QG&|d#8(B9Ue zJY92cUUCJ_3E*r&9JaBgd5xKOzsNgVtJa2>HSZ=fI`pn(`$tMgTlr8&I)*bk+Ujri zbhMQZc7)@M4*F32HMmx-588Ej7gWB!J#<%HVB23@xH~Avg;_le#KqaTq>`SvJnKr% z_6{v2)yr{L@+<%TfBSb&&BooXxReZBjw`cqk819wjoOy=P>oCWFc|mF#v4>J5ckb$ z1C3VV4N#Q)mp}Rkf0)zjUIRqUs*9GA!OL;K+b|n%RQa~c@ut~$KoIVaqggkgE&@FU zfYw7b-aLz%Qj(81%p%a-D+Yf{^2cwz@-}pAb(;<)(% zchn&p?R6$vxk1j#eIw`4C5L^Yt%eB2hAL8IXfiDEj-r@<>>0~Y9THstiv ztzMROHd#4nHOhS>W8W^=gMJty9?NYG5p8o`Y}-V!@%@elszYv@VZm>@+kps7e((VZ z#wm2NAx}_klIQlD$rDV5#3c6Kq;xvzXkDI8GI8t3)5<%@vnC~|!5(a;MV?3pIT{f- zcF_sn*Oh0@t6Dp!GOpisQ+XN{s5I8QQ+hk(P$#(GE>BcAmI^YAHp`QmG`E+TrkLJR z4%;c;B|&O89=gziCDQpA@^sK?@@tYOs`t_5Sx0o2Fd33M?Ud3Znjr_1n&erNOls@M z)5<%@bElMGC+xv0TI7i%_!#nxp%YbW=Tye^k0DRfThs~eH_Njw8a8f9Z!^f5+`K%k zykVd9M#eq@Zs|e`mPqGg$WwZ2@@tZ(d9uvk-pW3kU&j58@-&&m>&Vl}JIWI! z*a>^EiWYeyosS_;>8*J#Mxe}-8K*L?e++q=-l9%$zoR^BQo-6f^0e}f@`Qc6&|7q& zMV?6KW5`o_Yw~N7r+Kp5kSCZBg&6lc%F|@>as6!?JJjTkTI7jzK88G{x8}K;U?UFQrk4QpRhY9VPN*u5@OryNA%FFQDNg85!whaWrBP&;>sR?kLx=KhGEC;H^5%vJShDN9Thi%v z1*0=$H#@;uVK(klkd0&WHH^;Cs#Kv9RV$=%H&uJ3$Gs`ndliI7q1azOQ9k}qdt zyb$6Q2pJN*ssfMl0kI1N*a)w_!H(NVzC98}geVb0@&V$2Vre4!E{OnW)T}Ix6Q&28 zP3V^Y*eremgG!LyEi8ag6i<57P+zY!JpqLx#!Pw1ZIT8V`m_aop2}{(+A?JkYIVv= zbXcZCb`Kj2uoUH6D9%${a%C${n{`thP;3CUU8djhXozRy2hS1DgFG8@JePT98Urp+ z$a9tFYUQ8p49i!e$;Mwc^!{o1aBmPza`)DUPd(ZjBtc(=2WDJha^*rTN|?XA_u^!n zd$2d3?5O3f>iZvlSbC&4V5W5s+k4BG9*=`_my+-T4a4Pg$){&)`AWj;bG4BA`6OCS z-ucT+(V14o1@%!Y@)Pt#)T$K$6C~TH7G65X?;Q_qU#_vSx)$U>)fM6b2PeQKF1rX^ z;b?S@`|u0g7ie9sg|2Ks;ZDuUdh!)Vs{yL1et~`3cV0^J7eJsI7e2dPrQ4}m0^M54 z6&~!3dV)%&!I)@LbpIH-3rW@3k^Cel@+uE+kE0}aDGo1jY9n{yL68WNt4kst0hf~8 zGUf07rt%RXZ+@&hksUE>jf)TUhNxI5UE>A~fCVCdcl7>?laKa-Zm&Iw-Y^w~ib2r@sU--z z5I!c@CtP6iy1iD)O}bL8?1~UcvCge%g3DZzhBT9Jrt&$!a%3**uVXVces2g&FCyWC}s9^LVknJGIU~^(3$simR&xjv(%sy z#TK0^!H~^{QT0+?MzxZ%i|7x9x@f#8`%5Y+aD@Xp&*kNV4z=nB@I4beH$Wb4$i~Xl zd$2c$9&)7jGHf-L>ugeZ+-s}CQ^Rx{zIjpj-?KGLpl82{fH}1LH0F^k%!=m;B02Za zb^@>%ys4l9)qU=plNy&@_{jEgJv7i+(Hmr$yu*;K*h6L!XgtKGlxk*d!rx0_8EboT zK3ovY7ifuQ(1R;H3Ze7`mnfBk`*6p2^rH2TtN8wJ3K3quqZ@NEcs#pTpfDc&q^bPJ zcK^PWulyC&_jTU4@(-;1IV<;dKCtp<|BB$eVCBBfvv&WymA_>7zRvS@|Du&&YpV02 z-M?(*uQt_r+3o>;-8%Y~Rd~Ir^0(~Xmw(&tzioBC+jRdOyZ?@ruQuI(&+flxpyfu6hB zWH$j9wO3YF@IB9eb5hYD0l#JeI3dkOFvT9mP+47Ol>*`l4`b_?BM;?SW**+az2#?S zPSB;;yl&nRL!O6+J!cJaxC@oaD_D;jDS4KodbpQJUd@@-q(Ng!AS2R8CJKs*$%$FT zBv;Il%Gb>@=O`u6!02qwG~r0JT*!@zVjsmIt&T;$`UhsD-dbrkH0xbjLZkwx5>SS3 zjNuC1!iRcGTuKTao*@Ryg9fGNaHl@fvT!F5@#&}&x`0uq+%oDAfbw2JR>6-tI_K|$ z$slQrI*O__h^mMVe3%6r8>0?5!C!@0HqN9|C0!t=+(Q&*+5;KJe1s~L8)0H8>B~5X zg4pnMu0|{>tw7}->dpCkVz7sL^H3Ga!xx#&C7~h~$$${BSk-hPnpIP>%&}_Km^kRQ z=Mtk;J?JJaLg*U+Evk>q5j-lk@Q9I|_u^5}^9W!mkD?|XMYT{ovLdFNo=0k#O($z8 zTWIXi9Hc%ZYh+d&-c!&Z#QnJ#K4MJXBs9cPh_;$)ipvfBvnd4{h6)z}-~;JUhT>rr zlRoTBu+nf|Hw3UkC2h`~>8(`BbO$;>#fSaeKm;=uYmR8LIg47^oJH?FBTFF$PLd4Y55Mu{cu*cLRWf573-ApqRp4cX^4;BBc(vV6lSV9Olzj5 zLB7q~MG_Uohuj(m_fi;dI`*Oev*RoAz_H+qg0L@G&{TL| zZ~?m|E~e&L_Fd$Tt&rLJ(pTewiCo!ja+`nYSoc@mz|&SUF1~i`!|*G|ieI(%alsy+ z{v{W^_7^a|oZCd@iCmRC`1W4ta!KyBWA3Y5-wS77dv+rCg39tLBO#eXkSctRLb}S~ z5%stdCjaWIs{zj|i1gdvSPiPfL8Urb;a8$LkukR3A!0tJy)Rf|w+okrS}zqPiFngQ?z&p_ z9u0n|YuC}>Pjs!go(C6`XfduF4Sqxc$nnpmc;~@8>os2$i(HrYeX8g=N-lAet5j`@ z&{m53$=wL@uEtgfu2pvC6x^ojMgU4OXh&iWnfCUhzCNmE9CgCfCg<3Z6cm zKIjh66FGPBH3=RF7r<-NvGD1M+*zqb&d`=J8ro>w2dG)HH-+&2^E~`Oinlw!%`?z{ zxvXh%kbEngmBf;t1hchnZ!-flLI|srY)ISd9!Z-1IzEHrbu~BX2C$FXrsOIR!%L5o zStV?txJWiKyu`G86B=}N_O-8=jmp7vUwW+ug(k^DiUrEd%pAW)*pqxuSPnw1?+sO5 zkKdI#uVq#!t7m<0raOqF7eYw!cXR2$dhyY85!Z0tL~$Tod?1usic;6*3RqT=+;nff z^#-Y3U;QytSO1Nh%ud8UOsd#wQWS!u|K3DaQ{RKJdtA&u45DV&AB8nOX|yl+PhoAN zu0IUH0(Z0rH08$@cqkqXu5j6SH24yis_XYwR6QCz3HtBBkp`k>MJWyW_d;X-#>|YU zN&_kv-GHlRX0)a>HKUa%wWIe!42fV1%dT!rEvYC=%CWnrXqMDxD#^-{X{0QSLDcNX z6gJ~9YZe3MKg?}Z*@SWnsF-oE60$2TaD76%t|vi&qocIZQ3^|j^4!y{c;=h%tms=j z4S*G()=L_|DgYL}Zqw6}!Py8i!u?`J=!xvod~zAz{0JG~0%1Aoj6%OvS7>9%N`$!N zTHb52QHiU?Mol8^)XB@pj~N4U)rOIyTbl6aW2=gePJPuZb+XCGmnL*|G4IPCMoy8`=ktiS5VdNrX1 z?pb#KLNvQe8I*te8+fh#$@^TYhk{D->tYbfhJN*GjpN=)@-zNk)ytA*yeS)~eXkPd7cT>=`7m zlCplVl47G&omfXUU@1PutpvKpT{pfMOYDxlobaXFYeaXZ5&i0=+f~=m2t#6HM@o(U zHp%zmbSoWpxwed?hFCX<5mYuzM6gVZ8-(sg$7iDAKEEfrFH<;47^YO!F5_~hdF$v* zNf6|ZFt&BhQQfX3IoXIFL-%{s{ ziKh1>a7F@XmYoFCfnOU?eCQP#Y9Zm6;!AIO)Ix7EIHos}4aiAs|2_2v2avH=2xNR| zh@!4Hsp5JBj5~dq-eihX=-0CzW0Ki+n~s>LXyXlAUPNz49l42bVeQ+H zGgB;PrfA41XoRnsUS*Ki@hzEDQDGtx-%R#$6CFvF)FaS=J0P%*j-;J+vX%ah1f}wB zzlaAa5{Tp$|Or<;Mh}li1BkE8;rX%8Y>f$@bC_OjlLT4Q@uh6C=OrbJw zq3%DbA7P;q9Vz(nrxEZ-fp3FH5(VK!Hj>{$N6e4(rwEviv_^2ZsUw{TT2DtTy!h{{ zBc1(7J%E(pT6Dys79R$by&oxOeng`l;pI&&rL&IIyUQ$FmX3yOP&k#9wdjaNg?f|u z1R?1MLi8RTsdyc!WPZeuyGnh##Q3 zCLa0e{D`-aPJV=Cmxgy~))9G;eiK5;Ye*^Mfd(P%U0lbPD#G*8XB$;EA8GRlq%kIo`fsis>Rvsld((|p zzOaFF9loR!#7<*OeX`k%^^M20zQ#;gcGX5~BXDZwr?sh8{$XFPay5J=RAT=19Qor$ zV%{g|n3=PI!&%08J4*pBJa&~5@)hBw*N$C%`pM^BiMt-{%|(+eCTZ5st`_b-Ah!)d z%ABGAN_lAAtDFERB(vzq6?6fUjSt<`qB6(Y=^MJPb5^%YX~L&{6L#|g&vl=p9^zEi zicMR|XGpTp6QD}To6q@_VPv=-z7Y4=o+MUEYQ5S(S5`(rizZ}FmAiC3bggX~Vfx;I zvQ}0UPetXtGs+EeL1L?xu%zcynHzlQ1@B3p5|-40IrY$!ZyqGM3VA;V4`$- znru>fP%LRKA)i%lNG!=6`Ug2Uv)(gpIrm>i4PkPc^*)qy-> zGt!hrP|=6fdu`_|W|n*yJmS|S;WD`oHdrEeP+)zH-G#-fcDAitomE>U|J43i{%Nwu z12ql_t@HGTRj7(*iJQEm&P!uoA7Uv9@3(x31A!f zN?~n{wFN81wKdijtd!Q)$O`(QwoCcMs?RqE%37kF$~|-6%`!Ve??M?7i`Es8Y=z8y zTww>1wiuKo9<7S-rEslqj3TsRthBgZv=H277nT0D79 z&zZ4ZS0&c>USmyaMHT4K1v;a!fmM_|1;AG_GF|(S>7qhSD>L$ilr*hF%3o3}McVMlI#hujy~&P6 zSJGmmWc`Bw4<_tLh@P+IutbiCNFsq&uJDk(S}7q@F20nKAJ)A%Lp2$4*IW7b8*)7i z-?pxPoptr7)9@|*unh<-9ry%%%(hU>S1y*lA_15w83=0HK!Uf`0O3Ndv}@f?n^0Jw z&O-;auZ6aeR6I}tYrNWH#KLb}vJcDC8Jp0l&IJw`yyUyT%J8y>;U%(VpHNHFb;(7@ zw@nwUaIHF6;aZIVXFzFHdciRureKGK|5*kwP+J5UE=#p?lr61B$u*X}3|_XnimIU5 zkzi;r3oq0);_E78z3n+&H=x~GF32yUWQAH@BCmj;!I@r0ZhGOYn7Jn>#we(ri`d*sG!=0!v^YcOMPD9x*7f zw5W^7I~Kq&ejAD17?VyzNPqJh3OIfuS;iTZ*hbza$4Ov36A>u~JZ5N{N2(F!^D^4qYb+AFL7m5n1+Fx9MPCU=k9f z?m+mUm|Ff~nIkKW82>7{w?i=O<7Sncl-i61yvfwS7{qf=8ckdyfYAXB$<8qlz00Dw z+*-(<2b@vHk90-1l?_HXd;XfqhUv!`dcD(V(47^tbUyf8*zuK_ha7!+a zZWJ<`kdauQzEn10%5jZ#@C;u5bv>lk;i4-4ACY#w-S47fY(-tEPDHyX(f=4V9*t3W zjl$wYbXcI0UATOlhyUxlip+314R$T8a%#}vLBa16`Mjium%g@Yvz%M4EGO7Nbu6g- zs5k1pT6#)c(b{!6&S_WgDno-v$^J6Qp>)McU2q`W5a;dONMwf{=-&s0&SW za>R)tA7}t)yI9tM9b*jSIxSKbkkA8BVN@@S(n3Y|(IsR{AmsP&6x95|-U8KE5BCwQkVr;dBE-!mxHnXm>zfJA>j4h9#S6`;t7P zL3ewwigWKnxyrRJ9BP+566=h*R^yF6`IN1~S?)`3oDKqW9VXDV+@anm)1jJ;Y@$*% zDv3AWc+|2$ZgWz!|BmNIPq^j}b$)@27CkD<(wyWW$ZP5)d)uLxEm<#H(q8x~R0>m> zspciDR+!n_j|WpUD_^Rsai#^&>}>~_{TZJ9&3J0Hw*C*9bTKAL17)|`kjQMEzx}N% zH0nh;vVK^{D-n{RlfJsKU8>DVUpBClY2Yk!t)S^5(dCW5BvG$Lrh{nuZ~OwPz?Yi% z6_7zwyNy}9jTsTD)vsFoQfi4mWwuX*i5Hm}+QwJ*C0~#~!49J&PsoO0 zcJpJq*`uWALXA^Lz69Wvi@j3tt@MM3uwvBUG0X@)EGY_+p9CN@VQ1(^Z&Mib}oZUdRI8>0QSlL;! zfPkqEAllZy28`nck^0Z66FVJY=|$^MCneaTEp%xwDB#7J4G%r!!8%|7&Jb+iM6YU^ z9VcmPM1Ya3z}<+C3Mew+q+I$@fgrhuCw){P=RYct%RVZgEpJRLIB3aUQhfj+S3gKD z$@!ziys~4y^lpcPg*JyWxo{Ms$6p8y(&}~&m@~&p^9HKXasoiq`V#;lC>hxjN}XJw zq(>n94MGxKV8Q{Vn9vNDU|iMtK)DCQk`o0uuRRZw)wYvGp(&gR9*NmWCV0nlo~TU+ zPIRg$OivYoBE5M0?ZknhsJl9fy7Q9pC((riN4jJ92y!}nB8fvTEb4ebC#NZ6>j51Q zYaBkoV}oDI0Ug94-omZ2?-V`}%#s9=!d|)H0%>o)mV(yz9zcihJ>>lN9-0hPq`P4`n!I5-|Tis_}AqJw0-sGHCcE%x~W5T>@B-Q~hlrVMge zv=p0kQWG}@6z&|pA(~V%hpFs@rz(P|7HVn`YY2FcN7;uv?3WIf(pX%`n)>tiwTPyW zV+V@VG)$YaTawSJ@GEVK<5yG(cD)lG3EXJSWGWc+*x3mkV@Bw?aK)G zF%Qi^%Y1R2oWqqzebish5w+7x$;t1qjWa!hp&A+iO5%yR8CdI|>=+hrP!tG3H5Cy- z4}O{fg=Gs0*jZ5>paGAZYxki)*{Aem%LScukaSs!kfSA}HvdPAEC?u*);iL2ZPgzC zRSB33>llO^jj4@d)N6CTP1wq|G_MdQHfDhhSop(G8ef3H`RY0frB*2Dj7DKQ&?S^r zy;0fl&5l742B{xTG{KIh<9MlSGP_TYQt1>NEHyM9RrX{($et4^w2TQwnBY%%+2JT7 z?vdU+W$+kV(hpM;3{V<^qwpssOxJRf4XG~nCv$A)LY(uuhT4QHv{wyi_CWNJUxoS>_xus*djG`8jw=60DOb#vaj!8D=X@R*2zJgRAu(}Gv3E|{HHMkwB~m%ZWVj(a?sEyr z2_4nUz51(@q{U#PJ-uX43oQ8+m;m_rF=ttnjtdRlX;YFF37OmGN3t%>k6LJ8kZI}g zYfRFE@qoUq>NP~0rsi@<$kK!CT`fFP*Lf{WM2xL2h%!r{W~Y)wM2Ebx6@pzJ6oKYb zuT}_WT2>l}D5!9qqzWlk*00mr!#a>bnwZ~Tk1Rz@kjKMwk+-Il0;xbvP4OuTq8+D>((TZp=90VEK$TDmnMq1 zQ=*8A5=C5gDgJFeJ7AwKDVuF7tq?N~=;%#pH__&1)IABN zzgGNT4N|)Jg@M_2f9AcrYwl@bByHBg?{L`Glh6rg(?Iu&4D)Ig2n;j5tV1h ze^XaPv4RVI%LzD5k+#*QV~en(`p~DS#`mNMTPBE0rJY)XQ0C!ylqqU7Xkf-bts;=Y z&8g{HT7(2>hY|-FjY*G-PU^Pl4rf3JqZhlWO!@Pmg$sy;GI3D8$vS_x>4 zpTP>IDSpoR_!$(^_?gw4rq8ORlr(=<1r2G$&qSmtRw<5autLP3)G`y1R%B<9Gp%b; z3Xz9WD2%lkf}%M=X_+i|C{kn?kcLFuc_r(sRIV5JKpcCPRs*7}bg*ML0WQFQV67`* zdaQt&Jy)~jCe#5PTAH;?E4ZJ8X$^FCH6Velu#Mwek_`K9d~4pVXgO5h?xLN&V|Cde ztDW8r`F8rKJ3}S&`Rt^w=8e2}*H3mm%B290h;#BQPmvbQ7-Y!c9e3N!Xw z?No-YPE;ahLwu+-dPorDz+|_tD*_)Ckyh<7Nqrg2M(_U>8^wx?HFNR0!g4_ z>w{$W^|2Ya%;CBOyCcBRFb&`FzKSPdC6(Fz!!Do=Z2ReD_kz&w!EE^PXoC_mQ*uzN zYBxciIHit!ZY}b}-zmZx;{$X@wB$T3ossw7$uWK_*~f^}%}QU)+yR_@dSl08ja=KF%m zYTj#j^<13y3XYoLNaV>_ThiwVaY3wzm@hXUb4gPIWs3Ydi5@JXFTqD#g3X9To{?Bs zO|Jetuj%&9bM$FCwvm4f;t`QutSfmXejMpSTnW`{s+Kh#i70QO% zYt;-SHM{Xp*_)=kl@&0;DRrS*1z(F1Ta%vXKoG*mw5GU9O+Is=nhgJ}$1yWwNkG0T zF+426s@aJ}{RxLJ2^V$;*t3++XFhFyCW|N5_71zOEFr7A2LcMg08Ck!Fn0RSsWE;S zBvnwzVHNbbxlRyDqJi1EkH}obl5pb8fiQ32dFj%ZB$SmPOkWZeV!IE78@N;+2&Ma& zympW@u^iy*aaxY60E1G_;Gw-X1R`GPOf(QRD^&hrJ}NOO@Uwr6Z-pf)CWD(VIZU^d z(nqQP(wxovtT4jY1fC^fnz|K(v(+TZvehKZhXbpU>;?QAn9TN+{miPfT&iFiujtCS z$(gltKK(55)L%Dps{tzA%??W3cXyKIRv9c&D$A{wMN%H6;1bjVv>QjL%&M0H+gQXR z3elXRllC!XS7sG`v4gpR3-&PX_PipdeE8&rJ(z8p7*0upCkiuJok&TOuTwLMz59UImA{eCMK-52@df*Wvo_6+ zx}<5(K7U&&NW=I#e!MhetMJ41mZfccNCvFgWM;Tv18=GPm$kpQdLy!-TyoXC2EK!J0pD(PpUjEzs29pxz=uQ7#O*dc};r}R5R?0#L;R(Y2le7DRbG-p7Nqj6MMS!QtP8dJ9 z$gpZGnE`&xWN^mnXu$~B0+8=Uu6(0FCv>Nxh{xBnn7=vD8{ z(IWx;Wx-nmil}5gP#eHz11nxu$5!6{cmL%5=q~`a=ayhe*{uh*TVSuf_*=hr zXC3g$U;CSX^0)sm6zFvPxe3O&{`Ei?1$yP#|MoZk!+-2=0KfLn-@N|jAO7l9fzRe6 zF5iI{!TOLRyi9Z~6;%FM*Pln7L4&<}d<96qtv zUSS$X)@=ixeOV!hwMA?J-$=Zu{dWO=Qb8Tb+;saLB^KuSW~7eO)DR<$Dr#3B$L)hO zxfCf;tSvG{_%ccNSI z5}#EuUgqFF4gpuCS*)^H`eZ}(XCz~*zXHDv#zHpz@jS54RJ{BRKmYlEB32!xhtVW% z@b^CCa}H$l`FUhJAL>>bUe95g-RG)2=q%F%pEX5cmFTB6?OBA-C!~CUTno(o*nS0# z=K4;WA}n%@E;wi3tRs4cB(OZE?pR8(5T7jpz|tqy6kICA)mTZa=`-OxYACa>S7fPr zrW%TagL)+iM9y8g+^iBMq_Qy$X8h1AFh1v`s?1I0qHR=W-fmsnM~vkWtFtl#3;|-B z(I9vhiy3{%Zrhpx{c1*HDTXY8o~!m~NM!%6L|yrM&Js?V85q}$bM-H@?&iz5ITe8n zC`yO?m$JcZfaIEe_9L`;hc-6}VP!AKzVwJ>;wMo^lUaUY2@#+3Wfapgc!-~-*z5)b z2jWnl`lU?r%>bi3q=m%|rzJJR0~A_5h|8=d(9Q}HGiZwDER10Ka&2TEC6+*y z3q^a$rZ}4yLcoS=p2HGuA$PE>uiCd)J-Ie$@y0j- zNj4u5A8ZMl7k!lstWr@Hiq}7+`<0xZ;JuB3;|AeHS(%Dj4tvn9Fl~)tmWk67A3$B9 zlR|x#G%oE6-um^)xV*~5E%+RyC_&FcyR@jluCCeS85vIz&-IxL$x6r$NX=n2yW`tp zm$*=(EJsm1Vp237eZFC)Z{a21cz%_2Rn16sDu|}4&u56lQ8s)qag(#<1BMqoDy7vr zp29`!Eoy?f79}Mu8Yt;$9hCU|V<(iJW8uP3qG=r^A%u=$YgnCT=b%jX1F1%n$r}p$ zUCQ4#vfY|mE7%oO{&j>skVj&coeo7(H`1-iAM5`ONV;oinvZ2LHLQ`CV~#$!oL3sQ zK0e|avrtC1Ac=GsFgG9{2Gwu>fgER2^j=E4$$_^3I7Kp0kwIeC1*Ci@NZr6uBr}Ju zM0{A=G$fyN0ax~Bo+i=%L8tK?kB>8IMz#ZPDGXamW9*cx@&L5V%&ys%aU55ah^6Xb zHtEKvR{LO;%o|k>D&G@d|EEmm=9Drq6+r!9KTpclnx{dR%+tV3yrUeQ#Oo_qE1s!P z`K>4liL22|*PQ(Uio}bD)8R+^5TPO?t!Se|)|3nGM>z$VLyvuaa4>!H-zKakzrV67 zr2>PT4hc3|8jh?b*w+l#Z~uROd2M@ne!zUh_W$+f?QeqlnwJ0C*rcPYtAvszWy$b$ zvUV2lxwAv4a0|l~KeU^8CFB^X&X1CcBrfYPM=V<`=qotR1~omxvGmcdE1`@|f6;q= zCX)*%H0dzW_w1PLYi!E;)yceSFQXt;bmD?4T3yNMlWF!JLa_2JtXwghd^O_FKp|vF z_y%c2ie{*!>FlTtxGga%2}d85VW|x9Ve&({bIUj4c6t7O^YdzuyqP|83|7-RuYMYu zJ#xD%)K<2w@8i_AYT*d+b6ASZ@Vg*VS= zrjZRI_;X+p?xUW^`u-f#>in_%bA$;wIrK33!LNg)nC9Cu%JpY}Ne!+C_&@x*rXIHm zSWeJ+m-aUzwzh~B%;!w#;whsnqoVc-p#>CFRS;Q0l1uU`S|KsQ6x^};Lmk`$%p}zG zK^?kT-Qj_1Cp+YcihUTbL2+c>H*UE_I}^jso5 z_X92Md;^9m8ywU^?tFxhx1S+GtR9H5Qvz!{RX z%0Z5)@}?BW=63I4kbyMEPI|RxVsw5_Z{K8O$?gv~ZuuHzwml%`T%1)M#9yWJ5VhD% z>vTUrtYAu!3h+B%XpO4gHP}=-|@}7*87xw>*aCWe-dMv2C6NX{aJ}MW=%q%1Zp7vmq`@UfN_^MsRt*?D;HAp^F z{fvaH4WHIAXsByq0-QeG3NKE0F!h@x0sT*ZgC^^LvG2FTY+PHd2fZ)TL^C09G-MNB zkWDDCKZqc@Prhvxb-v|C)EwgaePTfko>3CokcA?`MSXEshgx{j?J$LW<(F^vM#-SP z+|6B-miQ0!y5x6$c@;)s>dBA4fs#lkaY-Z9zc&B+4ccf~Whj}>iDsb4(U0|y?qNyJt>Q}=M zqngxT$syn(>1PSMs9)11O7edG(Xw|J{0sxXH9|Y>LYkYfHJwbq&Ztd9$om$;d*0(A zI$yOx7A8pbD;aLmK(xUaktzrbQBk^`SX0_m`83OALfM2*Xv(KV@JGVtOHuA>zMgrs z2;c8r_Po(dfLIGYo90SoSv&AYT!Rt5OH8<=^R0@XbWXJBD7}EwL~d$^3v>m2Df(Gg z;_J{)|9vTV`n~4GFjtp{1mT$>0YoXGvNEO+;=Nf}upQm;w3>ry;(hdysJ%j$UKc|} zIbtyipe}RSG8iQUHd=2DFG+h+Lhvi9&2YUh;BH^gRlQsKqQQIRikYwskDO`J)uG~L zbkBF3--Wz2LH-sShLs2!*B!VpV`l~`B}f2&?VFR{K6Xbhvqe?abd}O*$0`#-8)&do zG$rgWFoX!~Q8hHhtUeOJ=koxB6kDO>wWUfDc^tpd!DUnJ1cIyGe$OIX>1uu)HZ@eJ z4g1ZM?4k@d3Sq_G!VGdyIomL>RI&Z`ciZj4Nncplyn!$jYSK8E{P9fafAKDbpuT|1 zXE+yKXrRQvYKFwX`lk?t!RDd#|1HUj4N~HLRzZ)|XEd`bpS!+ef`&V&1!Vsbegmbw7Wh_0C<{}>tu;d@O+wB6x?p8* zoo-W#RmzQ{!E47l4`T!>K#)_i1d^1LDjzDHc4%}!;Nb_SI$?xE!8gGx`3 zyrJ(FZ&x4%SASm*|2)tWl|ZBVv-P`&R0EQK=QpS`SS{07VBj3#X$JGN1{06QhSF~m zbg3(O$e>ctS;r#Ry({syH%QN$HZgG0~;^N%&X*V_JrWY6I7v06V=_?D~HNYw?W^}PAyEGnpuAGK`}`s ztC4;mSiHEjeCojT*zn;aV-qLOoE{w>o;ou&divzS(b3aWXNJ#Aojx-?F+Dnd`ryga zBL~jUoLrn*eDc85`SbIq4lFJ5v%EMn_xS$BC1B0ZFCW-{YHIQE`2&m7kI(SxNvmT# z+ka|tYI%BT|IGZpkt1VAPMtY?YW(D(lZOsXoH=lLYUzo6C#m-M^xVGT{iFLwMh{xc z=_||Ai*t+9XM6`gEwYPKmychXK6T)U`O|fP2To5fEFCy~{t2VW_da5sXJtPnaVO7D zotr+mZ)E@B{f7)LOccm_PR%bodHl@c{1eAbMUJ|q`HPF_t5=v*TXwJR-Q!NrPcON- z`DORS)bgpb?)3E1^30sKfHjrFJ;9*Vbm`*B<;Ce#IT4WQ{YH0c9?O}TGrfOeehC%Y z;7&e?SlsS2QhT=wfFSsCSPdTGdIqDE{h8KGEiX?$v4CN@<$1RZO>X}3-1Oqo*_j2I z%@+}lyEJwF;PN>}F>C>lD4RrIDrWengpT7*;{ZTdeMXp2q5Pbzt z%$#E+`ya|;)jkI$bu!-IQ5!Z|tZE-oRy z%QJYPGbqscnJ-NrpPQdMF3ZheK7}(LZNlj*Ja2D${vOZr(OLa%^x6D=@0#b|rhYSz zF7v*N_NTBnkU87|S3myaI3ICpt*YGL=D6Ar(QJ3oQao$v~U!d^UcX6Doke&1z! zbZ@a5YzB`0(1rbom-=(7XGa}a9aQ-l9v&VU9vwb7JT`o2czk$b`0((Nk>Qb%k zA3l8K2q+$*d;BPXj!=mreCF7p@rlDnrcRzZJ$+{QN~0t>_On0t;6oY5zp$V&WG%tXn@%|nU-T#AqjdOZe&&nQXMT42{CPKZesOy0^pkbVZVXG=Fto6QX1P!QjI--r z_r%QHahd&D+Fiw#?*rc70l4&Fz1E*EYZ(P+FLunPa4I3xZ}2 zN(0;C^r?%BOEZ^D@8zRVtND|&Lb3fd=4$;P>#0C+PE8e#DdP`8U(&AbI;QuLhO8_!jKR&rLls9me6OcN{-{IUU`O zPn|w}e0lyj(a5QD$5XU}PjxLU&Y!+`3U3T6Wb8ZnWK_Hiw=XUg85GC&kM%SU Kl%;f_{Qm(B void) | null, + * postMessage: (message: unknown) => void }} + */ +const workerScope = /** @type {any} */ (self); + +/** + * Memoized WASM module (in-flight or settled), `default()` already run. + * Reset on failure so a later message can retry a transient load error. + * @type {Promise | null} + */ +let _wasmPromise = null; + +/** @returns {Promise} */ +function getWasm() { + if (!_wasmPromise) { + _wasmPromise = import(WASM_GLUE_URL) + .then(async (mod) => { + await mod.default(); + return mod; + }) + .catch((err) => { + _wasmPromise = null; + throw err; + }); + } + return _wasmPromise; +} + +workerScope.onmessage = async (event) => { + const file = /** @type {{ file: File }} */ (event.data).file; + try { + const wasm = await getWasm(); + const hasher = new wasm.Blake3Hasher(); + try { + for (let offset = 0; offset < file.size; offset += SLICE_BYTES) { + const end = Math.min(offset + SLICE_BYTES, file.size); + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const buffer = await file.slice(offset, end).arrayBuffer(); + hasher.update(new Uint8Array(buffer)); + } + workerScope.postMessage({ ok: true, hash: hasher.finalizeHex() }); + } finally { + hasher.free(); + } + } catch (err) { + workerScope.postMessage({ + ok: false, + error: err instanceof Error ? err.message : String(err) + }); + } +}; diff --git a/wasm/oxicloud-hash/Cargo.lock b/wasm/oxicloud-hash/Cargo.lock new file mode 100644 index 00000000..b053178b --- /dev/null +++ b/wasm/oxicloud-hash/Cargo.lock @@ -0,0 +1,184 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxicloud-hash-wasm" +version = "0.1.0" +dependencies = [ + "blake3", + "wasm-bindgen", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] diff --git a/wasm/oxicloud-hash/Cargo.toml b/wasm/oxicloud-hash/Cargo.toml new file mode 100644 index 00000000..0c54d4d1 --- /dev/null +++ b/wasm/oxicloud-hash/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "oxicloud-hash-wasm" +version = "0.1.0" +edition = "2021" +description = "BLAKE3 hashing for the OxiCloud web frontend — compiled from the exact same crate the server uses, so client-side hashes match server-side content addressing bit for bit." +publish = false + +# Standalone workspace root: this crate is built only by +# scripts/build-wasm.sh (wasm32 target) and must not join the server +# workspace — `cargo test --workspace` / clippy on the host have nothing +# useful to do with a wasm cdylib. +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Same major as the server's Cargo.toml. `wasm32_simd` enables the WASM +# SIMD128 kernels (~3-4× over the portable implementation); every +# evergreen browser since 2023 supports it, and the frontend falls back +# to a plain byte upload when instantiation fails. +blake3 = { version = "1.8.4", default-features = false, features = ["wasm32_simd"] } +wasm-bindgen = "0.2" + +[profile.release] +# Hashing throughput is the whole point of this module. +opt-level = 3 +lto = "fat" +codegen-units = 1 +strip = true diff --git a/wasm/oxicloud-hash/src/lib.rs b/wasm/oxicloud-hash/src/lib.rs new file mode 100644 index 00000000..91cb8d55 --- /dev/null +++ b/wasm/oxicloud-hash/src/lib.rs @@ -0,0 +1,98 @@ +//! BLAKE3 for the OxiCloud web frontend. +//! +//! Compiled from the same `blake3` crate the server uses, so a hash +//! computed in the browser equals the server's content address bit for +//! bit — the property the instant-upload path depends on. +//! +//! The API is incremental on purpose: the worker feeds the file in +//! slices (`Blob.slice().arrayBuffer()`), keeping RAM constant no matter +//! how large the file is. + +use wasm_bindgen::prelude::*; + +/// Incremental BLAKE3 hasher. +/// +/// ```js +/// const h = new Blake3Hasher(); +/// h.update(chunkBytes); // repeat per slice +/// const hex = h.finalizeHex(); +/// ``` +#[wasm_bindgen] +pub struct Blake3Hasher { + inner: blake3::Hasher, +} + +#[wasm_bindgen] +impl Blake3Hasher { + /// Create a fresh hasher. + #[wasm_bindgen(constructor)] + pub fn new() -> Blake3Hasher { + Blake3Hasher { + inner: blake3::Hasher::new(), + } + } + + /// Feed one slice of the file. + pub fn update(&mut self, data: &[u8]) { + self.inner.update(data); + } + + /// Finish and return the lowercase hex digest (64 chars). The hasher + /// can keep receiving `update` calls afterwards (BLAKE3 finalization + /// is non-destructive), but the frontend treats it as terminal. + #[wasm_bindgen(js_name = finalizeHex)] + pub fn finalize_hex(&self) -> String { + self.inner.finalize().to_hex().to_string() + } + + /// Bytes hashed so far — lets the worker report progress without + /// tracking its own counter. + pub fn count(&self) -> f64 { + self.inner.count() as f64 + } +} + +impl Default for Blake3Hasher { + fn default() -> Self { + Self::new() + } +} + +/// One-shot convenience for small buffers. +#[wasm_bindgen(js_name = blake3Hex)] +pub fn blake3_hex(data: &[u8]) -> String { + blake3::hash(data).to_hex().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The vector the frontend smoke test uses — also proves the wasm + /// build hashes identically to the server (same crate, same output). + #[test] + fn hello_world_vector() { + let hasher = { + let mut h = Blake3Hasher::new(); + h.update(b"Hello, "); + h.update(b"World!"); + h + }; + assert_eq!( + hasher.finalize_hex(), + "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + ); + assert_eq!( + blake3_hex(b"Hello, World!"), + "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + ); + } + + #[test] + fn empty_input_vector() { + assert_eq!( + blake3_hex(b""), + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + ); + } +} From 44967da7f1fa1667edd933fe7bd3a20e583a612e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 14:34:02 +0000 Subject: [PATCH 4/6] Delta-upload protocol: negotiate chunks by hash, upload only what changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the delta-sync plan, server side. The CDC store already shares unchanged chunks between file versions after the bytes arrive; these three stateless endpoints move that detection to the client, so editing a few bytes of a large file uploads ~1 MiB instead of the whole file: - POST /api/files/delta/negotiate — given the file's chunk hashes, answer which ones the caller must upload. User-scoped and purely advisory. - PUT /api/files/delta/chunks — missing chunks as [u32 BE len][bytes] frames (streaming parse, ≤1 MiB per frame, chunk_max_bytes per request). Every hash is recomputed server-side; chunks land as ref_count=0 orphans that a commit pins or the periodic GC sweeps — no session table. - POST /api/files/delta/commit — pin one reference per distinct chunk with a single UPDATE…RETURNING restricted to chunks the caller is entitled to (reachable through their own non-trashed files, or unreferenced orphans); anything else returns 409 {still_missing} for the client to upload and retry. The pinned sequence is then RE-READ and the whole-file BLAKE3 recomputed before any manifest exists — a declared file_hash is never trusted, because a forged manifest would poison future whole-file dedup hits for other users. The manifest accounting is shared with the byte path (attach_manifest, extracted from store_from_stream); the file row is created (201) or its content swapped by file_id (200). Owners of the exact file_hash short-circuit to a pure reference bump. Supporting pieces: GIN index on chunk_manifests.chunk_hashes (containment probes were sequential scans), claimable/pin/release/store-loose/verify primitives on DedupService, update-by-id with Update-permission AuthZ on FileUploadService, per-caller rate limiter (240/min), audit events with stable reasons (rate_limited, chunk_verification_failed, file_hash_mismatch), OpenAPI + docs/delta-upload-protocol.md, framing parser unit tests and a PG-gated integration suite covering the entitlement matrix (owned/foreign/orphan/unknown), orphan registration and the verification read. Verified end-to-end against PostgreSQL 16 with a node client hashing via the vendored WASM: a 24 MB file delta-committed in 96 fixed-size chunks; a 3-byte edit then negotiated missing 1/96 and synced with 278 KB on the wire vs 24 MB (98.9% saved), downloading byte-identical. A second user probing the same chunks got nothing (negotiate: all missing; commit: 409 with all 96 still withheld); a forged file_hash returned 400 plus the audit line; a commit referencing one never-uploaded chunk returned 409 naming exactly that hash; the GIN index serves containment probes (Bitmap Index Scan) once the planner favors it. https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- docs/delta-upload-protocol.md | 139 ++++ .../20260628000000_delta_upload_gin_index.sql | 13 + .../services/delta_upload_service.rs | 477 ++++++++++++++ .../services/file_upload_service.rs | 65 ++ src/application/services/mod.rs | 1 + src/common/di.rs | 25 + src/infrastructure/services/dedup_service.rs | 609 +++++++++++++++++- .../api/handlers/delta_upload_handler.rs | 322 +++++++++ src/interfaces/api/handlers/mod.rs | 1 + src/interfaces/api/mod.rs | 10 + src/interfaces/api/routes.rs | 6 + 11 files changed, 1641 insertions(+), 27 deletions(-) create mode 100644 docs/delta-upload-protocol.md create mode 100644 migrations/20260628000000_delta_upload_gin_index.sql create mode 100644 src/application/services/delta_upload_service.rs create mode 100644 src/interfaces/api/handlers/delta_upload_handler.rs diff --git a/docs/delta-upload-protocol.md b/docs/delta-upload-protocol.md new file mode 100644 index 00000000..31fd58ff --- /dev/null +++ b/docs/delta-upload-protocol.md @@ -0,0 +1,139 @@ +# Delta-Upload Protocol + +Upload only what changed. The server's dedup store already splits every +file into content-defined chunks (FastCDC, 64 KB – 1 MiB, avg 256 KB, +BLAKE3-addressed) and shares unchanged chunks between file versions — +but a classic upload still transfers every byte just for the server to +discard the known ones. This protocol moves the "which chunks are new?" +question to the client, so unchanged bytes never cross the wire. + +Editing a few bytes of a 500 MB file re-uploads ~1 MiB instead of +500 MB. + +## Who can use it + +Any authenticated API client. The OxiCloud web frontend adopts it in a +later phase; generic WebDAV/NextCloud clients cannot (their protocols +have no delta concept) — they keep uploading full bytes, and the server +keeps deduplicating those on write. + +Chunk boundaries are the **client's choice**: matching the server's +FastCDC parameters maximizes cross-version sharing, but any split with +chunks of 1 byte … 1 MiB is valid — correctness is guaranteed by +server-side verification, not by the chunking scheme. + +## The three steps + +### 1. `POST /api/files/delta/negotiate` + +```json +{ "chunks": [ { "h": "", "s": 262144 }, … ] } +``` + +Response — the distinct chunk hashes the caller must upload, in +first-occurrence order: + +```json +{ "missing": [ "", … ] } +``` + +The answer is **user-scoped**: a chunk counts as available only when one +of the *caller's own* (non-trashed) files already references it. The +endpoint is purely advisory — the commit re-checks entitlement +atomically, so a stale or spoofed answer can never leak content. + +### 2. `PUT /api/files/delta/chunks` + +Body: `application/octet-stream`, a sequence of frames + +``` +[u32 length, big-endian][length bytes] …repeated… +``` + +- one frame per chunk, each 1 byte … 1 MiB (the CDC maximum), +- whole request capped by `OXICLOUD_CHUNK_MAX_BYTES` (default 100 MB) — + split larger deltas across requests. + +The server **recomputes BLAKE3 of every frame itself** (a declared hash +is never trusted for content addressing) and registers the chunks as +unreferenced orphans (`ref_count = 0`). Response: + +```json +{ "received": [ { "h": "", "s": 262144 }, … ] } +``` + +Compare against your own hashes to catch corruption before committing. +Abandoned uploads need no cleanup call: the periodic GC sweeps +zero-reference chunks. + +### 3. `POST /api/files/delta/commit` + +```json +{ + "file_hash": "", + "chunks": [ { "h": "…", "s": 262144 }, … ], // full sequence, in order + "name": "video.mp4", "folder_id": "" // create mode + // — or — + "file_id": "" // update (replace content) +} +``` + +Server-side, in order: + +1. **AuthZ** — `Create` on the folder (create mode) or `Update` on the + file (update mode); quota on the logical size. +2. **Pin** — one atomic `UPDATE … RETURNING` takes a reference on every + distinct chunk the caller is *entitled* to: chunks reachable through + the caller's own files, or unreferenced orphans (the just-uploaded + state). Anything else → `409 { "still_missing": […] }`: upload + exactly those and retry the same commit. +3. **Verify** — the pinned sequence is re-read and the whole-file BLAKE3 + recomputed. A mismatch releases the pins and returns 400 (and an + audit event): the declared `file_hash` is never trusted, because a + forged manifest would poison future whole-file dedup hits for *other + users* uploading the genuine content. +4. **Attach** — the manifest is inserted with the same accounting as the + streaming byte path (a concurrent identical commit resolves via + `ON CONFLICT`: the loser's references are released and it becomes a + dedup hit). +5. **Row** — the file is created (`201`, body = FileDto) or its content + swapped (`200`). + +If the caller already owns the exact `file_hash`, the commit +short-circuits to a pure reference bump — chunks aren't even looked at +(same as `POST /api/files/by-hash`). + +## Security model + +- **No content oracle.** Possession is proven per chunk: without bytes + you can only claim what your own files already reference. Probing + someone else's chunk hashes yields `still_missing`, indistinguishable + from the hash never existing. +- **No manifest poisoning.** `file_hash` and every chunk hash are + recomputed server-side before becoming addressable. +- **Bounded resources.** Per-frame cap 1 MiB, per-request cap + `OXICLOUD_CHUNK_MAX_BYTES`, whole-file cap `OXICLOUD_MAX_UPLOAD_SIZE`, + per-caller rate limit (240 delta requests/min), quota enforced at + commit. Orphan chunks are GC-swept. +- **Audit.** Rejections emit `delta_upload.rejected` with stable + `reason` keys: `rate_limited`, `chunk_verification_failed`, + `file_hash_mismatch`. AuthZ denials surface as the engine's standard + `authz.denied`. + +## Error summary + +| Status | Meaning | Client action | +|---|---|---| +| 400 | malformed framing/hashes/sizes, or `file_hash` mismatch | fix and retry from step 1 | +| 404 | folder/file not found or not accessible | — | +| 409 | `{"still_missing": […]}` | PUT those chunks, retry the commit | +| 429 | rate limited | back off | +| 507 | quota exceeded | — | + +## Cost notes + +- `negotiate` is one indexed query (GIN over manifest chunk arrays). +- `commit` performs one sequential server-side read of the full logical + file for verification — cheap on local backends, a full object read on + S3/Azure. Still strictly cheaper than receiving the bytes, and the + client's bandwidth saving is unaffected. diff --git a/migrations/20260628000000_delta_upload_gin_index.sql b/migrations/20260628000000_delta_upload_gin_index.sql new file mode 100644 index 00000000..ef742b16 --- /dev/null +++ b/migrations/20260628000000_delta_upload_gin_index.sql @@ -0,0 +1,13 @@ +-- Delta-upload protocol: chunk-level ownership lookups. +-- +-- The negotiate/commit endpoints answer "which of these N chunk hashes may +-- this caller claim without uploading bytes?" — a chunk is claimable when a +-- manifest of one of the caller's (non-trashed) files contains it. That +-- containment test (`chunk_hashes @> ARRAY[hash]`) would be a sequential +-- scan over storage.chunk_manifests without an index; GIN makes each probe +-- an index lookup. +-- +-- Plan B if this disappoints at scale: a normalized +-- storage.manifest_chunks(file_hash, chunk_hash) join table. +CREATE INDEX IF NOT EXISTS idx_chunk_manifests_chunk_hashes_gin + ON storage.chunk_manifests USING GIN (chunk_hashes); diff --git a/src/application/services/delta_upload_service.rs b/src/application/services/delta_upload_service.rs new file mode 100644 index 00000000..68a463e0 --- /dev/null +++ b/src/application/services/delta_upload_service.rs @@ -0,0 +1,477 @@ +//! Delta-upload protocol — "upload only what changed". +//! +//! The CDC dedup store already shares unchanged chunks between file +//! versions *after* the bytes arrive; this protocol moves that detection +//! to the client side so unchanged bytes never cross the wire: +//! +//! 1. `negotiate`: the client sends the chunk hashes that compose its +//! file; the server answers which of them it cannot claim — only those +//! need uploading. +//! 2. `chunks`: the client uploads the missing chunks (raw frames). The +//! server recomputes every hash itself and registers the chunks as +//! unreferenced (`ref_count = 0`) orphans — pinned by the commit that +//! follows, or swept by the periodic GC if the client never returns. +//! 3. `commit`: the server pins one reference per distinct chunk (only +//! chunks the caller owns or unreferenced orphans — see the security +//! notes in `dedup_service.rs`), **re-reads the proposed sequence and +//! recomputes the whole-file BLAKE3** (a declared hash is never +//! trusted: a forged manifest would poison future whole-file dedup +//! hits for other users), attaches the manifest with the same +//! accounting as the streaming ingest, and creates or updates the +//! file row. +//! +//! Stateless by design: there is no session table. Every step re-derives +//! its facts from the chunk store, and the GC reclaims anything a client +//! abandons mid-protocol. + +use std::collections::HashSet; +use std::sync::Arc; + +use bytes::Bytes; +use futures::Stream; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; +use uuid::Uuid; + +use crate::application::dtos::file_dto::FileDto; +use crate::application::ports::authorization_ports::AuthorizationEngine; +use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; +use crate::application::ports::storage_ports::StorageUsagePort; +use crate::application::services::file_upload_service::FileUploadService; +use crate::application::services::storage_usage_service::StorageUsageService; +use crate::common::errors::DomainError; +use crate::common::mime_detect::{MAGIC_BYTES_LEN, refine_content_type}; +use crate::domain::services::authorization::{Permission, Resource, Subject}; +use crate::infrastructure::services::dedup_service::{CDC_MAX_CHUNK, DedupService}; +use crate::infrastructure::services::pg_acl_engine::PgAclEngine; + +// ── Wire DTOs ──────────────────────────────────────────────────────────────── + +/// One chunk reference: `h` = BLAKE3 hex (64 chars), `s` = size in bytes. +/// Field names are deliberately terse — a 10 GB file is ~40 000 of these. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ChunkRef { + /// BLAKE3 hash of the chunk (64 hex chars). + pub h: String, + /// Chunk size in bytes (1 ..= 1 MiB). + pub s: u64, +} + +/// Request body of `POST /api/files/delta/negotiate`. +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeltaNegotiateRequest { + /// The file's chunks, in order (duplicates allowed — repeated content). + pub chunks: Vec, +} + +/// Response of `POST /api/files/delta/negotiate`. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaNegotiateResponse { + /// Distinct chunk hashes the caller must upload (first-occurrence order). + pub missing: Vec, +} + +/// Response of `PUT /api/files/delta/chunks` — the server-computed identity +/// of every received frame, in wire order. Clients compare against their +/// own hashes to detect corruption before committing. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaChunksResponse { + pub received: Vec, +} + +/// Request body of `POST /api/files/delta/commit`. +/// +/// Exactly one of (`name` + `folder_id`) or `file_id` selects the mode: +/// create a new file, or replace an existing file's content. +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeltaCommitRequest { + /// BLAKE3 of the complete file (verified server-side, never trusted). + pub file_hash: String, + /// Full chunk sequence, in file order (per occurrence). + pub chunks: Vec, + /// Create mode: file name (basename). + pub name: Option, + /// Create mode: target folder (caller needs Create permission). + pub folder_id: Option, + /// Update mode: file whose content is replaced (caller needs Write). + pub file_id: Option, +} + +/// Resolved commit mode after request validation. +enum CommitMode { + Create { name: String, folder_id: String }, + Update { file_id: String }, +} + +/// Outcome of a commit attempt. +pub enum DeltaCommitOutcome { + /// The file row exists; `created` distinguishes 201 from 200. + Done { file: FileDto, created: bool }, + /// Some chunks could not be pinned (GC race, skipped negotiate, or + /// chunks the caller may not claim). The client uploads exactly these + /// and retries the same commit. + StillMissing(Vec), +} + +// ── Service ────────────────────────────────────────────────────────────────── + +/// Orchestrates the three delta-upload steps. All authorization lives here +/// (service layer), per the project's AuthZ rule; handlers only +/// authenticate, rate-limit and translate the wire format. +pub struct DeltaUploadService { + dedup: Arc, + uploads: Arc, + quota: Arc, + authz: Arc, + /// Whole-file ceiling — same `max_upload_size` that bounds byte uploads. + max_total_size: u64, +} + +impl DeltaUploadService { + pub fn new( + dedup: Arc, + uploads: Arc, + quota: Arc, + authz: Arc, + max_total_size: u64, + ) -> Self { + Self { + dedup, + uploads, + quota, + authz, + max_total_size, + } + } + + /// Most chunks a single request may reference: the whole-file ceiling + /// divided by the smallest possible CDC chunk, with headroom for + /// fixed-size client chunkers. + fn max_chunk_count(&self) -> usize { + (self.max_total_size as usize + / crate::infrastructure::services::dedup_service::CDC_MIN_CHUNK) + .saturating_mul(2) + .max(1024) + } + + /// Shape-validate a chunk list: hash format, per-chunk size bounds, + /// count and total ceilings. Returns the total size. + fn validate_chunk_list(&self, chunks: &[ChunkRef]) -> Result { + if chunks.len() > self.max_chunk_count() { + return Err(DomainError::validation_error(format!( + "Too many chunks: {} (maximum {})", + chunks.len(), + self.max_chunk_count() + ))); + } + let mut total: u64 = 0; + for chunk in chunks { + if !is_valid_hash(&chunk.h) { + return Err(DomainError::validation_error( + "Invalid chunk hash format. Expected BLAKE3 (64 hex characters)", + )); + } + if chunk.s == 0 || chunk.s > CDC_MAX_CHUNK as u64 { + return Err(DomainError::validation_error(format!( + "Chunk size {} out of bounds (1 ..= {CDC_MAX_CHUNK})", + chunk.s + ))); + } + total = total.saturating_add(chunk.s); + } + if total > self.max_total_size { + return Err(DomainError::validation_error(format!( + "Declared total of {total} bytes exceeds the {}-byte upload ceiling", + self.max_total_size + ))); + } + Ok(total) + } + + /// Step 1: which of these chunks must the caller upload? + /// + /// Purely advisory and user-scoped — the commit re-checks entitlement + /// atomically, so a stale answer can never leak content. + pub async fn negotiate_with_perms( + &self, + caller_id: Uuid, + request: &DeltaNegotiateRequest, + ) -> Result { + self.validate_chunk_list(&request.chunks)?; + + let distinct = distinct_hashes(&request.chunks); + let claimable = self.dedup.claimable_chunks(caller_id, &distinct).await?; + let missing = distinct + .into_iter() + .filter(|h| !claimable.contains(h)) + .collect(); + Ok(DeltaNegotiateResponse { missing }) + } + + /// Step 2: store uploaded chunk frames. Hashes are computed + /// server-side; chunks land as unreferenced orphans awaiting a commit. + pub async fn receive_chunks(&self, frames: S) -> Result + where + S: Stream> + Send, + { + let received = self.dedup.store_loose_chunks(frames).await?; + Ok(DeltaChunksResponse { + received: received + .into_iter() + .map(|(h, s)| ChunkRef { h, s }) + .collect(), + }) + } + + /// Step 3: pin → verify → attach manifest → create/update the file row. + pub async fn commit_with_perms( + &self, + caller_id: Uuid, + request: DeltaCommitRequest, + ) -> Result { + // ── Shape ───────────────────────────────────────────────── + if !is_valid_hash(&request.file_hash) { + return Err(DomainError::validation_error( + "Invalid file_hash format. Expected BLAKE3 (64 hex characters)", + )); + } + let total_size = self.validate_chunk_list(&request.chunks)?; + + let mode = match (&request.name, &request.folder_id, &request.file_id) { + (Some(name), Some(folder_id), None) => { + let name = sanitize_file_name(name)?; + CommitMode::Create { + name, + folder_id: folder_id.clone(), + } + } + (None, None, Some(file_id)) => CommitMode::Update { + file_id: file_id.clone(), + }, + _ => { + return Err(DomainError::validation_error( + "Provide either name + folder_id (create) or file_id (update)", + )); + } + }; + + // ── AuthZ first: nothing is pinned for callers who may not write ── + match &mode { + CommitMode::Create { folder_id, .. } => { + let folder_uuid = Uuid::parse_str(folder_id) + .map_err(|_| DomainError::not_found("Folder", folder_id.clone()))?; + self.authz + .require( + Subject::User(caller_id), + Permission::Create, + Resource::Folder(folder_uuid), + ) + .await?; + } + CommitMode::Update { file_id } => { + let file_uuid = Uuid::parse_str(file_id) + .map_err(|_| DomainError::not_found("File", file_id.clone()))?; + self.authz + .require( + Subject::User(caller_id), + Permission::Update, + Resource::File(file_uuid), + ) + .await?; + } + } + + // ── Quota on the logical size (same semantics as a byte upload) ── + self.quota + .check_storage_quota(caller_id, total_size) + .await?; + + // ── Whole-file fast path: caller already owns this exact content ── + // Mirrors the instant-upload endpoint: a reference bump, no chunk + // work at all. Ownership is required — an existing-but-foreign + // manifest must be earned through the pin + verify path below. + if self + .dedup + .user_owns_blob_reference(&request.file_hash, &caller_id.to_string()) + .await + && let Some(metadata) = self.dedup.get_blob_metadata(&request.file_hash).await + { + self.dedup.add_reference(&request.file_hash).await?; + let blob = StoredBlob { + hash: request.file_hash.clone(), + size: metadata.size, + is_new_blob: false, + }; + let file = self + .register_row(caller_id, &mode, metadata.content_type, blob) + .await?; + return Ok(DeltaCommitOutcome::Done { + file, + created: matches!(mode, CommitMode::Create { .. }), + }); + } + + // ── Pin: atomically take one reference per distinct entitled chunk ── + let distinct = distinct_hashes(&request.chunks); + let pinned = self + .dedup + .pin_claimable_chunks(caller_id, &distinct) + .await?; + if pinned.len() != distinct.len() { + let still_missing: Vec = distinct + .iter() + .filter(|h| !pinned.contains(*h)) + .cloned() + .collect(); + let pinned_vec: Vec = pinned.into_iter().collect(); + self.dedup.release_pinned_chunks(&pinned_vec).await; + tracing::debug!( + "Delta commit: {} of {} chunks not claimable — client must upload them", + still_missing.len(), + distinct.len() + ); + return Ok(DeltaCommitOutcome::StillMissing(still_missing)); + } + + // ── Verify: the declared file_hash is recomputed from the pinned + // bytes before any manifest row can exist. ── + let verification = self + .dedup + .hash_chunk_sequence( + &request + .chunks + .iter() + .map(|c| (c.h.clone(), c.s)) + .collect::>(), + MAGIC_BYTES_LEN, + ) + .await; + let (computed_hash, head) = match verification { + Ok(v) => v, + Err(e) => { + self.dedup.release_pinned_chunks(&distinct).await; + tracing::info!( + target: "audit", + event = "delta_upload.rejected", + reason = "chunk_verification_failed", + caller_id = %caller_id, + file_hash = %request.file_hash, + "👮🏻‍♂️ Delta commit rejected: chunk sequence failed verification read", + ); + return Err(e); + } + }; + if computed_hash != request.file_hash { + self.dedup.release_pinned_chunks(&distinct).await; + tracing::info!( + target: "audit", + event = "delta_upload.rejected", + reason = "file_hash_mismatch", + caller_id = %caller_id, + declared_hash = %request.file_hash, + computed_hash = %computed_hash, + "👮🏻‍♂️ Delta commit rejected: declared file_hash does not match the chunk sequence", + ); + return Err(DomainError::validation_error( + "file_hash does not match the chunk sequence", + )); + } + + // ── Attach the manifest (shared accounting with the byte path) ── + let display_name = match &mode { + CommitMode::Create { name, .. } => name.clone(), + CommitMode::Update { file_id } => file_id.clone(), + }; + let content_type = match refine_content_type(&head, &display_name, "") { + ct if ct.is_empty() => "application/octet-stream".to_string(), + ct => ct, + }; + let chunk_hashes: Vec = request.chunks.iter().map(|c| c.h.clone()).collect(); + let chunk_sizes: Vec = request.chunks.iter().map(|c| c.s).collect(); + let attached = self + .dedup + .attach_manifest( + &request.file_hash, + &chunk_hashes, + &chunk_sizes, + total_size, + Some(content_type.clone()), + &distinct, + ) + .await?; + + let blob = StoredBlob { + hash: request.file_hash.clone(), + size: attached.size(), + is_new_blob: !attached.was_deduplicated(), + }; + let file = self + .register_row(caller_id, &mode, Some(content_type), blob) + .await?; + Ok(DeltaCommitOutcome::Done { + file, + created: matches!(mode, CommitMode::Create { .. }), + }) + } + + /// Create or update the file row against a blob reference the commit + /// already holds (the registration paths release it on failure). + async fn register_row( + &self, + caller_id: Uuid, + mode: &CommitMode, + content_type: Option, + blob: StoredBlob, + ) -> Result { + match mode { + CommitMode::Create { name, folder_id } => { + let content_type = + content_type.unwrap_or_else(|| "application/octet-stream".to_string()); + self.uploads + .upload_file_streaming( + name.clone(), + Some(folder_id.clone()), + content_type, + blob, + ) + .await + } + CommitMode::Update { file_id } => { + self.uploads + .update_file_content_by_id_with_perms(caller_id, file_id, blob) + .await + } + } + } +} + +/// 64 lowercase/uppercase hex characters. +fn is_valid_hash(hash: &str) -> bool { + hash.len() == 64 && hash.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// Basename only — same path-traversal guard as the upload handlers. +fn sanitize_file_name(name: &str) -> Result { + let base = name + .rsplit('/') + .next() + .unwrap_or(name) + .rsplit('\\') + .next() + .unwrap_or(name) + .trim(); + if base.is_empty() { + return Err(DomainError::validation_error("File name must not be empty")); + } + Ok(base.to_string()) +} + +/// Distinct hashes in first-occurrence order. +fn distinct_hashes(chunks: &[ChunkRef]) -> Vec { + let mut seen = HashSet::new(); + chunks + .iter() + .filter(|c| seen.insert(c.h.as_str())) + .map(|c| c.h.clone()) + .collect() +} diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 02ac99a0..fabcb5c1 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -238,6 +238,71 @@ impl FileUploadService { Ok(dto) } + /// Swap an existing file's content to an already-ingested blob — the + /// update mode of the delta-upload commit. The caller needs `Write` + /// permission on the file; the blob reference is consumed (released on + /// failure by the write port, like every other registration path). + pub async fn update_file_content_by_id_with_perms( + &self, + caller_id: Uuid, + file_id: &str, + blob: StoredBlob, + ) -> Result { + let Some(InstantUploadDeps { authz, .. }) = &self.instant_upload else { + return Err(DomainError::internal_error( + "FileUpload", + "instant upload is not wired (authz/dedup/quota missing)", + )); + }; + let Some(file_read) = &self.file_read else { + return Err(DomainError::internal_error( + "FileUpload", + "read port is not wired", + )); + }; + + let file_uuid = Uuid::parse_str(file_id) + .map_err(|_| DomainError::not_found("File", file_id.to_string()))?; + authz + .require( + Subject::User(caller_id), + Permission::Update, + Resource::File(file_uuid), + ) + .await?; + + let file = file_read.get_file(file_id).await?; + let (new_hash, updated_at) = self + .file_write + .update_file_content_with_blob(file_id, &blob.hash, blob.size, None) + .await?; + // The file maps to a different blob now — stale cached content must + // never be served for the rest of its TTI window. + if let Some(cc) = &self.content_cache { + cc.invalidate(file_id).await; + } + + let parts = file.into_parts(); + let updated = crate::domain::entities::file::File::with_timestamps_and_blob_hash( + parts.id, + parts.name, + parts.storage_path, + blob.size, + parts.mime_type, + parts.folder_id, + parts.created_at, + updated_at as u64, + parts.owner_id, + new_hash, + ) + .map_err(|e| DomainError::internal_error("FileUpload", format!("rebuild entity: {e}")))?; + let dto = FileDto::from(updated); + if let Some(hook) = &self.file_lifecycle_hook { + hook.on_file_updated(file_id, &dto.content_hash, &dto.mime_type); + } + Ok(dto) + } + // ── private helpers ────────────────────────────────────────── /// Optionally update storage usage after a successful upload. diff --git a/src/application/services/mod.rs b/src/application/services/mod.rs index 7fa82755..7006ef9f 100644 --- a/src/application/services/mod.rs +++ b/src/application/services/mod.rs @@ -5,6 +5,7 @@ pub mod batch_operations; pub mod blob_lifecycle_service; pub mod calendar_service; pub mod contact_service; +pub mod delta_upload_service; pub mod device_auth_service; pub mod external_identity_service; pub mod favorites_service; diff --git a/src/common/di.rs b/src/common/di.rs index 49e9bf87..4e9a94ca 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -461,6 +461,18 @@ impl AppServiceFactory { ), ); + // Delta-upload protocol — chunk negotiation over the same dedup + // store. Bounded by the same whole-file ceiling as byte uploads. + let delta_upload_service = Arc::new( + crate::application::services::delta_upload_service::DeltaUploadService::new( + core.dedup_service.clone(), + file_upload_service.clone(), + storage_usage.clone(), + authz.clone(), + self.config.storage.max_upload_size as u64, + ), + ); + let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( repos.file_read_repository.clone(), core.file_content_cache.clone(), @@ -505,6 +517,7 @@ impl AppServiceFactory { // Traits for abstraction folder_service, file_upload_service, + delta_upload_service, file_retrieval_service, file_management_service, file_use_case_factory, @@ -1040,6 +1053,13 @@ impl AppServiceFactory { user_profile_rate_limiter: Arc::new( crate::interfaces::middleware::rate_limit::RateLimiter::new(60, 60, 50_000), ), + // Delta upload: 240 requests / minute / caller. Generous for a + // real client (chunk PUTs carry up to 100 MB each) while + // stopping pin/negotiate floods; 50 000 tracked callers bound + // the memory like the other limiters. + delta_upload_rate_limiter: Arc::new( + crate::interfaces::middleware::rate_limit::RateLimiter::new(240, 60, 50_000), + ), // PR 12 — per-sharer email-invite ceiling: caller_id-keyed. // Defends against a compromised account spamming external // invites (each invite mints a new external user + email). @@ -1377,6 +1397,8 @@ pub struct ApplicationServices { // Traits for abstraction pub folder_service: Arc, pub file_upload_service: Arc, + pub delta_upload_service: + Arc, pub file_retrieval_service: Arc, pub file_management_service: Arc, pub file_use_case_factory: Arc, @@ -1496,6 +1518,9 @@ pub struct AppState { /// authenticated caller covers any legitimate UI rendering while /// throttling enumeration. pub user_profile_rate_limiter: Arc, + /// Per-caller flood guard for the delta-upload endpoints + /// (negotiate / chunks / commit share one budget). + pub delta_upload_rate_limiter: Arc, /// Per-sharer ceiling on `POST /api/grants` invitations whose /// subject is `{ type: "email" }`. 50 per hour keyed on /// `caller_id`. Anonymous attackers can't reach this code path diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 8f7ff957..5b6105d9 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -65,11 +65,11 @@ use crate::domain::errors::{DomainError, ErrorKind}; // ── CDC Constants ──────────────────────────────────────────────────────────── /// Minimum CDC chunk size (64 KB). -const CDC_MIN_CHUNK: usize = 65_536; +pub const CDC_MIN_CHUNK: usize = 65_536; /// Average CDC chunk size (256 KB). -const CDC_AVG_CHUNK: usize = 262_144; +pub const CDC_AVG_CHUNK: usize = 262_144; /// Maximum CDC chunk size (1 MB). -const CDC_MAX_CHUNK: usize = 1_048_576; +pub const CDC_MAX_CHUNK: usize = 1_048_576; // ── CDC helper types ───────────────────────────────────────────────────────── @@ -402,10 +402,42 @@ impl DedupService { S: Stream> + Send, { let outcome = self.ingest_chunks_from_stream(source).await?; + tracing::debug!( + "CDC stream ingested: {} ({} bytes, {} chunks, {} written)", + &outcome.file_hash[..12], + outcome.total_size, + outcome.chunk_hashes.len(), + outcome.newly_written, + ); let distinct = outcome.distinct_hashes(); - let total_size = outcome.total_size; - let file_hash = outcome.file_hash.clone(); + self.attach_manifest( + &outcome.file_hash, + &outcome.chunk_hashes, + &outcome.chunk_sizes, + outcome.total_size, + content_type, + &distinct, + ) + .await + } + /// Attach a manifest to chunk references the caller already holds (one + /// per distinct chunk hash) — the shared accounting tail of both + /// [`store_from_stream`] and the delta-upload commit. + /// + /// On a lost insert race or an already-existing manifest, the existing + /// manifest's ref_count is bumped FIRST and only then are the held chunk + /// references released (`distinct_held`); the reverse order could leave + /// the caller's file row without any manifest reference behind it. + pub async fn attach_manifest( + &self, + file_hash: &str, + chunk_hashes: &[String], + chunk_sizes: &[u64], + total_size: u64, + content_type: Option, + distinct_held: &[String], + ) -> Result { // A bounded retry covers the rare interleaving where the manifest // that beat our INSERT is deleted again before our ref bump lands. for _ in 0..3 { @@ -415,17 +447,11 @@ impl DedupService { VALUES ($1, $2, $3, $4, $5, $6, 1) ON CONFLICT (file_hash) DO NOTHING", ) - .bind(&file_hash) - .bind(&outcome.chunk_hashes) - .bind( - outcome - .chunk_sizes - .iter() - .map(|s| *s as i64) - .collect::>(), - ) + .bind(file_hash) + .bind(chunk_hashes) + .bind(chunk_sizes.iter().map(|s| *s as i64).collect::>()) .bind(total_size as i64) - .bind(outcome.chunk_hashes.len() as i32) + .bind(chunk_hashes.len() as i32) .bind(&content_type) .execute(self.pool.as_ref()) .await @@ -436,46 +462,253 @@ impl DedupService { if inserted > 0 { tracing::info!( - "NEW BLOB (CDC stream): {} ({} bytes, {} chunks, {} written)", + "NEW BLOB (CDC): {} ({} bytes, {} chunks)", &file_hash[..12], total_size, - outcome.chunk_hashes.len(), - outcome.newly_written, + chunk_hashes.len(), ); - self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()); + self.fire_blob_creation_hooks(file_hash, content_type.as_deref()); return Ok(DedupResultDto::NewBlob { - hash: file_hash, + hash: file_hash.to_string(), size: total_size, }); } // The manifest already exists — either this exact content was // stored before or an identical concurrent upload just won the - // race. Bump ITS ref_count first and only then hand back this - // session's chunk references; the reverse order could leave the - // caller's file row without any manifest reference behind it. - if let Some(existing_size) = self.bump_manifest_if_exists(&file_hash).await? { - self.release_chunk_refs(self.pool.as_ref(), &distinct).await; + // race. Bump ITS ref_count, then hand back the held references. + if let Some(existing_size) = self.bump_manifest_if_exists(file_hash).await? { + self.release_chunk_refs(self.pool.as_ref(), distinct_held) + .await; tracing::info!( "DEDUP HIT (manifest): {} ({} bytes saved)", &file_hash[..12], existing_size, ); return Ok(DedupResultDto::ExistingBlob { - hash: file_hash, + hash: file_hash.to_string(), size: existing_size as u64, saved_bytes: existing_size as u64, }); } } - self.release_chunk_refs(self.pool.as_ref(), &distinct).await; + self.release_chunk_refs(self.pool.as_ref(), distinct_held) + .await; Err(DomainError::internal_error( "Dedup", format!("Manifest insert/bump kept racing for {file_hash}"), )) } + // ── Delta-upload primitives ────────────────────────────────── + // + // The delta protocol ("upload only what changed") lets a client claim + // chunks by hash instead of sending their bytes. Two invariants keep + // that from becoming a content oracle or a poisoning vector: + // + // 1. **Ownership**: without bytes, a caller may only claim chunks that + // are already reachable through their own (non-trashed) files, or + // unreferenced orphans (ref_count = 0 — i.e. "I just uploaded it"). + // Everything else must be uploaded; the store dedups it on write. + // 2. **Verification**: a declared file_hash is never trusted — the + // commit re-reads the proposed chunk sequence server-side and + // recomputes BLAKE3 before any manifest row exists. A forged hash + // would otherwise poison future whole-file dedup hits for OTHER + // users uploading the genuine content. + + /// Of `hashes` (distinct), the subset `caller_id` may claim without + /// uploading bytes: chunks referenced by manifests of the caller's + /// non-trashed files, or directly referenced as (legacy) whole-file + /// blobs. Backed by the GIN index on `chunk_manifests.chunk_hashes`. + pub async fn claimable_chunks( + &self, + caller_id: uuid::Uuid, + hashes: &[String], + ) -> Result, DomainError> { + if hashes.is_empty() { + return Ok(HashSet::new()); + } + sqlx::query_scalar::<_, String>( + "SELECT c.h FROM UNNEST($1::text[]) AS c(h) + WHERE EXISTS ( + SELECT 1 + FROM storage.files f + JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash + WHERE f.user_id = $2 AND NOT f.is_trashed + AND m.chunk_hashes @> ARRAY[c.h] + ) + OR EXISTS ( + SELECT 1 FROM storage.files f2 + WHERE f2.user_id = $2 AND NOT f2.is_trashed + AND f2.blob_hash = c.h + )", + ) + .bind(hashes) + .bind(caller_id) + .fetch_all(self.pool.as_ref()) + .await + .map(|rows| rows.into_iter().collect()) + .map_err(|e| DomainError::internal_error("Dedup", format!("claimable_chunks query: {e}"))) + } + + /// Pin one reference on each of `hashes` (distinct) that the caller is + /// entitled to claim — owned chunks (see [`claimable_chunks`]) or + /// unreferenced orphans (`ref_count = 0`, the just-uploaded state). + /// One statement: entitlement check and bump are atomic per row, so a + /// concurrent last-reference delete can never be resurrected and a + /// non-entitled hash is simply not returned. + /// + /// Returns the set actually pinned; the caller compares against its + /// input and reports the difference as `still_missing`. + pub async fn pin_claimable_chunks( + &self, + caller_id: uuid::Uuid, + hashes: &[String], + ) -> Result, DomainError> { + if hashes.is_empty() { + return Ok(HashSet::new()); + } + sqlx::query_scalar::<_, String>( + "UPDATE storage.blobs b + SET ref_count = ref_count + 1 + WHERE b.hash = ANY($1) + AND ( b.ref_count = 0 + OR EXISTS ( + SELECT 1 + FROM storage.files f + JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash + WHERE f.user_id = $2 AND NOT f.is_trashed + AND m.chunk_hashes @> ARRAY[b.hash::text] + ) + OR EXISTS ( + SELECT 1 FROM storage.files f2 + WHERE f2.user_id = $2 AND NOT f2.is_trashed + AND f2.blob_hash = b.hash + ) ) + RETURNING b.hash", + ) + .bind(hashes) + .bind(caller_id) + .fetch_all(self.pool.as_ref()) + .await + .map(|rows| rows.into_iter().collect()) + .map_err(|e| { + DomainError::internal_error("Dedup", format!("pin_claimable_chunks query: {e}")) + }) + } + + /// Release one reference per distinct hash — the public counterpart of + /// [`pin_claimable_chunks`] for aborted commits. Best-effort. + pub async fn release_pinned_chunks(&self, hashes: &[String]) { + self.release_chunk_refs(self.pool.as_ref(), hashes).await; + } + + /// Store client-provided loose chunks (delta upload, step 2). + /// + /// Each element of `frames` is one chunk's raw bytes (the wire framing + /// is the interface layer's concern). The hash is ALWAYS computed + /// server-side — a declared hash is never trusted for content + /// addressing. Chunks are written unsynced, made durable with one + /// batched sweep, then registered at `ref_count = 0`: unreferenced + /// orphans that either get pinned by a following commit or swept by + /// the periodic GC if the client never returns. `ON CONFLICT DO + /// NOTHING` keeps existing rows' reference counts untouched. + /// + /// Returns `(hash, size)` per frame, in input order. + pub async fn store_loose_chunks(&self, frames: S) -> Result, DomainError> + where + S: Stream> + Send, + { + futures::pin_mut!(frames); + + let mut received: Vec<(String, u64)> = Vec::new(); + let mut new_rows: Vec<(String, i64)> = Vec::new(); + let mut seen: HashSet = HashSet::new(); + + while let Some(frame) = frames.next().await { + let data = frame?; + if data.len() > CDC_MAX_CHUNK { + return Err(DomainError::validation_error(format!( + "Chunk frame of {} bytes exceeds the {CDC_MAX_CHUNK}-byte maximum", + data.len() + ))); + } + let hash = blake3::hash(&data).to_hex().to_string(); + received.push((hash.clone(), data.len() as u64)); + if seen.insert(hash.clone()) { + let len = data.len() as i64; + self.backend + .put_blob_from_bytes_unsynced(&hash, data) + .await?; + new_rows.push((hash, len)); + } + } + + if !new_rows.is_empty() { + // Durability before visibility — same invariant as the ingest + // engine: no PG row may ever point at unsynced bytes. + let hashes: Vec = new_rows.iter().map(|(h, _)| h.clone()).collect(); + let sizes: Vec = new_rows.iter().map(|(_, s)| *s).collect(); + self.backend.sync_blobs(&hashes).await?; + sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) + SELECT h, s, 0 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) + ON CONFLICT (hash) DO NOTHING", + ) + .bind(&hashes) + .bind(&sizes) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to register chunks: {e}")) + })?; + } + + Ok(received) + } + + /// Verification read for the delta commit: stream the proposed chunk + /// sequence from the backend, recompute the whole-file BLAKE3 and + /// capture the first bytes for MIME sniffing. The caller must hold a + /// pin on every chunk (so a concurrent GC cannot pull bytes out from + /// under the read). Also validates each chunk's actual size against + /// the declared one — the manifest's Range arithmetic depends on it. + pub async fn hash_chunk_sequence( + &self, + chunks: &[(String, u64)], + sniff_len: usize, + ) -> Result<(String, Vec), DomainError> { + let mut hasher = blake3::Hasher::new(); + let mut head: Vec = Vec::with_capacity(sniff_len.min(16 * 1024)); + + for (hash, declared_size) in chunks { + let mut stream = self.backend.get_blob_stream(hash).await?; + let mut actual: u64 = 0; + while let Some(part) = stream.next().await { + let part = part.map_err(|e| { + DomainError::internal_error( + "Dedup", + format!("Verification read of chunk {hash}: {e}"), + ) + })?; + actual += part.len() as u64; + hasher.update(&part); + if head.len() < sniff_len { + let take = (sniff_len - head.len()).min(part.len()); + head.extend_from_slice(&part[..take]); + } + } + if actual != *declared_size { + return Err(DomainError::validation_error(format!( + "Chunk {hash} is {actual} bytes, manifest declares {declared_size}" + ))); + } + } + + Ok((hasher.finalize().to_hex().to_string(), head)) + } + /// Bump a manifest's ref_count if it exists; returns its total_size. /// Single statement — no window between the existence check and the bump. async fn bump_manifest_if_exists(&self, file_hash: &str) -> Result, DomainError> { @@ -2702,3 +2935,325 @@ mod rechunk_integration_tests { cleanup(&pool, &hash, &files).await; } } + +// ───────────────────────────────────────────────────────────────────────────── +// Integration tests for the delta-upload primitives — the entitlement and +// verification rules the chunk-negotiation protocol stands on. Same gating +// and DB conventions as the re-chunk suite above. +// ───────────────────────────────────────────────────────────────────────────── +#[cfg(integration_tests)] +#[allow(dead_code)] +mod delta_upload_integration_tests { + use super::*; + use crate::infrastructure::services::local_blob_backend::LocalBlobBackend; + use crate::integration_test_support::{ensure_clean_test_db, test_db_url}; + use sqlx::Row; + use sqlx::postgres::PgPoolOptions; + use tempfile::TempDir; + use uuid::Uuid; + + async fn test_pool() -> Arc { + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&test_db_url()) + .await + .expect("connect to test DB — run tests/common/spawn-db.sh first"); + ensure_clean_test_db(&pool).await; + Arc::new(pool) + } + + async fn seed_user(pool: &PgPool) -> Uuid { + sqlx::query("SELECT id FROM auth.users LIMIT 1") + .fetch_one(pool) + .await + .map(|r| r.get::("id")) + .expect("auth.users must be seeded (init-test-schema.sh)") + } + + async fn local_svc(pool: &Arc, dir: &TempDir) -> DedupService { + let backend = Arc::new(LocalBlobBackend::new(&dir.path().join("blobs"))); + backend.initialize().await.expect("init backend"); + DedupService::new(backend, pool.clone(), pool.clone()) + } + + /// Store `data` through the streaming path and give `user_id` a file + /// row referencing it — making its chunks claimable by that user. + async fn seed_owned_content( + svc: &DedupService, + pool: &PgPool, + user_id: Uuid, + data: &[u8], + label: &str, + ) -> (String, Vec, Uuid) { + let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::copy_from_slice(data))]); + let stored = svc + .store_from_stream(source, Some("application/octet-stream".into())) + .await + .expect("store"); + let file_hash = stored.hash().to_string(); + let chunks: Vec = sqlx::query_scalar( + "SELECT UNNEST(chunk_hashes) FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(&file_hash) + .fetch_all(pool) + .await + .expect("chunks"); + + let file_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.files (name, user_id, blob_hash, size) + VALUES ($1, $2, $3, $4) RETURNING id", + ) + .bind(format!( + "rust-test-delta-{label}-{}", + &Uuid::new_v4().to_string()[..8] + )) + .bind(user_id) + .bind(&file_hash) + .bind(data.len() as i64) + .fetch_one(pool) + .await + .expect("file row"); + (file_hash, chunks, file_id) + } + + async fn blob_ref(pool: &PgPool, hash: &str) -> Option { + sqlx::query_scalar("SELECT ref_count FROM storage.blobs WHERE hash = $1") + .bind(hash) + .fetch_optional(pool) + .await + .expect("blob query") + } + + async fn cleanup(pool: &PgPool, file_hash: &str, file_id: Uuid, extra_hashes: &[String]) { + let chunks: Option> = sqlx::query_scalar( + "SELECT chunk_hashes FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(file_hash) + .fetch_optional(pool) + .await + .unwrap_or(None); + let _ = sqlx::query("DELETE FROM storage.files WHERE id = $1") + .bind(file_id) + .execute(pool) + .await; + let _ = sqlx::query("DELETE FROM storage.chunk_manifests WHERE file_hash = $1") + .bind(file_hash) + .execute(pool) + .await; + let mut to_drop = chunks.unwrap_or_default(); + to_drop.push(file_hash.to_string()); + to_drop.extend_from_slice(extra_hashes); + let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = ANY($1)") + .bind(&to_drop) + .execute(pool) + .await; + } + + fn content(len: usize, salt: u8) -> Vec { + let mut data: Vec = (0..len) + .map(|i| { + ((i % 251) as u8) + .wrapping_add(salt) + .wrapping_add((i / 7919) as u8) + }) + .collect(); + data.extend_from_slice(Uuid::new_v4().as_bytes()); + data + } + + // ── Entitlement: claimable vs pin ──────────────────────────── + #[tokio::test] + async fn claim_and_pin_respect_ownership_and_orphans() { + let pool = test_pool().await; + let dir = TempDir::new().unwrap(); + let svc = local_svc(&pool, &dir).await; + let user = seed_user(&pool).await; + + // Owned content (multi-chunk), one foreign chunk (ref 1, no file + // row for this user), one orphan (ref 0), one unknown hash. + let data = content(3 * 1024 * 1024, 21); + let (file_hash, owned_chunks, file_id) = + seed_owned_content(&svc, &pool, user, &data, "claim").await; + assert!(owned_chunks.len() >= 3, "3 MiB must split into ≥3 chunks"); + + let foreign = blake3::hash(format!("foreign-{}", Uuid::new_v4()).as_bytes()) + .to_hex() + .to_string(); + let orphan = blake3::hash(format!("orphan-{}", Uuid::new_v4()).as_bytes()) + .to_hex() + .to_string(); + sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 10, 1), ($2, 10, 0)", + ) + .bind(&foreign) + .bind(&orphan) + .execute(pool.as_ref()) + .await + .expect("seed foreign+orphan"); + let unknown = blake3::hash(format!("unknown-{}", Uuid::new_v4()).as_bytes()) + .to_hex() + .to_string(); + + let mut probe: Vec = owned_chunks.clone(); + probe.push(foreign.clone()); + probe.push(orphan.clone()); + probe.push(unknown.clone()); + + // claimable: only the owned chunks (advisory view — orphans are + // intentionally NOT advertised; the commit pin may still take them). + let claimable = svc.claimable_chunks(user, &probe).await.expect("claimable"); + for c in &owned_chunks { + assert!(claimable.contains(c), "owned chunk {c} must be claimable"); + } + assert!( + !claimable.contains(&foreign), + "foreign chunk must not be claimable" + ); + assert!( + !claimable.contains(&unknown), + "unknown chunk must not be claimable" + ); + + // pin: owned + orphan succeed; foreign and unknown are refused. + let pinned = svc.pin_claimable_chunks(user, &probe).await.expect("pin"); + for c in &owned_chunks { + assert!(pinned.contains(c), "owned chunk {c} must pin"); + } + assert!( + pinned.contains(&orphan), + "ref-0 orphan must pin (just-uploaded state)" + ); + assert!( + !pinned.contains(&foreign), + "foreign owned chunk must NOT pin" + ); + assert!(!pinned.contains(&unknown), "unknown hash must NOT pin"); + + // Ref counts moved exactly where they should. + assert_eq!(blob_ref(&pool, &orphan).await, Some(1), "orphan 0→1"); + assert_eq!( + blob_ref(&pool, &foreign).await, + Some(1), + "foreign untouched" + ); + assert_eq!( + blob_ref(&pool, &owned_chunks[0]).await, + Some(2), + "owned chunk 1→2 (manifest + pin)" + ); + + // Release restores the original counts (clamped at 0). + let pinned_vec: Vec = pinned.into_iter().collect(); + svc.release_pinned_chunks(&pinned_vec).await; + assert_eq!(blob_ref(&pool, &orphan).await, Some(0)); + assert_eq!(blob_ref(&pool, &owned_chunks[0]).await, Some(1)); + + cleanup(&pool, &file_hash, file_id, &[foreign, orphan]).await; + } + + // ── Loose chunk store ──────────────────────────────────────── + #[tokio::test] + async fn loose_chunks_register_as_orphans_without_touching_existing_refs() { + let pool = test_pool().await; + let dir = TempDir::new().unwrap(); + let svc = local_svc(&pool, &dir).await; + let user = seed_user(&pool).await; + + // An owned chunk that the client redundantly re-uploads. + let data = content(100 * 1024, 22); + let (file_hash, owned_chunks, file_id) = + seed_owned_content(&svc, &pool, user, &data, "loose").await; + let owned_chunk_bytes = { + let mut stream = svc.read_blob_stream(&file_hash).await.expect("stream"); + let mut out = Vec::new(); + while let Some(part) = stream.next().await { + out.extend_from_slice(&part.expect("part")); + } + out + }; + + let fresh = content(50 * 1024, 23); + let frames = stream::iter(vec![ + Ok::<_, DomainError>(Bytes::from(fresh.clone())), + Ok(Bytes::from(fresh.clone())), // duplicate frame + Ok(Bytes::from(owned_chunk_bytes.clone())), // already-referenced chunk + ]); + + let received = svc.store_loose_chunks(frames).await.expect("store loose"); + assert_eq!(received.len(), 3, "every frame is answered, in order"); + assert_eq!( + received[0].0, received[1].0, + "duplicate frames share a hash" + ); + let fresh_hash = received[0].0.clone(); + + assert_eq!( + blob_ref(&pool, &fresh_hash).await, + Some(0), + "fresh chunk lands as an unreferenced orphan" + ); + assert_eq!( + blob_ref(&pool, &owned_chunks[0]).await, + Some(1), + "re-uploading an existing chunk must not disturb its refs" + ); + + // The orphan's bytes are really there and addressable. + assert_eq!( + svc.backend().blob_exists(&fresh_hash).await.unwrap(), + true, + "orphan chunk bytes must exist in the backend" + ); + + cleanup(&pool, &file_hash, file_id, &[fresh_hash]).await; + } + + // ── Verification read ──────────────────────────────────────── + #[tokio::test] + async fn hash_chunk_sequence_recomputes_and_validates_sizes() { + let pool = test_pool().await; + let dir = TempDir::new().unwrap(); + let svc = local_svc(&pool, &dir).await; + let user = seed_user(&pool).await; + + let data = content(2 * 1024 * 1024 + 137, 24); + let (file_hash, _chunks, file_id) = + seed_owned_content(&svc, &pool, user, &data, "verify").await; + + let manifest: (Vec, Vec) = sqlx::query_as( + "SELECT chunk_hashes, chunk_sizes FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(&file_hash) + .fetch_one(pool.as_ref()) + .await + .expect("manifest"); + let sequence: Vec<(String, u64)> = manifest + .0 + .iter() + .cloned() + .zip(manifest.1.iter().map(|s| *s as u64)) + .collect(); + + let (computed, head) = svc + .hash_chunk_sequence(&sequence, 16) + .await + .expect("verification read"); + assert_eq!(computed, file_hash, "recomputed hash must match"); + assert_eq!( + &head[..], + &data[..16], + "sniff head must be the file's first bytes" + ); + + // A wrong declared size must be rejected — Range arithmetic + // depends on manifest sizes being true. + let mut lying = sequence.clone(); + lying[0].1 += 1; + assert!( + svc.hash_chunk_sequence(&lying, 0).await.is_err(), + "size lie must fail verification" + ); + + cleanup(&pool, &file_hash, file_id, &[]).await; + } +} diff --git a/src/interfaces/api/handlers/delta_upload_handler.rs b/src/interfaces/api/handlers/delta_upload_handler.rs new file mode 100644 index 00000000..e45765c0 --- /dev/null +++ b/src/interfaces/api/handlers/delta_upload_handler.rs @@ -0,0 +1,322 @@ +//! Delta-upload protocol endpoints — "upload only what changed". +//! +//! Wire surface of [`DeltaUploadService`]; see that module (and +//! `docs/delta-upload-protocol.md`) for the protocol and its security +//! model. These handlers only authenticate, rate-limit and translate +//! the wire formats — every decision lives in the application service. +//! +//! Chunk-frame wire format (`PUT /api/files/delta/chunks`): the body is a +//! sequence of `[u32 big-endian length][length bytes]` frames, one per +//! chunk, with `Content-Type: application/octet-stream`. Frames are capped +//! at the CDC maximum chunk size (1 MiB) and the whole request at the same +//! per-request ceiling as resumable chunk PUTs (`chunk_max_bytes`). + +use axum::{ + Json, + body::Body, + extract::State, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use bytes::{Buf, Bytes, BytesMut}; +use futures::Stream; +use std::sync::Arc; +use tokio_stream::StreamExt; + +use crate::application::services::delta_upload_service::{ + DeltaChunksResponse, DeltaCommitOutcome, DeltaCommitRequest, DeltaNegotiateRequest, + DeltaNegotiateResponse, +}; +use crate::common::di::AppState; +use crate::common::errors::DomainError; +use crate::infrastructure::services::dedup_service::CDC_MAX_CHUNK; +use crate::interfaces::errors::AppError; +use crate::interfaces::middleware::auth::AuthUser; +use http_body_util::BodyStream; +use serde::Serialize; +use utoipa::ToSchema; + +/// 409 body of `POST /api/files/delta/commit` when chunks vanished between +/// negotiate and commit (GC race) or were never claimable: the client +/// uploads exactly these hashes and retries the same commit. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaStillMissingResponse { + pub still_missing: Vec, +} + +/// Per-caller flood guard shared by the three delta endpoints. +fn check_rate_limit(state: &Arc, auth_user: &AuthUser) -> Result<(), AppError> { + if state + .delta_upload_rate_limiter + .check_and_increment(&auth_user.id.to_string()) + .is_err() + { + tracing::info!( + target: "audit", + event = "delta_upload.rejected", + reason = "rate_limited", + caller_id = %auth_user.id, + "👮🏻‍♂️ Delta upload rejected: per-caller rate limit exceeded", + ); + return Err(AppError::new( + StatusCode::TOO_MANY_REQUESTS, + "Too many delta-upload requests; please retry shortly", + "RateLimited", + )); + } + Ok(()) +} + +/// Parse a `[u32 BE length][bytes]` frame sequence from the request body. +/// +/// Streaming: peak RAM is one frame (≤ 1 MiB) plus one HTTP frame, +/// regardless of how many chunks the request carries. `max_total` bounds +/// the whole request body. +fn parse_chunk_frames( + body: Body, + max_total: usize, +) -> impl Stream> + Send { + async_stream::try_stream! { + let mut body_stream = BodyStream::new(body); + let mut buf = BytesMut::new(); + let mut expecting: Option = None; + let mut total: usize = 0; + + loop { + // Drain every complete frame already buffered. + loop { + match expecting { + None => { + if buf.len() < 4 { + break; + } + let len = + u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; + buf.advance(4); + if len == 0 || len > CDC_MAX_CHUNK { + Err(DomainError::validation_error(format!( + "Chunk frame of {len} bytes out of bounds (1 ..= {CDC_MAX_CHUNK})" + )))?; + } + expecting = Some(len); + } + Some(len) => { + if buf.len() < len { + break; + } + let frame = buf.split_to(len).freeze(); + expecting = None; + yield frame; + } + } + } + + match body_stream.next().await { + Some(Ok(http_frame)) => { + if let Some(data) = http_frame.data_ref() { + total += data.len(); + if total > max_total { + Err(DomainError::validation_error(format!( + "Request body exceeds the {max_total}-byte per-request cap; \ + split the chunk upload into several requests" + )))?; + } + buf.extend_from_slice(data); + } + } + Some(Err(e)) => { + Err(DomainError::validation_error(format!( + "Failed to read request body: {e}" + )))?; + } + None => { + if expecting.is_some() || !buf.is_empty() { + Err(DomainError::validation_error( + "Truncated chunk frame at end of body", + ))?; + } + break; + } + } + } + } +} + +#[utoipa::path( + post, + path = "/api/files/delta/negotiate", + request_body = DeltaNegotiateRequest, + responses( + (status = 200, description = "Chunks the caller must upload (the rest are claimable without bytes)", body = DeltaNegotiateResponse), + (status = 400, description = "Malformed chunk list (hash format, size bounds, count ceiling)"), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_negotiate( + State(state): State>, + auth_user: AuthUser, + Json(request): Json, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let response = state + .applications + .delta_upload_service + .negotiate_with_perms(auth_user.id, &request) + .await + .map_err(AppError::from)?; + Ok(Json(response)) +} + +#[utoipa::path( + put, + path = "/api/files/delta/chunks", + request_body(content_type = "application/octet-stream", + description = "Sequence of [u32 BE length][bytes] frames, one per chunk (each ≤ 1 MiB)"), + responses( + (status = 200, description = "Server-computed identity of every received frame, in order", body = DeltaChunksResponse), + (status = 400, description = "Malformed framing, oversized frame, or oversized request"), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_upload_chunks( + State(state): State>, + auth_user: AuthUser, + body: Body, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let frames = parse_chunk_frames(body, state.core.config.storage.chunk_max_bytes); + let response = state + .applications + .delta_upload_service + .receive_chunks(frames) + .await + .map_err(AppError::from)?; + Ok(Json(response)) +} + +#[utoipa::path( + post, + path = "/api/files/delta/commit", + request_body = DeltaCommitRequest, + responses( + (status = 201, description = "File created from the committed chunk sequence", body = crate::application::dtos::file_dto::FileDto), + (status = 200, description = "Existing file's content replaced", body = crate::application::dtos::file_dto::FileDto), + (status = 400, description = "Malformed request, or the declared file_hash does not match the chunk sequence"), + (status = 404, description = "Target folder/file not found or not accessible"), + (status = 409, description = "Chunks not claimable — upload them and retry", body = DeltaStillMissingResponse), + (status = 429, description = "Rate limited"), + (status = 507, description = "Storage quota exceeded"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_commit( + State(state): State>, + auth_user: AuthUser, + Json(request): Json, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let outcome = state + .applications + .delta_upload_service + .commit_with_perms(auth_user.id, request) + .await + .map_err(AppError::from)?; + Ok(match outcome { + DeltaCommitOutcome::Done { file, created } => { + let status = if created { + StatusCode::CREATED + } else { + StatusCode::OK + }; + (status, Json(file)).into_response() + } + DeltaCommitOutcome::StillMissing(still_missing) => ( + StatusCode::CONFLICT, + Json(DeltaStillMissingResponse { still_missing }), + ) + .into_response(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode frames the way a client would. + fn encode(frames: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for f in frames { + out.extend_from_slice(&(f.len() as u32).to_be_bytes()); + out.extend_from_slice(f); + } + out + } + + async fn collect(body: Body, max_total: usize) -> Result, DomainError> { + let stream = parse_chunk_frames(body, max_total); + futures::pin_mut!(stream); + let mut out = Vec::new(); + while let Some(item) = stream.next().await { + out.push(item?); + } + Ok(out) + } + + #[tokio::test] + async fn roundtrips_frames_in_order() { + let wire = encode(&[b"first", b"second chunk", &[0xAB; 1000]]); + let frames = collect(Body::from(wire), usize::MAX).await.unwrap(); + assert_eq!(frames.len(), 3); + assert_eq!(&frames[0][..], b"first"); + assert_eq!(&frames[1][..], b"second chunk"); + assert_eq!(frames[2].len(), 1000); + } + + #[tokio::test] + async fn empty_body_yields_no_frames() { + let frames = collect(Body::empty(), usize::MAX).await.unwrap(); + assert!(frames.is_empty()); + } + + #[tokio::test] + async fn rejects_zero_length_frame() { + let wire = encode(&[b""]); + assert!(collect(Body::from(wire), usize::MAX).await.is_err()); + } + + #[tokio::test] + async fn rejects_frame_above_cdc_max() { + let mut wire = Vec::new(); + wire.extend_from_slice(&((CDC_MAX_CHUNK as u32) + 1).to_be_bytes()); + // Header alone is enough — the length is rejected before any data. + assert!(collect(Body::from(wire), usize::MAX).await.is_err()); + } + + #[tokio::test] + async fn rejects_truncated_frame() { + let mut wire = encode(&[b"complete"]); + wire.extend_from_slice(&10u32.to_be_bytes()); + wire.extend_from_slice(b"only5"); // promises 10, delivers 5 + assert!(collect(Body::from(wire), usize::MAX).await.is_err()); + } + + #[tokio::test] + async fn rejects_request_above_total_cap() { + let wire = encode(&[&[0u8; 600], &[1u8; 600]]); + assert!(collect(Body::from(wire), 1000).await.is_err()); + } + + #[tokio::test] + async fn accepts_frame_exactly_at_cdc_max() { + let big = vec![7u8; CDC_MAX_CHUNK]; + let wire = encode(&[&big]); + let frames = collect(Body::from(wire), usize::MAX).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].len(), CDC_MAX_CHUNK); + } +} diff --git a/src/interfaces/api/handlers/mod.rs b/src/interfaces/api/handlers/mod.rs index 2033ed2c..aebeb707 100644 --- a/src/interfaces/api/handlers/mod.rs +++ b/src/interfaces/api/handlers/mod.rs @@ -7,6 +7,7 @@ pub mod carddav_handler; pub mod chunked_upload_handler; pub mod contacts_handler; pub mod dedup_handler; +pub mod delta_upload_handler; pub mod device_auth_handler; pub mod favorites_handler; pub mod file_handler; diff --git a/src/interfaces/api/mod.rs b/src/interfaces/api/mod.rs index 51eb8e6d..a655c3f1 100644 --- a/src/interfaces/api/mod.rs +++ b/src/interfaces/api/mod.rs @@ -81,6 +81,9 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; handlers::file_handler::list_files_query, handlers::file_handler::upload_file_with_thumbnails, handlers::file_handler::create_file_by_hash, + handlers::delta_upload_handler::delta_negotiate, + handlers::delta_upload_handler::delta_upload_chunks, + handlers::delta_upload_handler::delta_commit, handlers::file_handler::download_file, handlers::file_handler::get_thumbnail, handlers::file_handler::upload_thumbnail, @@ -262,6 +265,13 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; ResourceContentDto, // File schemas FileDto, + // Delta-upload schemas + crate::application::services::delta_upload_service::ChunkRef, + crate::application::services::delta_upload_service::DeltaNegotiateRequest, + crate::application::services::delta_upload_service::DeltaNegotiateResponse, + crate::application::services::delta_upload_service::DeltaChunksResponse, + crate::application::services::delta_upload_service::DeltaCommitRequest, + handlers::delta_upload_handler::DeltaStillMissingResponse, MoveFilePayload, PaginationDto, PaginationRequestDto, diff --git a/src/interfaces/api/routes.rs b/src/interfaces/api/routes.rs index 262dfa9c..4b0255b1 100644 --- a/src/interfaces/api/routes.rs +++ b/src/interfaces/api/routes.rs @@ -54,6 +54,9 @@ use crate::interfaces::api::handlers::batch_handler::{self, BatchHandlerState}; use crate::interfaces::api::handlers::chunked_upload_handler::{ cancel_upload, complete_upload, create_upload, get_upload_status, upload_chunk, }; +use crate::interfaces::api::handlers::delta_upload_handler::{ + delta_commit, delta_negotiate, delta_upload_chunks, +}; use crate::interfaces::api::handlers::file_handler::{ create_file_by_hash, delete_file, download_file, get_file_metadata, get_thumbnail, list_files_query, move_file_simple, rename_file, upload_file_with_thumbnails, upload_thumbnail, @@ -230,6 +233,9 @@ pub fn create_api_routes(app_state: &Arc) -> Router> { .route("/", get(list_files_query)) .route("/upload", post(upload_file_with_thumbnails)) .route("/by-hash", post(create_file_by_hash)) + .route("/delta/negotiate", post(delta_negotiate)) + .route("/delta/chunks", put(delta_upload_chunks)) + .route("/delta/commit", post(delta_commit)) .route("/{id}", get(download_file)) .route( "/{id}/thumbnail/{size}", From 5d034b0d09a399b85597c47ee0a4e19b1162bc79 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 15:44:44 +0000 Subject: [PATCH 5/6] Delta-upload client: FastCDC in WASM + overlapped worker pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 — the client side of "upload only what changed", closing the delta-sync plan. WASM (wasm/oxicloud-hash): DeltaChunker adds incremental FastCDC with the server's exact crate and parameters (64K/256K/1M) next to the BLAKE3 hasher. The incremental split is provably identical to a single pass: every chunk except the last ends on a content/max-size condition whose decision window was fully buffered, so only the tail is provisional and re-examined as slices arrive. A mirror test — the client twin of the server's stream≡slice test — chunks 4 MiB of xorshift noise with adversarial slice sizes (7 B … 8 MiB) and requires boundary-for-boundary equality with one FastCDC pass. Vendored artifacts rebuilt (55 KB wasm). Worker (static/js/workers/deltaWorker.js): the full protocol off the main thread with OVERLAPPED stages — 8 MiB file slices feed the chunker while earlier batches (256 hashes) negotiate and their missing chunks upload through a 2-deep PUT pool (≤8 MiB framed bodies, bytes re-sliced from the File at send time, never hoarded). Commit handles 409 still_missing by uploading exactly the named hashes and retrying. Orchestrator (features/files/deltaUpload.js): threshold (8 MiB), worker lifecycle + size-scaled timeout, progress relay to the upload bell, conclusive-outcome mapping (201/200, 507 quota, 409 name conflict) and silent fallback to the byte upload for everything else. Wired into uploadFiles and uploadFolderEntries, which now surface one batch summary of the bytes dedup saved. This subsumes the whole-file instant-upload module — a fully-known file negotiates to nothing missing and the commit short-circuits on possession — so instantUpload.js and hashWorker.js are removed (the /api/dedup/check and /api/files/by-hash endpoints remain for API clients). Verified end-to-end against PostgreSQL 16 — the cross-boundary proof the whole design hangs on, in both directions: a 24 MB file byte- uploaded (server-side CDC) then edited and delta-negotiated with WASM-computed chunks reported missing 1/74 (boundaries bit-identical), synced with 344 KB on the wire vs 24 MB (98.6% saved) and downloaded byte-identical; inversely, a file created via delta then byte-uploaded as identical content produced a server-side manifest DEDUP HIT with the same content_hash. Insertion at the head of the file (the adversarial CDC case) still negotiated missing 1/74. Chunk+hash throughput ≈275 MB/s in V8 with SIMD128. https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- docs/delta-upload-protocol.md | 19 +- static/js/features/files/deltaUpload.js | 160 +++++++++ static/js/features/files/fileOperations.js | 46 ++- static/js/features/files/instantUpload.js | 177 ---------- .../vendors/hash-wasm/oxicloud_hash_wasm.js | 148 ++++++-- .../hash-wasm/oxicloud_hash_wasm_bg.wasm | Bin 45242 -> 55809 bytes static/js/workers/deltaWorker.js | 330 ++++++++++++++++++ static/js/workers/hashWorker.js | 79 ----- wasm/oxicloud-hash/Cargo.lock | 7 + wasm/oxicloud-hash/Cargo.toml | 4 + wasm/oxicloud-hash/src/lib.rs | 221 ++++++++++++ 11 files changed, 885 insertions(+), 306 deletions(-) create mode 100644 static/js/features/files/deltaUpload.js delete mode 100644 static/js/features/files/instantUpload.js create mode 100644 static/js/workers/deltaWorker.js delete mode 100644 static/js/workers/hashWorker.js diff --git a/docs/delta-upload-protocol.md b/docs/delta-upload-protocol.md index 31fd58ff..171e1a1d 100644 --- a/docs/delta-upload-protocol.md +++ b/docs/delta-upload-protocol.md @@ -12,15 +12,20 @@ Editing a few bytes of a 500 MB file re-uploads ~1 MiB instead of ## Who can use it -Any authenticated API client. The OxiCloud web frontend adopts it in a -later phase; generic WebDAV/NextCloud clients cannot (their protocols -have no delta concept) — they keep uploading full bytes, and the server -keeps deduplicating those on write. +Any authenticated API client. The OxiCloud web frontend uses it +automatically for files ≥ 8 MiB (`features/files/deltaUpload.js` + +`workers/deltaWorker.js`, chunking with the vendored WASM build of the +server's own FastCDC+BLAKE3 crates, falling back to a plain byte upload +on any failure). Generic WebDAV/NextCloud clients cannot (their +protocols have no delta concept) — they keep uploading full bytes, and +the server keeps deduplicating those on write. Chunk boundaries are the **client's choice**: matching the server's -FastCDC parameters maximizes cross-version sharing, but any split with -chunks of 1 byte … 1 MiB is valid — correctness is guaranteed by -server-side verification, not by the chunking scheme. +FastCDC parameters (64 KB / 256 KB / 1 MiB, as the bundled WASM module +does) maximizes cross-version sharing — including against versions that +entered through plain byte uploads — but any split with chunks of +1 byte … 1 MiB is valid; correctness is guaranteed by server-side +verification, not by the chunking scheme. ## The three steps diff --git a/static/js/features/files/deltaUpload.js b/static/js/features/files/deltaUpload.js new file mode 100644 index 00000000..f6a17d14 --- /dev/null +++ b/static/js/features/files/deltaUpload.js @@ -0,0 +1,160 @@ +/** + * OxiCloud - Delta upload ("upload only what changed"). + * + * Main-thread orchestrator for `workers/deltaWorker.js`, which runs the + * whole client side of the delta protocol off the UI thread: FastCDC + * chunking + BLAKE3 (the same WASM crate and parameters as the server, + * so boundaries match bit for bit), per-batch negotiation, upload of + * only the missing chunks, and the commit. + * + * This SUBSUMES the previous whole-file instant upload: a fully known + * file negotiates to "nothing missing" and the commit short-circuits on + * possession of the file hash — same zero-byte outcome, one pipeline. + * + * Performance posture: + * - Stages overlap inside the worker (hash ‖ negotiate ‖ upload), so + * wall-clock approaches max(hash, upload) instead of their sum. + * - RAM stays flat: 8 MiB read slices; chunk bytes are re-sliced from + * the File at upload time, never hoarded. + * - Files below {@link DELTA_UPLOAD_MIN_SIZE} skip the pipeline: the + * round-trips cost more than the bytes. + * - Any failure falls back silently to the normal byte upload — delta + * is an optimization, never a gate. + */ + +import { getCsrfToken } from '../../core/csrf.js'; + +/** + * Files smaller than this upload normally: hashing + negotiation + * round-trips outweigh the transfer. + */ +export const DELTA_UPLOAD_MIN_SIZE = 8 * 1024 * 1024; + +// Absolute URL on purpose — works in dev and in the release IIFE bundle +// (same pattern as the pdf.js loader in thumbnail.js). +const DELTA_WORKER_URL = '/js/workers/deltaWorker.js'; + +/** Budget: 120 s base + 90 s per GB (hashing + uploading the delta). */ +const DELTA_TIMEOUT_BASE_MS = 120000; +const DELTA_TIMEOUT_PER_GB_MS = 90000; + +/** + * `false` once the environment proved unable to run the worker/WASM — + * later files skip straight to the byte upload. `null` = not yet known. + * @type {boolean | null} + */ +let _deltaUploadUsable = null; + +/** + * Result contract shared with the uploaders' `UploadAnswer`, plus the + * bandwidth accounting the UI surfaces. + * @typedef {Object} DeltaUploadAnswer + * @property {boolean} ok + * @property {any} [data] FileDto on success + * @property {string} [errorMsg] + * @property {boolean} [isQuotaError] + * @property {number} [savedBytes] bytes NOT transferred thanks to dedup + */ + +/** + * Try to upload `file` through the delta protocol. + * + * Resolves `null` whenever the plain byte upload should proceed (file too + * small, environment unusable, any transport/protocol failure). Resolves + * a {@link DeltaUploadAnswer} when the outcome is conclusive — success, + * quota exceeded, or a name conflict a byte upload would also hit. + * + * @param {File} file + * @param {string | null | undefined} folderId + * @param {(pct: number) => void} [onProgress] 0-99 while transferring + * @returns {Promise} + */ +export function tryDeltaUpload(file, folderId, onProgress) { + if (!folderId || file.size < DELTA_UPLOAD_MIN_SIZE || _deltaUploadUsable === false || typeof Worker === 'undefined') { + return Promise.resolve(null); + } + + return new Promise((resolve) => { + /** @type {Worker} */ + let worker; + try { + worker = new Worker(DELTA_WORKER_URL, { type: 'module' }); + } catch (_) { + _deltaUploadUsable = false; + resolve(null); + return; + } + + const sizeGB = file.size / (1024 * 1024 * 1024); + const timeoutMs = DELTA_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * DELTA_TIMEOUT_PER_GB_MS; + + let savedBytes = 0; + + /** @param {DeltaUploadAnswer | null} answer */ + const settle = (answer) => { + clearTimeout(timer); + worker.terminate(); + resolve(answer); + }; + const timer = setTimeout(() => settle(null), timeoutMs); + + worker.onmessage = (event) => { + const msg = /** @type {any} */ (event.data); + if (msg.type === 'progress') { + savedBytes = msg.reusedBytes; + if (onProgress && msg.totalBytes > 0) { + const pct = Math.min(99, Math.round((100 * (msg.reusedBytes + msg.uploadedBytes)) / msg.totalBytes)); + onProgress(pct); + } + return; + } + if (msg.type === 'fallback') { + settle(null); + return; + } + if (msg.type === 'done') { + if (msg.status === 201 || msg.status === 200) { + settle({ ok: true, data: msg.body, savedBytes }); + return; + } + /** @type {string} */ + const errorMsg = msg.body?.message || msg.body?.error || `Delta upload failed (HTTP ${msg.status})`; + if (msg.status === 507) { + settle({ ok: false, isQuotaError: true, errorMsg }); + return; + } + if (msg.status === 409 && !msg.body?.still_missing) { + // Duplicate name — a byte upload would hit the same wall. + settle({ ok: false, errorMsg }); + return; + } + // still_missing exhausted, 4xx/5xx oddities: byte upload is + // the safe road (the server dedups it on write anyway). + settle(null); + } + }; + worker.onerror = () => { + // Worker script failed to load/parse — permanent environment trait. + _deltaUploadUsable = false; + settle(null); + }; + + worker.postMessage({ + file, + folderId, + name: file.name, + csrfToken: getCsrfToken() || '' + }); + }); +} + +/** + * Bilingual one-line summary for the bandwidth saved by a batch. + * @param {number} savedBytes + * @param {string} locale + * @returns {string} + */ +export function formatSavedSummary(savedBytes, locale) { + const mb = (savedBytes / (1024 * 1024)).toFixed(1); + return locale.startsWith('es') ? `Deduplicación: ${mb} MB no necesitaron subirse` : `Deduplication: ${mb} MB didn't need uploading`; +} diff --git a/static/js/features/files/fileOperations.js b/static/js/features/files/fileOperations.js index a4bc092d..f62b52d3 100644 --- a/static/js/features/files/fileOperations.js +++ b/static/js/features/files/fileOperations.js @@ -12,7 +12,7 @@ import { i18n } from '../../core/i18n.js'; import { notifications } from '../../core/notifications.js'; import { invalidateFolderMeta } from '../../model/filesModel.js'; import { triggerBrowserDownload } from '../../utils/download.js'; -import { tryInstantUpload } from './instantUpload.js'; +import { formatSavedSummary, tryDeltaUpload } from './deltaUpload.js'; /** * @typedef {Object} BatchResult @@ -392,6 +392,7 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; let quotaStop = false; + let savedBytesTotal = 0; const targetFolderId = app.currentPath || app.userHomeFolderId; @@ -405,12 +406,19 @@ const fileOps = { if (quotaStop) return; const file = readableFiles[idx]; - // ── Instant upload: when the server already has this exact - // content for this user, register it by hash — zero bytes - // on the wire. Any miss/failure falls back to a byte upload. - /** @type {UploadAnswer | null} */ - let result = await tryInstantUpload(file, targetFolderId); + // ── Delta upload: chunk + hash locally (worker/WASM) and + // transfer only what the server doesn't already have for + // this user. Any miss/failure falls back to a byte upload. + /** @type {UploadAnswer & { savedBytes?: number } | null} */ + let result = await tryDeltaUpload(file, targetFolderId, (pct) => { + if (batchId) { + try { + notifications.updateFile(batchId, file.name, pct, 'uploading'); + } catch (_) {} + } + }); if (result) { + savedBytesTotal += result.savedBytes || 0; if (batchId) { try { notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error'); @@ -492,6 +500,14 @@ const fileOps = { // All done this._finishUploadToast(successCount, totalFiles); + if (savedBytesTotal > 0 && notifications) { + notifications.addNotification({ + icon: 'fa-bolt', + iconClass: 'upload', + title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload', + text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en') + }); + } // Refresh storage usage display try { @@ -643,6 +659,7 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; let quotaStop = false; + let savedBytesTotal = 0; // ── Concurrent upload with limited parallelism ────────── // FIFOs are pre-caught by the 0-byte arrayBuffer guard, @@ -672,13 +689,14 @@ const fileOps = { const parentPath = parts.slice(0, -1).join('/'); const targetFolderId = folderMap.get(parentPath) || currentFolderId; - // ── Instant upload (zero bytes on the wire) ── + // ── Delta upload (only changed bytes on the wire) ── // Same fallback contract as uploadFiles: a null result // means "do the byte upload". The shared accounting // after this try block handles both outcomes. - const instant = await tryInstantUpload(file, targetFolderId); - if (instant) { - result = instant; + const delta = await tryDeltaUpload(file, targetFolderId); + if (delta) { + result = delta; + savedBytesTotal += delta.savedBytes || 0; } else { // ── FIFO/pipe guard (0-byte files only) ── // Named pipes (runit supervise/control) report size=0 @@ -773,6 +791,14 @@ const fileOps = { await Promise.all(workers); this._finishUploadToast(successCount, totalFiles); + if (savedBytesTotal > 0 && notifications) { + notifications.addNotification({ + icon: 'fa-bolt', + iconClass: 'upload', + title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload', + text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en') + }); + } try { await refreshUserData(); diff --git a/static/js/features/files/instantUpload.js b/static/js/features/files/instantUpload.js deleted file mode 100644 index bda924e1..00000000 --- a/static/js/features/files/instantUpload.js +++ /dev/null @@ -1,177 +0,0 @@ -/** - * OxiCloud - Instant upload (zero-byte dedup upload) - * - * Before transferring a file's bytes, compute its BLAKE3 locally (in a - * worker, off the main thread) and ask the server whether the caller - * already owns that exact content (`GET /api/dedup/check/{hash}` — the - * check is user-scoped, never a global content oracle). On a hit, a - * single metadata call (`POST /api/files/by-hash`) registers the file - * with ZERO content bytes on the wire. - * - * Performance posture: - * - Hashing runs in a dedicated worker with WASM SIMD128 — the UI thread - * never blocks, RAM stays constant (8 MiB slices). - * - Files below {@link INSTANT_UPLOAD_MIN_SIZE} skip the whole dance: - * two extra round-trips cost more than just uploading them. - * - Any failure (no WASM support, worker error, server miss, races) - * falls back silently to the normal byte upload — instant upload is - * an optimization, never a gate. - */ - -import { getCsrfHeaders } from '../../core/csrf.js'; - -/** - * Files smaller than this upload normally: hashing + two round-trips - * outweigh the transfer. 8 MiB matches the chunked-upload threshold's - * order of magnitude. - */ -export const INSTANT_UPLOAD_MIN_SIZE = 8 * 1024 * 1024; - -// Absolute URL on purpose — works in dev and in the release IIFE bundle -// (same pattern as the pdf.js loader in thumbnail.js). -const HASH_WORKER_URL = '/js/workers/hashWorker.js'; - -/** Hashing budget: 60 s base + 30 s per GB (WASM SIMD does ~0.5-1 GB/s). */ -const HASH_TIMEOUT_BASE_MS = 60000; -const HASH_TIMEOUT_PER_GB_MS = 30000; - -/** - * `false` once the environment proved unable to run the worker/WASM - * (old browser, blocked worker) — later files skip straight to the byte - * upload instead of failing the same way again. `null` = not yet known. - * @type {boolean | null} - */ -let _instantUploadUsable = null; - -/** - * Hash a file in a one-shot worker. Resolves `null` on any failure — - * the caller falls back to a normal upload. - * @param {File} file - * @returns {Promise} - */ -function hashFileInWorker(file) { - return new Promise((resolve) => { - /** @type {Worker} */ - let worker; - try { - worker = new Worker(HASH_WORKER_URL, { type: 'module' }); - } catch (_) { - _instantUploadUsable = false; - resolve(null); - return; - } - - const sizeGB = file.size / (1024 * 1024 * 1024); - const timeoutMs = HASH_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * HASH_TIMEOUT_PER_GB_MS; - - /** @param {string | null} hash */ - const settle = (hash) => { - clearTimeout(timer); - worker.terminate(); - resolve(hash); - }; - const timer = setTimeout(() => settle(null), timeoutMs); - - worker.onmessage = (event) => { - const data = /** @type {{ ok: boolean, hash?: string, error?: string }} */ (event.data); - if (!data.ok) { - // The worker ran but WASM failed (e.g. no SIMD128 support): - // a permanent environment property, don't retry per file. - _instantUploadUsable = false; - } - settle(data.ok && data.hash ? data.hash : null); - }; - worker.onerror = () => { - // Worker script failed to load/parse — permanent. - _instantUploadUsable = false; - settle(null); - }; - - worker.postMessage({ file }); - }); -} - -/** - * Ask the server whether the caller already owns content with this hash. - * @param {string} hash - * @returns {Promise} - */ -async function callerOwnsHash(hash) { - try { - const response = await fetch(`/api/dedup/check/${hash}`, { - headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate' } - }); - if (!response.ok) return false; - const body = /** @type {import('../../core/types.js').HashCheckAnswer} */ (await response.json()); - return body.exists === true; - } catch (_) { - return false; - } -} - -/** - * Try to register `file` as a zero-byte instant upload. - * - * Returns `null` whenever the byte upload should proceed (file too - * small, environment unusable, hash miss, lost race, transient errors). - * Returns an upload-result object compatible with the uploaders' - * `UploadAnswer` shape when the attempt is conclusive — success, quota - * exceeded, or name conflict (a byte upload would fail identically). - * - * @param {File} file - * @param {string | null | undefined} folderId - * @returns {Promise<{ ok: boolean, data?: any, errorMsg?: string, isQuotaError?: boolean } | null>} - */ -export async function tryInstantUpload(file, folderId) { - if (!folderId || file.size < INSTANT_UPLOAD_MIN_SIZE || _instantUploadUsable === false || typeof Worker === 'undefined') { - return null; - } - - const hash = await hashFileInWorker(file); - if (!hash) return null; - - if (!(await callerOwnsHash(hash))) return null; - - try { - const response = await fetch('/api/files/by-hash', { - method: 'POST', - headers: { - ...getCsrfHeaders(), - 'Content-Type': 'application/json', - 'Cache-Control': 'no-cache, no-store, must-revalidate' - }, - body: JSON.stringify( - /** @type {import('../../core/types.js').CreateFileByHash} */ ({ - name: file.name, - folder_id: folderId, - hash - }) - ) - }); - - if (response.status === 201) { - return { ok: true, data: await response.json() }; - } - - /** @type {string} */ - let errorMsg = `Instant upload failed (HTTP ${response.status})`; - try { - const body = await response.json(); - errorMsg = body.message || body.error || errorMsg; - } catch (_) {} - - if (response.status === 507) { - return { ok: false, isQuotaError: true, errorMsg }; - } - if (response.status === 409) { - // Duplicate name in the folder — a byte upload would hit the - // exact same conflict; surface it without transferring. - return { ok: false, errorMsg }; - } - // 404 (ownership race with a delete+GC), 4xx/5xx: fall back to the - // byte upload — the server dedups it on write anyway. - return null; - } catch (_) { - return null; - } -} diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js index 2a341978..b706d36b 100644 --- a/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js +++ b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js @@ -70,6 +70,99 @@ export class Blake3Hasher { } if (Symbol.dispose) Blake3Hasher.prototype[Symbol.dispose] = Blake3Hasher.prototype.free; +/** + * Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload + * worker. Feed the file in slices; every call returns the chunks that + * became FINAL; `finish()` flushes the tail and returns the file hash. + * + * ```js + * const c = new DeltaChunker(); + * for (const slice of slices) { + * for (const [h, s] of JSON.parse(c.update(bytes))) { … } + * } + * const { chunks, file_hash } = JSON.parse(c.finish()); + * ``` + * + * Correctness of the incremental split: FastCDC decides each cut by + * scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When + * the chunker runs over the buffered prefix of a longer file, every + * produced chunk except the LAST ended on a content/max-size condition + * — its decision window was fully available, so the full-file chunker + * makes the same cut. Only the last chunk (cut by "end of buffer") is + * provisional: it stays buffered and is re-examined when more bytes + * arrive. By induction the emitted boundaries equal a single FastCDC + * pass over the whole file — the mirror test below proves it. + */ +export class DeltaChunker { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + DeltaChunkerFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_deltachunker_free(ptr, 0); + } + /** + * Flush the provisional tail and return + * `{"chunks":[["",size]…],"file_hash":"","total":N}`. + * `chunks` holds at most one entry (the tail); an empty file has none + * and its `file_hash` is BLAKE3 of the empty input. + * @returns {string} + */ + finish() { + let deferred1_0; + let deferred1_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.deltachunker_finish(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred1_0 = r0; + deferred1_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1); + } + } + /** + * Create a chunker with the server's CDC parameters. + */ + constructor() { + const ret = wasm.deltachunker_new(); + this.__wbg_ptr = ret; + DeltaChunkerFinalization.register(this, this.__wbg_ptr, this); + return this; + } + /** + * Feed one slice. Returns a JSON array of the chunks that became + * final: `[["", size], …]` (possibly empty). + * @param {Uint8Array} data + * @returns {string} + */ + update(data) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.deltachunker_update(retptr, this.__wbg_ptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1); + } + } +} +if (Symbol.dispose) DeltaChunker.prototype[Symbol.dispose] = DeltaChunker.prototype.free; + /** * One-shot convenience for small buffers. * @param {Uint8Array} data @@ -96,28 +189,26 @@ export function blake3Hex(data) { function __wbg_get_imports() { const import0 = { __proto__: null, - __wbg___wbindgen_throw_bbadd78c1bac3a77: (arg0, arg1) => { + __wbg___wbindgen_throw_bbadd78c1bac3a77: function(arg0, arg1) { throw new Error(getStringFromWasm0(arg0, arg1)); - } + }, }; return { __proto__: null, - './oxicloud_hash_wasm_bg.js': import0 + "./oxicloud_hash_wasm_bg.js": import0, }; } -const Blake3HasherFinalization = - typeof FinalizationRegistry === 'undefined' - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry((ptr) => wasm.__wbg_blake3hasher_free(ptr, 1)); +const Blake3HasherFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_blake3hasher_free(ptr, 1)); +const DeltaChunkerFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_deltachunker_free(ptr, 1)); let cachedDataViewMemory0 = null; function getDataViewMemory0() { - if ( - cachedDataViewMemory0 === null || - cachedDataViewMemory0.buffer.detached === true || - (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer) - ) { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { cachedDataViewMemory0 = new DataView(wasm.memory.buffer); } return cachedDataViewMemory0; @@ -177,13 +268,9 @@ async function __wbg_load(module, imports) { const validResponse = module.ok && expectedResponseType(module.type); if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { - console.warn( - '`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n', - e - ); - } else { - throw e; - } + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { throw e; } } } @@ -201,10 +288,7 @@ async function __wbg_load(module, imports) { function expectedResponseType(type) { switch (type) { - case 'basic': - case 'cors': - case 'default': - return true; + case 'basic': case 'cors': case 'default': return true; } return false; } @@ -213,11 +297,12 @@ async function __wbg_load(module, imports) { function initSync(module) { if (wasm !== undefined) return wasm; + if (module !== undefined) { if (Object.getPrototypeOf(module) === Object.prototype) { - ({ module } = module); + ({module} = module) } else { - console.warn('using deprecated parameters for `initSync()`; pass a single object instead'); + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') } } @@ -232,11 +317,12 @@ function initSync(module) { async function __wbg_init(module_or_path) { if (wasm !== undefined) return wasm; + if (module_or_path !== undefined) { if (Object.getPrototypeOf(module_or_path) === Object.prototype) { - ({ module_or_path } = module_or_path); + ({module_or_path} = module_or_path) } else { - console.warn('using deprecated parameters for the initialization function; pass a single object instead'); + console.warn('using deprecated parameters for the initialization function; pass a single object instead') } } @@ -245,11 +331,7 @@ async function __wbg_init(module_or_path) { } const imports = __wbg_get_imports(); - if ( - typeof module_or_path === 'string' || - (typeof Request === 'function' && module_or_path instanceof Request) || - (typeof URL === 'function' && module_or_path instanceof URL) - ) { + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { module_or_path = fetch(module_or_path); } @@ -258,4 +340,4 @@ async function __wbg_init(module_or_path) { return __wbg_finalize_init(instance, module); } -export { __wbg_init as default, initSync }; +export { initSync, __wbg_init as default }; diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm b/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm index 7ab00d463942f1b339a82028fd04fccc5e01d2c5..0f09d2a67647f0a5d3bed763fb86391889bf01c4 100644 GIT binary patch delta 21014 zcmcJ12Ut{B)BnA9E9@?;Vn^Zb+E78Tfg&mwMa3R_>Yo(NRq^JtH##|fctR2vSp2`*YIC(*O7on=8;Ugzuiv1V)Zw$vHS1ZLhXn;&UGKaSy61!l zrD_Wcb8VgLh6%5B+A`|y5YlsPZR!OXQ+MZaF;-sWr1agkdG)G#XXJ*qHR(mpBuJbO z=7d_gaHJv|XZZL{!1A-4wO>8*Dtl**jfJaWlCT?Cv z!SG?}1tDc?C{(}VKaj0a$VO8U-5Q1NW+73Y;6WwaXUP$Y0(` zi=j^7+M0N9SkwdxXeD01_YM?t8B~3s@Maf*N9QXlFqVP&yQ!QGF$)w{urjd{JI%@& zteojICWDn%v-Zs>%bcVPTFp8(u#TNhgEp{oHfulNG$xysb6NXQr!l##oX6TvJB`U> zWneay6;7}Lmv_sAzB2lRQ^$E$KFr^gav+@SkzLS-eF&kJpo3$Ts8k5b+xvc%D)0kXV&SUMToyO#`@(F5} zh09KZPO$QMHsR)nLBe@f&S&j+oyO#|asg|9;xwj!l^?S97fxdyvT`A7f5XZ>l|hB9 z@``m#C4rZeL9bYQN-kqNYgfjkFr|PlK%FpH8IZ~v^-_1HjnLZVl9!O3Yr7w6DbLhQ z)R2}%KqXN{MIU3)k{(u+{Gn8~&Qg$>!4v-*#(v4SI`r4hmCu z+lqyC>*!_FI~k%-t5$+VU^ZE!*krA!vD;)wS&?<(ELyTotOv>t1+3ikDACf570 zX`Rw57>BVK7Pi7H##nJ8WaipVg_ko}IIL+$q;0dZiq)3D0@&#h&8nDTNx>}ui?L#_ zR_EH*Hw%&(9LYU~EI$q-J3JbQp|Og=4aMGA`52rzG!ZC>V3x!KeKn1Wk$h01jZaCz z8IheEYE+Ts#L8>~9YCoMFeL0S3Mh{auS~}N%W#kXfog9# z+sqr~idbBTwW=#|WZC~W|3otXbm)RMIz-`FFdpEj#qlZfXfX-smjeYFrjHQ?QDxSZ z10UgFrMknh z>?NwhQYK?Z=s!6~=uAEHsi%Xeerx17p5r(*e$4pcQr;TWp%{Vg`mDQPomxM77Mb;kQNmu$hCd7AA>F|APkkw|OVy8Gz2vP8=tF$e24-^6bHkFUTZ*z>& zP>c{DSu~-sYD{aBi$OG)9njDJ4fM$rVA3`v~h?C|Zp@4*BApj`y61E!6+928nQjr24Tk?d4 z(5^BGOy$jb6PEJ9R3;y4h=-4Gm$yR2d%#1^fm=q9O?tgZtr>6k^E^!U&T*SX}@F##k#mj3z}!?nd=rGRc_` zh$}cRrOi}?44|>iZQu`(1bw259jvZ)vRsHPJtGT>4GKL01FEQ%vF8Z(L>EGh0MnkU zA=g9~Ab0<3$i4mRShT4uSRc)$S$mK`?KMMCW(7_npY z)q`>R80iN(NdTN3L4ut|J<+IRG{Ve)Tu+=q5he}BL3k-@RWa&d@C2d`>%>@O#KN>C zbZ*iSX+K0|I1H#_Dvy~Ob7iu6K$4C`+I@IGW}_8M127!5iYMVh0O%l#sl*|={U>2& z{h=SL!$&*)PXGvsCOM7T8O++eStJg&c75J9cej{GjMC5|c}UsR(GdCpyn&f?o(+U? zLYhEI5s7B$#T*Q+0j<6{7llF^Q9EaT-_O;&N6zEOX1rWbG)c-mqX1RO!6cJ?y>mpZ652IgINFu5o_cU<>>$KW>Oc!+g?es(zMYqADy! zFPg_rx0-`gdf2pL6T)1li{|lcq*b#e>^)z8N|<&!17KBCeaiR=qOhe0fT~1yTW0eb zvOBg`AG2jD92uS(Q8lw_3NL*5j+#;PV6fqII9 z1dP=1#^5TcBRs7dvj-3tL`|$!AfHNsb)KEW_-)m`l^qAZ(uI?hyoI8}RB08dRs6Iy zfLa5bTH&T}s!+HOc|-|S$d^cg(f5Pvu2UEY{QS6BoXU)=t}SUK*lBeSQWelN*I@1; zeO-dNpXrN)O?ZMNBv^I9+#@sqJN!@7D$v!$NVcrauchZ8SMX0~W_v^f6?!53)G8Q6 z)mu(2T8gu;+2VnN$=zI1U}h+Thv^JLssoMWHo&CO!Q~=pl478^#S!pq#(^Yyps>QW ztz|hGP4JUc(rTP%j&inB`8>+dC?}sv#m0dj3d->cr-*tBd|0VKajX=iQFEAAO^k(y z#B&Y#!Gv)b52v@EQO2Ro01D{@knn=U*!4l#wkM<$mq~DN%79Ekngk0ENs(5-d|*GM zLhOqx34T9|8S8?z6l4J~lQ}B8*vlf0N|HSx6uyNSa^BhEF3AANj~XBvY75aw|4>uVfQYuIG#Q8zq@9GOx6QMamz}ZNBBqk~L)H`? z!-5qFCmrcla)z7*wur~XFeXXR&H`c)=g-G9GH!CE$`3wg#?goShtv9b7#(Qcv{4v<|N z0KvJ{t_>-IT^$N}=mSIpqF^p5Qb5RCHKY*W8%3OoLg-K=h0qX}1v8EunFNm`qgcdM z=$o0eJBEJ6n#KPOSZFAh!oM|ew5UUYQ9v_n30f5tNefcP6Qc45OerG%3rt}t>~k4J z3I-f4bN~{?qWgH-hJFHc1a=c#gm947p$5wdquScU8#Uxsph=L9GGWC@q!N9I5h(~v znFba*X&`NxlLp$#wyh=?sV5}JsgEE5Eh6{>dSqP4%n|Y`^9RlTuc@%?#1~*|_JHLl zdtxq05Yy>27dgWtlVFl{AEg38t$$+2` zM+&Tzq>DjvfSZvL8puIUC9WEmk`fw(vLG2K1_p!3fnYTblgVFXk%!_i;?RZ|C}D&a zF@zu)5pGi>LI{Ck-ege-OOfIKA%ciUlM+CrfEY&rF%@NWz*Gea<5746hyt-nfNzBe zLPLdk3)>kVAqldHdNTYA3LMzUsBw0X6)(%-AgkeJ=>&Mv%)h0e(;m-h$jyRrVy-X% za?o6S^cFanl#3v12=Q0PN^Y&ee(*s;!a$-tgJT%JYM3WA)P)!~TVb4r@w6JZz|KdS z2$v0P1UDJGv}RY7{*qDu7e46-u_Ty%eBv4UC?qp>$>jX8oC)Y49dgj2MyP~QL^Y91 zf<^@4P}V4?6C1B!j{yo3?cCOU2uC#J(`w+igWE0+Kopk~v%zPXLJ_oMDMCbYaj=x4 ze>+Q^P+>SIOkL(+Dm-h>pIeg>+K^Lr!=Nc8c7PlJ z7g!u6c0ghhET`-d#SYGKw1I`r_Q+R|=D{NVW#lV6ht(g&5&t~*Z>Q4v{x^~PM@xVv znFdZXs0=c%gaxY+76?jMAcTS&$Tz0*%$ZTD?9fqxDNJLr4LoUXksHCx6VDDBdpb=Z zhk?g0r!fK4!u)5MOgxD#kM2;EvAwLUwDaUXYeam8AYXw`EVLM(!4?&bgT2Sug_0_m z8;lPeWf(^i!SDnmmW701ZU_*C0VS(xcgx{Iz^W?CspWWJ;+kOLvF$t}Fd|YR6pO$@ z(8^YI0!1Ap7ScpYp&+e_^c^Ksr0g^}Z*_5s+3o%RmKo$ne&EJGFazYmn*vJY!ZK#S z6ibTp%%}t~hw}`@2#`pTvt)p-yvsS}_&D&9~0`YUDkFls#5Z%G0)wFSLC?NcC5b2;CJ&t{6d_mn{Um#vYav7%> zs1RX}ZbFK9Q6@qxdg45mBIeskhE6%k0yIKN#@NXPm&0&cDgN~*ZSbx?CD;Z0`x`Pf zviN_b4t9gFkMy`7QKVy=Y+@;IYy^?)K(WvsSPFIn0!2b5T|{*VCJHPzVtVYLoMxgN z3Lzl)dS)j``H@4kD;!cGkPol|O?aPy41Xh71Ik+EeJ~9q4`3)82?hZp$N~hOTG7c$?%WFT^F78h2~>QXUEb53$g7ynUT)H5Y&sGOKcr(=^ zG`Nw$_3PG=bw0kAQ&t7RFGqSe$0H7?C@oK~S_I`r6U6@I4XTP|t0a2g;kZFXJV!U0 zJxARxOD`&L@B?9#M2?cggkH*STbI!_O=xq6n%i5_N7pP)qu>R=831+2{^Lsd{OJCC zX#@EDc27&sLvT0yYHV9I{7b>c%Z&*qNiI$6_K>SKDEHx% zBVP$TC0;^klno+rVr9EHk#IqrR1l|qvXeMLLICiAJk(tgC*+L@0t6~xv7hS)fwC=2 z+0XV)z}_h@l2ppx0kVDXc-koC;@Qeum>p$JMQ?NHHWYiee3Bns9+QaCqZ|kzOZifX z!wCEBu(A!%9*F0sLP-ypNQpq40wtpe3YDCo05)j>96Q4`4Z=w6L;>sh$bzp`0J~rU zgH-4wpR$q##5?{`gAa~Hy75**YGgj()`xKxclgz0FJV{i9lGTpr}Dg2Pi^vWnjpbLMQ+>5|$bOp#>RzdQ5OEXh*Wxln#^v z)>u?V1Oz4=5wEt}-8RaaSm7(Q#|u0c;GBhYs>UnUv~Sk$O&isJKY^ zO2rmDd0T0Fentn!O*%anmMD8R0M=eJMUYoh380}bB9tt}Ud6?49mEbAUQH<#bJ0Tx zN}=f$dQehZX@9^fKaRs&0vJl5$ig^uBDA6-4xgeRjwVys#@fJ^Kta|MM1q4r&KFlm z#4}}bti)5iOk)Lm5J*FzM+^*{FShuNqa4M?Y#0McF|fd35Oys5M<>{$1d;^&0}Poh zuSA}-Nyq@eFo=Q@;IerNP>E;pBSHKdymUo)*>S<^3A}hD2Cc~_93@wh21y%Zu85}` z6jMXsnVf4SR|?rRNKoPN2v0oFgj*>PL$rqp6m@GH5xa)bCJJ`S;8cI0ObzqGxFayx zV+09AkTM%w!f^H+hPZ@fWm=cFhwSXJriw-EI!bNmJTX|gjAY)WN~(oK0lSRkDPk8` zUVs=X3d&Y9CQFpsaO@GfpbNwav_P8eH-#)pKwMoF_tK#I9|PJP&uQ~je28sJYFqq% zotnmn`_c6fG6?VGvCn3!$b(n<<%=GrQy)l^!Hj+WtR!PK!S ztt@?xr!`jk)sBA4oJQ2qZw~7pH%Fq{p*eoS+qpUanRB0)BcP4UV;53LdCf?d#ygUk zN>IuYp$q4bqH%`hYL$Q}oTtmV0{+A0oS<(~dd8Ceol zA%@^6H%h5m110k9@fHkQFn5FQ8fb!oK{i#%3kK@AdLWnLyo{>jri>6~VHm{;_QZjT zkP`6AsU?6~SmMADGEtP0jkLLPZhaW)N5L;H9Io>$@TDdF5E)un;|=gqlyM1h9@i3- z%hpgD0gR=x^g=j~&(cSh$GJ!kuPqc<8Fr|6jgr7-SsGin z=oT`ISfHT#nEyZ7Q!oPAQYEpLL_s!7qGBiEVZa%dp}CADtoG3jyvpoLHU=3$I36gd z!5l|Mn%B%~qy>@o+L9T)?(l8Ixdgc{yB zyZ-o_RNVhPjkXSE!@+Y^CE8x71IQPU)mKSInJ$;mg0wy)p3?d%$Gaft#D}cD$|*b~^yT_B@udo>||H`uC8Y+n4o*<_nE zv9xSYPfB)HsQx}1`L`Lxk75mY0-wez_Kf396`TRG;zmU!h%(WZI)6Pq*RQo8%8U>W ze>Olt0R7nj{)}Pmf)@5S{FvqVTY@(WB;jbft%4v=3`$ovx9dYgsD zoV0cSi!muMj^^67;Dr%YHmL-a%xo$>c;G5=gcCixp(l<^(m3fnZ<~}+LRK+*$Vqv0 zUuZ8QEmTx4B#Q^dci31`Tb7N18Ah{W=?Zv&S5k5urMft2HKsYTa-6ipVK(7RIe~); z$tn7!qQOa_vL59FI!bm<>DtqrMtn5tIr>oMhwCm&On( zZ_i+X1cBwrILVh@5|TpFGZVUrgO#K|9f9*mjHjbqPuREP3?~F$#XKBbMF~a!L>{6K z{CPmYBUxr?kamHh(>p8}b9~d?5kVlr!#a>MJeDOtg@Z~U_QInhm?o462{Jnzdq~Dv z0d9n}*j9AcPl~h|FT|uK=GOl_)b{39B+Uv`t-7$79MQrxy4r$m82)E%7rQaBjD$Gd^u;oUxO4w>{68P_4io z21s;ul7d?R=Q@bUv)9+!wk_)ht4%JSO~3p*TgK8hh7wk&Ga{T=b8*|#rCy}T?#nD? zE~jH6WgkdpDQJ+2DPvkL^AfJIG4kJyv5>a~or1J+hD?W4VY43S5F<#gBs2)n5Pm?Y z(42mj8-@c2pE$bce*Ydhw!3J;DW2rdG3OBBI3!+;=D1j80G||O@6$u(@Ygw;@2Cm7 z)}FI~5CC92;1MWzLJm*@d5toI3L%zKj$HE}W#p(+V*#EKGIR<;Gi}47OUb~DUd^-o9JS)1Cbkc+<}r{x6mY~a zYEoqlQ)SAm!UiB$6Tu#QDwpPvBT{ADY%)6lU}R2|r-s-NyKS)>9u6pwqf^cK0`PxS zb6eRjCL4(x4rRBk|H3N!qUGP1h#kB@VFElNE!=t?;{ywtpe<1H*b!Us#{mzp6v4ioHE2 zzOx4qEX_(UF7~{JKC`hw*P!p!EqWGGkeLUSvrG6 zi~b;khyB8q@rKZ%jm2d&HXTn&#p%+{xp^GD7iF1ItVJ8QmGBTK`YC3?-eP%7;0+Bv z0)W#8${|pmgmjsekYeFtic7BSgt1T?o66ZHW>)&>fZCF2_5O#IZT@S+`Np=9Yul8| zK(>MYRE8%xr{IR{5DqkRHQSA~zCK+zLkUg@wVN&&7+7F)OU_V?Bn)UKA&CvE)zId@ zuB@?`p)8}4lcvtJbzav-rZ+->ESQU7f;1nRBGr*r(5rIr7A=p-eW)nC`OtEoj?@~` zRMMDGT#8|tiX2!}ku$ikCCyzqgOQUS((QvMIUW#~DYze!&T&-3AW&j%&q{7o>_|g^ z>Y@2I&-I>iCpI1CF_=4$ixmhebIJ@ll+6B|L9eXCAuoZ$3OLNF2XmSN!7#k2JI>1f zG(6E7+_Sf@b>NV==)Ux);MQQP)z_LdJgI-2HMmq_+>pp<5r3u~oh&9M^-oAj6#FNM zc_wkd$ehCUE`n{t#zwkP(UJZ`hez2OZ){R}VE@FV$f!uax-|o8*78qGi1Z&-JD_$z z&XbK6-rPrJ=3F_BTZ*45__2*#-?stRoHui=Q5$IBI5&L56B33e;M+_vb8YZ#fc8rG z&PV-o*3NOB_}Z+SItb5s+vrVJxfCFGL|ZWdKlnQ8UZC9DsooHEqfos_PwLJH4@AGkso#_5ICoaJ$(w5y zci`1a{t#H8V3)R0j_qN2qCW5lu13Bzsf z+k9n?yBRc5oDkBx;q-vw^Y;yIo1BqQGu1t{nDz{FfFjaWrJtF2iND_y227qe8bgpq)=gArMXeHn@?Bm+`HgR zqjOnjx+a$ixnI2Cx7}R%$(Q~ZkUDF)`^fr<{u_oHPJMm6QFpIr>zgjw9bRkfV_nP8 zU)*K&Y`dl%AryZf>mvu0Mkx%be3EYC;3se&>l^qn*?*KqFA&5d412W2$!{k2B3 z=s$0JUa!^lSf(kU#L`}!kISbXW!zu8pxyC_=ExS6VunP&v2;Jar_kC`8&`G|->2hq z>0!^7zqMa7ZN-6(FCYE%XtRI4{F8GZ)q8yN!m#S@gNxnVH|53gXT{GwDxLN=WJJ{3 znteiYXKq~ZQ>C}P25v1m^OXAD(K=5)OM3TR-|c&wEVA@nc)|Pawfv_|>rN}!QtRTK z+j3;an?9A#g>{|7?f%)UJ~6g(WXrSZS-$rAFQ~Q_5J>LGa6l4(IVD= zPoqCB-EoP$`iFPc>v@UC29McRxruxwww0^t`t;8I25mifIC8O}?d=lk$EEV~_wOn7 zWxd#EnXi7l`fgj+G(X#emT!%Z_LVQCpIx|c$01#pX*-t6m)@^yS8=}Zyk4c&=i;|i z^*azfa^uX4KQ2A_K4@x}xuJO_N3@Q=xL9%<_~y+1nff`!7u+1w&eUhgeg-%%R^>2UD9+`?^*w8f9rSEW$9dn7hIEm9p|bZmuUIf)jQo!@7wmpw)oqv=Z5>g z3*Gp#t@NZ@vl4ZRe^$c(zV@`dU|-^e66S;bMqlZ3r_A#oD^ELl>yhc_RjmhzaSv{P z5kBzhozjcTJlT3`N{>xi- zc;uUJ^d3~JYDQ#UeyQ1wpY7Y{x#-{u^H+1X^s4{5r&_uYFu{A<>aR~#zjau3Yi8SK z_y3qzCU?ZQXrZ^g>je$%#$eBu3GUNm%Bo$j5xSp2&-kE}EERg1HE`Jv18_bMj0T(D^9 zfr;Z2?(U0^cr-d;UXKp@Pi)t|So0>{Jm%S=S>8Q0(}nL=ZBt)r`5@psc~fZTFLzZx z`MR=St&vwU>+Z;WVNOh%sP0lBAl>Wp{&A@Zkqx!&nvLHh)U9@H!lO%>b-lyK9rMgD zA?J6mHRSu{J;TBxRQs>jzW0Z}>dw6}mf5k|tpoB$%<(tg4quelITEq|A%&-w1) z*}e<8HE}zZ9@-x@co8D4RIaB$;fTqoNtV{?wobPoy zr`wsA`J+pAu3YI}waC13Td&L-oHZc))$%F>D_3%Dy?%427UgD_3Yl2*{nKqTf4^~h zcte)~b>eooq+K-p_OOjD>tT<*elc8Hxke!aOg)$MOsi9`LGONLMucpM^LV+t=eqn? zylzb=_dy|#9$f5^wy9dyX1VHTn|FowojFnrc{whvS4t~>;;MtmwewE=@#4{UJD$&f zTs1>|_36RkUr&DV`P~|6Sv``D75HA?8+|=v$Gj}pk_}VheE$flk?iK)?$O%LX|)4f zjgNN!u_!4yu4&dP+k@{yXT{8W7klB)i%qIbuaVJg%Dt(7Gso3J|QZq=^eH5HS0v{-goYAk)WW7Q>(%F+D{*G}(< z@SNq9v#7jhn^T@|8$CGbzUQ~?wWeIJ>wmp|#a|n=OBUr9ZMCf$^)GYitfB9n@AIF% z=`(F>$??C}Sz&%$d~Z%_$l71F#eP>aX3-3{Ed{$YtIN&Zf3&LR8rP-SX-ibuZgD>~ z_$Bc4^ovJ5CRLw3snMFwU2kscf5GGC_R+ig%<21Mv-XK=X2!KDJ#j=oFH207h{Y2w zr?1>yqjsi8&yY8}Ub!qD_`Cew_csr9*?iyf^UzZLYwYg2e8t2u4fYP4_hQ-B`+0xV zeqJwo&WvR-O%hKZ(SKe3ZvQVEeb#n$?HLkvshFW*MB8#n8(&vwTji|wW?<6I zUoUK|TD?ubQYne?SDxF3JqoN)vFw~FONylq9A&(mt(Yy6KniU%E`M zk~1)|f8)~4->t6awzXLM$&DXPSbbr`>C8UGW-r(6-#V{L(t?e)9M!A>>oBj^=jROh z>PFwlpL+#&@UcaAI+t?o%5blopU!wR>wn1-)_AV(nrE^3=j9SlZ}g6Pb}1nL_HA{^ z@ai=VX5|il6Z&WJzEQ^hm8>H#EGrDS*URi}Uc0Q_(${rXlwWnF>+v{GqUO&g*@6+MOQj4d|8J8V$r~QnYLuU_oan(IN z<)(L3w|z1tST`)-^&#O*eCd_z_I7HNdM;pG_q*Lv z<&!VI3EC04c~*pHKcCurH>54ORO?Ku%3q|t6_3wtdHLeWS533ihTRTNH|E!VA{@xP zKjr+M^q*JNG&C5q_Q}O>FMZxOCVNr&OR-n?e)U6Vuk+U2k-A57?o?iR$-m6&F?UY1 z>U|(N-lJpXZzKNPd%9Qk;!9jk&K#7#=HVOL;Q@(q^`?hppRbzyxh?zFjKQv7_l(Ru z_RY5ee;jPJa@yu{eB2d_KK-{=$I}butZQHQS?AQAtxo>@+t~((yX+eE)_q)7gLzl` z3>)|B`ePo)H$QLE`FHOJv(Eq8r}e~lS=SbpD7VA3G4E`zaq_pmrI#J?(O>n;-adM` zZ&>WrI}y(gZhqUhi=k(gbLoG|WiO6w;A<{*yIS-F_bk__I_uxJ{i3;5eLuG4;+401 zwCj<4>gSFgYY)^}^nSR0Q~8vX8g~{KWcp1mfBMz((H_OhRj#!6+=_AsYTo#^!1}Dy zz2km)9vd!|no)B`iTx#RtUC0IZo`z74IlCo^D@ipu2xH5v-#NejFEjC_PKpzN%`4* z-lWJK|Fe8@{s@pYxP*J zdc9oE&_zQ-U#?o&<6O}0!j~(KuW3^2N$Uu`&Jl!zS-)+^9P!dqY-OpUr8<2Nr)eZ^fkSNrA;qwkS-mRO3eS+gEo_ zB;8(c@#$T!U8_gUk2F0D|D!bby=5ww*P|nT_0`r`VCA&qCfoE4N^EqnWb zi#vn%Or5$wJ1Qsoyu#r)}I9hkf{F z*Q4r882z2wba|3Xcv_>tu5q1j&-~--?d4{dbN_9^qC>T3x%>9*{^-w+a@oVfUw#{L z^3kkCGu~XBxNg~_3-ZK;cTCMrHu%wPY_o{;vZndm^44dMPEU(!bT4^gcB70z+m`)j z`!Cfuv}$>>)A5vgy*8D+(D~S`!l1UNTeU2)xZUUL=hiLdm;9#JyIXxdp6*_HYgF;0 z1M{Oi%B(H@^M$2Vx?I?m(DO1^rry}(qvdwj++Ai+Ov1jrfIG=Ou2*?7bWfY@&t#Qb zv-MtQlP{iIyQ@sMj-5YmzovWaq(j9YhbBMyy-Y5*NO;qD+_R{Z`OhC+vPIvoc&&Ya z%jR2SZ%x)b$~n=kOqX2`XMO$oj)YS?YDIf@DOr6KcfI?s!@Cxi_gZnW^~N$${YsR% zIP`gqhlTmmJ2$C%Zo#AySG?X{Tk~SA+{V}~V5atz?D9PErQz7RI!}81;#u!$$i7V% zJnL<{chu#~=8gBYYfgT4;dF#%DR+8izY9-FY>2dWUtM~Aq3c%lquI~I>9cnC`0Dbz zQBCjE-1qF(kg-cQJy_;{Gv?3G;V0T{UN9rHFlt`ijZ?-?UK7=1UDZ}6^Ga=98f6PC z(P&_e27wuigZ>~UnIHfKnm9V43GO0a;3#QPaji%o8D;+wE>YnEAKPcGUz{Njmq6-$j;u&nOGj=wit)@Fq;H@Ycg>IA zU$*c0{o}l)-_!#2PaQA!Tz%+hl$YV!>}l`D^*!9^_bY`hqb)8b&zrlRco^FbJahlZ z_?l^Nq}cMktSi+MF1`-^s>9NVgVs_(?xT8pZ+cj%Q`LdG5?_bk40*ZQ^HTFYhmT** zO-%SabHt_ktj=A> zc5$S6$|G}97P$sTy(=;ywLYm=bE@}1f7MUwR;PL^nqTLW24?{6J^?^=0$^*nttPTE zk8LsAn(&WpS=+`1EkMeuAG)UEhq5=0`T~0$|0e-Ekz`XNeZq0o)WiA@jpnUP+njDoP)SA>vQR zpZkyqQPBZTG80&InQ_pe4k%6-MMaxBIJGV9;52PwU5l6By7Iy6DL>C zY@9mn?97h8SM&Q`RAVnzN4v~B(_7TQ24i3PQ+1@vym4t`?$E}5QShZJ^pTC`_*?n{IEQS0F#QLPv6WshrF*#g`$? zON9%4Fk?zBX2nRmL18IqkSSYbpBu}a}FH7=>VR=k& z+4x#oJ2kv^%1hJQM1iy{kC#RlO8(7dH)g}NC9Q4FYnnkexw+|n>g{jZL%nBau69B> z>|r^)%V(F5%c*Q^>E)M?pB$=vTn_7<4D|+SNbzK-_)jmtV>GEETy=dOI*>ZYyG+fj zM%B@67R{Pa-_cEGVMBCwV-H;6%6Zid3LRQE98#n=v^xVE&CXfV)cS7o(X6UqeRnh2 z@QVozs*9&dH>d*}X})BG8lVQ70!HQb5x7r-Iv%4R;r2167Y*@;Jja-Rg6W-14|z^7 zy~EHv>rFrdLZ%L8TE{PXxE(UB<8~L*dt*FZ+}^_UZ84rL+{WyKywX-?*coHq%I%OI zZ0F8)X4)NV@8b3@ruWC%d$_%a>F>qbd%3-r>HA~tecay1^aHW>0JjGMelZw!xP6e@ z2f2MX);`4TLrg!)?IYYi9-|-O_A#axDOPmIbByUHnBK|skmp2b|Bf!+41N(Zb#Qwf z(|fob@~q=_7t?!VJYC%0!t`x1o-N$o%JiKvo~_*8-qjyqpLWNXwsU(I59lv3foB)D z_b~mv7|$MV?`8V_7|&jA?_>Ib7|%X#4={ZY?N*^frU8C&kY5~*F&(5f#RB37_6!*g zF)^)gF8GiQ!C2OEo?6#!{z!Z0AfS8~KBl+R=I{ zwx04Nt;Zx^MF>kHvEKZvE*W>6&5p$gs(^L{A4p&9Hs8N$BJzoO^Q!S72y18#*!}RT z3mo7gjD$G~KDG!iKZb&ur3YZvl(0B%K}me4a5!>ThF_8U3YWN z0veKyd3#E{m|hWAGiFd)Lbn7o2^l#r5F=zEIlu_v4!52^1^mEbwSdzd8cTg-kFQel zvN;nH|Lj#X&91_{0TQv186TfPYY zOjzw#wTK&OhC0#a!5!K1zqPv9rH`|u;K+VO4pgymM6F?UW5DqRaE+|ki!bxIu|Ry^}np? z7w8vZ>NLNTf(l$ZqQ|kO}RTZju_%5@IXlUg?#Q!=M0w)gf(i zsh@Ne#g%V64wSk`x%N^&P4=+#ep$e@2TVmTU?i*hm5tUQjmkHP!}Kg`68}Zd zteb0X*D7fDvrXa?P+%qe=SG4iAZTgXgu^B&)ADLqffz#@x_MYP>u{NH8dYUo>!e7y z-dKJ*-Z)@Rr8AoRqDJ*fzf3oxcDTqDZkbN5EckhMEC6Pg#H{1c*b43NMiv^tjP=$_ zQ`}w^fQN0^3(J<;J7ew8;+-;XJ>n~p&Sb_1Lo>3+RjOt^$UdD6HA3%1M!*+<76me; z7BgiV1gUJ14>wv%asl?I8gAPmZQIwho3c>IluTlq?Qr>^p{jiN^gKqmt z^nsFJg^7M;xfJ3Qz}f6)uYfXiM_^(SQjy@$j+3;Kr3IWZGwhs;aS0+$S} zoi8~Eb|DYZ{R%gO$FK)8Uy*hUrelvoFb}fzr^)u2lAMopbIaT$0>*aB-Tct#a!eJt z6+``!Q`#ezKnFpC5FHUYAVNxeWZ>A*RE}%a!S6kOej>e5)ce2{8a5wUE=V z3L338^z0__AReQe#CP#<+))%Q?Y`3_{sIadg-s9!k)#|fkrs)glI=aZ6fI!I05IEb zmFtwErk zo58w(hIYmWXZ+XA%Ik}wJed{3tsuc8=fEZlKp3dQD@Bu0d~`lMQqV+9(CpAqo-m79 zhbY67Wihd+4Xh<@<-?Q$mMBCIpcTObXQ6mxyiEgl=jCNCnT3yol-B!(+?Y*LRM;^J z&7-lqeP5 z2s1Mc!H9%|GyxD*g-HSj)L(oL7&TQN!^62$WOc`S?(f2b`tS5TbzR!A-|HlFFq^(} za!%gMBTwt~py*1yn_eJy0!7MnQRxHB2pGW`T^C(IHV#nO6JTOmA0j+FP8>CT2XFmW zLOOj7cU&dC!$X7gP77kW`Ru=!TR_-H)k7179#m#kZ>2eZ<@x5d-mw8LH+m!i8x?=Z z(+MahIggMup1?D8;n8U2~M2sbCG0c0|)*9x`Y6P{QU$`flreTwkx$7}Xgq#sD=00MIf{(Q)F4hyf{4(;mk*7{o z9eeUs8r!kder9@}q9(#8R5PH+T}Zki7O?SokxK#o;Q;!*5z&ll{7mBc4-Mxa^6OYU zQy^z3#&CuSLyaS@4KgAY5bcc?YD9!)ljxyms!437XU4T#;p`|Tn#3j$$lqYVAqORA zg^jJYli38eH={G{UNV5*z^^1uxx)t*se5F>xtOXSGERTyz zJU2<4Q`jhaFl)~Fg%tVT_u3tCDKIEqp-a#7rc9Y2FE=E-1~g`9|H%1z)nj79>qWR!w4 zbWjg7aE3B)kg0Z4NWm734O4K6Q!q46Iyk0yd!E%nyPIu(7nOZ{J@5Oz=@(U zxW2x;SQp?f*-hZ80Z%Fi0KKQ&O5mi6rRi2ey0{&nTM1YU)AF(q6!%puTXWdJs<0Bh z#zlvZjYv|PZ%bOK9j;Ed7D-(@HN=HN2h zw_vL9%1MP+K+rh40vdVcWD(4dymG=>k*lNcrVsQ%4h5l6?vS;wA)la22Z;$x5?gbsOEV&EYyOMI&%I^x10KqO&@#mN~Pq zV;bGlZ0J}U-R$#M1vBw+oyj5aX#{SCmOuW-vrehkzQSpSeP%?xw{gvsEMNJASBR!@ z&4K_$q{_aUvTT%JlFF}MTn$;15Ej@z6&@WmU<3)d3|@phNyya$=xxC+Xo+KmMa7h+ zBlfl@n*o<}Q$W2Q?zQQJm*nIg9!?1KP&MTQ|1nQ&&jy)j5WPaF74HHb<6RUc@Q0C0 ztwkh>20^h-ti3p9MtKx|&?KH>=N}ShkWQflUQh$4&;)w5C?2TdDP6CS{9jTEf@nUv zvns$VSYbrzhnT_wgFrhzK@R5%oH0?!;r9u}MI2BHolZr^&N_%j>(?UXd)Nlt&~W15 z6DSggMF|9n1NT0hFD%AwDL43V}!uXJBMm_1))`_EAN}#Lhv2k*^mWd zR62$7=`(`wuoJ;|GzZ`t``WM&_>Jqj5`JUzBmB08@f*WS_)XG;f5Y`GuE7Ax5rm^f z5RL~yI3APC8_&&_W#;qe<^(Z#rEBB}UhPxi6#|ZgS4%&8hVc3okrJMaD(R>+5`Km0 zTysSUdXaGA&>Mb^`)JLDi0iVo)g|a<7>%Hpd=b!V0ebb?81#lim#@9MVt}C+*@l81 z;5+Tby38>68n5$B`oj6^&M*6H$5kt8;@D?96eI*^{u6bs4g4gGreVfGPcAsc(L~?zwT%*)WAL*8}VN zh2H%HWxpRSd|Fj+qb9DTu(k`CI-@S`887Nn8#h?;8>Xv!0e(O3UMIiNchC20q`i%g zl6`;q1FO_FFZ5iaI-WL1d&bivyJ@Q3@pSV{ku#rd8bg#n+0+zL(k04{O;^y1PdD+1 z^yXRNdr^9S+h$GgpWj@k9)G%TVDoQe-7GpoQrWJG5)Bmcs;zYJRVc}#yMrxsJyBy4 z&mINH|Nhzgayd#jIK)ylZGQEmx9C&thOKuv$XfVhOrNCblQA|p`TW(<_gm~q={xc* z&ug{s8T0J(-#jZaq|*Da)?TvYakivMz2FhgMK9d^B|M!}G1r*QFU$@8RmlWxjkDBO z5!z!Vf!z6@sd>oO84utQ9xCCqaNG6&B$oRLdMsP&;~VW4s5uUSgyZ$fX(F^SOhJ{@ zA+UrDL$AtJGv@eW`^O2I-1Y*&CoI4yCTzoeG!;OrG*z0XN_^Q=8BA1s=ijC+_Yzup zR0Ct~gf7FxE>cZ+a z@u!=c_=Xr42~^S%LqTU;vDB%6u@p-ANeB`1!Czhd^-;Xs47~VU{zE_~q(hRkoO-kR zCzn*dy(&h5li|X^Jn@sd{Qt!1#@c==W!CmxGH-sj{gDd}4}}qK1&cTm#ryD+_qcEN zymZ~&o*g$SXM*|tj=wXX@4P{6d(td;#Zx^`nzmQomn~-8uE%m^fRr6; zA<=ASwt0S+e}R{+z!?VsFrsJ)p?DaH%@2PzcGec$Ur~Jxe+%pDkdOlb&;#eu`0`bS z$986*vB$6#m@SRbJv#hun#DPsEq^p8{MA36{oI@6(| zW$t^`pNWC#dDW<(FHmr6$^zjBk{T?HZZ2d+&yy}zArS$yB(P-$U&UPkH2oSam%X;^ zOxQku?LIsFj%DuKeZ!fIIK2CMdA^zPdJTRTy*{RCE6Q&;I8IAwK^>G=*M#?K8h6LN z-OZu)HEIjWVPc~vpkiCM+4=gp0T}oW)0rpmIc=FBgXrK|1QJZfM<5+00}fOAHy>nI zjk3ncgr8t7$MR$r<0K7oE!sa6mhrJesE3Y0Bxv{asa925y2-9vE#x0r95-#nLm`Sv$&K`r0^=2&yXo{8C5 z?3iGWziD~g|7!8y+?duvL~T#>D!Z3sWvy3%REXU@w-b`)6`mS2f< zkJ`81^uIMeSDqcu!V0}@t@)3)=GRd*hnbaAjsg0}TEyo>U8FP%TY&Q0XF!>(*HO@1 zYTDi&bJ?{bn-h93T_jgKvqcvE@*#D8DX3RQvkB+Qq1GJYfE+J<&%FJ1_3W#8H1->p zCEW;q@JHL5Ld%&XvT2sv5ja=?r$sjVn^8F4lbqT8qs)|_-`aQiU!Sk~((hEsF{`8_ z<_RJ0%nDJCXKQ==J?&=i&%c>OXR0z|_xb@^;zi|%8}Y(4}=PAVn- z7W7BQ`g<`x6YK9pe{8J(JM?Qt_U{tYM-oOX@G%CAi@jfh1@$TYxWISfy%&35ErqDy zzB&8dsiRAB-va$Zxy`rVZMm&PekdWt#j(+=(XWp6_n@DT^~YnH`1{GH^j`#jD82c= zcdt_y+I`pk;!mX%=AFNtVu}Z*%D}vGU^ad~JMd8BKVq*IV)73BPb!uA8>7C^`%w#L Z6PQzo+QjljcQ2N{yuu*qo4p^*`hWPz0t5g6 diff --git a/static/js/workers/deltaWorker.js b/static/js/workers/deltaWorker.js new file mode 100644 index 00000000..3a86f650 --- /dev/null +++ b/static/js/workers/deltaWorker.js @@ -0,0 +1,330 @@ +/** + * OxiCloud — delta-upload worker ("upload only what changed"). + * + * Runs the whole client side of the delta protocol off the main thread: + * + * read 8 MiB slices ─► FastCDC chunk + BLAKE3 (WASM, same crate and + * parameters as the server) ─► negotiate hash batches ─► upload only + * the missing chunks (framed, bounded concurrency) ─► commit. + * + * The stages OVERLAP: negotiation of batch N and uploads of its missing + * chunks run while batch N+1 is still being hashed, so wall-clock time + * approaches max(hash time, upload time) instead of their sum. RAM stays + * flat: chunk bytes are re-sliced from the File at upload time, never + * hoarded. + * + * Protocol with the spawner: + * in : { file: File, folderId: string, name: string, csrfToken: string } + * out : { type: 'progress', hashedBytes, reusedBytes, uploadedBytes, totalBytes } + * { type: 'done', status, body } — conclusive HTTP outcome + * { type: 'fallback', reason } — do a plain byte upload + */ + +// Absolute URLs on purpose: vendors/workers are served verbatim in both +// dev and the release IIFE bundle (same pattern as the pdf.js loader). +const WASM_GLUE_URL = '/js/vendors/hash-wasm/oxicloud_hash_wasm.js'; + +/** File read granularity — large enough to amortize Blob→ArrayBuffer. */ +const SLICE_BYTES = 8 * 1024 * 1024; +/** Negotiate after this many freshly hashed chunks (~64 MiB of content). */ +const NEGOTIATE_BATCH = 256; +/** Group missing chunks into PUT bodies of at most this many bytes. */ +const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024; +/** Concurrent chunk-PUT requests. */ +const UPLOAD_CONCURRENCY = 2; +/** Re-commit attempts when the server answers 409 still_missing. */ +const COMMIT_RETRIES = 2; + +/** + * Typed view of the dedicated-worker global scope (jsconfig targets the + * DOM lib, where `self` is a Window — cast to what this worker uses). + * @type {{ onmessage: ((event: MessageEvent) => void) | null, + * postMessage: (message: unknown) => void }} + */ +const workerScope = /** @type {any} */ (self); + +/** + * One chunk occurrence, in file order. + * @typedef {{ h: string, s: number, offset: number }} WorkerChunk + */ + +/** @returns {Promise} the initialized WASM module */ +async function loadWasm() { + const mod = await import(WASM_GLUE_URL); + await mod.default(); + return mod; +} + +workerScope.onmessage = async (event) => { + const { file, folderId, name, csrfToken } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string }} */ (event.data); + + /** @param {string} reason */ + const fallback = (reason) => workerScope.postMessage({ type: 'fallback', reason }); + + /** @type {Record} */ + const mutHeaders = { 'Content-Type': 'application/json' }; + if (csrfToken) mutHeaders['X-CSRF-Token'] = csrfToken; + + let wasm; + try { + wasm = await loadWasm(); + } catch (err) { + fallback(`wasm unavailable: ${err instanceof Error ? err.message : String(err)}`); + return; + } + + // ── Shared pipeline state ───────────────────────────────────── + /** @type {WorkerChunk[]} */ + const chunks = []; // every occurrence, in file order + /** @type {Set} */ + const seenForNegotiate = new Set(); // distinct hashes already sent to negotiate + let reusedBytes = 0; + let uploadedBytes = 0; + let hashedBytes = 0; + let failed = /** @type {string | null} */ (null); + + let lastProgress = 0; + const progress = (force = false) => { + const now = Date.now(); + if (!force && now - lastProgress < 150) return; + lastProgress = now; + workerScope.postMessage({ + type: 'progress', + hashedBytes, + reusedBytes, + uploadedBytes, + totalBytes: file.size + }); + }; + + // ── Upload stage: bounded-concurrency drain of uploadByHash ── + /** @type {WorkerChunk[]} */ + const uploadQueue = []; + /** @type {Promise[]} */ + const uploadWorkers = []; + let uploadsClosed = false; + /** @type {(() => void) | null} */ + let wakeUploader = null; + const signalUploaders = () => { + if (wakeUploader) { + const w = wakeUploader; + wakeUploader = null; + w(); + } + }; + + /** Encode a batch of chunks as [u32 BE len][bytes] frames. */ + const encodeFrames = async (/** @type {WorkerChunk[]} */ batch) => { + const total = batch.reduce((n, c) => n + 4 + c.s, 0); + const wire = new Uint8Array(total); + const view = new DataView(wire.buffer); + let at = 0; + for (const c of batch) { + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const bytes = new Uint8Array(await file.slice(c.offset, c.offset + c.s).arrayBuffer()); + view.setUint32(at, c.s, false); + wire.set(bytes, at + 4); + at += 4 + c.s; + } + return wire; + }; + + const uploadLoop = async () => { + while (!failed) { + // Take up to UPLOAD_BATCH_BYTES from the queue. + /** @type {WorkerChunk[]} */ + const batch = []; + let bytes = 0; + while (uploadQueue.length > 0 && bytes < UPLOAD_BATCH_BYTES) { + const c = /** @type {WorkerChunk} */ (uploadQueue.shift()); + batch.push(c); + bytes += c.s; + } + if (batch.length === 0) { + if (uploadsClosed) return; + // eslint-disable-next-line no-await-in-loop -- queue wait + await new Promise((resolve) => { + wakeUploader = /** @type {() => void} */ (resolve); + }); + continue; + } + try { + // eslint-disable-next-line no-await-in-loop -- bounded by pool size + const wire = await encodeFrames(batch); + // eslint-disable-next-line no-await-in-loop -- bounded by pool size + const response = await fetch('/api/files/delta/chunks', { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + ...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {}) + }, + body: wire + }); + if (!response.ok) { + failed = `chunk PUT failed (HTTP ${response.status})`; + return; + } + for (const c of batch) uploadedBytes += c.s; + progress(); + } catch (err) { + failed = `chunk PUT failed: ${err instanceof Error ? err.message : String(err)}`; + return; + } + } + }; + for (let i = 0; i < UPLOAD_CONCURRENCY; i++) uploadWorkers.push(uploadLoop()); + + // ── Negotiate stage ─────────────────────────────────────────── + /** @type {Promise[]} */ + const negotiations = []; + const negotiate = (/** @type {WorkerChunk[]} */ fresh) => { + if (fresh.length === 0 || failed) return; + negotiations.push( + (async () => { + try { + const response = await fetch('/api/files/delta/negotiate', { + method: 'POST', + headers: mutHeaders, + body: JSON.stringify({ chunks: fresh.map(({ h, s }) => ({ h, s })) }) + }); + if (!response.ok) { + failed = failed || `negotiate failed (HTTP ${response.status})`; + return; + } + const missing = new Set(/** @type {{missing: string[]}} */ (await response.json()).missing); + for (const c of fresh) { + if (missing.has(c.h)) { + uploadQueue.push(c); + } else { + reusedBytes += c.s; + } + } + signalUploaders(); + progress(); + } catch (err) { + failed = failed || `negotiate failed: ${err instanceof Error ? err.message : String(err)}`; + } + })() + ); + }; + + // ── Chunking stage (drives the other two) ──────────────────── + try { + const chunker = new wasm.DeltaChunker(); + /** @type {WorkerChunk[]} */ + let freshBatch = []; + let offset = 0; + + /** @param {[string, number][]} emitted */ + const onChunks = (emitted) => { + for (const [h, s] of emitted) { + /** @type {WorkerChunk} */ + const chunk = { h, s, offset }; + offset += s; + chunks.push(chunk); + if (seenForNegotiate.has(h)) { + // Repeated content inside the same file: the first + // occurrence decides upload vs reuse; later ones are + // pure reuse for accounting. + reusedBytes += s; + } else { + seenForNegotiate.add(h); + freshBatch.push(chunk); + if (freshBatch.length >= NEGOTIATE_BATCH) { + negotiate(freshBatch); + freshBatch = []; + } + } + } + }; + + for (let read = 0; read < file.size && !failed; read += SLICE_BYTES) { + const end = Math.min(read + SLICE_BYTES, file.size); + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const slice = new Uint8Array(await file.slice(read, end).arrayBuffer()); + onChunks(JSON.parse(chunker.update(slice))); + hashedBytes = end; + progress(); + } + const fin = JSON.parse(chunker.finish()); + chunker.free(); + onChunks(fin.chunks); + negotiate(freshBatch); + const fileHash = /** @type {string} */ (fin.file_hash); + hashedBytes = file.size; + progress(true); + + // ── Drain: negotiations → uploads → commit ─────────────── + await Promise.all(negotiations); + uploadsClosed = true; + signalUploaders(); + await Promise.all(uploadWorkers); + if (failed) { + fallback(failed); + return; + } + + const commitBody = { + file_hash: fileHash, + chunks: chunks.map(({ h, s }) => ({ h, s })), + name, + folder_id: folderId + }; + for (let attempt = 0; ; attempt++) { + // eslint-disable-next-line no-await-in-loop -- retry loop + const response = await fetch('/api/files/delta/commit', { + method: 'POST', + headers: mutHeaders, + body: JSON.stringify(commitBody) + }); + /** @type {any} */ + let body = null; + try { + // eslint-disable-next-line no-await-in-loop -- retry loop + body = await response.json(); + } catch (_) {} + + const stillMissing = response.status === 409 && Array.isArray(body?.still_missing); + if (stillMissing && attempt < COMMIT_RETRIES) { + // GC race or a chunk we wrongly assumed claimable: upload + // exactly what the server names and try again. + const byHash = new Map(chunks.map((c) => [c.h, c])); + /** @type {WorkerChunk[]} */ + const retry = []; + for (const h of body.still_missing) { + const c = byHash.get(h); + if (!c) { + fallback('server requested an unknown chunk'); + return; + } + retry.push(c); + } + const wire = await encodeFrames(retry); + // eslint-disable-next-line no-await-in-loop -- retry loop + const put = await fetch('/api/files/delta/chunks', { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + ...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {}) + }, + body: wire + }); + if (!put.ok) { + fallback(`retry chunk PUT failed (HTTP ${put.status})`); + return; + } + for (const c of retry) uploadedBytes += c.s; + progress(true); + continue; + } + + // Conclusive: 201 created, or a real error (quota, name + // conflict, validation). The spawner maps it to the uploaders' + // UploadAnswer contract. + workerScope.postMessage({ type: 'done', status: response.status, body }); + return; + } + } catch (err) { + fallback(err instanceof Error ? err.message : String(err)); + } +}; diff --git a/static/js/workers/hashWorker.js b/static/js/workers/hashWorker.js deleted file mode 100644 index d2061ecd..00000000 --- a/static/js/workers/hashWorker.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * OxiCloud — BLAKE3 hashing worker (instant-upload support). - * - * Hashes a File off the main thread, reading it in fixed-size slices so - * RAM stays constant regardless of file size. The WASM module is compiled - * from the exact same `blake3` crate the server uses, so the digest - * computed here equals the server's content address bit for bit. - * - * Protocol: receives `{ file: File }`, answers - * `{ ok: true, hash: string }` or `{ ok: false, error: string }`. - * The spawner terminates the worker after one file. - */ - -// Absolute URL on purpose: vendors are served verbatim at /js/vendors/ in -// both dev and release mode (the release IIFE bundle would break a -// relative import) — same pattern as the pdf.js loader in thumbnail.js. -const WASM_GLUE_URL = '/js/vendors/hash-wasm/oxicloud_hash_wasm.js'; - -/** - * 8 MiB slices — large enough to amortize the per-slice Blob→ArrayBuffer - * round-trip, small enough that peak worker RAM stays flat for any size. - */ -const SLICE_BYTES = 8 * 1024 * 1024; - -/** - * Typed view of the dedicated-worker global scope. The project's - * jsconfig targets the DOM lib, where `self` is a Window — cast to the - * two members this worker actually uses. - * @type {{ onmessage: ((event: MessageEvent) => void) | null, - * postMessage: (message: unknown) => void }} - */ -const workerScope = /** @type {any} */ (self); - -/** - * Memoized WASM module (in-flight or settled), `default()` already run. - * Reset on failure so a later message can retry a transient load error. - * @type {Promise | null} - */ -let _wasmPromise = null; - -/** @returns {Promise} */ -function getWasm() { - if (!_wasmPromise) { - _wasmPromise = import(WASM_GLUE_URL) - .then(async (mod) => { - await mod.default(); - return mod; - }) - .catch((err) => { - _wasmPromise = null; - throw err; - }); - } - return _wasmPromise; -} - -workerScope.onmessage = async (event) => { - const file = /** @type {{ file: File }} */ (event.data).file; - try { - const wasm = await getWasm(); - const hasher = new wasm.Blake3Hasher(); - try { - for (let offset = 0; offset < file.size; offset += SLICE_BYTES) { - const end = Math.min(offset + SLICE_BYTES, file.size); - // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM - const buffer = await file.slice(offset, end).arrayBuffer(); - hasher.update(new Uint8Array(buffer)); - } - workerScope.postMessage({ ok: true, hash: hasher.finalizeHex() }); - } finally { - hasher.free(); - } - } catch (err) { - workerScope.postMessage({ - ok: false, - error: err instanceof Error ? err.message : String(err) - }); - } -}; diff --git a/wasm/oxicloud-hash/Cargo.lock b/wasm/oxicloud-hash/Cargo.lock index b053178b..5675d509 100644 --- a/wasm/oxicloud-hash/Cargo.lock +++ b/wasm/oxicloud-hash/Cargo.lock @@ -65,6 +65,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fastcdc" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -88,6 +94,7 @@ name = "oxicloud-hash-wasm" version = "0.1.0" dependencies = [ "blake3", + "fastcdc", "wasm-bindgen", ] diff --git a/wasm/oxicloud-hash/Cargo.toml b/wasm/oxicloud-hash/Cargo.toml index 0c54d4d1..b95fd253 100644 --- a/wasm/oxicloud-hash/Cargo.toml +++ b/wasm/oxicloud-hash/Cargo.toml @@ -20,6 +20,10 @@ crate-type = ["cdylib"] # evergreen browser since 2023 supports it, and the frontend falls back # to a plain byte upload when instantiation fails. blake3 = { version = "1.8.4", default-features = false, features = ["wasm32_simd"] } +# Same crate AND parameters as the server's CDC dedup engine — chunk +# boundaries computed in the browser must equal the server's bit for bit, +# or cross-version dedup between byte uploads and delta uploads collapses. +fastcdc = "4.0.0" wasm-bindgen = "0.2" [profile.release] diff --git a/wasm/oxicloud-hash/src/lib.rs b/wasm/oxicloud-hash/src/lib.rs index 91cb8d55..627c379b 100644 --- a/wasm/oxicloud-hash/src/lib.rs +++ b/wasm/oxicloud-hash/src/lib.rs @@ -64,6 +64,137 @@ pub fn blake3_hex(data: &[u8]) -> String { blake3::hash(data).to_hex().to_string() } +// ── Delta-upload chunker ───────────────────────────────────────────────────── + +/// CDC parameters — MUST mirror `dedup_service.rs` on the server +/// (`CDC_MIN_CHUNK` / `CDC_AVG_CHUNK` / `CDC_MAX_CHUNK`). Identical +/// parameters + identical crate ⇒ identical boundaries, which is what +/// makes a chunk hashed in the browser deduplicate against a chunk the +/// server cut from a byte upload. +const CDC_MIN_CHUNK: usize = 65_536; +const CDC_AVG_CHUNK: usize = 262_144; +const CDC_MAX_CHUNK: usize = 1_048_576; + +/// Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload +/// worker. Feed the file in slices; every call returns the chunks that +/// became FINAL; `finish()` flushes the tail and returns the file hash. +/// +/// ```js +/// const c = new DeltaChunker(); +/// for (const slice of slices) { +/// for (const [h, s] of JSON.parse(c.update(bytes))) { … } +/// } +/// const { chunks, file_hash } = JSON.parse(c.finish()); +/// ``` +/// +/// Correctness of the incremental split: FastCDC decides each cut by +/// scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When +/// the chunker runs over the buffered prefix of a longer file, every +/// produced chunk except the LAST ended on a content/max-size condition +/// — its decision window was fully available, so the full-file chunker +/// makes the same cut. Only the last chunk (cut by "end of buffer") is +/// provisional: it stays buffered and is re-examined when more bytes +/// arrive. By induction the emitted boundaries equal a single FastCDC +/// pass over the whole file — the mirror test below proves it. +#[wasm_bindgen] +pub struct DeltaChunker { + /// Provisional tail: bytes after the last FINAL cut. + buf: Vec, + file_hasher: blake3::Hasher, + total: u64, +} + +/// Append one `["",len]` item to a hand-rolled JSON array — hashes +/// are hex and sizes are integers, so manual JSON is unambiguous and +/// keeps a serde dependency out of the wasm binary. +fn push_chunk_json(out: &mut String, hash: &str, len: usize) { + if !out.ends_with('[') { + out.push(','); + } + out.push_str("[\""); + out.push_str(hash); + out.push_str("\","); + out.push_str(&len.to_string()); + out.push(']'); +} + +#[wasm_bindgen] +impl DeltaChunker { + /// Create a chunker with the server's CDC parameters. + #[wasm_bindgen(constructor)] + pub fn new() -> DeltaChunker { + DeltaChunker { + buf: Vec::with_capacity(2 * CDC_MAX_CHUNK), + file_hasher: blake3::Hasher::new(), + total: 0, + } + } + + /// Feed one slice. Returns a JSON array of the chunks that became + /// final: `[["", size], …]` (possibly empty). + pub fn update(&mut self, data: &[u8]) -> String { + self.file_hasher.update(data); + self.total += data.len() as u64; + self.buf.extend_from_slice(data); + + let mut out = String::from("["); + let mut consumed = 0usize; + { + let chunks: Vec = fastcdc::v2020::FastCDC::new( + &self.buf, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ) + .collect(); + // Every chunk but the last ended on a content/max condition → + // final. The last one ended because the buffer did → keep it. + for chunk in chunks.iter().take(chunks.len().saturating_sub(1)) { + let bytes = &self.buf[chunk.offset..chunk.offset + chunk.length]; + push_chunk_json( + &mut out, + &blake3::hash(bytes).to_hex().to_string(), + chunk.length, + ); + consumed = chunk.offset + chunk.length; + } + } + if consumed > 0 { + self.buf.drain(..consumed); + } + out.push(']'); + out + } + + /// Flush the provisional tail and return + /// `{"chunks":[["",size]…],"file_hash":"","total":N}`. + /// `chunks` holds at most one entry (the tail); an empty file has none + /// and its `file_hash` is BLAKE3 of the empty input. + pub fn finish(&mut self) -> String { + let mut out = String::from("{\"chunks\":["); + if !self.buf.is_empty() { + push_chunk_json( + &mut out, + &blake3::hash(&self.buf).to_hex().to_string(), + self.buf.len(), + ); + self.buf.clear(); + } + out.push_str("],\"file_hash\":\""); + out.push_str(&self.file_hasher.finalize().to_hex().to_string()); + out.push_str("\",\"total\":"); + out.push_str(&self.total.to_string()); + out.push('}'); + out + } +} + +impl Default for DeltaChunker { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use super::*; @@ -95,4 +226,94 @@ mod tests { "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" ); } + + // ── DeltaChunker mirror test ───────────────────────────────── + // + // The client-side twin of the server's + // `test_stream_chunking_matches_slice_chunking`: incremental chunking + // with adversarial slice sizes must produce exactly the boundaries of + // one FastCDC pass over the whole buffer — the property cross-version + // dedup between byte uploads and delta uploads hangs on. + + fn run_chunker(data: &[u8], slice: usize) -> (Vec<(String, usize)>, String) { + let mut chunker = DeltaChunker::new(); + let mut chunks: Vec<(String, usize)> = Vec::new(); + let mut parse = |json: &str, into: &mut Vec<(String, usize)>| { + // items look like ["",N] — split on '[' groups. + for item in json.split("[\"").skip(1) { + let hash = &item[..64]; + let size: usize = item[66..item.find(']').unwrap()].parse().unwrap(); + into.push((hash.to_string(), size)); + } + }; + for piece in data.chunks(slice.max(1)) { + let emitted = chunker.update(piece); + parse(&emitted, &mut chunks); + } + let fin = chunker.finish(); + let tail_json = &fin[fin.find('[').unwrap()..=fin.find(']').unwrap()]; + parse(tail_json, &mut chunks); + let file_hash = fin.split("\"file_hash\":\"").nth(1).unwrap()[..64].to_string(); + (chunks, file_hash) + } + + #[test] + fn incremental_chunking_matches_single_pass() { + // 4 MiB of xorshift noise — genuinely content-defined cut points + // (a byte-periodic generator would only ever hit max-size cuts). + let mut state: u64 = 0x243F_6A88_85A3_08D3; + let mut data = Vec::with_capacity(4 * 1024 * 1024); + while data.len() < 4 * 1024 * 1024 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.extend_from_slice(&state.to_le_bytes()); + } + + let reference: Vec<(String, usize)> = + fastcdc::v2020::FastCDC::new(&data, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK) + .map(|c| { + ( + blake3::hash(&data[c.offset..c.offset + c.length]) + .to_hex() + .to_string(), + c.length, + ) + }) + .collect(); + assert!(reference.len() > 4, "test data must span several chunks"); + + // Slice sizes chosen to stress every refill path: tiny (7 B), + // typical worker slice (8 MiB > file), page-ish, and exactly the + // CDC max so provisional tails land on boundaries. + for slice in [7usize, 4096, CDC_MAX_CHUNK, 8 * 1024 * 1024] { + let (chunks, file_hash) = run_chunker(&data, slice); + assert_eq!( + chunks, reference, + "boundaries must not depend on slicing (slice={slice})" + ); + assert_eq!( + file_hash, + blake3_hex(&data), + "file hash must match one-shot BLAKE3 (slice={slice})" + ); + } + } + + #[test] + fn delta_chunker_empty_and_tiny_inputs() { + let (chunks, file_hash) = run_chunker(b"", 1024); + assert!(chunks.is_empty()); + assert_eq!( + file_hash, + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + ); + + let tiny = b"below the CDC minimum"; + let (chunks, file_hash) = run_chunker(tiny, 4); + assert_eq!(chunks.len(), 1, "tiny input is one (tail) chunk"); + assert_eq!(chunks[0].1, tiny.len()); + assert_eq!(chunks[0].0, blake3_hex(tiny)); + assert_eq!(file_hash, blake3_hex(tiny)); + } } From ed9a204e4955ceebd3fed188c9107a3981f61905 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 16:38:02 +0000 Subject: [PATCH 6/6] Delta download: file manifest + user-scoped chunk fetch for sync clients MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 of the delta-sync plan — the inverse direction, so a future client app holding an older local version can fetch only what changed: - GET /api/files/{id}/manifest returns the file's chunk recipe ({file_hash, total_size, chunks}). Owner-scoped like the rest of the delta surface (Read permission through the authz engine first, then the chunk layer's possession standard; shared files use the regular download endpoints). A manifest is immutable for a given file_hash, so it is served with ETag = file_hash and If-None-Match answers 304 — polling sync clients pay one header round-trip per unchanged file. Legacy pre-CDC blobs are presented as a single-chunk manifest of themselves, so clients need no special case. - POST /api/files/delta/download streams the requested chunks as [u32 BE length][bytes] frames in request order — the same wire format the upload direction uses. Entitlement is the same possession rule as negotiate/commit (chunks reachable through the caller's own files); anything else returns 404 {not_available} — deliberately indistinguishable from "never existed" — with a delta_download.rejected audit event. Batches are bounded by the chunk_max_bytes budget; Content-Length is exact (sizes come from the dedup index) and peak RAM is one backend read frame. Both endpoints share the delta rate limiter. New DedupService primitives: manifest_chunk_list (with legacy fallback), chunk_sizes, chunk_stream. OpenAPI regenerated; protocol doc gains the download section; types.js maps the new wire shapes (plus the delta-upload typedefs that a container reset had silently dropped from a previous commit). Verified end-to-end against PostgreSQL 16 with a simulated two-device sync: device A uploaded 24 MB by bytes and delta-updated it (2 edits → 2 chunks); device B diffed the manifest against its WASM-chunked local copy, needed 2/79 chunks, fetched 970 KB instead of 24 MB (96.1% saved) and rebuilt the file byte-identical with the BLAKE3 verifying. If-None-Match revalidation returned 304; a second user got 404 on both the manifest and the chunk batch (with the not_available list and audit lines); an unknown hash was indistinguishable from a denied one; an empty hash list returned 400. https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- docs/delta-upload-protocol.md | 32 ++- .../services/delta_upload_service.rs | 187 +++++++++++++++++- src/common/di.rs | 2 + src/infrastructure/services/dedup_service.rs | 70 +++++++ .../api/handlers/delta_upload_handler.rs | 129 +++++++++++- src/interfaces/api/mod.rs | 5 + src/interfaces/api/routes.rs | 4 +- static/js/core/types.js | 51 +++++ 8 files changed, 471 insertions(+), 9 deletions(-) diff --git a/docs/delta-upload-protocol.md b/docs/delta-upload-protocol.md index 171e1a1d..686198d5 100644 --- a/docs/delta-upload-protocol.md +++ b/docs/delta-upload-protocol.md @@ -108,6 +108,33 @@ If the caller already owns the exact `file_hash`, the commit short-circuits to a pure reference bump — chunks aren't even looked at (same as `POST /api/files/by-hash`). +## Delta download (sync clients) + +The inverse direction, for a client app that already holds an older +version locally and wants the server's current one: + +1. `GET /api/files/{id}/manifest` → `{ file_hash, total_size, chunks }` + — the file's chunk recipe. **Owner-scoped** like the rest of the + delta surface (shared files use the regular download endpoints). + Served with `ETag: ""`; a manifest is immutable for a + given hash, so `If-None-Match` revalidation answers 304 — polling + sync clients pay one header round-trip per unchanged file. +2. Diff the manifest against the local chunk inventory (chunk the local + copy with the same WASM module the upload direction ships). +3. `POST /api/files/delta/download` with `{ "hashes": […] }` → the + requested chunks as `[u32 BE length][bytes]` frames in request order + (the same wire format as the upload direction). Entitlement is the + same possession rule as negotiate/commit: chunks must be reachable + through the caller's own files; anything else → 404 + `{ "not_available": […] }` — deliberately indistinguishable from + "never existed". Batches are bounded by `OXICLOUD_CHUNK_MAX_BYTES`; + split large deltas across requests. +4. Reassemble locally per the manifest order and verify the whole-file + BLAKE3 against `file_hash`. + +Editing 3 bytes of a 24 MB file on one device costs a second device one +manifest GET plus ~1 chunk (~256 KB) instead of 24 MB. + ## Security model - **No content oracle.** Possession is proven per chunk: without bytes @@ -122,8 +149,9 @@ short-circuits to a pure reference bump — chunks aren't even looked at commit. Orphan chunks are GC-swept. - **Audit.** Rejections emit `delta_upload.rejected` with stable `reason` keys: `rate_limited`, `chunk_verification_failed`, - `file_hash_mismatch`. AuthZ denials surface as the engine's standard - `authz.denied`. + `file_hash_mismatch` — and `delta_download.rejected` with + `manifest_not_owner` / `chunks_not_owned`. AuthZ denials surface as + the engine's standard `authz.denied`. ## Error summary diff --git a/src/application/services/delta_upload_service.rs b/src/application/services/delta_upload_service.rs index 68a463e0..07e4312a 100644 --- a/src/application/services/delta_upload_service.rs +++ b/src/application/services/delta_upload_service.rs @@ -36,12 +36,13 @@ use uuid::Uuid; use crate::application::dtos::file_dto::FileDto; use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; -use crate::application::ports::storage_ports::StorageUsagePort; +use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort}; use crate::application::services::file_upload_service::FileUploadService; use crate::application::services::storage_usage_service::StorageUsageService; use crate::common::errors::DomainError; use crate::common::mime_detect::{MAGIC_BYTES_LEN, refine_content_type}; use crate::domain::services::authorization::{Permission, Resource, Subject}; +use crate::infrastructure::repositories::pg::FileBlobReadRepository; use crate::infrastructure::services::dedup_service::{CDC_MAX_CHUNK, DedupService}; use crate::infrastructure::services::pg_acl_engine::PgAclEngine; @@ -97,6 +98,36 @@ pub struct DeltaCommitRequest { pub file_id: Option, } +/// Response of `GET /api/files/{id}/manifest` — the recipe to rebuild the +/// file from chunks. Immutable for a given `file_hash`, so clients cache +/// it keyed by hash (the endpoint also serves it with `ETag: file_hash`). +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaManifestResponse { + /// BLAKE3 of the whole file (verify the local reassembly against it). + pub file_hash: String, + /// Total size in bytes. + pub total_size: u64, + /// Full chunk sequence, in file order (per occurrence). + pub chunks: Vec, +} + +/// Request body of `POST /api/files/delta/download` — distinct chunk +/// hashes to fetch, served back as `[u32 BE length][bytes]` frames in +/// request order (the same wire format the upload direction uses). +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeltaDownloadRequest { + pub hashes: Vec, +} + +/// Outcome of a chunk-download authorization. +pub enum DeltaDownloadOutcome { + /// Every requested chunk is servable: `(hash, size)` in request order. + Ready(Vec<(String, u64)>), + /// Some chunks are not available to this caller (not reachable through + /// their files, or unknown — deliberately indistinguishable). + NotAvailable(Vec), +} + /// Resolved commit mode after request validation. enum CommitMode { Create { name: String, folder_id: String }, @@ -121,26 +152,35 @@ pub enum DeltaCommitOutcome { pub struct DeltaUploadService { dedup: Arc, uploads: Arc, + file_read: Arc, quota: Arc, authz: Arc, /// Whole-file ceiling — same `max_upload_size` that bounds byte uploads. max_total_size: u64, + /// Per-request ceiling for batched chunk downloads — same budget as + /// the chunk-upload requests (`chunk_max_bytes`). + max_download_batch: u64, } impl DeltaUploadService { + #[allow(clippy::too_many_arguments)] pub fn new( dedup: Arc, uploads: Arc, + file_read: Arc, quota: Arc, authz: Arc, max_total_size: u64, + max_download_batch: u64, ) -> Self { Self { dedup, uploads, + file_read, quota, authz, max_total_size, + max_download_batch, } } @@ -414,6 +454,151 @@ impl DeltaUploadService { }) } + // ── Delta download ("download only what changed") ──────────── + + /// The chunk recipe of a file the caller owns — step 1 of a delta + /// download. Owner-scoped like the rest of the delta protocol (shared + /// files use the regular download endpoints); a non-owned or unknown + /// file is `NotFound`, and the denial is audited. + pub async fn file_manifest_with_perms( + &self, + caller_id: Uuid, + file_id: &str, + ) -> Result { + let file_uuid = Uuid::parse_str(file_id) + .map_err(|_| DomainError::not_found("File", file_id.to_string()))?; + // Engine check first (Read on the file): owners always pass and + // the engine audits denials; the ownership constraint below is the + // chunk layer's own entitlement standard. + self.authz + .require( + Subject::User(caller_id), + Permission::Read, + Resource::File(file_uuid), + ) + .await?; + + let file = self.file_read.get_file(file_id).await?; + let file_hash = file.content_hash().to_string(); + if !self + .dedup + .user_owns_blob_reference(&file_hash, &caller_id.to_string()) + .await + { + // Read permission without ownership (e.g. a grant): the delta + // surface is owner-scoped — the regular download endpoints + // serve shared content. + tracing::info!( + target: "audit", + event = "delta_download.rejected", + reason = "manifest_not_owner", + caller_id = %caller_id, + file_id = %file_id, + "👮🏻‍♂️ Delta download rejected: manifest requested by a non-owner", + ); + return Err(DomainError::not_found("File", file_id.to_string())); + } + + let Some((chunks, total_size)) = self.dedup.manifest_chunk_list(&file_hash).await? else { + return Err(DomainError::not_found("File", file_id.to_string())); + }; + Ok(DeltaManifestResponse { + file_hash, + total_size, + chunks: chunks.into_iter().map(|(h, s)| ChunkRef { h, s }).collect(), + }) + } + + /// Authorize a batched chunk download — step 2. Every hash must be + /// reachable through the caller's own files; otherwise the full list of + /// unavailable hashes is returned (the same information N individual + /// requests would reveal, in one round-trip). The total payload is + /// bounded by the per-request budget so clients split large deltas. + pub async fn authorize_chunk_download_with_perms( + &self, + caller_id: Uuid, + request: &DeltaDownloadRequest, + ) -> Result { + if request.hashes.is_empty() { + return Err(DomainError::validation_error("hashes must not be empty")); + } + if request.hashes.len() > self.max_chunk_count() { + return Err(DomainError::validation_error(format!( + "Too many chunks: {} (maximum {})", + request.hashes.len(), + self.max_chunk_count() + ))); + } + let mut distinct_seen = HashSet::new(); + for hash in &request.hashes { + if !is_valid_hash(hash) { + return Err(DomainError::validation_error( + "Invalid chunk hash format. Expected BLAKE3 (64 hex characters)", + )); + } + if !distinct_seen.insert(hash.as_str()) { + return Err(DomainError::validation_error( + "Duplicate hashes in download request", + )); + } + } + + let entitled = self + .dedup + .claimable_chunks(caller_id, &request.hashes) + .await?; + let not_available: Vec = request + .hashes + .iter() + .filter(|h| !entitled.contains(*h)) + .cloned() + .collect(); + if !not_available.is_empty() { + tracing::info!( + target: "audit", + event = "delta_download.rejected", + reason = "chunks_not_owned", + caller_id = %caller_id, + requested = request.hashes.len(), + denied = not_available.len(), + "👮🏻‍♂️ Delta download rejected: caller requested chunks outside their files", + ); + return Ok(DeltaDownloadOutcome::NotAvailable(not_available)); + } + + let sizes = self.dedup.chunk_sizes(&request.hashes).await?; + let mut ordered = Vec::with_capacity(request.hashes.len()); + let mut total: u64 = 0; + for hash in &request.hashes { + // Entitled implies indexed; a vanished row between the two + // queries surfaces as unavailable rather than a 500. + let Some(size) = sizes.get(hash) else { + return Ok(DeltaDownloadOutcome::NotAvailable(vec![hash.clone()])); + }; + total = total.saturating_add(*size); + ordered.push((hash.clone(), *size)); + } + if total > self.max_download_batch { + return Err(DomainError::validation_error(format!( + "Requested chunks total {total} bytes; the per-request ceiling is {} — split the download into smaller batches", + self.max_download_batch + ))); + } + Ok(DeltaDownloadOutcome::Ready(ordered)) + } + + /// Stream one authorized chunk's bytes (entitlement was established by + /// [`authorize_chunk_download_with_perms`]). + pub async fn chunk_stream( + &self, + hash: &str, + ) -> Result< + std::pin::Pin> + Send>>, + DomainError, + > { + self.dedup.chunk_stream(hash).await + } + /// Create or update the file row against a blob reference the commit /// already holds (the registration paths release it on failure). async fn register_row( diff --git a/src/common/di.rs b/src/common/di.rs index 4e9a94ca..07227509 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -467,9 +467,11 @@ impl AppServiceFactory { crate::application::services::delta_upload_service::DeltaUploadService::new( core.dedup_service.clone(), file_upload_service.clone(), + repos.file_read_repository.clone(), storage_usage.clone(), authz.clone(), self.config.storage.max_upload_size as u64, + self.config.storage.chunk_max_bytes as u64, ), ); diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 5b6105d9..867c0767 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -516,6 +516,76 @@ impl DedupService { // recomputes BLAKE3 before any manifest row exists. A forged hash // would otherwise poison future whole-file dedup hits for OTHER // users uploading the genuine content. + // + // The download direction reuses invariant 1: a chunk's bytes are only + // served to callers whose own files already reference it. + + /// The ordered chunk list composing `file_hash`, for the delta-download + /// manifest: `(chunks[(hash, size)], total_size)`. + /// + /// Legacy whole-file blobs (pre-CDC, not yet re-chunked) are presented + /// as a single-chunk manifest of themselves — the chunk download path + /// can serve them directly, so sync clients need no special case. + pub async fn manifest_chunk_list( + &self, + file_hash: &str, + ) -> Result, u64)>, DomainError> { + let manifest = sqlx::query_as::<_, (Vec, Vec, i64)>( + "SELECT chunk_hashes, chunk_sizes, total_size + FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {e}")))?; + + if let Some((hashes, sizes, total)) = manifest { + let chunks = hashes + .into_iter() + .zip(sizes.into_iter().map(|s| s as u64)) + .collect(); + return Ok(Some((chunks, total as u64))); + } + + // Legacy fallback: the blob is its own single chunk. + let legacy = sqlx::query_scalar::<_, i64>("SELECT size FROM storage.blobs WHERE hash = $1") + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Legacy blob lookup: {e}")) + })?; + Ok(legacy.map(|size| (vec![(file_hash.to_string(), size as u64)], size as u64))) + } + + /// Sizes of the given chunk hashes from the dedup index, keyed by hash. + /// Hashes without a row are simply absent from the result. + pub async fn chunk_sizes( + &self, + hashes: &[String], + ) -> Result, DomainError> { + if hashes.is_empty() { + return Ok(std::collections::HashMap::new()); + } + sqlx::query_as::<_, (String, i64)>( + "SELECT hash, size FROM storage.blobs WHERE hash = ANY($1)", + ) + .bind(hashes) + .fetch_all(self.pool.as_ref()) + .await + .map(|rows| rows.into_iter().map(|(h, s)| (h, s as u64)).collect()) + .map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}"))) + } + + /// Stream one chunk's raw bytes from the backend. The caller is + /// responsible for entitlement (see [`claimable_chunks`]). + pub async fn chunk_stream( + &self, + hash: &str, + ) -> Result> + Send>>, DomainError> + { + self.backend.get_blob_stream(hash).await + } /// Of `hashes` (distinct), the subset `caller_id` may claim without /// uploading bytes: chunks referenced by manifests of the caller's diff --git a/src/interfaces/api/handlers/delta_upload_handler.rs b/src/interfaces/api/handlers/delta_upload_handler.rs index e45765c0..6912d37f 100644 --- a/src/interfaces/api/handlers/delta_upload_handler.rs +++ b/src/interfaces/api/handlers/delta_upload_handler.rs @@ -14,8 +14,8 @@ use axum::{ Json, body::Body, - extract::State, - http::StatusCode, + extract::{Path, State}, + http::{HeaderMap, StatusCode, header}, response::{IntoResponse, Response}, }; use bytes::{Buf, Bytes, BytesMut}; @@ -24,8 +24,8 @@ use std::sync::Arc; use tokio_stream::StreamExt; use crate::application::services::delta_upload_service::{ - DeltaChunksResponse, DeltaCommitOutcome, DeltaCommitRequest, DeltaNegotiateRequest, - DeltaNegotiateResponse, + DeltaChunksResponse, DeltaCommitOutcome, DeltaCommitRequest, DeltaDownloadOutcome, + DeltaDownloadRequest, DeltaManifestResponse, DeltaNegotiateRequest, DeltaNegotiateResponse, }; use crate::common::di::AppState; use crate::common::errors::DomainError; @@ -44,7 +44,15 @@ pub struct DeltaStillMissingResponse { pub still_missing: Vec, } -/// Per-caller flood guard shared by the three delta endpoints. +/// 404 body of `POST /api/files/delta/download` when some requested chunks +/// are not reachable through the caller's files (or don't exist — the two +/// are deliberately indistinguishable). +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaNotAvailableResponse { + pub not_available: Vec, +} + +/// Per-caller flood guard shared by the delta endpoints. fn check_rate_limit(state: &Arc, auth_user: &AuthUser) -> Result<(), AppError> { if state .delta_upload_rate_limiter @@ -243,6 +251,117 @@ pub async fn delta_commit( }) } +#[utoipa::path( + get, + path = "/api/files/{id}/manifest", + params(("id" = String, Path, description = "File ID")), + responses( + (status = 200, description = "Chunk recipe of the file (immutable per file_hash; served with ETag = file_hash)", body = DeltaManifestResponse), + (status = 304, description = "Not modified (If-None-Match matched the current file_hash)"), + (status = 404, description = "File not found, not accessible, or not owned by the caller"), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_file_manifest( + State(state): State>, + auth_user: AuthUser, + Path(file_id): Path, + headers: HeaderMap, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let manifest = state + .applications + .delta_upload_service + .file_manifest_with_perms(auth_user.id, &file_id) + .await + .map_err(AppError::from)?; + + // A manifest is immutable for a given file_hash, so the hash IS the + // strong validator: sync clients polling a file revalidate for free. + let etag = format!("\"{}\"", manifest.file_hash); + if headers + .get(header::IF_NONE_MATCH) + .and_then(|v| v.to_str().ok()) + .is_some_and(|inm| inm == etag) + { + return Ok(Response::builder() + .status(StatusCode::NOT_MODIFIED) + .header(header::ETAG, etag) + .body(Body::empty()) + .unwrap()); + } + Ok(( + StatusCode::OK, + [ + (header::ETAG, etag), + (header::CACHE_CONTROL, "private, no-cache".to_string()), + ], + Json(manifest), + ) + .into_response()) +} + +#[utoipa::path( + post, + path = "/api/files/delta/download", + request_body = DeltaDownloadRequest, + responses( + (status = 200, description = "Requested chunks as [u32 BE length][bytes] frames, in request order", + content_type = "application/octet-stream"), + (status = 400, description = "Malformed hashes, duplicates, or batch above the per-request ceiling"), + (status = 404, description = "Some chunks are not available to this caller", body = DeltaNotAvailableResponse), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_download_chunks( + State(state): State>, + auth_user: AuthUser, + Json(request): Json, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let service = state.applications.delta_upload_service.clone(); + let outcome = service + .authorize_chunk_download_with_perms(auth_user.id, &request) + .await + .map_err(AppError::from)?; + + let ordered = match outcome { + DeltaDownloadOutcome::NotAvailable(not_available) => { + return Ok(( + StatusCode::NOT_FOUND, + Json(DeltaNotAvailableResponse { not_available }), + ) + .into_response()); + } + DeltaDownloadOutcome::Ready(ordered) => ordered, + }; + let total: u64 = ordered.iter().map(|(_, s)| 4 + s).sum(); + + // Stream the frames: 4-byte length headers come from the (entitled) + // index sizes; bytes stream straight from the blob backend. Peak RAM + // is one backend read frame, independent of batch size. + let body_stream: std::pin::Pin> + Send>> = + Box::pin(async_stream::try_stream! { + for (hash, size) in ordered { + yield Bytes::copy_from_slice(&(size as u32).to_be_bytes()); + let mut chunk = service.chunk_stream(&hash).await.map_err(std::io::Error::other)?; + while let Some(part) = chunk.next().await { + yield part?; + } + } + }); + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header(header::CONTENT_LENGTH, total.to_string()) + .body(Body::from_stream(body_stream)) + .unwrap()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/interfaces/api/mod.rs b/src/interfaces/api/mod.rs index a655c3f1..e30113f3 100644 --- a/src/interfaces/api/mod.rs +++ b/src/interfaces/api/mod.rs @@ -84,6 +84,8 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; handlers::delta_upload_handler::delta_negotiate, handlers::delta_upload_handler::delta_upload_chunks, handlers::delta_upload_handler::delta_commit, + handlers::delta_upload_handler::delta_file_manifest, + handlers::delta_upload_handler::delta_download_chunks, handlers::file_handler::download_file, handlers::file_handler::get_thumbnail, handlers::file_handler::upload_thumbnail, @@ -272,6 +274,9 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; crate::application::services::delta_upload_service::DeltaChunksResponse, crate::application::services::delta_upload_service::DeltaCommitRequest, handlers::delta_upload_handler::DeltaStillMissingResponse, + handlers::delta_upload_handler::DeltaNotAvailableResponse, + crate::application::services::delta_upload_service::DeltaManifestResponse, + crate::application::services::delta_upload_service::DeltaDownloadRequest, MoveFilePayload, PaginationDto, PaginationRequestDto, diff --git a/src/interfaces/api/routes.rs b/src/interfaces/api/routes.rs index 4b0255b1..14c9d017 100644 --- a/src/interfaces/api/routes.rs +++ b/src/interfaces/api/routes.rs @@ -55,7 +55,7 @@ use crate::interfaces::api::handlers::chunked_upload_handler::{ cancel_upload, complete_upload, create_upload, get_upload_status, upload_chunk, }; use crate::interfaces::api::handlers::delta_upload_handler::{ - delta_commit, delta_negotiate, delta_upload_chunks, + delta_commit, delta_download_chunks, delta_file_manifest, delta_negotiate, delta_upload_chunks, }; use crate::interfaces::api::handlers::file_handler::{ create_file_by_hash, delete_file, download_file, get_file_metadata, get_thumbnail, @@ -236,6 +236,8 @@ pub fn create_api_routes(app_state: &Arc) -> Router> { .route("/delta/negotiate", post(delta_negotiate)) .route("/delta/chunks", put(delta_upload_chunks)) .route("/delta/commit", post(delta_commit)) + .route("/delta/download", post(delta_download_chunks)) + .route("/{id}/manifest", get(delta_file_manifest)) .route("/{id}", get(download_file)) .route( "/{id}/thumbnail/{size}", diff --git a/static/js/core/types.js b/static/js/core/types.js index f0b49ae1..aee0f564 100644 --- a/static/js/core/types.js +++ b/static/js/core/types.js @@ -541,3 +541,54 @@ * @property {string} hash BLAKE3 of the owned content (64 hex chars) */ +// ------------------- Delta sync (chunk negotiation) + +/** + * One chunk reference on the delta wire: terse on purpose (a 10 GB file + * is ~40 000 of these). Mirrors `ChunkRef` on the server + * (`delta_upload_service.rs`). + * @typedef {Object} DeltaChunkRef + * @property {string} h BLAKE3 of the chunk (64 hex chars) + * @property {number} s chunk size in bytes (1 ..= 1 MiB) + */ + +/** + * Response of `POST /api/files/delta/negotiate` — the distinct chunk + * hashes the caller must upload (user-scoped, advisory). + * @typedef {Object} DeltaNegotiateAnswer + * @property {string[]} missing + */ + +/** + * Request body of `POST /api/files/delta/commit`. Exactly one of + * (`name` + `folder_id`) or `file_id` selects create vs update mode. + * 201/200 responses carry a {@link FileItem}; 409 carries + * `{still_missing: string[]}` (upload those chunks and retry). + * @typedef {Object} DeltaCommitRequest + * @property {string} file_hash BLAKE3 of the whole file (verified server-side) + * @property {DeltaChunkRef[]} chunks full sequence, in file order + * @property {string} [name] create mode: file name + * @property {string} [folder_id] create mode: target folder + * @property {string} [file_id] update mode: file whose content is replaced + */ + +/** + * Response of `GET /api/files/{id}/manifest` — the recipe to rebuild a + * file from chunks (delta download, step 1). Immutable per `file_hash`; + * the endpoint serves it with `ETag: file_hash` so polling sync clients + * revalidate with a 304 for free. + * @typedef {Object} DeltaManifestAnswer + * @property {string} file_hash BLAKE3 of the whole file + * @property {number} total_size bytes + * @property {DeltaChunkRef[]} chunks full sequence, in file order + */ + +/** + * Request body of `POST /api/files/delta/download` (delta download, + * step 2). Responds with `[u32 BE length][bytes]` frames in request + * order, or 404 `{not_available: string[]}` for chunks outside the + * caller's files. + * @typedef {Object} DeltaDownloadRequest + * @property {string[]} hashes distinct chunk hashes to fetch + */ +