diff --git a/Cargo.lock b/Cargo.lock index 00cfd3d6..05fbf7b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1906,6 +1906,11 @@ name = "fastcdc" version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" +dependencies = [ + "async-stream", + "tokio", + "tokio-stream", +] [[package]] name = "fastrand" @@ -3824,7 +3829,6 @@ dependencies = [ "lightningcss", "lru", "md-5 0.11.0", - "memmap2", "mimalloc", "mime_guess", "mockall", diff --git a/Cargo.toml b/Cargo.toml index 42fa985e..29f5ce19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,8 +68,7 @@ azure_storage = { version = "0.21", default-features = false, features = ["enabl azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } aes-gcm = "0.10.3" lru = "0.16.4" -fastcdc = "4.0.0" -memmap2 = "0.9.10" +fastcdc = { version = "4.0.0", features = ["tokio"] } lettre = { version = "0.11.18", default-features = false, features = ["smtp-transport", "tokio1-rustls-tls", "rustls-native-certs", "builder"] } idna = "1.1" smol_str = { version = "0.3.2", features = ["serde"] } diff --git a/docs/config/env.md b/docs/config/env.md index f131ccd4..87fcfd7f 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -14,8 +14,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_MAX_UPLOAD_SIZE` | `10737418240` | Whole-file size ceiling, in bytes (10 GB on 64-bit, 1 GB on 32-bit). Applies to BOTH direct PUTs (per-request body) and chunked uploads (declared `total_size`, checked upfront at session creation). | | `OXICLOUD_DIRECT_PUT_MAX_BYTES` | `1073741824` | Per-request cap for non-chunked PUT bodies, in bytes (1 GiB). Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol (resumable on failure). See [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_CHUNK_MAX_BYTES` | `104857600` | Maximum size of a single chunked-upload PUT in bytes (100 MB). Per-chunk cap, independent of `OXICLOUD_MAX_UPLOAD_SIZE` (whole-file cap). See [Storage Fine Tuning](./storage-fine-tuning.md). | -| `OXICLOUD_UPLOAD_TMPDIR` | (OS temp dir) | Spool directory for non-chunked PUT bodies (`/api/files/upload`, WebDAV PUT). Point at a real-disk path on the same FS as `.blobs/` to avoid tmpfs OOMKill and make blob promotion an atomic rename. See [Storage Fine Tuning](./storage-fine-tuning.md). | -| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Same-FS / NVMe placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | +| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Direct (non-chunked) uploads stream straight into the blob store and need no spool directory. Placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | ## Database diff --git a/docs/config/storage-fine-tuning.md b/docs/config/storage-fine-tuning.md index ea51d3e1..dfd59849 100644 --- a/docs/config/storage-fine-tuning.md +++ b/docs/config/storage-fine-tuning.md @@ -1,62 +1,56 @@ # Storage Fine Tuning -This page is for sysadmins who want to tune **where** OxiCloud spools -upload bodies and **why** the placement matters for throughput and -memory. The defaults work; the gains from a tuned layout are +This page is for sysadmins who want to tune **where** OxiCloud places +upload data on disk and **why** the placement matters for throughput +and memory. The defaults work; the gains from a tuned layout are significant on busy instances or constrained containers. ## The upload lifecycle in 30 seconds -Every upload moves through two stages: - ``` - ┌─── direct (single-PUT) upload ────────┐ -client ─┤ ├──► OxiCloud accepts the - └─── multi-chunk upload │ bytes into a SPOOL on - (`/api/uploads` / │ local disk. - `/dav/uploads/...`) │ - │ Direct upload → OXICLOUD_UPLOAD_TMPDIR - │ Chunked upload → OXICLOUD_CHUNK_DIR - │ - ▼ + ┌─── direct (single-PUT / multipart) ───┐ +client ─┤ │ Streamed DIRECTLY into the + │ ├──► content-addressable store: + │ │ CDC chunking + BLAKE3 + + │ │ dedup checks happen while + │ │ the bytes arrive. No spool + │ │ file, no re-read; chunks + │ │ the store already has are + │ │ never written at all. + │ │ + └─── multi-chunk upload ────────────────┤ Chunk PARTS accumulate on + (`/api/uploads` / │ disk under OXICLOUD_CHUNK_DIR + `/dav/uploads/...`) │ until /complete, which + │ streams them (in order) + ▼ through the same CDC path. ┌─────────────────────────┐ - │ Once the upload is │ - │ complete (and verified │ - │ if a checksum was │ - │ supplied), OxiCloud │ - │ MOVES the assembled │ - │ blob into the configured│ - │ STORAGE BACKEND: │ - │ │ - │ • local FS (.blobs/) │ - │ • S3-compatible │ - │ • Azure Blob │ + │ STORAGE BACKEND │ + │ • local FS (.blobs/) │ + │ • S3-compatible │ + │ • Azure Blob │ └─────────────────────────┘ ``` Two practical consequences: -- **The spool/chunk directories see write-heavy churn** during uploads — - fast disk (NVMe) and sufficient free space matter more here than on - the final storage backend. -- **The promotion from spool → storage is a `rename(2)` whenever - source and destination share a filesystem** (i.e. when the backend - is `local` and the spool dir is on the same FS as `.blobs/`). On - remote backends (S3, Azure) the promotion is always a network - upload from the local spool; placement of the spool still matters - for intake throughput but the "same FS" rule doesn't apply. +- **Direct uploads no longer use a spool directory.** Each uploaded + byte is written at most once — straight into the blob backend as a + CDC chunk. Re-uploads and edited files write only the chunks the + store doesn't already have. +- **The chunk-session directory sees write-heavy churn** during + multi-chunk uploads — fast disk (NVMe) and sufficient free space + matter more here than on the final storage backend. ## Upload size caps — what each one bounds Three independent caps control how large an upload OxiCloud will -accept. Pick them with disk and tmpfs sizing in mind: the spool/chunk -directories must be able to hold the worst case (cap × concurrent -uploads). +accept. Pick them with disk sizing in mind: the chunk directory must +be able to hold the worst case (cap × concurrent sessions). | Variable | Default | What it caps | When it fires | |---|---|---|---| | `OXICLOUD_MAX_UPLOAD_SIZE` | 10 GB | **Whole-file ceiling.** Applies to both direct PUT (per-body) and chunked uploads (declared `total_size`). The absolute upper bound on any single file in OxiCloud. | Chunked: at `POST /api/uploads` against the JSON-declared `total_size`, before any chunk is uploaded. Direct PUT: indirectly via `OXICLOUD_DIRECT_PUT_MAX_BYTES`, which is expected to be ≤ `OXICLOUD_MAX_UPLOAD_SIZE`. | -| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `POST /api/files/upload`, `PUT /webdav/...`, and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | +| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `PUT /webdav/...` and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | | `OXICLOUD_CHUNK_MAX_BYTES` | 100 MB | **Per-chunk body** in a chunked-upload session (`PATCH /api/uploads/{id}` or `PUT /remote.php/dav/uploads/.../chunk`). Independent of the whole-file cap — a 5 GB file in 100 MB chunks is 50 PATCHes each bounded by this. | During chunk-body streaming. Excess → 413. | ### Recommendation: prefer chunked uploads for large files @@ -64,244 +58,117 @@ uploads). The defaults (`OXICLOUD_DIRECT_PUT_MAX_BYTES` = 1 GiB, well below `OXICLOUD_MAX_UPLOAD_SIZE` = 10 GB) are deliberately asymmetric. Files between those two caps can only succeed via the chunked -protocol. Three reasons to keep them that way: +protocol. The reason is **resilience**: a direct PUT at 95 % of 5 GB +that drops loses everything (the partially ingested chunks are +reclaimed by GC, but the client must restart from byte 0). The same +drop on a chunked upload loses one ~5 MB chunk; the client retries +that chunk and continues. NextCloud desktop and the OxiCloud web UI +already switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). -- **Resilience.** A direct PUT at 95 % of 5 GB that drops loses - everything. The same drop on a chunked upload loses one ~5 MB - chunk; the client retries that chunk and continues. -- **Memory + disk pressure.** Direct PUT spools the full body to - disk per request. Ten concurrent 5 GB direct PUTs use up to 50 GB - of transient spool disk. Chunked spreads each upload across many - small PATCHes; per-request resource use stays bounded by - `OXICLOUD_CHUNK_MAX_BYTES`. -- **Convention.** NextCloud desktop and the OxiCloud web UI already - switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). +### Disk sizing -### Why caps matter for tmpfs sizing +OxiCloud streams bodies frame-by-frame, so **RAM** is bounded +(~10 MB per in-flight upload for the CDC ingest buffers) regardless +of the caps. **Disk space** scales with the caps: -OxiCloud streams bodies frame-by-frame, so **RAM** is bounded to one -HTTP frame (~64 KB) per request regardless of the caps. **Disk space**, -however, scales with the caps: +- **Direct PUT / multipart**: no transient spool. Bytes land directly + in the blob backend as deduplicated chunks; worst-case extra disk + per upload is the file's own (deduplicated) size — the same space + the stored file occupies afterwards. +- **Chunked upload**: each in-flight session accumulates its chunk + parts under `OXICLOUD_CHUNK_DIR` until `/complete` streams them + into the blob store and the session is cleaned up. Worst case disk + per session = **file_size** (the parts); total = + `OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. -- **Direct PUT**: each in-flight upload spools the full body to disk - under `OXICLOUD_UPLOAD_TMPDIR` until promotion. Worst case disk = - `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs`. -- **Chunked upload**: each in-flight session accumulates chunks - under `OXICLOUD_CHUNK_DIR`, then assembles them into a single temp - file before promotion. Worst case disk per session = **2 × - file_size** (chunks + assembled file); total disk = - `2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. +| Settings | Chunked worst case (5 sessions) | Safe on 4 GB volume? | +|---|---|---| +| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB | 50 GB | ❌ overflows | +| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB | 2.5 GB | ✅ fits | -The chunked formula uses `OXICLOUD_MAX_UPLOAD_SIZE` because that's -what bounds the declared `total_size` at session creation. The -direct-PUT formula uses the smaller `OXICLOUD_DIRECT_PUT_MAX_BYTES` -since that's what bounds each direct PUT body. +### Don't put `OXICLOUD_CHUNK_DIR` on tmpfs -### Sizing examples - -A 4 GB tmpfs serving a small team (5 concurrent direct PUTs OR 5 -concurrent chunked sessions): - -| Settings | Direct-PUT worst case | Chunked worst case | Safe on 4 GB tmpfs? | -|---|---|---|---| -| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=100 MB | 5 GiB (5 × 1 GiB) | 100 GB (5 × 2 × 10 GB) | ❌ chunked overflows | -| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=100 MB, `OXICLOUD_CHUNK_MAX_BYTES`=20 MB | 500 MB | 5 GB | ⚠ direct PUT fits, chunked still overflows | -| `OXICLOUD_MAX_UPLOAD_SIZE`=300 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=50 MB, `OXICLOUD_CHUNK_MAX_BYTES`=10 MB | 250 MB | 3 GB | ✅ both fit | - -A real-disk volume (cheap, large): - -| Settings | Direct-PUT worst case | Chunked worst case | Comment | -|---|---|---|---| -| Defaults (see row above) | 5 GiB | 100 GB | Fine on a 200+ GB volume; almost any real-disk setup | -| `OXICLOUD_MAX_UPLOAD_SIZE`=100 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=5 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=500 MB | 25 GiB | 1 TB | Plausible for video archives; needs a dedicated upload volume | - -### Choosing tmpfs vs real disk - -| Constraint | Choice | -|---|---| -| `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs + 2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions ≤ free RAM × 0.5` | tmpfs OK (fast, atomic with `.blobs/` if also tmpfs) | -| Worst case exceeds half free RAM | **real disk** — same filesystem as `.blobs/` ideal | -| Container with cgroup memory limit | **real disk** — tmpfs spool counts against the cgroup limit and triggers OOMKill | -| Multi-GB uploads expected | **real disk** — even small concurrency on tmpfs runs out of space | -| Small-file workload only (≤ 50 MB), high concurrency | tmpfs gives a noticeable intake speedup | - -The defaults (`OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, -`OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, -`OXICLOUD_CHUNK_MAX_BYTES`=100 MB) assume **real disk**. Don't run -the defaults against tmpfs unless you've sized it for the worst case. +In many container setups the OS temp dir is **tmpfs** — RAM-backed +storage that counts against the cgroup memory limit. A few concurrent +multi-GB chunked sessions on tmpfs will wake the OOMKiller long +before the uploads finish. Point `OXICLOUD_CHUNK_DIR` at a real-disk +directory in containers. ## TL;DR | Variable | Default | Purpose | |---|---|---| | `OXICLOUD_STORAGE_PATH` | `./storage` | Where `.blobs/` lives (the canonical content store) | -| `OXICLOUD_UPLOAD_TMPDIR` | OS temp dir | Where non-chunked PUT bodies are spooled | | `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Where chunked-upload sessions accumulate | The two rules that matter most: -1. **Put all three on the same filesystem.** Blob promotion is an - atomic `rename(2)` when source and destination share an FS — cheap - and crash-safe. Across filesystems it becomes a full `read + write + - unlink`, multiplying the IO and widening the durability window. -2. **Don't leave the spool dir on tmpfs** (the default in many - containers). Spool bodies count against the cgroup memory limit +1. **Keep `OXICLOUD_CHUNK_DIR` off tmpfs** (the default in many + containers) — chunk parts count against the cgroup memory limit and can trigger OOMKill on multi-GB uploads. +2. **NVMe for the chunk dir pays off** on deployments with heavy + large-file traffic: each chunk PUT writes a file and the progress + bitmap, and `/complete` reads them all back in order. -## Where each upload surface spools - -OxiCloud has several entry points that accept request bodies. They -land in different places by default: +## Where each upload surface writes | Surface | Default destination | Configurable via | |---|---|---| | REST chunked PUT (`PATCH /api/uploads/{id}`) | `{STORAGE_PATH}/.uploads/{upload_id}/chunk_NNNNNN` | `OXICLOUD_CHUNK_DIR` | -| REST chunked assemble (during `/complete`) | `{STORAGE_PATH}/.uploads/{upload_id}/assembled` | `OXICLOUD_CHUNK_DIR` | | NextCloud chunked PUT (`PUT /dav/uploads/.../chunk`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/{chunk_name}` | `OXICLOUD_CHUNK_DIR` | -| NextCloud chunked assemble (during `MOVE`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/.assembled` | `OXICLOUD_CHUNK_DIR` | -| Native WebDAV PUT (`PUT /webdav/{path}`) | OS temp dir (`/tmp`) | `OXICLOUD_UPLOAD_TMPDIR` | -| NextCloud single-file PUT (`PUT /dav/files/.../{path}`) | OS temp dir | `OXICLOUD_UPLOAD_TMPDIR` | -| REST multipart upload (`POST /api/files/upload`) | `{STORAGE_PATH}/.dedup_temp/upload-{uuid}` | `OXICLOUD_STORAGE_PATH` (subdir is hard-wired) | -| Final blob storage (after fsync + rename) | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | +| Direct PUT / multipart / WOPI / chunked `/complete` | straight into the blob backend (CDC chunks) | `OXICLOUD_STORAGE_PATH` (local backend) | +| Final blob storage | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | -## Why placement matters - -### 1. Same filesystem ⇒ promotion is a rename - -OxiCloud uses **content-addressable storage**: the final blob path is -derived from the file's BLAKE3 hash, which can only be known after the -last byte arrives. So every upload writes to a temp location first, -then **promotes** the temp file to `.blobs/{ab}/{abc…}.blob` by way of -a `rename(2)` call. - -- **Same FS:** `rename` is atomic, O(1), no data copy. Total upload - cost = body bytes received + one rename syscall. Crash-safe — the - blob either exists at the final path or doesn't. -- **Cross-FS:** the kernel can't `rename(2)` across filesystems. The - blob backend falls back to `fs::copy + fs::remove_file` (visible in - `local_blob_backend.rs` as the EXDEV handler). Total cost = body - bytes received + one full file copy. Doubles the IO bandwidth used - per upload and widens the durability window. - -### 2. Spool off tmpfs - -`tempfile::NamedTempFile::new()` (used when `OXICLOUD_UPLOAD_TMPDIR` -is unset) honors `$TMPDIR`, which in many container setups points at -**tmpfs** — RAM-backed storage. A 2 GB upload spool then consumes 2 GB -of memory until the rename promotes it to disk. - -In a Kubernetes pod with a 4 GB memory limit, the OOMKiller wakes up -long before the upload finishes. With `OXICLOUD_UPLOAD_TMPDIR` pointed -at a real-disk directory, the spool's memory footprint stays at ~one -HTTP frame regardless of file size. - -### 3. NVMe for the hot path - -The chunked-upload session directory sees a LOT of small writes — -each chunk PUT writes a file, the progress bitmap is rewritten after -each PUT, the assemble step reads them all back in order. Pointing -`OXICLOUD_CHUNK_DIR` at an NVMe device is a substantial win on -deployments that handle large file uploads, even if the final blob -storage is on slower disk. - -The same applies to `OXICLOUD_UPLOAD_TMPDIR` (single-file PUTs). - -A common high-throughput layout: - -- **NVMe** (small, fast): `OXICLOUD_CHUNK_DIR`, `OXICLOUD_UPLOAD_TMPDIR` -- **HDD or NAS** (large, cheap): `OXICLOUD_STORAGE_PATH`/`.blobs/` - -Trade-off: the rename optimization (rule 1) DOESN'T apply across -filesystems. If you split the hot path off the blob filesystem, every -upload pays a full file copy on promotion. You have to choose -between **fast intake** and **zero-copy promotion**. - -| Goal | Layout | Cost per upload | -|---|---|---| -| Fastest possible intake | NVMe chunk dir + HDD blobs | 1× write to NVMe + 1× read NVMe + 1× write to HDD (copy) | -| Lowest IO + crash safety | NVMe everything OR HDD everything | 1× write to disk + 1 rename (~0 cost) | -| Default (do nothing) | Everything under `STORAGE_PATH` on whatever FS that is | Depends on `STORAGE_PATH` placement | - -For most deployments **"same FS everywhere"** wins. The NVMe-split is -useful when intake latency dominates the user experience and you can -afford the doubled IO. +The local blob backend stages each chunk write under +`{STORAGE_PATH}/.dedup_temp/` and promotes it with an atomic +`rename(2)` — both directories live under `OXICLOUD_STORAGE_PATH`, +so same-filesystem placement (and therefore atomic promotion) is +automatic and not separately configurable. ## Recommended layouts ### Single-disk box (most common) -Defaults are fine. Optionally set `OXICLOUD_UPLOAD_TMPDIR` to keep -the PUT spool off `/tmp`: +Defaults are fine: ```bash OXICLOUD_STORAGE_PATH=/var/lib/oxicloud -OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/.spool # OXICLOUD_CHUNK_DIR unset → /var/lib/oxicloud/.uploads ``` -All three on the same filesystem → rename promotion → atomic and fast. - ### Container with constrained memory -Critical: make sure neither spool sits on tmpfs. +Critical: make sure the chunk dir doesn't sit on tmpfs. ```bash OXICLOUD_STORAGE_PATH=/data -OXICLOUD_UPLOAD_TMPDIR=/data/.spool OXICLOUD_CHUNK_DIR=/data/.uploads ``` -If you can't mount a writable `/data`, at minimum bind-mount a real -volume at the spool dirs. - ### Split-disk (NVMe intake + HDD blobs) ```bash OXICLOUD_STORAGE_PATH=/mnt/hdd/oxicloud # .blobs/ + .dedup_temp/ -OXICLOUD_UPLOAD_TMPDIR=/mnt/nvme/oxi-spool OXICLOUD_CHUNK_DIR=/mnt/nvme/oxi-chunks ``` -Faster intake; pays a copy on promotion. Worth it when uploads are -many small files (NVMe IOPS dominates) or when intake latency directly -hits user-visible UX. +Chunk parts land on NVMe (fast PUTs, fast `/complete` read-back); +the deduplicated chunks are written once to the HDD-backed blob +store as `/complete` streams through them. -## Sharing the spool and chunk directories +## Sharing the chunk directory -Pointing `OXICLOUD_UPLOAD_TMPDIR` and `OXICLOUD_CHUNK_DIR` at the -**same directory** is supported by design. Each writer tags its -output so the surfaces never interfere with each other: +The REST and NC chunked surfaces can share `OXICLOUD_CHUNK_DIR` by +design. Each writer tags its output so they never interfere: | Writer | On-disk name pattern | |---|---| -| PUT spool (single-file uploads) | `.tmpXXXXXXXX` — files (not directories), random suffix | | REST chunked sessions | `oxi-chunk-{uuid}/` — directories with a well-known prefix | | NC chunked subtree | `nextcloud/{user}/{uuid}/` — under its own root subdir | The 24-hour orphan-session cleanup loop filters strictly on the `oxi-chunk-` prefix, so it can NEVER delete a non-OxiCloud directory -that happens to live alongside chunked sessions. The PUT spool's -`.tmpXXXX` files are files (not directories) and the NC subtree's -`nextcloud/` root has its own name — both are invisible to the -cleanup loop. - -**Recommendation:** for new deployments, use separate directories -anyway (the defaults `.spool/` and `.uploads/` already do this) — -it makes disk-usage attribution clearer and keeps IOPS isolated when -both are busy. Shared directories are safe to use when disk layout -forces it. - -## What's NOT yet configurable - -- **REST multipart upload directory** (`POST /api/files/upload`) is - hard-wired to `{STORAGE_PATH}/.dedup_temp/`. It can't be moved - separately. Same-FS placement is automatic. -- **WOPI PutFile spool** (Office editor saves) uses the bare OS temp - dir without honoring `OXICLOUD_UPLOAD_TMPDIR`. This is a known - inconsistency and on the hardening backlog. -- **Per-user / per-drive spool directories** — all users share the - same `OXICLOUD_CHUNK_DIR` root today. Multi-tenant isolation - through separate spool dirs isn't supported. +that happens to live alongside chunked sessions. ## Quick verification diff --git a/example.env b/example.env index c1dffd63..2084135a 100644 --- a/example.env +++ b/example.env @@ -55,22 +55,14 @@ OXICLOUD_SERVER_HOST=127.0.0.1 #OXICLOUD_CHUNK_MAX_BYTES=104857600 # ── Upload spool directories ───────────────────────────────────────── -# Where in-flight uploads land BEFORE being promoted into final blob -# storage. Same-filesystem placement (with `.blobs/`) makes the -# promotion an atomic rename(2); NVMe placement speeds up intake. +# Direct (non-chunked) uploads stream straight into the blob store — +# no spool directory. Only chunked-upload sessions accumulate on disk. # See docs/config/storage-fine-tuning.md for layout examples. -# Directory for non-chunked PUT spool tempfiles. Default: OS temp dir -# ($TMPDIR / /tmp), often tmpfs (RAM) in containers — writing a large -# upload there fills page-cache that counts against the cgroup memory -# limit and can OOMKill the process. Point this at a real-disk path -# (same FS as the storage backend is ideal). -#OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/tmp - # Root directory for chunked-upload sessions (REST + NextCloud chunked -# share this root). Default: {STORAGE_PATH}/.uploads. Pointing this at -# NVMe accelerates the chunk-write + assembly loop; pointing it at the -# same FS as `.blobs/` makes blob promotion atomic. +# share this root). Default: {STORAGE_PATH}/.uploads. Avoid tmpfs in +# containers (parts count against the cgroup memory limit); NVMe +# placement accelerates chunk PUTs and the /complete streaming pass. #OXICLOUD_CHUNK_DIR=/var/lib/oxicloud/.uploads # How often (seconds) the background sweep reconciles each user's cached diff --git a/src/application/ports/chunked_upload_ports.rs b/src/application/ports/chunked_upload_ports.rs index 5d7d5427..098dd276 100644 --- a/src/application/ports/chunked_upload_ports.rs +++ b/src/application/ports/chunked_upload_ports.rs @@ -98,6 +98,21 @@ pub struct UploadStatusResponseDto { pub is_complete: bool, } +/// A completed upload session, ready to be streamed into the blob store. +/// +/// `chunk_paths` lists every chunk file in assembly order — the caller +/// concatenates them as one byte stream (CDC chunking + hashing happen in +/// that single pass). The files stay on disk until `finalize_upload`, so a +/// failed completion (e.g. checksum mismatch) remains retryable. +#[derive(Debug, Clone)] +pub struct CompletedUploadParts { + pub chunk_paths: Vec, + pub filename: String, + pub folder_id: Option, + pub content_type: String, + pub total_size: u64, +} + /// Port for chunked/resumable upload operations. /// /// Implementations manage upload sessions, chunk storage, reassembly, @@ -137,16 +152,15 @@ pub trait ChunkedUploadPort: Send + Sync + 'static { user_id: Uuid, ) -> Result; - /// Assemble all chunks into the final file. + /// Validate completion and hand back the ordered chunk parts. /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. - /// The hash is computed during assembly (hash-on-write), eliminating a - /// second sequential read of the assembled file. + /// No assembled file is written — the caller streams the parts straight + /// into the content-addressable store. async fn complete_upload( &self, upload_id: &str, user_id: Uuid, - ) -> Result<(PathBuf, String, Option, String, u64, String), DomainError>; + ) -> Result; /// Finalize upload: clean up the session and temporary files. async fn finalize_upload(&self, upload_id: &str, user_id: Uuid) -> Result<(), DomainError>; diff --git a/src/application/ports/dedup_ports.rs b/src/application/ports/dedup_ports.rs index 95f219ea..23ddc962 100644 --- a/src/application/ports/dedup_ports.rs +++ b/src/application/ports/dedup_ports.rs @@ -80,18 +80,6 @@ pub struct DedupStatsDto { /// duplicate storage automatically. Multiple file references can /// point to the same physical blob. pub trait DedupPort: Send + Sync + 'static { - /// Store content with deduplication (streaming from file). - /// - /// If `pre_computed_hash` is provided (e.g. hash-on-write from the handler), - /// the file will NOT be re-read to calculate the hash — saving one full - /// sequential read of the file. - async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result; - /// Check if a blob with the given hash exists. async fn blob_exists(&self, hash: &str) -> bool; diff --git a/src/application/ports/file_ports.rs b/src/application/ports/file_ports.rs index b40e0a35..19d9e926 100644 --- a/src/application/ports/file_ports.rs +++ b/src/application/ports/file_ports.rs @@ -1,6 +1,5 @@ use bytes::Bytes; use futures::Stream; -use std::path::Path; use std::pin::Pin; use std::sync::Arc; use uuid::Uuid; @@ -17,74 +16,51 @@ use crate::domain::services::authorization::Permission; // Upload port // ───────────────────────────────────────────────────── +/// A blob already stored in the content-addressable chunk store, carrying +/// exactly ONE reference that the receiving method takes ownership of. +/// +/// Produced by the upload-ingest layer (`interfaces::upload_ingest`), which +/// streams the request body straight into the CDC dedup store. Methods that +/// accept a `StoredBlob` either attach the reference to a file row or +/// release it on failure — callers never need to compensate themselves. +#[derive(Debug, Clone)] +pub struct StoredBlob { + /// BLAKE3 of the full content (manifest / blob key). + pub hash: String, + /// Content size in bytes. + pub size: u64, + /// `false` when the content already existed (dedup hit) — forwarded to + /// lifecycle hooks so e.g. thumbnails aren't regenerated for known blobs. + pub is_new_blob: bool, +} + /// Primary port for file upload operations. /// -/// **All upload paths converge on streaming-to-disk** — no method accepts -/// `Vec` for content. Even `create_file` / `update_file` (WebDAV -/// helpers that receive `&[u8]`) spool to a temp file internally so that -/// peak RAM stays at ~256 KB regardless of file size. -/// -/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming` -/// - Chunked uploads: chunks already on disk → `upload_file_from_path` -/// - WebDAV PUT (new): handler streams to temp file → `update_file_streaming` -/// - WebDAV PUT (small/compat): `create_file` / `update_file` spool internally +/// **All upload paths converge on streaming-into-the-chunk-store** — content +/// never passes through this port; it is ingested by the interface layer +/// (CDC chunking + hashing while the body arrives, no spool file) and only +/// the resulting [`StoredBlob`] reference travels through here. pub trait FileUploadUseCase: Send + Sync + 'static { - /// Upload from a temp file already on disk (true streaming, ~256 KB RAM). + /// Register a new file row pointing at an already-ingested blob. /// - /// When `pre_computed_hash` is `Some`, the blob store skips the hash - /// re-read — the handler already computed it during the multipart spool. + /// Takes ownership of the blob's reference (released on failure). async fn upload_file_streaming( &self, name: String, folder_id: Option, content_type: String, - temp_path: &Path, - size: u64, - pre_computed_hash: Option, + blob: StoredBlob, ) -> Result; - /// Upload from a file already assembled on disk (chunked uploads). + /// Replace the content of the file at `path` with an already-ingested + /// blob, or create the file when it doesn't exist (WebDAV/WOPI PUT). /// - /// Same as `upload_file_streaming` but with a separate name for clarity. - async fn upload_file_from_path( - &self, - name: String, - folder_id: Option, - content_type: String, - file_path: &Path, - pre_computed_hash: Option, - ) -> Result; - - /// Creates a new file at the specified path (for WebDAV) - async fn create_file( - &self, - parent_path: &str, - filename: &str, - content: &[u8], - content_type: &str, - ) -> Result; - - /// Updates the content of an existing file (for WebDAV) - async fn update_file( - &self, - path: &str, - content: &[u8], - content_type: &str, - modified_at: Option, - ) -> Result; - - /// Streaming update — spools body to a temp file with incremental hash, - /// then atomically replaces the file content via dedup store. - /// - /// Peak RAM: ~256 KB regardless of file size. - /// Used by WebDAV PUT for large files. + /// Takes ownership of the blob's reference (released on failure). async fn update_file_streaming( &self, path: &str, - temp_path: &Path, - size: u64, + blob: StoredBlob, content_type: &str, - pre_computed_hash: Option, modified_at: Option, ) -> Result; } diff --git a/src/application/ports/storage_ports.rs b/src/application/ports/storage_ports.rs index 213cb759..33307c35 100644 --- a/src/application/ports/storage_ports.rs +++ b/src/application/ports/storage_ports.rs @@ -251,21 +251,21 @@ pub struct CopyFolderTreeResult { /// Secondary port for file **writing**. /// -/// Covers: upload (buffered + streaming), move, delete, update, -/// and deferred registration for the write-behind cache. +/// Covers: upload registration, move, delete, update, and deferred +/// registration for the write-behind cache. pub trait FileWritePort: Send + Sync + 'static { - /// Streaming upload — saves a file from a temp file already on disk. + /// Register a file row pointing at a blob already stored in the + /// content-addressable chunk store. /// - /// When `pre_computed_hash` is provided, the dedup service skips the - /// hash re-read — zero extra I/O beyond the initial spool. - async fn save_file_from_temp( + /// Takes ownership of one blob reference: on any failure the reference + /// is released before the error is returned. + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result; /// Moves a file to another folder. @@ -281,22 +281,20 @@ pub trait FileWritePort: Send + Sync + 'static { /// Deletes a file. async fn delete_file(&self, id: &str) -> Result<(), DomainError>; - /// Streaming update — replaces file content from a temp file on disk. + /// Atomically swap a file's content to a blob already stored in the + /// content-addressable chunk store. /// - /// When `pre_computed_hash` is provided, the dedup service skips the - /// hash re-read — zero extra I/O beyond the initial spool. - /// Peak RAM: ~256 KB regardless of file size. + /// Takes ownership of one blob reference (released on failure); the + /// previous content's reference is dropped after the swap. /// /// Returns `(new_blob_hash, updated_at_epoch)` — everything a caller /// needs to rebuild the fresh entity/ETag from a `File` it already /// holds, without re-reading the row it just updated. - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, file_id: &str, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - content_type: Option, - pre_computed_hash: Option, modified_at: Option, ) -> Result<(String, i64), DomainError>; diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 92ec9215..49f00a95 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -1,15 +1,13 @@ -use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::application::dtos::file_dto::FileDto; use crate::application::ports::file_lifecycle::FileLifecycleHook; -use crate::application::ports::file_ports::FileUploadUseCase; +use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; use crate::application::services::storage_usage_service::StorageUsageService; use crate::common::errors::DomainError; use crate::infrastructure::repositories::pg::FileBlobReadRepository; use crate::infrastructure::repositories::pg::FileBlobWriteRepository; -use crate::infrastructure::services::dedup_service::DedupService; use crate::infrastructure::services::file_content_cache::FileContentCache; use tracing::{debug, info, warn}; @@ -34,20 +32,16 @@ fn extract_username_from_path(path: &str) -> Option { /// Service for file upload operations. /// -/// **Every upload path converges on streaming-to-disk** — there is no -/// `Vec` buffer path. -/// -/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming` -/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path` -/// - **WebDAV PUT (large)**: handler streams body to temp file → `update_file_streaming` -/// - **WebDAV PUT (small / compat)**: `create_file` / `update_file` spool `&[u8]` -/// to a temp file internally, then call the streaming path. -/// -/// Peak RAM usage during any upload: ~256 KB (streaming hash) regardless of file size. +/// Content never passes through this service: the interface layer streams +/// the request body straight into the CDC chunk store (no spool file, no +/// full-body buffering) and hands over a [`StoredBlob`] reference. This +/// service registers the metadata row, keeps caches coherent and fires +/// lifecycle hooks. Blob-reference ownership is consumed by the write +/// port, which releases it on failure — callers never compensate. pub struct FileUploadService { - /// Write port — handles save, streaming, deferred registration + /// Write port — registers file rows against ingested blobs. file_write: Arc, - /// Read port — needed for WebDAV create_file / update_file + /// Read port — needed for WebDAV/WOPI update-by-path. file_read: Option>, /// Optional storage usage tracking storage_usage_service: Option>, @@ -55,9 +49,6 @@ pub struct FileUploadService { content_cache: Option>, /// Single lifecycle dispatcher — fires on_file_created / on_file_updated. file_lifecycle_hook: Option>, - /// Directory for spool temp files (`&[u8]` upload variants). When `Some`, - /// keeps spools off tmpfs/RAM so they don't count against the cgroup limit. - upload_temp_dir: Option, } impl FileUploadService { @@ -69,7 +60,6 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, - upload_temp_dir: None, } } @@ -84,16 +74,9 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, - upload_temp_dir: None, } } - /// Configures the spool directory for the `&[u8]` upload variants. - pub fn with_upload_temp_dir(mut self, dir: Option) -> Self { - self.upload_temp_dir = dir; - self - } - /// Configures the content cache for invalidation on file updates. pub fn with_content_cache(mut self, cache: Arc) -> Self { self.content_cache = Some(cache); @@ -117,11 +100,6 @@ impl FileUploadService { // ── private helpers ────────────────────────────────────────── - /// Create a spool temp file, honoring the configured upload temp dir. - fn new_temp(&self) -> std::io::Result { - crate::common::temp::new_spool_temp_file(self.upload_temp_dir.as_deref()) - } - /// Optionally update storage usage after a successful upload. fn maybe_update_storage_usage(&self, file: &FileDto) { if let Some(storage_service) = &self.storage_usage_service { @@ -146,172 +124,37 @@ impl FileUploadService { } impl FileUploadUseCase for FileUploadService { - /// Streaming upload from a temp file on disk. - /// - /// Peak RAM: ~256 KB (hash calculation) regardless of file size. - /// The temp file is consumed (moved/deleted) by the blob store. + /// Register a new file row pointing at an already-ingested blob. async fn upload_file_streaming( &self, name: String, folder_id: Option, content_type: String, - temp_path: &Path, - size: u64, - pre_computed_hash: Option, + blob: StoredBlob, ) -> Result { - let (file, is_new_blob) = self + let file = self .file_write - .save_file_from_temp_with_dedup( - name.clone(), - folder_id, - content_type, - temp_path, - size, - pre_computed_hash, - ) + .save_file_with_blob(name.clone(), folder_id, content_type, &blob.hash, blob.size) .await?; let dto = FileDto::from(file); info!( "📡 STREAMING UPLOAD: {} ({} bytes, ID: {})", - name, size, dto.id + name, blob.size, dto.id ); self.maybe_update_storage_usage(&dto); if let Some(hook) = &self.file_lifecycle_hook { - hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, is_new_blob); + hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, blob.is_new_blob); } Ok(dto) } - /// Upload from a file already on disk (chunked uploads). - async fn upload_file_from_path( - &self, - name: String, - folder_id: Option, - content_type: String, - file_path: &Path, - pre_computed_hash: Option, - ) -> Result { - let size = tokio::fs::metadata(file_path) - .await - .map_err(|e| { - DomainError::internal_error( - "FileUpload", - format!("Failed to read file metadata: {}", e), - ) - })? - .len(); - - self.upload_file_streaming( - name, - folder_id, - content_type, - file_path, - size, - pre_computed_hash, - ) - .await - } - - /// Creates a file at a specific path (for WebDAV PUT on new resource). - /// - /// Spools the in-memory `&[u8]` to a temp file with hash-on-write, - /// then delegates to the streaming path. Peak RAM: the caller's - /// buffer + ~256 KB for the hasher. - async fn create_file( - &self, - parent_path: &str, - filename: &str, - content: &[u8], - content_type: &str, - ) -> Result { - // Look up the folder ID by folder path - let parent_id = if !parent_path.is_empty() { - if let Some(file_read) = &self.file_read { - file_read.get_folder_id_by_path(parent_path).await.ok() - } else { - None - } - } else { - None - }; - - // Spool to temp file + hash - let temp = self - .new_temp() - .map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?; - tokio::fs::write(temp.path(), content) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = DedupService::hash_file(temp.path()) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("hash: {e}")))?; - - let (file, is_new_blob) = self - .file_write - .save_file_from_temp_with_dedup( - filename.to_string(), - parent_id, - content_type.to_string(), - temp.path(), - content.len() as u64, - Some(hash), - ) - .await?; - let dto = FileDto::from(file); - self.maybe_update_storage_usage(&dto); - if let Some(hook) = &self.file_lifecycle_hook { - hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, is_new_blob); - } - Ok(dto) - } - - /// Updates an existing file's content, or creates it if not found (for WebDAV PUT). - /// - /// Spools the in-memory `&[u8]` to a temp file with hash-on-write, - /// then delegates to the streaming update/create path. - async fn update_file( - &self, - path: &str, - content: &[u8], - content_type: &str, - modified_at: Option, - ) -> Result { - // Spool to temp file + hash - let temp = self - .new_temp() - .map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?; - tokio::fs::write(temp.path(), content) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = DedupService::hash_file(temp.path()) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("hash: {e}")))?; - - self.update_file_streaming( - path, - temp.path(), - content.len() as u64, - content_type, - Some(hash), - modified_at, - ) - .await - } - - /// Streaming update — replaces file content from a temp file on disk. - /// - /// Uses `update_file_content_from_temp` which passes the pre-computed hash - /// to dedup, avoiding a second full read of the file. - /// For new files (not found at `path`), falls back to `upload_file_streaming`. - /// - /// Peak RAM: ~256 KB regardless of file size. + /// Swap the content of the file at `path` to an already-ingested blob, + /// creating the file when it doesn't exist (WebDAV/NextCloud/WOPI PUT). async fn update_file_streaming( &self, path: &str, - temp_path: &Path, - size: u64, + blob: StoredBlob, content_type: &str, - pre_computed_hash: Option, modified_at: Option, ) -> Result { // Try to find the existing file first @@ -321,14 +164,7 @@ impl FileUploadUseCase for FileUploadService { let file_id = file.id().to_string(); let (new_hash, updated_at) = self .file_write - .update_file_content_from_temp( - &file_id, - temp_path, - size, - Some(content_type.to_string()), - pre_computed_hash, - modified_at, - ) + .update_file_content_with_blob(&file_id, &blob.hash, blob.size, modified_at) .await?; // Invalidate content cache — file content has changed. if let Some(cc) = &self.content_cache { @@ -343,7 +179,7 @@ impl FileUploadUseCase for FileUploadService { parts.id, parts.name, parts.storage_path, - size, + blob.size, parts.mime_type, parts.folder_id, parts.created_at, @@ -381,15 +217,15 @@ impl FileUploadUseCase for FileUploadService { None }; - let (created, is_new_blob) = self + let is_new_blob = blob.is_new_blob; + let created = self .file_write - .save_file_from_temp_with_dedup( + .save_file_with_blob( filename.to_string(), parent_id, content_type.to_string(), - temp_path, - size, - pre_computed_hash, + &blob.hash, + blob.size, ) .await?; let dto = FileDto::from(created); diff --git a/src/application/services/idor_protection_test.rs b/src/application/services/idor_protection_test.rs index 8682ae28..33d1dcf7 100644 --- a/src/application/services/idor_protection_test.rs +++ b/src/application/services/idor_protection_test.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use futures::Stream; use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::pin::Pin; use std::sync::Mutex; use uuid::Uuid; @@ -169,14 +169,13 @@ impl MockFileWritePort { } impl FileWritePort for MockFileWritePort { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> Result { unimplemented!() } @@ -205,13 +204,11 @@ impl FileWritePort for MockFileWritePort { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result<(String, i64), DomainError> { Ok((String::new(), 0)) diff --git a/src/application/services/trash_service_test.rs b/src/application/services/trash_service_test.rs index 951abf39..675bb300 100644 --- a/src/application/services/trash_service_test.rs +++ b/src/application/services/trash_service_test.rs @@ -554,14 +554,13 @@ impl FileReadPort for MockFileRepository { } impl FileWritePort for MockFileRepository { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &std::path::Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> std::result::Result { unimplemented!() } @@ -586,13 +585,11 @@ impl FileWritePort for MockFileRepository { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &std::path::Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> std::result::Result<(String, i64), DomainError> { Ok((String::new(), 0)) diff --git a/src/common/config.rs b/src/common/config.rs index ffc89a93..6a6217e6 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -228,12 +228,6 @@ pub struct StorageConfig { /// returns 413 with a "use chunked upload" hint when a direct PUT /// exceeds this cap. Env: `OXICLOUD_DIRECT_PUT_MAX_BYTES`. pub direct_put_max_bytes: usize, - /// Directory for upload spool temp files. When `Some`, large uploads are - /// spooled here instead of the OS default temp dir (often tmpfs/RAM in - /// containers, where the spool's page-cache counts against the cgroup - /// memory limit and can trigger OOMKill on large files). Env: - /// `OXICLOUD_UPLOAD_TMPDIR`. - pub upload_temp_dir: Option, /// Root directory for chunked-upload sessions. When `Some`, chunks land /// under `{chunk_dir}/{upload_id}/` (REST) and /// `{chunk_dir}/nextcloud/{user}/{upload_id}/` (NC). When `None`, falls @@ -401,7 +395,6 @@ impl Default for StorageConfig { max_upload_size: MAX_UPLOAD_SIZE, chunk_max_bytes: 100 * 1024 * 1024, // 100 MB — sane upper bound for a single chunked-upload PUT direct_put_max_bytes: 1024 * 1024 * 1024, // 1 GiB — pushes larger uploads onto the chunked protocol - upload_temp_dir: None, chunk_dir: None, usage_reconcile_secs: 600, // 10 minutes tree_etag_flush_ms: 500, @@ -1281,18 +1274,10 @@ impl AppConfig { config.storage.direct_put_max_bytes = val; } - // Upload spool directory — keep large upload temp files off tmpfs/RAM - // (otherwise their page-cache counts against the cgroup memory limit). - if let Ok(dir) = env::var("OXICLOUD_UPLOAD_TMPDIR") - && !dir.trim().is_empty() - { - config.storage.upload_temp_dir = Some(PathBuf::from(dir.trim())); - } - // Chunked-upload session root — separate from the PUT spool because - // chunked sessions accumulate disk on long uploads (multi-chunk - // resumable transfers) while PUT spool is short-lived. Sysadmins - // commonly want one of them on fast/local storage (NVMe) and the - // other on bulk storage; this knob lets that be expressed. + // Chunked-upload session root — chunked sessions accumulate disk on + // long uploads (multi-chunk resumable transfers); sysadmins commonly + // want them on fast/local storage (NVMe). This knob lets that be + // expressed. if let Ok(dir) = env::var("OXICLOUD_CHUNK_DIR") && !dir.trim().is_empty() { diff --git a/src/common/di.rs b/src/common/di.rs index 08b483f0..809611f1 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -452,8 +452,7 @@ impl AppServiceFactory { repos.file_read_repository.clone(), ) .with_content_cache(core.file_content_cache.clone()) - .with_file_lifecycle_hook(core.file_lifecycle.clone()) - .with_upload_temp_dir(self.config.storage.upload_temp_dir.clone()), + .with_file_lifecycle_hook(core.file_lifecycle.clone()), ); let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( diff --git a/src/common/mime_detect.rs b/src/common/mime_detect.rs index 8cecd2c9..d2fcf118 100644 --- a/src/common/mime_detect.rs +++ b/src/common/mime_detect.rs @@ -8,21 +8,25 @@ //! //! Performance: < 1µs for the `infer` check (reads only header bytes, no allocation). -use std::path::Path; -use tokio::io::AsyncReadExt; - -/// Maximum bytes to read for magic-byte detection. -const MAGIC_BYTES_LEN: usize = 8192; +/// Maximum bytes needed for magic-byte detection. Upload ingestion peeks +/// this many bytes off the stream before forwarding them unchanged. +pub const MAGIC_BYTES_LEN: usize = 8192; /// Extract the filename component from a `/`-separated path. pub fn filename_from_path(path: &str) -> &str { path.rsplit('/').next().unwrap_or(path) } +/// Whether a claimed Content-Type is too generic to trust — these trigger +/// magic-byte detection on the upload path. +pub fn is_generic_mime(claimed: &str) -> bool { + claimed.is_empty() || claimed == "application/octet-stream" || claimed == "binary/octet-stream" +} + /// Refine a claimed MIME type using magic bytes and filename extension. /// /// This is a synchronous function — the caller should already have the first -/// bytes of the file available (or call the async wrapper below). +/// bytes of the content available (upload ingestion peeks them in-flight). /// /// # Arguments /// * `buf` — first bytes of the file (at least 8192 for best results) @@ -30,10 +34,7 @@ pub fn filename_from_path(path: &str) -> &str { /// * `claimed` — the Content-Type sent by the client pub fn refine_content_type(buf: &[u8], filename: &str, claimed: &str) -> String { // If the client sent a specific type (not generic), trust it - if !claimed.is_empty() - && claimed != "application/octet-stream" - && claimed != "binary/octet-stream" - { + if !is_generic_mime(claimed) { return claimed.to_string(); } @@ -52,49 +53,9 @@ pub fn refine_content_type(buf: &[u8], filename: &str, claimed: &str) -> String claimed.to_string() } -/// Async helper: reads the first bytes of a file on disk and refines the MIME type. -/// -/// Designed for the upload path where the file has been spooled to a temp path. -pub async fn refine_content_type_from_file( - temp_path: &Path, - filename: &str, - claimed: &str, -) -> String { - // Fast path: if the client gave us a specific type, trust it - if !claimed.is_empty() - && claimed != "application/octet-stream" - && claimed != "binary/octet-stream" - { - return claimed.to_string(); - } - - // Read only the first bytes needed for magic detection (not the whole file). - match tokio::fs::File::open(temp_path).await { - Ok(mut file) => { - let mut buf = vec![0u8; MAGIC_BYTES_LEN]; - let n = file.read(&mut buf).await.unwrap_or(0); - refine_content_type(&buf[..n], filename, claimed) - } - Err(e) => { - tracing::warn!( - "MIME detection: failed to read {} for magic bytes: {}", - temp_path.display(), - e - ); - // Fall back to extension - let guess = mime_guess::from_path(filename); - if let Some(mime) = guess.first() { - return mime.to_string(); - } - claimed.to_string() - } - } -} - #[cfg(test)] mod tests { use super::*; - use std::io::Write; // ── refine_content_type (sync) ────────────────────────────── @@ -145,47 +106,15 @@ mod tests { assert_eq!(result, "image/png"); } - // ── refine_content_type_from_file (async) ─────────────────── + // ── is_generic_mime ───────────────────────────────────────── - #[tokio::test] - async fn from_file_detects_png() { - let mut tmp = tempfile::NamedTempFile::new().unwrap(); - let png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"; - tmp.write_all(png).unwrap(); - tmp.flush().unwrap(); - - let result = - refine_content_type_from_file(tmp.path(), "photo", "application/octet-stream").await; - assert_eq!(result, "image/png"); - } - - #[tokio::test] - async fn from_file_falls_back_to_extension() { - let mut tmp = tempfile::NamedTempFile::new().unwrap(); - tmp.write_all(b"not magic").unwrap(); - tmp.flush().unwrap(); - - let result = - refine_content_type_from_file(tmp.path(), "doc.css", "application/octet-stream").await; - assert_eq!(result, "text/css"); - } - - #[tokio::test] - async fn from_file_trusts_specific_claimed() { - let result = - refine_content_type_from_file(Path::new("/nonexistent"), "file", "image/webp").await; - assert_eq!(result, "image/webp"); - } - - #[tokio::test] - async fn from_file_missing_file_falls_back_to_extension() { - let result = refine_content_type_from_file( - Path::new("/nonexistent/file"), - "photo.jpg", - "application/octet-stream", - ) - .await; - assert_eq!(result, "image/jpeg"); + #[test] + fn generic_mime_detection() { + assert!(is_generic_mime("")); + assert!(is_generic_mime("application/octet-stream")); + assert!(is_generic_mime("binary/octet-stream")); + assert!(!is_generic_mime("image/png")); + assert!(!is_generic_mime("text/plain")); } // ── filename_from_path ────────────────────────────────────── diff --git a/src/common/mod.rs b/src/common/mod.rs index ddb9e2c6..b343f797 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -4,4 +4,3 @@ pub mod errors; pub mod locale; pub mod mime_detect; pub mod stubs; -pub mod temp; diff --git a/src/common/stubs.rs b/src/common/stubs.rs index c2c3957e..e857085a 100644 --- a/src/common/stubs.rs +++ b/src/common/stubs.rs @@ -24,6 +24,7 @@ use crate::application::dtos::search_dto::{ }; use crate::application::ports::file_ports::{ FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, OptimizedFileContent, + StoredBlob, }; use crate::application::ports::folder_ports::FolderUseCase; @@ -149,14 +150,13 @@ impl FileReadPort for StubFileReadPort { pub struct StubFileWritePort; impl FileWritePort for StubFileWritePort { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> Result { Ok(File::default()) } @@ -185,13 +185,11 @@ impl FileWritePort for StubFileWritePort { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result<(String, i64), DomainError> { Ok((String::new(), 0)) @@ -478,40 +476,7 @@ impl FileUploadUseCase for StubFileUploadUseCase { _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, - _size: u64, - _pre_computed_hash: Option, - ) -> Result { - Ok(FileDto::default()) - } - - async fn upload_file_from_path( - &self, - _name: String, - _folder_id: Option, - _content_type: String, - _file_path: &Path, - _pre_computed_hash: Option, - ) -> Result { - Ok(FileDto::default()) - } - - async fn create_file( - &self, - _parent_path: &str, - _filename: &str, - _content: &[u8], - _content_type: &str, - ) -> Result { - Ok(FileDto::default()) - } - - async fn update_file( - &self, - _path: &str, - _content: &[u8], - _content_type: &str, - _modified_at: Option, + _blob: StoredBlob, ) -> Result { Ok(FileDto::default()) } @@ -519,10 +484,8 @@ impl FileUploadUseCase for StubFileUploadUseCase { async fn update_file_streaming( &self, _path: &str, - _temp_path: &Path, - _size: u64, + _blob: StoredBlob, _content_type: &str, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result { Ok(FileDto::default()) @@ -756,25 +719,11 @@ impl SearchUseCase for StubSearchUseCase { // DedupPort // --------------------------------------------------------------------------- -use crate::application::ports::dedup_ports::{ - BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, -}; +use crate::application::ports::dedup_ports::{BlobMetadataDto, DedupPort, DedupStatsDto}; pub struct StubDedupPort; impl DedupPort for StubDedupPort { - async fn store_from_file( - &self, - _source_path: &Path, - _content_type: Option, - _pre_computed_hash: Option, - ) -> Result { - Err(DomainError::internal_error( - "DedupService", - "DedupService not initialized", - )) - } - async fn blob_exists(&self, _hash: &str) -> bool { false } diff --git a/src/common/temp.rs b/src/common/temp.rs deleted file mode 100644 index b83729df..00000000 --- a/src/common/temp.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Shared helper for creating upload spool temp files. -//! -//! Upload paths spool the request body to a temp file before deduplication. -//! By default `tempfile` uses the OS temp dir (`std::env::temp_dir()`, i.e. -//! `$TMPDIR` / `/tmp`), which in many container setups is **tmpfs (RAM)**. -//! Writing a multi-hundred-MB upload there fills page-cache that counts -//! against the cgroup memory limit and can OOMKill the process. Pointing the -//! spool at a real-disk directory (`OXICLOUD_UPLOAD_TMPDIR`) keeps the upload -//! footprint proportional to the streaming buffer, not the file size. - -use std::path::Path; -use tempfile::NamedTempFile; - -/// Create a [`NamedTempFile`], honoring an optional configured spool directory. -/// -/// When `dir` is `Some`, the temp file is created there (the directory is -/// created if missing); otherwise the OS default temp dir is used. -pub fn new_spool_temp_file(dir: Option<&Path>) -> std::io::Result { - match dir { - Some(d) => { - std::fs::create_dir_all(d)?; - NamedTempFile::new_in(d) - } - None => NamedTempFile::new(), - } -} diff --git a/src/domain/repositories/file_repository.rs b/src/domain/repositories/file_repository.rs index 4fba9aae..d9878f2e 100644 --- a/src/domain/repositories/file_repository.rs +++ b/src/domain/repositories/file_repository.rs @@ -71,15 +71,15 @@ pub trait FileWriteRepository: Send + Sync + 'static { content: Vec, ) -> Result; - /// Streaming upload — saves a file from a temp file already on disk. - async fn save_file_from_temp( + /// Registers a file row pointing at a blob already stored in the + /// content-addressable chunk store (one blob reference is consumed). + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result; /// Moves a file to another folder. diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs index 185509bf..c83208fe 100644 --- a/src/infrastructure/repositories/pg/file_blob_write_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -220,26 +220,30 @@ impl FileBlobWriteRepository { Ok((new_hash.to_string(), updated_at)) } - /// Like [`FileWritePort::save_file_from_temp`] but also returns whether the - /// blob was genuinely new (`true`) or a dedup hit (`false`). - /// Used by [`FileUploadService`] to pass `is_new_blob` to lifecycle hooks. - pub async fn save_file_from_temp_with_dedup( + /// Register a file row pointing at a blob already stored in the chunk + /// store (the upload-ingest layer streamed the content in). Consumes the + /// caller's blob reference: any failure releases it before returning. + async fn save_file_with_blob_impl( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, - ) -> Result<(File, bool), DomainError> { - let user_id = self.resolve_user_id(folder_id.as_deref()).await?; - - let dedup_result = self - .dedup - .store_from_file(temp_path, Some(content_type.clone()), pre_computed_hash) - .await?; - let is_new_blob = !dedup_result.was_deduplicated(); - let blob_hash = dedup_result.hash().to_string(); + ) -> Result { + let user_id = match self.resolve_user_id(folder_id.as_deref()).await { + Ok(user_id) => user_id, + Err(e) => { + if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await { + tracing::error!( + "Blob orphaned after owner resolution failure — hash: {}, err: {}", + &blob_hash[..12], + rollback_err + ); + } + return Err(e); + } + }; // Deadlock victims (40P01) retry before the compensation below runs — // a successful retry must keep the blob reference alive. The final @@ -259,7 +263,7 @@ impl FileBlobWriteRepository { .bind(&name) .bind(&folder_id) .bind(user_id) - .bind(&blob_hash) + .bind(blob_hash) .bind(size as i64) .bind(&content_type) .bind(category_order_for(&name, &content_type)) @@ -269,7 +273,7 @@ impl FileBlobWriteRepository { { Ok(row) => row, Err(e) => { - if let Err(rollback_err) = self.dedup.remove_reference(&blob_hash).await { + if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await { tracing::error!( "Blob orphaned after failed INSERT — hash: {}, err: {}", &blob_hash[..12], @@ -309,32 +313,23 @@ impl FileBlobWriteRepository { row.1, row.2, Some(user_id), - blob_hash, + blob_hash.to_string(), )?; - Ok((file, is_new_blob)) + Ok(file) } } impl FileWritePort for FileBlobWriteRepository { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result { - self.save_file_from_temp_with_dedup( - name, - folder_id, - content_type, - temp_path, - size, - pre_computed_hash, - ) - .await - .map(|(file, _)| file) + self.save_file_with_blob_impl(name, folder_id, content_type, blob_hash, size) + .await } async fn move_file( @@ -532,24 +527,18 @@ impl FileWritePort for FileBlobWriteRepository { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, file_id: &str, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - content_type: Option, - pre_computed_hash: Option, modified_at: Option, ) -> Result<(String, i64), DomainError> { - // Streaming: pass pre-computed hash so dedup skips re-reading the file. - let dedup_result = self - .dedup - .store_from_file(temp_path, content_type, pre_computed_hash) - .await?; - let new_hash = dedup_result.hash().to_string(); - + // The content was already ingested into the chunk store by the + // upload-ingest layer; swap_blob_hash consumes its reference and + // releases it on failure. let swapped = self - .swap_blob_hash(file_id, &new_hash, size as i64, modified_at) + .swap_blob_hash(file_id, blob_hash, size as i64, modified_at) .await?; // The file now maps to a different blob — drop the read-side cache // entry so streaming downloads cannot serve the previous content diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index 39aaa7d8..ffc9b8e5 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -6,14 +6,14 @@ //! (updated atomically on each chunk) are stored alongside the chunk files. //! On boot the service scans `temp_base_dir` and recovers any active sessions. //! - Parallel chunk transfers (up to 6 concurrent) -//! - Automatic reassembly with hash-on-write (BLAKE3) +//! - Completion streams the ordered parts straight into the CDC blob store //! - Expiration cleanup (24 h) //! //! Protocol: //! 1. POST /api/uploads → Create upload session, get upload_id //! 2. PATCH /api/uploads/:id → Upload chunks (parallel OK) //! 3. HEAD /api/uploads/:id → Check progress -//! 4. POST /api/uploads/:id/complete → Finalize and assemble +//! 4. POST /api/uploads/:id/complete → Stream parts into the blob store use chrono::{DateTime, Utc}; use dashmap::DashMap; @@ -28,7 +28,8 @@ use tokio::io::AsyncWriteExt; use uuid::Uuid; use crate::application::ports::chunked_upload_ports::{ - ChunkUploadResponseDto, ChunkedUploadPort, CreateUploadResponseDto, UploadStatusResponseDto, + ChunkUploadResponseDto, ChunkedUploadPort, CompletedUploadParts, CreateUploadResponseDto, + UploadStatusResponseDto, }; use crate::domain::errors::{DomainError, ErrorKind}; @@ -603,7 +604,7 @@ impl ChunkedUploadService { /// /// Used by the streaming REST PUT path: the handler calls /// `prepare_chunk` → streams body to disk via - /// `interfaces::upload_spool::stream_body_to_path` → calls + /// `interfaces::upload_ingest::stream_body_to_path` → calls /// `commit_chunk` to finalise. This lets the body bypass the /// in-memory `Bytes` allocation entirely (peak heap ~one HTTP /// frame instead of "chunk size"). @@ -936,24 +937,23 @@ impl ChunkedUploadService { }) } - /// Assemble chunks into final file and return the path + pre-computed BLAKE3 hash. + /// Validate completion and return the chunk parts in assembly order. /// - /// **Hash-on-Write**: BLAKE3 is computed while copying chunks into the - /// assembled file, eliminating the second sequential read that dedup_service - /// would otherwise need. - /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. + /// No assembled file is written and nothing is hashed here — the caller + /// streams the parts straight into the CDC chunk store, which computes + /// BLAKE3 and dedup-checks each CDC chunk in that single read pass. The + /// part files stay on disk until `finalize_upload`, so a completion + /// that fails downstream (e.g. client checksum mismatch) is retryable. async fn complete_upload_inner( &self, upload_id: &str, user_id: &str, - ) -> Result<(PathBuf, String, Option, String, u64, String), String> { - // Verify ownership before assembly + ) -> Result { + // Verify ownership before completion self.verify_session_owner(upload_id, user_id)?; // Get session and validate completion. - // Clone the session data and drop the DashMap ref immediately - // so the shard is not held during the expensive assembly step. + // Clone the session data and drop the DashMap ref immediately. let session = { let entry = self .sessions @@ -971,118 +971,29 @@ impl ChunkedUploadService { entry.clone() }; - // Assemble file with hash-on-write. - // - // The entire loop is offloaded to spawn_blocking because BLAKE3 - // hashing is CPU-bound and would otherwise block a Tokio worker, - // starving all other connections. - // Synchronous I/O is used inside the blocking thread — it avoids - // the async reactor overhead and is actually faster for this - // sequential workload. - let assembled_path = session.temp_dir.join("assembled"); - let chunks_meta: Vec<(usize, PathBuf)> = session - .chunks + // Chunk indices are unique (upload_chunk rejects duplicates) but may + // have arrived out of order — sort to recover the assembly order. + let mut indices: Vec = session.chunks.iter().map(|c| c.index).collect(); + indices.sort_unstable(); + let chunk_paths: Vec = indices .iter() - .map(|c| { - ( - c.index, - session.temp_dir.join(format!("chunk_{:06}", c.index)), - ) - }) + .map(|index| session.temp_dir.join(format!("chunk_{:06}", index))) .collect(); - let total_size = session.total_size; - - let hash = tokio::task::spawn_blocking(move || -> Result { - use std::io::{BufWriter as StdBufWriter, Read, Write}; - - let raw_output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&assembled_path) - .map_err(|e| format!("Failed to create assembled file: {e}"))?; - - // Pre-allocate assembled file to reduce fragmentation - let _ = raw_output.set_len(total_size); - - // 512 KB I/O buffers — 8× fewer syscalls than 64 KB - let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = blake3::Hasher::new(); - - // For files >10 MB, use multithreaded BLAKE3 hashing (all cores) - const RAYON_THRESHOLD: u64 = 10 * 1024 * 1024; - let use_rayon = total_size > RAYON_THRESHOLD; - - // Single 512 KB read buffer reused across all chunks (avoids N allocations) - let mut buf = vec![0u8; 524_288]; - for (index, chunk_path) in &chunks_meta { - let mut chunk_file = std::fs::File::open(chunk_path) - .map_err(|e| format!("Failed to open chunk {index}: {e}"))?; - loop { - let n = chunk_file - .read(&mut buf) - .map_err(|e| format!("Failed to read chunk {index}: {e}"))?; - if n == 0 { - break; - } - if use_rayon { - hasher.update_rayon(&buf[..n]); - } else { - hasher.update(&buf[..n]); - } - output.write_all(&buf[..n]).map_err(|e| { - format!("Failed to write chunk {index} to assembled file: {e}") - })?; - } - } - - output - .flush() - .map_err(|e| format!("Failed to flush assembled file: {e}"))?; - // ── Durability boundary ──────────────────────────────────── - // `flush` drains BufWriter's userspace buffer but leaves the - // bytes in the kernel page cache. Without `sync_all`, a - // power loss between this `complete_upload` returning 2xx - // and the OS writeback timer firing (~5 s default) loses - // the merged blob — and PG's metadata row references a hash - // that no longer exists on disk. Reclaim the BufWriter's - // inner File via `into_inner` so we can `sync_all` it; the - // BufWriter would otherwise drop without flushing on the - // inner handle. - let raw_output = output - .into_inner() - .map_err(|e| format!("into_inner on BufWriter failed: {e}"))?; - raw_output - .sync_all() - .map_err(|e| format!("Failed to fsync assembled file: {e}"))?; - - // Clean up chunk files (keep assembled) — already on a blocking thread - for (_index, chunk_path) in &chunks_meta { - let _ = std::fs::remove_file(chunk_path); - } - - Ok(hasher.finalize().to_hex().to_string()) - }) - .await - .map_err(|e| format!("Assembly task panicked: {e}"))??; - - let assembled_path = session.temp_dir.join("assembled"); tracing::info!( - "✅ Assembled chunked upload: {} ({} bytes from {} chunks)", + "✅ Chunked upload complete: {} ({} bytes in {} chunks, streamed to CAS)", session.filename, session.total_size, - session.chunks.len() + chunk_paths.len() ); - Ok(( - assembled_path, - session.filename.clone(), - session.folder_id.clone(), - session.content_type.clone(), - session.total_size, - hash, - )) + Ok(CompletedUploadParts { + chunk_paths, + filename: session.filename.clone(), + folder_id: session.folder_id.clone(), + content_type: session.content_type.clone(), + total_size: session.total_size, + }) } /// Finalize upload: remove session from RAM, then clean disk OUTSIDE lock. @@ -1187,7 +1098,7 @@ impl ChunkedUploadPort for ChunkedUploadService { &self, upload_id: &str, user_id: Uuid, - ) -> Result<(PathBuf, String, Option, String, u64, String), DomainError> { + ) -> Result { self.complete_upload_inner(upload_id, &user_id.to_string()) .await .map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e)) @@ -1458,18 +1369,21 @@ mod tests { assert_eq!(status.completed_chunks, 2); assert!(status.pending_chunks.is_empty()); - // 4. Complete (assemble) - let (path, filename, _folder, _ct, size, hash) = service + // 4. Complete — returns the ordered chunk parts, no assembled file + let parts = service .complete_upload_inner(&id, "test-user") .await .expect("complete"); - assert_eq!(filename, "test.txt"); - assert_eq!(size, 1024); - assert!(!hash.is_empty()); - assert!(path.exists()); + assert_eq!(parts.filename, "test.txt"); + assert_eq!(parts.total_size, 1024); + assert_eq!(parts.chunk_paths.len(), 2); - // 5. Verify assembled content - let content = fs::read(&path).await.expect("read assembled"); + // 5. Verify the parts concatenate to the original content in order + let mut content = Vec::new(); + for path in &parts.chunk_paths { + assert!(path.exists(), "chunk part must remain until finalize"); + content.extend_from_slice(&fs::read(path).await.expect("read part")); + } assert_eq!(&content[..512], &[b'A'; 512]); assert_eq!(&content[512..], &[b'B'; 512]); diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 2d06f6ec..8f7ff957 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -18,33 +18,41 @@ //! blobs in `storage.blobs`) are served transparently — when no manifest //! row exists for a hash, the service falls back to direct blob reads. //! -//! **Write-first strategy** (store_from_file): -//! 1. CDC-analyse the file (mmap → FastCDC boundaries + per-chunk BLAKE3). -//! 2. Batch-check which chunk hashes already exist in PG (dedup skip). -//! 3. Bump ref_count for existing chunks (no disk I/O). -//! 4. Read + write only *new* chunks to the blob backend (idempotent, -//! no per-chunk fsync). -//! 5. One batched fsync sweep makes the new chunks durable, then ONE -//! batched INSERT registers them — durability before visibility. -//! 6. Single manifest INSERT (~few ms total). -//! 7. PG connection is never held during disk I/O. +//! **Single-pass streaming ingest** (store_from_stream): +//! 1. FastCDC boundaries, per-chunk BLAKE3 and the whole-file BLAKE3 are +//! all computed WHILE the bytes arrive — no spool file, no mmap +//! re-read. Peak RAM stays bounded (current chunk + one small batch). +//! 2. Per batch of distinct chunks, ONE `UPDATE … RETURNING` bumps +//! ref_count on already-known chunks (pinning them against concurrent +//! reclaim for the rest of the upload) and atomically classifies the +//! rest as new — no check-then-bump TOCTOU window. +//! 3. Only *new* chunks are written to the blob backend (unsynced, +//! bounded concurrency). Bytes the store already knows never touch +//! disk — a full dedup hit performs zero content writes. +//! 4. At end of stream ONE batched fsync sweep makes the new chunks +//! durable, then ONE batched INSERT registers them — durability +//! before visibility. +//! 5. Single manifest INSERT (~few ms). An identical concurrent upload +//! is resolved via ON CONFLICT: the loser releases its chunk +//! references and turns into a dedup hit. +//! 6. PG connections are never held during disk I/O. //! //! Benefits: +//! - Each uploaded byte hits the disk at most ONCE (dedup hits: zero) //! - Sub-file dedup: edited files share unchanged chunks //! - ACID durability — crash-safe, zero orphaned index entries -//! - PG connections never blocked by disk I/O (write-first) //! - 60-80% storage reduction for versioned / edited files -//! - Faster uploads when chunks already exist use bytes::Bytes; use futures::stream::{self, StreamExt}; use futures::{Stream, TryStreamExt}; use sqlx::PgPool; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use tokio::fs; +use tokio_util::io::StreamReader; use crate::application::ports::blob_lifecycle::BlobLifecycleHook; use crate::application::ports::blob_storage_ports::BlobStorageBackend; @@ -65,11 +73,163 @@ const CDC_MAX_CHUNK: usize = 1_048_576; // ── CDC helper types ───────────────────────────────────────────────────────── -/// Metadata for a single CDC chunk (offset + length + BLAKE3 hash). -struct ChunkMeta { - hash: String, - offset: usize, - length: usize, +/// Everything a streaming chunk ingest learned about its byte stream. +/// +/// Produced by [`DedupService::ingest_chunks_from_stream`]. On success the +/// ingest session holds exactly ONE `storage.blobs.ref_count` reference per +/// *distinct* chunk hash; the caller must either attach those references to +/// a manifest or hand them back via `release_chunk_refs`. +struct ChunkIngestOutcome { + /// BLAKE3 of the complete byte stream (the future manifest key). + file_hash: String, + /// Total bytes consumed from the stream. + total_size: u64, + /// Per-occurrence chunk hashes, in file order (the manifest layout). + chunk_hashes: Vec, + /// Per-occurrence chunk sizes, in file order. + chunk_sizes: Vec, + /// How many distinct chunks were actually written to the backend. + newly_written: usize, +} + +impl ChunkIngestOutcome { + /// Distinct chunk hashes — the set this ingest holds one reference on each. + fn distinct_hashes(&self) -> Vec { + let mut seen = HashSet::new(); + self.chunk_hashes + .iter() + .filter(|h| seen.insert(h.as_str())) + .cloned() + .collect() + } +} + +/// Compensation guard for an in-flight ingest session. +/// +/// Tracks the two side effects a session accumulates before its chunks are +/// fully registered: ref_count pins taken on pre-existing chunks and freshly +/// written (still unregistered) chunk files. If the session future is dropped +/// mid-stream — a client disconnect aborts the whole handler future — the +/// guard spawns a rollback so pinned chunks don't leak references forever and +/// written files become GC-collectible rows instead of invisible orphans. +struct IngestGuard { + pool: Arc, + backend: Arc, + /// Pre-existing chunks whose ref_count this session bumped (distinct). + pinned: Vec, + /// Chunks written to the backend but not yet registered: (hash, size). + written: Vec<(String, i64)>, + armed: bool, +} + +impl IngestGuard { + fn new(pool: Arc, backend: Arc) -> Self { + Self { + pool, + backend, + pinned: Vec::new(), + written: Vec::new(), + armed: true, + } + } + + /// The session's chunks are fully registered — references now belong to + /// the caller, nothing to compensate. + fn disarm(mut self) { + self.armed = false; + } + + /// Deterministic rollback for handled errors (awaited inline, unlike the + /// spawned Drop path). + async fn rollback(mut self) { + self.armed = false; + let pinned = std::mem::take(&mut self.pinned); + let written = std::mem::take(&mut self.written); + Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await; + } + + /// Release pins and surface written-but-unregistered chunk files to GC. + /// + /// Best-effort: every step logs instead of failing — the worst outcome of + /// a failed rollback is a bounded ref_count over-count (storage leak), + /// never data loss. + async fn run_rollback( + pool: Arc, + backend: Arc, + pinned: Vec, + written: Vec<(String, i64)>, + ) { + if !pinned.is_empty() + && let Err(e) = sqlx::query( + "UPDATE storage.blobs SET ref_count = GREATEST(ref_count - 1, 0) + WHERE hash = ANY($1)", + ) + .bind(&pinned) + .execute(pool.as_ref()) + .await + { + tracing::warn!( + "Ingest rollback: failed to release {} chunk pins: {e}", + pinned.len() + ); + } + + if written.is_empty() { + return; + } + // Durability first, then visibility at ref_count 0 so the existing GC + // sweep can reclaim the bytes — a backend file with no PG row would be + // invisible to it. ON CONFLICT DO NOTHING keeps a concurrent + // uploader's row (and its references) intact. + let hashes: Vec = written.iter().map(|(h, _)| h.clone()).collect(); + let sizes: Vec = written.iter().map(|(_, s)| *s).collect(); + if let Err(e) = backend.sync_blobs(&hashes).await { + tracing::warn!( + "Ingest rollback: sync of {} chunks failed: {e}", + hashes.len() + ); + } + if let Err(e) = sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) + SELECT h, s, 0 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) + ON CONFLICT (hash) DO NOTHING", + ) + .bind(&hashes) + .bind(&sizes) + .execute(pool.as_ref()) + .await + { + tracing::warn!( + "Ingest rollback: failed to register {} orphan chunks for GC: {e}", + hashes.len() + ); + } + } +} + +impl Drop for IngestGuard { + fn drop(&mut self) { + if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) { + return; + } + let pinned = std::mem::take(&mut self.pinned); + let written = std::mem::take(&mut self.written); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + let pool = self.pool.clone(); + let backend = self.backend.clone(); + handle.spawn(async move { + Self::run_rollback(pool, backend, pinned, written).await; + }); + } + Err(_) => tracing::warn!( + "Ingest guard dropped outside a runtime: {} pins / {} written chunks \ + stay leaked until the next GC sweep", + pinned.len(), + written.len() + ), + } + } } /// Content-Addressable Storage Service with CDC (PostgreSQL-backed) @@ -82,7 +242,7 @@ pub struct DedupService { /// Pluggable blob storage backend (local FS, S3, …). backend: Arc, /// PostgreSQL connection pool (dedup index in `storage.blobs`) — primary, - /// used by request-path operations (store_from_file, etc.). + /// used by request-path operations (store_from_stream, etc.). pool: Arc, /// Isolated maintenance pool for long-running operations /// (verify_integrity, garbage_collect) that must never starve the primary. @@ -192,86 +352,12 @@ impl DedupService { .unwrap_or_else(|| PathBuf::from(format!("remote://{}", hash))) } - // ── CDC analysis ─────────────────────────────────────────── - - /// Single-pass CDC: compute whole-file BLAKE3 hash + chunk boundaries + per-chunk hashes. - /// - /// Memory-maps the file and runs FastCDC boundary detection - /// concurrently with BLAKE3 hashing — all in one pass. - async fn cdc_hash_and_chunk_file(path: &Path) -> std::io::Result<(String, Vec)> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path)?; - let file_size = file.metadata()?.len(); - - if file_size == 0 { - return Ok((blake3::hash(b"").to_hex().to_string(), vec![])); - } - - // SAFETY: file is opened read-only; no concurrent writers expected - // (source is a temp upload file owned exclusively by this request). - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let chunker = - fastcdc::v2020::FastCDC::new(&mmap, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK); - - let mut file_hasher = blake3::Hasher::new(); - let mut chunks = Vec::new(); - - for chunk in chunker { - let data = &mmap[chunk.offset..chunk.offset + chunk.length]; - file_hasher.update(data); - chunks.push(ChunkMeta { - hash: blake3::hash(data).to_hex().to_string(), - offset: chunk.offset, - length: chunk.length, - }); - } - - Ok((file_hasher.finalize().to_hex().to_string(), chunks)) - }) - .await - .expect("cdc_hash_and_chunk_file: spawn_blocking panicked") - } - - /// CDC analysis without file-hash computation (when hash is pre-computed). - async fn cdc_chunk_file(path: &Path) -> std::io::Result> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path)?; - let file_size = file.metadata()?.len(); - - if file_size == 0 { - return Ok(vec![]); - } - - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let chunker = - fastcdc::v2020::FastCDC::new(&mmap, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK); - - let chunks: Vec = chunker - .map(|chunk| { - let data = &mmap[chunk.offset..chunk.offset + chunk.length]; - ChunkMeta { - hash: blake3::hash(data).to_hex().to_string(), - offset: chunk.offset, - length: chunk.length, - } - }) - .collect(); - - Ok(chunks) - }) - .await - .expect("cdc_chunk_file: spawn_blocking panicked") - } - // ── Hash helpers ───────────────────────────────────────────── /// Calculate BLAKE3 hash of a file (~5× faster than SHA-256). /// - /// Uses memory-mapped I/O with rayon parallelism. Kept for callers - /// that only need the hash (e.g. upload handlers pre-computing the hash - /// before calling `store_from_file`). + /// Uses memory-mapped I/O with rayon parallelism. Used by + /// `verify_integrity` to re-hash local blob files. pub async fn hash_file(path: &Path) -> std::io::Result { let path = path.to_path_buf(); tokio::task::spawn_blocking(move || { @@ -283,329 +369,238 @@ impl DedupService { .expect("hash_file: spawn_blocking task panicked") } - // ── Core store operations ──────────────────────────────────── - - /// Store content with CDC deduplication (from file). - /// - /// **Fast path**: if `pre_computed_hash` is `Some`, the manifest / - /// legacy-blob index is checked *before* running CDC — returning - /// instantly on a full-file dedup hit. - /// - /// **New-file path**: CDC-analyses the file (single mmap pass), - /// stores unique chunks via the blob backend, then inserts the - /// manifest in PostgreSQL. - pub async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result { - // ── Fast path: pre-computed hash → check before CDC ────── - if let Some(ref hash) = pre_computed_hash - && let Some(result) = self.try_dedup_hit(hash, source_path).await? - { - return Ok(result); - } - - // ── CDC analysis ───────────────────────────────────────── - let (file_hash, chunks) = if let Some(hash) = pre_computed_hash { - let chunks = Self::cdc_chunk_file(source_path) - .await - .map_err(DomainError::from)?; - (hash, chunks) - } else { - let (hash, chunks) = Self::cdc_hash_and_chunk_file(source_path) - .await - .map_err(DomainError::from)?; - // Check dedup with newly computed hash - if let Some(result) = self.try_dedup_hit(&hash, source_path).await? { - return Ok(result); - } - (hash, chunks) - }; - - let file_size = fs::metadata(source_path) - .await - .map_err(DomainError::from)? - .len(); - - // ── Store chunks (write-first — no PG connection held) ─── - let (chunk_hashes, chunk_sizes) = self.store_chunks(source_path, &chunks).await?; - - // ── Insert manifest ────────────────────────────────────── - sqlx::query( - "INSERT INTO storage.chunk_manifests - (file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, content_type, ref_count) - VALUES ($1, $2, $3, $4, $5, $6, 1)", - ) - .bind(&file_hash) - .bind(&chunk_hashes) - .bind(chunk_sizes.iter().map(|s| *s as i64).collect::>()) - .bind(file_size as i64) - .bind(chunk_hashes.len() as i32) - .bind(&content_type) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to insert manifest: {}", e)) - })?; - - // ── Clean up source file ───────────────────────────────── - let _ = fs::remove_file(source_path).await; - - tracing::info!( - "NEW BLOB (CDC): {} ({} bytes, {} chunks)", - &file_hash[..12], - file_size, - chunk_hashes.len() - ); - - self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()); - - Ok(DedupResultDto::NewBlob { - hash: file_hash, - size: file_size, - }) - } - - /// Check manifest or legacy blob for a dedup hit. - /// - /// Returns `Some(ExistingBlob)` if the exact file was already stored. - /// Bumps the appropriate ref_count and removes the source file. - async fn try_dedup_hit( - &self, - hash: &str, - source_path: &Path, - ) -> Result, DomainError> { - // ── CDC manifest hit ───────────────────────────────────── - let manifest = sqlx::query_as::<_, (i64,)>( - "SELECT total_size FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to check manifest: {}", e)) - })?; - - if let Some((total_size,)) = manifest { - sqlx::query( - "UPDATE storage.chunk_manifests SET ref_count = ref_count + 1 WHERE file_hash = $1", - ) - .bind(hash) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to bump manifest ref_count: {}", e), - ) - })?; - - let _ = fs::remove_file(source_path).await; - - tracing::info!( - "DEDUP HIT (manifest): {} ({} bytes saved)", - &hash[..12], - total_size - ); - return Ok(Some(DedupResultDto::ExistingBlob { - hash: hash.to_owned(), - size: total_size as u64, - saved_bytes: total_size as u64, - })); - } - - // ── Legacy whole-file blob hit ─────────────────────────── - let legacy = sqlx::query_as::<_, (i64,)>("SELECT size FROM storage.blobs WHERE hash = $1") - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to check legacy blob: {}", e)) - })?; - - if let Some((size,)) = legacy { - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1") - .bind(hash) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to bump legacy ref_count: {}", e), - ) - })?; - - let _ = fs::remove_file(source_path).await; - tracing::info!( - "DEDUP HIT (legacy blob): {} ({} bytes saved)", - &hash[..12], - size - ); - return Ok(Some(DedupResultDto::ExistingBlob { - hash: hash.to_owned(), - size: size as u64, - saved_bytes: size as u64, - })); - } - - Ok(None) - } + // ── Core store operations (streaming CDC) ─────────────────── /// Maximum concurrent chunk uploads to the blob backend. const CHUNK_UPLOAD_CONCURRENCY: usize = 8; + /// Flush the pending distinct-chunk batch after this many chunks… + const FLUSH_MAX_CHUNKS: usize = 32; + /// …or after this many buffered bytes, whichever comes first. Together + /// with the ≤ 1 MiB chunk in flight this bounds peak RAM per upload to + /// ~9 MiB regardless of file size. + const FLUSH_MAX_BYTES: usize = 8 * 1024 * 1024; - /// Store CDC chunks via the blob backend + upsert in PG. + /// Store content with CDC deduplication, straight from a byte stream — + /// the single write path for every upload surface (REST multipart, + /// WebDAV PUT, NextCloud PUT, chunked-upload assembly, WOPI PutFile). /// - /// Phase 0: Batch-queries PG to discover which chunk hashes already - /// exist in `storage.blobs`. - /// Phase 1: Bumps `ref_count` for chunks that already exist (one - /// batched UPDATE, no disk I/O — the biggest saving for versioned - /// files where most chunks are unchanged). - /// Phase 2: Reads + writes only *new* chunks, with up to - /// [`CHUNK_UPLOAD_CONCURRENCY`] writes in flight and **no per-chunk - /// fsync**. - /// Phase 3: One batched `sync_blobs` sweep makes every new chunk - /// durable (no-op for remote backends, which are durable on PUT). - /// Phase 4: ONE batched INSERT registers the new chunks in PG. The - /// sweep runs first so a crash can never leave a `storage.blobs` row - /// pointing at bytes that were still in the page cache. + /// One pass over the incoming bytes: FastCDC boundary detection, + /// per-chunk BLAKE3, the whole-file BLAKE3, dedup lookups and blob + /// writes all happen while the stream is still arriving. There is no + /// spool file and no re-read — each uploaded byte touches the disk at + /// most once, and not at all when the store already has its chunk. /// - /// `ref_count` is incremented once per *distinct* chunk (one reference per - /// manifest), staying symmetric with `remove_manifest_reference` so a file - /// that repeats a chunk cannot over-count and leak the blob forever. - async fn store_chunks( + /// Identical-content races (two clients uploading the same file + /// concurrently) are resolved at the manifest INSERT via ON CONFLICT: + /// the loser releases its chunk references and returns `ExistingBlob`. + pub async fn store_from_stream( &self, - source_path: &Path, - chunks: &[ChunkMeta], - ) -> Result<(Vec, Vec), DomainError> { - let pool = &self.pool; - let backend = &self.backend; + source: S, + content_type: Option, + ) -> Result + where + S: Stream> + Send, + { + let outcome = self.ingest_chunks_from_stream(source).await?; + let distinct = outcome.distinct_hashes(); + let total_size = outcome.total_size; + let file_hash = outcome.file_hash.clone(); - // ── Phase 0: de-duplicate chunk hashes, then batch-check existence ── - // A single file can legitimately repeat the same chunk many times - // (zero-filled regions in disk/VM images, repeated document structures, - // concatenated archives). `ref_count` is tracked per *distinct* chunk — - // one reference per manifest — to stay symmetric with - // `remove_manifest_reference`, which decrements via - // `WHERE hash = ANY(chunk_hashes)` (matching each row once). Counting - // per-occurrence here would over-increment and leak the blob forever. - // Keep the first occurrence of each hash so new chunks know where to - // read their bytes. - let mut seen = std::collections::HashSet::new(); - let unique_chunks: Vec<&ChunkMeta> = chunks - .iter() - .filter(|c| seen.insert(c.hash.as_str())) - .collect(); - let unique_hashes: Vec = unique_chunks.iter().map(|c| c.hash.clone()).collect(); + // A bounded retry covers the rare interleaving where the manifest + // that beat our INSERT is deleted again before our ref bump lands. + for _ in 0..3 { + let inserted = sqlx::query( + "INSERT INTO storage.chunk_manifests + (file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, content_type, ref_count) + VALUES ($1, $2, $3, $4, $5, $6, 1) + ON CONFLICT (file_hash) DO NOTHING", + ) + .bind(&file_hash) + .bind(&outcome.chunk_hashes) + .bind( + outcome + .chunk_sizes + .iter() + .map(|s| *s as i64) + .collect::>(), + ) + .bind(total_size as i64) + .bind(outcome.chunk_hashes.len() as i32) + .bind(&content_type) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to insert manifest: {}", e)) + })? + .rows_affected(); - let existing_hashes: std::collections::HashSet = - sqlx::query_scalar::<_, String>("SELECT hash FROM storage.blobs WHERE hash = ANY($1)") - .bind(&unique_hashes) - .fetch_all(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( + if inserted > 0 { + tracing::info!( + "NEW BLOB (CDC stream): {} ({} bytes, {} chunks, {} written)", + &file_hash[..12], + total_size, + outcome.chunk_hashes.len(), + outcome.newly_written, + ); + self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()); + return Ok(DedupResultDto::NewBlob { + hash: file_hash, + size: total_size, + }); + } + + // The manifest already exists — either this exact content was + // stored before or an identical concurrent upload just won the + // race. Bump ITS ref_count first and only then hand back this + // session's chunk references; the reverse order could leave the + // caller's file row without any manifest reference behind it. + if let Some(existing_size) = self.bump_manifest_if_exists(&file_hash).await? { + self.release_chunk_refs(self.pool.as_ref(), &distinct).await; + tracing::info!( + "DEDUP HIT (manifest): {} ({} bytes saved)", + &file_hash[..12], + existing_size, + ); + return Ok(DedupResultDto::ExistingBlob { + hash: file_hash, + size: existing_size as u64, + saved_bytes: existing_size as u64, + }); + } + } + + self.release_chunk_refs(self.pool.as_ref(), &distinct).await; + Err(DomainError::internal_error( + "Dedup", + format!("Manifest insert/bump kept racing for {file_hash}"), + )) + } + + /// Bump a manifest's ref_count if it exists; returns its total_size. + /// Single statement — no window between the existence check and the bump. + async fn bump_manifest_if_exists(&self, file_hash: &str) -> Result, DomainError> { + sqlx::query_scalar::<_, i64>( + "UPDATE storage.chunk_manifests SET ref_count = ref_count + 1 + WHERE file_hash = $1 + RETURNING total_size", + ) + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to bump manifest ref_count: {e}")) + }) + } + + /// Stream → chunk store, WITHOUT creating a manifest. + /// + /// Splits the stream with FastCDC while computing per-chunk and + /// whole-stream BLAKE3 hashes, then settles each batch of distinct + /// chunks against PG: + /// + /// 1. ONE `UPDATE … RETURNING` per batch pins every already-known chunk + /// (`ref_count + 1` — protecting it from a concurrent last-reference + /// delete for the rest of the upload) and atomically classifies the + /// remaining hashes as new. No check-then-bump TOCTOU window. + /// 2. New chunks are written to the backend unsynced with bounded + /// concurrency; chunks the store already has are dropped from RAM + /// without any disk I/O. + /// 3. At end of stream, ONE `sync_blobs` sweep makes the new chunks + /// durable, then ONE batched INSERT registers them (`ON CONFLICT` + /// bumps instead — a concurrent identical upload may have registered + /// the same brand-new chunk first). Durability before visibility. + /// + /// `ref_count` is taken once per *distinct* chunk — symmetric with + /// `remove_manifest_reference`, which decrements via + /// `WHERE hash = ANY(chunk_hashes)` (each row once). A repeated chunk + /// (zero-filled regions, concatenated archives) must not over-count or + /// the blob leaks forever. + /// + /// If the returned references are not attached to a manifest, the caller + /// must hand them back via `release_chunk_refs`. If this future is + /// dropped mid-stream (client disconnect), the internal guard rolls the + /// session back in a spawned task. + async fn ingest_chunks_from_stream( + &self, + source: S, + ) -> Result + where + S: Stream> + Send, + { + let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone()); + + let reader = StreamReader::new(Box::pin(source)); + let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( + reader, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ); + let chunk_stream = chunker.as_stream(); + futures::pin_mut!(chunk_stream); + + let mut file_hasher = blake3::Hasher::new(); + let mut total_size: u64 = 0; + let mut chunk_hashes: Vec = Vec::new(); + let mut chunk_sizes: Vec = Vec::new(); + let mut session_seen: HashSet = HashSet::new(); + let mut pending: Vec<(String, Bytes)> = Vec::new(); + let mut pending_bytes: usize = 0; + + while let Some(item) = chunk_stream.next().await { + let chunk = match item { + Ok(chunk) => chunk, + Err(e) => { + guard.rollback().await; + return Err(DomainError::internal_error( "Dedup", - format!("Failed to check existing chunks: {}", e), - ) - })? - .into_iter() - .collect(); - - // ── Phase 1: bump ref_count for every existing chunk in ONE query ── - // (was one UPDATE per occurrence — now a single batched round-trip). - let existing: Vec = unique_hashes - .iter() - .filter(|h| existing_hashes.contains(*h)) - .cloned() - .collect(); - if !existing.is_empty() { - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = ANY($1)") - .bind(&existing) - .execute(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to bump ref_count: {}", e)) - })?; - } - - // ── Phase 2: upload each NEW chunk once, concurrently ────────────── - // Read each new chunk by positioned I/O immediately before its upload - // instead of materializing every chunk's *data* up front. Peak heap for - // file content stays bounded to ~CHUNK_UPLOAD_CONCURRENCY × CDC_MAX_CHUNK - // (≈ 8 MiB) — proportional to the chunk size, never the file size. - // - // Owned metadata (hash + offset + length, no file data) so the stream - // below does not borrow `chunks` across an `.await` (which would make - // this future non-`Send` and break the upload handlers). - let new_ops: Vec<(String, u64, usize)> = unique_chunks - .iter() - .filter(|c| !existing_hashes.contains(&c.hash)) - .map(|c| (c.hash.clone(), c.offset as u64, c.length)) - .collect(); - - let source = Arc::new(std::fs::File::open(source_path).map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to open source file: {}", e)) - })?); - - // Writes are *unsynced*: no per-chunk fsync. Durability comes from - // the single batched sweep below, BEFORE any PG row references the - // new chunks — so a crash can never leave storage.blobs claiming a - // chunk whose bytes didn't reach the platter. - let results: Vec> = stream::iter(new_ops) - .map(|(hash, offset, length)| { - let source = source.clone(); - let backend = backend.clone(); - async move { - // Positioned read of just this chunk (≤ CDC_MAX_CHUNK) off - // the async runtime, then upload. - let bytes = tokio::task::spawn_blocking(move || { - use std::os::unix::fs::FileExt; - let mut buf = vec![0u8; length]; - source.read_exact_at(&mut buf, offset)?; - Ok::, std::io::Error>(buf) - }) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Read task failed: {}", e)) - })? - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to read chunk: {}", e)) - })?; - - backend - .put_blob_from_bytes_unsynced(&hash, Bytes::from(bytes)) - .await?; - Ok((hash, length as i64)) + format!("Upload stream failed: {e}"), + )); } - }) - .buffer_unordered(Self::CHUNK_UPLOAD_CONCURRENCY) - .collect() - .await; + }; - let mut new_rows: Vec<(String, i64)> = Vec::with_capacity(results.len()); - for result in results { - new_rows.push(result?); + let data = chunk.data; + total_size += data.len() as u64; + // Per-chunk hashing is ≤ 1 MiB of BLAKE3 (< 1 ms) — cheaper than + // a spawn_blocking round-trip per chunk. + file_hasher.update(&data); + let hash = blake3::hash(&data).to_hex().to_string(); + chunk_sizes.push(data.len() as u64); + chunk_hashes.push(hash.clone()); + + if session_seen.insert(hash.clone()) { + pending_bytes += data.len(); + pending.push((hash, Bytes::from(data))); + if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES + { + if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + guard.rollback().await; + return Err(e); + } + pending_bytes = 0; + } + } } - if !new_rows.is_empty() { - // ── Phase 3: durability barrier — one batched fsync sweep ────── - // (was 2 fsyncs per chunk: ~8 200 for a 1 GB upload; now one - // parallel sweep over the new files + ≤256 prefix dirs). - // Remote backends are durable on PUT — sync_blobs is a no-op. - let new_hashes: Vec = new_rows.iter().map(|(h, _)| h.clone()).collect(); - backend.sync_blobs(&new_hashes).await?; + if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + guard.rollback().await; + return Err(e); + } - // ── Phase 4: register all new chunks in ONE batched INSERT ───── - // (was one round-trip per chunk). `new_rows` is built from - // `unique_chunks`, so no hash repeats within the batch — safe for - // ON CONFLICT DO UPDATE, which covers a concurrent uploader - // inserting the same brand-new chunk between the existence check - // in Phase 0 and this INSERT. - let new_sizes: Vec = new_rows.iter().map(|(_, s)| *s).collect(); - sqlx::query( + // ── Durability before visibility for the new chunks ────── + // One batched fsync sweep (no-op for remote backends, durable on + // PUT), then one batched INSERT. A crash before the INSERT leaves + // only unreferenced files; never a row pointing at unsynced bytes. + if !guard.written.is_empty() { + let new_hashes: Vec = guard.written.iter().map(|(h, _)| h.clone()).collect(); + let new_sizes: Vec = guard.written.iter().map(|(_, s)| *s).collect(); + + if let Err(e) = self.backend.sync_blobs(&new_hashes).await { + guard.rollback().await; + return Err(e); + } + + let registered = sqlx::query( "INSERT INTO storage.blobs (hash, size, ref_count) SELECT h, s, 1 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) ON CONFLICT (hash) DO UPDATE @@ -613,19 +608,102 @@ impl DedupService { ) .bind(&new_hashes) .bind(&new_sizes) - .execute(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to upsert chunks: {}", e)) - })?; + .execute(self.pool.as_ref()) + .await; + + if let Err(e) = registered { + guard.rollback().await; + return Err(DomainError::internal_error( + "Dedup", + format!("Failed to register chunks: {e}"), + )); + } } - // chunk_hashes/chunk_sizes keep the full per-occurrence CDC sequence — - // the manifest needs every chunk, in order, to reassemble the file. - let chunk_hashes: Vec = chunks.iter().map(|c| c.hash.clone()).collect(); - let chunk_sizes: Vec = chunks.iter().map(|c| c.length as u64).collect(); + let newly_written = guard.written.len(); + guard.disarm(); - Ok((chunk_hashes, chunk_sizes)) + Ok(ChunkIngestOutcome { + file_hash: file_hasher.finalize().to_hex().to_string(), + total_size, + chunk_hashes, + chunk_sizes, + newly_written, + }) + } + + /// Settle one batch of distinct in-RAM chunks against PG + the backend. + /// + /// Successfully pinned hashes and written chunks are recorded on the + /// guard as they happen, so a failure mid-batch leaves nothing + /// untracked for rollback. + async fn flush_pending( + &self, + guard: &mut IngestGuard, + pending: &mut Vec<(String, Bytes)>, + ) -> Result<(), DomainError> { + if pending.is_empty() { + return Ok(()); + } + let batch = std::mem::take(pending); + let hashes: Vec = batch.iter().map(|(h, _)| h.clone()).collect(); + + // Pin-or-classify in one statement: rows that exist take this + // session's reference NOW; hashes not returned don't exist and are + // ours to write. + let pinned: HashSet = sqlx::query_scalar::<_, String>( + "UPDATE storage.blobs SET ref_count = ref_count + 1 + WHERE hash = ANY($1) + RETURNING hash", + ) + .bind(&hashes) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}")) + })? + .into_iter() + .collect(); + + let mut to_write: Vec<(String, Bytes)> = Vec::with_capacity(batch.len()); + for (hash, data) in batch { + if pinned.contains(&hash) { + guard.pinned.push(hash); + } else { + to_write.push((hash, data)); + } + } + if to_write.is_empty() { + return Ok(()); + } + + // Unsynced writes — durability comes from the single end-of-stream + // sweep, before any PG row references these chunks. + let backend = self.backend.clone(); + let results: Vec> = stream::iter(to_write) + .map(|(hash, data)| { + let backend = backend.clone(); + async move { + let len = data.len() as i64; + backend.put_blob_from_bytes_unsynced(&hash, data).await?; + Ok((hash, len)) + } + }) + .buffer_unordered(Self::CHUNK_UPLOAD_CONCURRENCY) + .collect() + .await; + + let mut first_err: Option = None; + for result in results { + match result { + Ok(row) => guard.written.push(row), + Err(e) => first_err = first_err.or(Some(e)), + } + } + match first_err { + Some(e) => Err(e), + None => Ok(()), + } } // ── Reference counting ─────────────────────────────────────── @@ -879,7 +957,7 @@ impl DedupService { DomainError::internal_error("Dedup", format!("Failed to begin transaction: {}", e)) })?; - // Lock the row exclusively — prevents concurrent store_from_file from + // Lock the row exclusively — prevents a concurrent ingest from // incrementing ref_count while we might be deleting let row = sqlx::query_as::<_, (i32, i64)>( "SELECT ref_count, size FROM storage.blobs WHERE hash = $1 FOR UPDATE", @@ -917,7 +995,7 @@ impl DedupService { })?; // Delete blob from backend AFTER committing PG — the row is gone, - // so no concurrent store_from_file can resurrect a reference. + // so no concurrent ingest can resurrect a reference. if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete blob file {}: {}", hash, e); } @@ -1532,15 +1610,16 @@ impl DedupService { // report `legacy re-chunk: nothing to do`). // // Per-hash algorithm: - // 1. Spool the blob to a temp file via the normal read path (this - // decrypts it when encryption is on), verifying BLAKE3 == hash. - // 2. CDC-chunk the spool + store chunks (`store_chunks` bumps each - // distinct chunk once — the manifest's reference). - // 3. One short accounting TX with the blob row locked: + // 1. Stream the blob through the normal read path (this decrypts it + // when encryption is on) straight into the chunk-ingest engine — + // no spool file — verifying BLAKE3 == hash before keeping the + // chunks (each distinct chunk bumped once — the manifest's + // reference). + // 2. One short accounting TX with the blob row locked: // manifest INSERT with ref_count = N (current file rows referencing // the hash), blob ref_count -= N (those references now live on the // manifest), DELETE the blob row only if it hits exactly 0. - // 4. Physically delete the whole-file blob only when its row was + // 3. Physically delete the whole-file blob only when its row was // removed. Single-chunk files (chunk hash == file hash) keep the // physical blob — it IS the chunk; only the bookkeeping moves. // @@ -1548,7 +1627,7 @@ impl DedupService { // and the legacy dedup-hit path. A racing identical upload can land one // legacy reference after our commit; the blob row then survives (> 0) // and that file stays readable through the legacy fallback — a bounded - // space leak, never data loss. A crash between step 2 and 3 leaks one + // space leak, never data loss. A crash between step 1 and 2 leaks one // +1 on that file's chunk refs (re-run re-bumps); also a bounded leak, // never data loss. @@ -1688,22 +1767,12 @@ impl DedupService { hash: &str, content_type: Option, ) -> Result { - // ── 1. Spool + verify (decrypts via the normal read path) ── - // The spooled, hash-verified plaintext is the source of truth for - // sizes — `storage.blobs.size` is legacy metadata we don't trust + // ── 1. Stream + verify (decrypts via the normal read path) ── + // The chunk store is fed directly from the blob read stream — no + // spool file. Sizes come from the CDC pass over the hash-verified + // plaintext; `storage.blobs.size` is legacy metadata we don't trust // for the manifest's Range arithmetic. - // - // The path carries a per-attempt UUID: two processes sharing a temp - // dir and racing on the same hash must never truncate or delete each - // other's in-flight spool. - let spool = std::env::temp_dir().join(format!( - "oxicloud-rechunk-{}-{}.tmp", - &hash[..hash.len().min(16)], - uuid::Uuid::new_v4() - )); - let result = self.spool_and_chunk(hash, &spool).await; - let _ = fs::remove_file(&spool).await; - let (chunk_hashes, chunk_sizes) = result?; + let (chunk_hashes, chunk_sizes) = self.ingest_legacy_blob(hash).await?; let total_size: u64 = chunk_sizes.iter().sum(); // ── 2. Accounting TX: move the file references onto the manifest ── @@ -1756,10 +1825,11 @@ impl DedupService { if inserted == 0 { // A manifest appeared concurrently — only possible if the same - // content was re-uploaded and fully stored during our spool. + // content was re-uploaded and fully stored while we streamed. // Their bookkeeping is already correct; drop ours. tx.rollback().await.ok(); - self.release_chunk_refs(&chunk_hashes).await; + self.release_chunk_refs(self.maintenance_pool.as_ref(), &chunk_hashes) + .await; return Ok(0); } @@ -1828,63 +1898,36 @@ impl DedupService { Ok(freed) } - /// Spool a legacy blob to `spool`, verify its BLAKE3 matches `hash`, - /// CDC-chunk it and store the chunks. Returns (chunk_hashes, chunk_sizes). - async fn spool_and_chunk( - &self, - hash: &str, - spool: &Path, - ) -> Result<(Vec, Vec), DomainError> { - use tokio::io::AsyncWriteExt; - - let mut stream = self.read_blob_stream(hash).await?; - let file = fs::File::create(spool) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk spool: {e}")))?; - let mut writer = tokio::io::BufWriter::with_capacity(512 * 1024, file); - let mut hasher = blake3::Hasher::new(); - - while let Some(chunk) = stream.next().await { - let chunk = chunk - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk read: {e}")))?; - hasher.update(&chunk); - writer - .write_all(&chunk) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk write: {e}")))?; - } - writer - .flush() - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk flush: {e}")))?; - - let actual = hasher.finalize().to_hex().to_string(); - if actual != hash { + /// Re-chunk one legacy whole-file blob straight from the backend read + /// stream (no spool file), verifying that the streamed content still + /// matches its recorded BLAKE3 before the chunks are kept. + /// + /// On mismatch the freshly taken chunk references are released — the + /// written chunk bytes become unreferenced rows the GC sweeps — and an + /// error is returned; the legacy blob itself stays untouched. + async fn ingest_legacy_blob(&self, hash: &str) -> Result<(Vec, Vec), DomainError> { + let stream = self.read_blob_stream(hash).await?; + let outcome = self.ingest_chunks_from_stream(stream).await?; + if outcome.file_hash != hash { + let distinct = outcome.distinct_hashes(); + self.release_chunk_refs(self.maintenance_pool.as_ref(), &distinct) + .await; return Err(DomainError::internal_error( "Dedup", - format!("Blob content does not match its hash (expected {hash}, got {actual})"), + format!( + "Blob content does not match its hash (expected {hash}, got {})", + outcome.file_hash + ), )); } - - // Empty blobs can't be mmap'd by the CDC analyser; they become an - // empty manifest (the chunked read path streams zero chunks). - let spooled_len = fs::metadata(spool) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk stat: {e}")))? - .len(); - if spooled_len == 0 { - return Ok((Vec::new(), Vec::new())); - } - - let chunks = Self::cdc_chunk_file(spool) - .await - .map_err(DomainError::from)?; - self.store_chunks(spool, &chunks).await + Ok((outcome.chunk_hashes, outcome.chunk_sizes)) } - /// Best-effort compensation: drop the per-manifest chunk references - /// taken by `store_chunks` when the manifest insert was abandoned. - async fn release_chunk_refs(&self, chunk_hashes: &[String]) { + /// Best-effort compensation: drop one reference per *distinct* chunk + /// hash (clamped at 0). Used whenever an ingest session's references end + /// up not being attached to a manifest — dedup hit, lost insert race, or + /// content-verification failure. + async fn release_chunk_refs(&self, pool: &PgPool, chunk_hashes: &[String]) { if chunk_hashes.is_empty() { return; } @@ -1893,10 +1936,10 @@ impl DedupService { WHERE hash = ANY($1)", ) .bind(chunk_hashes) - .execute(self.maintenance_pool.as_ref()) + .execute(pool) .await { - tracing::warn!("Legacy re-chunk: failed to release chunk refs: {e}"); + tracing::warn!("Dedup: failed to release chunk refs: {e}"); } } } @@ -1915,16 +1958,6 @@ pub struct LegacyRechunkReport { // ─── Port implementation ───────────────────────────────────────────────────── impl DedupPort for DedupService { - async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result { - self.store_from_file(source_path, content_type, pre_computed_hash) - .await - } - async fn blob_exists(&self, hash: &str) -> bool { self.blob_exists(hash).await } @@ -2002,20 +2035,95 @@ mod tests { file } + /// One chunk as seen by the streaming analyser. + struct TestChunk { + hash: String, + offset: usize, + length: usize, + } + + /// Run the exact same streaming chunker the ingest engine uses + /// (`AsyncStreamCDC` + the production CDC parameters) over an in-memory + /// buffer, feeding it in `frame`-sized pieces to exercise the refill + /// logic the same way HTTP body frames do. + /// + /// Returns the whole-stream BLAKE3 plus per-chunk metadata. + async fn stream_cdc(data: &[u8], frame: usize) -> (String, Vec) { + let frames: Vec> = data + .chunks(frame.max(1)) + .map(|c| Ok(Bytes::copy_from_slice(c))) + .collect(); + let reader = StreamReader::new(Box::pin(stream::iter(frames))); + let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( + reader, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ); + let chunk_stream = chunker.as_stream(); + futures::pin_mut!(chunk_stream); + + let mut file_hasher = blake3::Hasher::new(); + let mut chunks = Vec::new(); + while let Some(item) = chunk_stream.next().await { + let chunk = item.expect("in-memory stream cannot fail"); + file_hasher.update(&chunk.data); + chunks.push(TestChunk { + hash: blake3::hash(&chunk.data).to_hex().to_string(), + offset: chunk.offset as usize, + length: chunk.length, + }); + } + (file_hasher.finalize().to_hex().to_string(), chunks) + } + + const TEST_FRAME: usize = 64 * 1024; // typical HTTP body frame size + + // ── Stream chunking ≡ slice chunking ───────────────────────── + // + // The whole dedup index hinges on this invariant: the boundaries (and + // therefore the chunk hashes) produced by the streaming chunker must be + // identical to FastCDC over the full in-memory slice, regardless of how + // the bytes were framed on the wire. Pre-streaming blobs were chunked + // via mmap + slice FastCDC — their chunks must keep deduplicating + // against newly streamed uploads. + + #[tokio::test] + async fn test_stream_chunking_matches_slice_chunking() { + let data: Vec = (0..4 * 1024 * 1024) + .map(|i| ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1)) as u8) + .collect(); + + let slice_chunks: Vec<(usize, usize)> = + fastcdc::v2020::FastCDC::new(&data, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK) + .map(|c| (c.offset, c.length)) + .collect(); + + for frame in [7usize, 4096, TEST_FRAME, data.len()] { + let (_, streamed) = stream_cdc(&data, frame).await; + assert_eq!( + streamed.len(), + slice_chunks.len(), + "chunk count must not depend on framing (frame={frame})" + ); + for (s, (offset, length)) in streamed.iter().zip(slice_chunks.iter()) { + assert_eq!((s.offset, s.length), (*offset, *length), "frame={frame}"); + let expected = blake3::hash(&data[*offset..*offset + *length]) + .to_hex() + .to_string(); + assert_eq!(s.hash, expected, "frame={frame}"); + } + } + } + // ── Determinism ────────────────────────────────────────────── #[tokio::test] async fn test_cdc_deterministic_same_content() { let data = vec![42u8; 512 * 1024]; // 512 KB of 0x2A - let f1 = write_temp_file(&data).await; - let f2 = write_temp_file(&data).await; - let (hash1, chunks1) = DedupService::cdc_hash_and_chunk_file(f1.path()) - .await - .unwrap(); - let (hash2, chunks2) = DedupService::cdc_hash_and_chunk_file(f2.path()) - .await - .unwrap(); + let (hash1, chunks1) = stream_cdc(&data, TEST_FRAME).await; + let (hash2, chunks2) = stream_cdc(&data, 4096).await; assert_eq!(hash1, hash2, "same content must produce same file hash"); assert_eq!( @@ -2030,16 +2138,13 @@ mod tests { } } - // ── Empty file ─────────────────────────────────────────────── + // ── Empty stream ───────────────────────────────────────────── #[tokio::test] - async fn test_cdc_empty_file() { - let f = write_temp_file(b"").await; - let (hash, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + async fn test_cdc_empty_stream() { + let (hash, chunks) = stream_cdc(b"", TEST_FRAME).await; - assert!(chunks.is_empty(), "empty file must produce zero chunks"); + assert!(chunks.is_empty(), "empty stream must produce zero chunks"); assert_eq!(hash, blake3::hash(b"").to_hex().to_string()); } @@ -2048,10 +2153,7 @@ mod tests { #[tokio::test] async fn test_cdc_small_file_single_chunk() { let data = b"Hello, OxiCloud CDC dedup!"; - let f = write_temp_file(data).await; - let (hash, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (hash, chunks) = stream_cdc(data, TEST_FRAME).await; assert_eq!(chunks.len(), 1, "tiny file must be a single chunk"); assert_eq!(chunks[0].offset, 0); @@ -2067,11 +2169,8 @@ mod tests { let data: Vec = (0..4 * 1024 * 1024) .map(|i| ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1)) as u8) .collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; assert!(chunks.len() > 1, "4 MB should produce multiple chunks"); @@ -2104,78 +2203,22 @@ mod tests { let data: Vec = (0..1024 * 1024).map(|i| (i % 251) as u8).collect(); let f = write_temp_file(&data).await; - let (cdc_hash, _) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (cdc_hash, _) = stream_cdc(&data, TEST_FRAME).await; let standalone_hash = DedupService::hash_file(f.path()).await.unwrap(); assert_eq!( cdc_hash, standalone_hash, - "CDC file hash must match standalone hash_file()" + "streamed file hash must match standalone hash_file()" ); } - // ── Chunk hashes are correct BLAKE3 of chunk data ──────────── - - #[tokio::test] - async fn test_cdc_chunk_hashes_are_correct() { - let data: Vec = (0..2 * 1024 * 1024) - .map(|i| ((i as u64).wrapping_mul(2862933555777941757).wrapping_add(3)) as u8) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - - for chunk in &chunks { - let chunk_data = &data[chunk.offset..chunk.offset + chunk.length]; - let expected_hash = blake3::hash(chunk_data).to_hex().to_string(); - assert_eq!( - chunk.hash, expected_hash, - "chunk at offset {} has wrong hash", - chunk.offset - ); - } - } - - // ── Reassembly matches original ────────────────────────────── - - #[tokio::test] - async fn test_cdc_reassembly_matches_original() { - let data: Vec = (0..3 * 1024 * 1024) - .map(|i| ((i as u64).wrapping_mul(1103515245).wrapping_add(12345)) as u8) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - - // Reassemble from chunks - let mut reassembled = Vec::with_capacity(data.len()); - for chunk in &chunks { - reassembled.extend_from_slice(&data[chunk.offset..chunk.offset + chunk.length]); - } - - assert_eq!( - reassembled.len(), - data.len(), - "reassembled length must match" - ); - assert_eq!(reassembled, data, "reassembled content must match original"); - } - - // ── Chunks cover entire file (no gaps, no overlaps) ────────── + // ── Reassembly: chunks are contiguous and cover the file ───── #[tokio::test] async fn test_cdc_chunks_are_contiguous() { let data: Vec = (0..2 * 1024 * 1024).map(|i| (i % 199) as u8).collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; let mut expected_offset = 0usize; for (i, chunk) in chunks.iter().enumerate() { @@ -2205,15 +2248,8 @@ mod tests { *b = b.wrapping_add(1); } - let f_base = write_temp_file(&base).await; - let f_mod = write_temp_file(&modified).await; - - let (hash_base, chunks_base) = DedupService::cdc_hash_and_chunk_file(f_base.path()) - .await - .unwrap(); - let (hash_mod, chunks_mod) = DedupService::cdc_hash_and_chunk_file(f_mod.path()) - .await - .unwrap(); + let (hash_base, chunks_base) = stream_cdc(&base, TEST_FRAME).await; + let (hash_mod, chunks_mod) = stream_cdc(&modified, TEST_FRAME).await; // File hashes must differ assert_ne!( @@ -2241,28 +2277,6 @@ mod tests { ); } - // ── cdc_chunk_file matches cdc_hash_and_chunk_file ─────────── - - #[tokio::test] - async fn test_cdc_chunk_file_matches_full() { - let data: Vec = (0..1024 * 1024) - .map(|i| (i as u8).wrapping_mul(7)) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks_full) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - let chunks_only = DedupService::cdc_chunk_file(f.path()).await.unwrap(); - - assert_eq!(chunks_full.len(), chunks_only.len()); - for (a, b) in chunks_full.iter().zip(chunks_only.iter()) { - assert_eq!(a.hash, b.hash); - assert_eq!(a.offset, b.offset); - assert_eq!(a.length, b.length); - } - } - // ── Large file produces expected chunk count ────────────────── #[tokio::test] @@ -2271,11 +2285,8 @@ mod tests { let data: Vec = (0..8 * 1024 * 1024) .map(|i| ((i as u64).wrapping_mul(2862933555777941757).wrapping_add(3)) as u8) .collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; // With 256KB avg, expect 20-60 chunks for 8MB assert!( @@ -2306,15 +2317,8 @@ mod tests { let mut with_prefix = prefix; with_prefix.extend_from_slice(&base); - let f_base = write_temp_file(&base).await; - let f_prefix = write_temp_file(&with_prefix).await; - - let (_, chunks_base) = DedupService::cdc_hash_and_chunk_file(f_base.path()) - .await - .unwrap(); - let (_, chunks_prefix) = DedupService::cdc_hash_and_chunk_file(f_prefix.path()) - .await - .unwrap(); + let (_, chunks_base) = stream_cdc(&base, TEST_FRAME).await; + let (_, chunks_prefix) = stream_cdc(&with_prefix, TEST_FRAME).await; let base_set: HashSet<&str> = chunks_base.iter().map(|c| c.hash.as_str()).collect(); let prefix_set: HashSet<&str> = chunks_prefix.iter().map(|c| c.hash.as_str()).collect(); @@ -2331,6 +2335,20 @@ mod tests { chunks_prefix.len() ); } + + // ── ChunkIngestOutcome helpers ─────────────────────────────── + + #[test] + fn test_distinct_hashes_deduplicates_preserving_order() { + let outcome = ChunkIngestOutcome { + file_hash: String::new(), + total_size: 0, + chunk_hashes: vec!["a".into(), "b".into(), "a".into(), "c".into(), "b".into()], + chunk_sizes: vec![1, 2, 1, 3, 2], + newly_written: 0, + }; + assert_eq!(outcome.distinct_hashes(), vec!["a", "b", "c"]); + } } // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/infrastructure/services/local_blob_backend.rs b/src/infrastructure/services/local_blob_backend.rs index 632701cb..8ef8cf3b 100644 --- a/src/infrastructure/services/local_blob_backend.rs +++ b/src/infrastructure/services/local_blob_backend.rs @@ -244,12 +244,13 @@ impl BlobStorageBackend for LocalBlobBackend { // Atomic rename (same filesystem). Falls back to copy+delete for // cross-device moves (EXDEV errno 18). // - // Durability boundary: the caller is responsible for - // having sync_all'd the source file before invoking this - // function — both `interfaces::upload_spool::spool_body_to_temp` - // and `chunked_upload_service::complete_upload_inner` - // (the two production producers of `source_path`) now do - // so. We fsync the parent of `blob_path` AFTER the rename + // Durability boundary: the caller is responsible for having + // sync_all'd the source file before invoking this function. + // (The streaming upload path writes chunks via + // `put_blob_from_bytes_unsynced` + a batched `sync_blobs` + // sweep instead; this move-based entry point remains for + // whole-file producers such as migration tooling and tests.) + // We fsync the parent of `blob_path` AFTER the rename // so the dirent change itself becomes durable; without // that, a power loss can resurrect the old (unrenamed) // name even when the file contents survive. diff --git a/src/infrastructure/services/nextcloud_chunked_upload_service.rs b/src/infrastructure/services/nextcloud_chunked_upload_service.rs index 34e6cb52..fff15b5e 100644 --- a/src/infrastructure/services/nextcloud_chunked_upload_service.rs +++ b/src/infrastructure/services/nextcloud_chunked_upload_service.rs @@ -73,7 +73,7 @@ impl NextcloudChunkedUploadService { /// Store a chunk in the session directory. Buffers `data` in memory — /// use [`safe_chunk_path`](Self::safe_chunk_path) + the - /// `interfaces/upload_spool::stream_body_to_path` helper to stream the + /// `interfaces/upload_ingest::stream_body_to_path` helper to stream the /// HTTP body directly to disk and avoid materialising the whole chunk /// in RAM. pub async fn store_chunk( @@ -93,24 +93,14 @@ impl NextcloudChunkedUploadService { Ok(()) } - /// Assemble all chunks in numeric order into a temp file, computing - /// the BLAKE3 of the concatenated stream **during** the same read/ - /// write pass (hash-on-write). + /// List the session's chunk files in assembly (numeric) order. /// - /// Returns `(temp_path, total_size, blake3_hex)`. The caller passes - /// the hash to the upload service as `pre_computed_hash` so the - /// downstream dedup layer never has to re-read the assembled file - /// to compute it — saving one full file-sized read pass per upload. - /// - /// The read/hash/write loop runs inside `spawn_blocking` because - /// BLAKE3 is CPU-bound and would otherwise starve the Tokio worker - /// running other connections; synchronous I/O is used inside the - /// blocking thread because the workload is sequential and the - /// async reactor overhead would only slow it down. For files larger - /// than ~10 MB BLAKE3's Rayon mode parallelises across cores — - /// mirrors what `ChunkedUploadService::complete_upload_inner` does - /// for the REST chunked path. - pub async fn assemble(&self, user: &str, upload_id: &str) -> Result<(PathBuf, u64, String)> { + /// The caller streams these directly into the CDC chunk store + /// (`interfaces::upload_ingest::stream_from_files`) — chunking, BLAKE3 + /// hashing and dedup checks happen in that single read pass, so no + /// assembled temp file is ever written. The chunk parts stay on disk + /// until [`cleanup`](Self::cleanup), keeping completion retryable. + pub async fn ordered_chunk_paths(&self, user: &str, upload_id: &str) -> Result> { let session_dir = self.safe_session_dir(user, upload_id)?; let mut entries: Vec = Vec::new(); @@ -124,8 +114,11 @@ impl NextcloudChunkedUploadService { .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))? { let name = entry.file_name().to_string_lossy().to_string(); - if name == ".file" { - continue; // Skip the assembly marker. + // `.file` is the NC-protocol assembly marker; `.assembled` is + // the staging file older releases wrote — sessions in flight + // across an upgrade may still contain one. + if name == ".file" || name == ".assembled" { + continue; } entries.push(name); } @@ -133,74 +126,7 @@ impl NextcloudChunkedUploadService { // Sort chunks numerically (Nextcloud sends them as "00001", "00002", ...). entries.sort(); - let temp_path = session_dir.join(".assembled"); - let chunk_paths: Vec = entries.iter().map(|n| session_dir.join(n)).collect(); - let assembled_for_blocking = temp_path.clone(); - - // Read/hash/write loop runs synchronously on the blocking pool. - // BLAKE3 is computed in the same pass that copies bytes from chunk - // files into the assembled file — no second read after the fact. - let (total_size, hash) = - tokio::task::spawn_blocking(move || -> std::io::Result<(u64, String)> { - use std::io::{BufWriter as StdBufWriter, Read, Write}; - - let raw_output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&assembled_for_blocking)?; - - // 512 KB write buffer — 8× fewer syscalls than 64 KB. - let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = blake3::Hasher::new(); - let mut buf = vec![0u8; 524_288]; - let mut total: u64 = 0; - - // Files >10 MB benefit from BLAKE3's multi-threaded mode. - // The threshold matches the REST chunked path's heuristic. - const RAYON_THRESHOLD_PER_FRAME: usize = 128 * 1024; - - for chunk_path in &chunk_paths { - let mut chunk_file = std::fs::File::open(chunk_path)?; - loop { - let n = chunk_file.read(&mut buf)?; - if n == 0 { - break; - } - if n >= RAYON_THRESHOLD_PER_FRAME { - hasher.update_rayon(&buf[..n]); - } else { - hasher.update(&buf[..n]); - } - output.write_all(&buf[..n])?; - total += n as u64; - } - } - - output.flush()?; - // ── Durability boundary ───────────────────────────────── - // sync_all is the actual fsync; without it, a power loss - // before the kernel writeback timer (~5 s) loses - // acknowledged data. Pull the inner File out of the - // BufWriter so we can sync the underlying handle — - // dropping the BufWriter wouldn't trigger fsync. macOS - // caveat: fsync there flushes to the disk controller - // only; true durability needs F_FULLFSYNC, not exposed - // by std. - let raw_output = output - .into_inner() - .map_err(|e| std::io::Error::other(format!("into_inner: {e}")))?; - raw_output.sync_all()?; - - Ok((total, hasher.finalize().to_hex().to_string())) - }) - .await - .map_err(|e| { - DomainError::internal_error("ChunkedUpload", format!("assemble task: {e}")) - })? - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - - Ok((temp_path, total_size, hash)) + Ok(entries.iter().map(|n| session_dir.join(n)).collect()) } /// Delete the upload session directory. @@ -260,9 +186,10 @@ impl NextcloudChunkedUploadService { let name = entry.file_name().to_string_lossy().to_string(); // Filter internal markers — `.file` is the NC-protocol // assembly trigger target (it never reaches the disk - // because MOVE redirects it), `.assembled` is our own - // staging file from `assemble()`. Surfacing either to - // the client would confuse its chunk-count check. + // because MOVE redirects it), `.assembled` is the staging + // file pre-streaming releases wrote (kept for sessions in + // flight across an upgrade). Surfacing either to the + // client would confuse its chunk-count check. if name == ".file" || name == ".assembled" { continue; } @@ -331,8 +258,18 @@ mod tests { assert!(!svc.session_exists("alice", "upload-999").await); } + /// Concatenate the session's chunk files in assembly order — mirrors + /// what `upload_ingest::stream_from_files` feeds the CDC store. + async fn concat_chunks(svc: &NextcloudChunkedUploadService, user: &str, id: &str) -> Vec { + let mut out = Vec::new(); + for path in svc.ordered_chunk_paths(user, id).await.unwrap() { + out.extend_from_slice(&fs::read(&path).await.unwrap()); + } + out + } + #[tokio::test] - async fn test_store_and_assemble_chunks() { + async fn test_store_and_order_chunks() { let (svc, _dir) = test_service(); svc.create_session("alice", "upload-002").await.unwrap(); @@ -343,20 +280,14 @@ mod tests { .await .unwrap(); - let (temp_path, size, hash) = svc.assemble("alice", "upload-002").await.unwrap(); - let assembled = fs::read(&temp_path).await.unwrap(); - assert_eq!(assembled, b"Hello, World!"); - assert_eq!(size, 13); - // BLAKE3("Hello, World!") — proves hash-on-write happens during - // the assemble pass, not via a re-read. assert_eq!( - hash, - "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + concat_chunks(&svc, "alice", "upload-002").await, + b"Hello, World!" ); } #[tokio::test] - async fn test_assemble_chunks_in_sorted_order() { + async fn test_chunk_paths_sorted_regardless_of_upload_order() { let (svc, _dir) = test_service(); svc.create_session("alice", "upload-003").await.unwrap(); @@ -371,16 +302,29 @@ mod tests { .await .unwrap(); - let (temp_path, size, hash) = svc.assemble("alice", "upload-003").await.unwrap(); - let assembled = fs::read(&temp_path).await.unwrap(); - assert_eq!(assembled, b"ABC"); - assert_eq!(size, 3); - // BLAKE3("ABC") — confirms sort happened (chunks were stored in - // order 3,1,2 but the hash matches "ABC", not "CAB" or "BAC"). - assert_eq!( - hash, - "d1717274597cf0289694f75d96d444b992a096f1afd8e7bbfa6ebb1d360fedfc" - ); + // Chunks were stored in order 3,1,2 but must concatenate as "ABC". + assert_eq!(concat_chunks(&svc, "alice", "upload-003").await, b"ABC"); + } + + #[tokio::test] + async fn test_internal_markers_excluded_from_chunk_paths() { + let (svc, _dir) = test_service(); + svc.create_session("alice", "upload-005").await.unwrap(); + + svc.store_chunk("alice", "upload-005", "00001", b"data") + .await + .unwrap(); + // Stale staging file from a pre-streaming release. + svc.store_chunk("alice", "upload-005", ".assembled", b"old") + .await + .unwrap(); + + let paths = svc + .ordered_chunk_paths("alice", "upload-005") + .await + .unwrap(); + assert_eq!(paths.len(), 1, "markers must be filtered out"); + assert!(paths[0].ends_with("00001")); } #[tokio::test] diff --git a/src/interfaces/api/handlers/chunked_upload_handler.rs b/src/interfaces/api/handlers/chunked_upload_handler.rs index f2276e50..0998803b 100644 --- a/src/interfaces/api/handlers/chunked_upload_handler.rs +++ b/src/interfaces/api/handlers/chunked_upload_handler.rs @@ -4,7 +4,7 @@ //! - POST /api/uploads → Create upload session //! - PATCH /api/uploads/:id → Upload a chunk //! - HEAD /api/uploads/:id → Get upload status -//! - POST /api/uploads/:id/complete → Assemble and finalize +//! - POST /api/uploads/:id/complete → Stream parts into the blob store //! - DELETE /api/uploads/:id → Cancel upload use axum::{ @@ -27,7 +27,7 @@ use crate::common::di::AppState; use crate::domain::services::authorization::Permission; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; -use crate::interfaces::upload_spool::stream_body_to_path; +use crate::interfaces::upload_ingest::{self, stream_body_to_path}; /// Request body for creating an upload session #[derive(Debug, Deserialize, ToSchema)] @@ -64,35 +64,32 @@ pub struct CompleteUploadResponse { /// Optional body for `POST /api/uploads/{id}/complete`. /// /// When the client supplies `checksum`, the server compares it against -/// the assembled file's hash BEFORE promoting the blob to storage — -/// failure aborts the upload atomically (no orphaned blob, no DB row). -/// This is the end-to-end integrity check: per-chunk MD5 proves each -/// chunk arrived intact, but only the final hash catches assembly / -/// promotion bugs and mis-ordered chunks. +/// the streamed content's hash BEFORE the file row is created — failure +/// releases the blob reference and returns 400, with the chunk parts +/// kept on disk for a retry. This is the end-to-end integrity check: +/// per-chunk MD5 proves each chunk arrived intact, but only the final +/// hash catches mis-ordered or corrupted assemblies. /// -/// **`blake3` is highly recommended** — it's the algorithm the server -/// already runs over the assembled file during hash-on-write -/// assembly, so verification is a string comparison with zero extra -/// I/O and zero extra CPU. It's also the same algorithm the server -/// uses for blob-storage addressing, so the value the client sends -/// equals the `content_hash` they'd later read back from -/// `GET /api/files/{id}`. `md5` and `sha256` are accepted for -/// compatibility with legacy client tooling but each triggers a -/// second hash pass over the assembled file (~30–100 ms depending -/// on size). +/// **`blake3` is highly recommended** — it's the content-addressing +/// algorithm of the blob store itself, so verification is a string +/// comparison against the hash the store already computed, and the +/// value the client sends equals the `content_hash` they'd later read +/// back from `GET /api/files/{id}`. `md5` and `sha256` are accepted +/// for legacy client tooling; they are computed by an in-flight tee +/// during the same streaming pass — no extra disk read either way. /// /// `Default` keeps the existing wire shape: clients that POST with no /// body get today's behavior (no verification, server just returns /// what it computed). #[derive(Debug, Default, Deserialize, ToSchema)] pub struct CompleteUploadRequest { - /// Lowercase hex digest the client expects the assembled file to + /// Lowercase hex digest the client expects the streamed content to /// hash to. Compared case-insensitively. Omit to skip verification. pub checksum: Option, /// Algorithm name. `blake3` is the recommended choice (default — - /// matches the server's hash-on-write algorithm, zero extra cost). - /// `md5`, `sha256` / `sha-256` are accepted but trigger an extra - /// hash pass. Unknown values return 400. + /// matches the blob store's content-addressing algorithm). `md5`, + /// `sha256` / `sha-256` are accepted and computed in-flight. + /// Unknown values return 400. pub checksumalg: Option, } @@ -307,70 +304,15 @@ impl ChunkedUploadHandler { } } - /// Compute the requested checksum of the assembled file. - /// - /// For `Blake3` the server already has the hash from hash-on-write - /// assembly — we just return it (zero I/O, zero CPU). For `Md5` and - /// `Sha256` we re-read the assembled file on the blocking pool and - /// hash it; the cost (~30–100 ms for typical files) is the trade-off - /// for accepting non-default algorithms. - async fn compute_assembled_hash( - assembled_path: &std::path::Path, - alg: ChecksumAlg, - blake3_already_computed: &str, - ) -> Result { - match alg { - ChecksumAlg::Blake3 => Ok(blake3_already_computed.to_string()), - ChecksumAlg::Md5 | ChecksumAlg::Sha256 => { - let path = assembled_path.to_path_buf(); - tokio::task::spawn_blocking(move || -> Result { - use std::io::Read; - let mut file = std::fs::File::open(&path)?; - let mut buf = vec![0u8; 524_288]; - match alg { - ChecksumAlg::Md5 => { - use md5::Digest as _; - let mut h = md5::Md5::new(); - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - h.update(&buf[..n]); - } - Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) - } - ChecksumAlg::Sha256 => { - use sha2::Digest as _; - let mut h = sha2::Sha256::new(); - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - h.update(&buf[..n]); - } - Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) - } - // Blake3 handled above — this branch is unreachable but - // keeps the match exhaustive without an else-clause. - ChecksumAlg::Blake3 => unreachable!(), - } - }) - .await - .map_err(|e| std::io::Error::other(format!("hash task join failed: {e}")))? - } - } - } - /// POST /api/uploads/:upload_id/complete - Finalize upload /// - /// Assembles all chunks into the final file and creates the file record. - /// When `body.checksum` is supplied, the assembled file's hash is - /// verified before the blob is promoted to storage — mismatch - /// returns 400 and the assembled temp is removed (the session - /// itself is kept so the client can re-issue complete after - /// diagnosing). + /// Streams the uploaded chunk parts, in order, straight into the CDC + /// chunk store and creates the file record — no assembled temp file is + /// ever written. When `body.checksum` is supplied it is verified from + /// the same streaming pass (BLAKE3 comes from the store itself; + /// MD5/SHA-256 are computed by an in-flight tee) — mismatch returns 400 + /// with the blob reference released, and the chunk parts stay on disk + /// so the client can re-issue complete after diagnosing. pub(super) async fn complete_upload_impl( State(state): State>, auth_user: AuthUser, @@ -379,10 +321,11 @@ impl ChunkedUploadHandler { ) -> impl IntoResponse { let chunked_service = &state.core.chunked_upload_service; let upload_service = &state.applications.file_upload_service; + let dedup = &state.core.dedup_service; - // ── Parse the optional algorithm BEFORE assembly so a bad - // `checksumalg` doesn't waste the (potentially expensive) - // hash work on a request we'll reject anyway. + // ── Parse the optional algorithm BEFORE completion so a bad + // `checksumalg` doesn't waste any work on a request we'll + // reject anyway. let alg = match body.checksumalg.as_deref() { Some(name) => match ChecksumAlg::parse(name) { Some(a) => Some(a), @@ -397,37 +340,54 @@ impl ChunkedUploadHandler { }; let expected_checksum = body.checksum.as_deref(); - // Assemble chunks (hash-on-write: BLAKE3 computed during assembly) - let (assembled_path, filename, folder_id, content_type, total_size, hash) = - match chunked_service - .complete_upload(&upload_id, auth_user.id) - .await - { - Ok(result) => result, - Err(e) => { - return AppError::from(e).into_response(); - } - }; + // Validate completion and get the chunk parts in assembly order. + let parts = match chunked_service + .complete_upload(&upload_id, auth_user.id) + .await + { + Ok(result) => result, + Err(e) => { + return AppError::from(e).into_response(); + } + }; + + // MD5/SHA-256 verification taps the stream while it is ingested; + // BLAKE3 needs no tee — the store's own content hash IS BLAKE3. + let alg = expected_checksum.map(|_| alg.unwrap_or(ChecksumAlg::Blake3)); + let tee = match alg { + Some(ChecksumAlg::Md5) | Some(ChecksumAlg::Sha256) => { + Some(upload_ingest::checksum_tee(alg.unwrap())) + } + _ => None, + }; + + // ── Stream the parts into the CDC chunk store ─────────────── + let ingested = match upload_ingest::ingest_stream_to_cas( + upload_ingest::stream_from_files(parts.chunk_paths), + dedup, + &parts.filename, + &parts.content_type, + usize::MAX, + tee.clone(), + ) + .await + { + Ok(ingested) => ingested, + Err(e) => return e.into_response(), + }; // ── End-to-end integrity verification ─────────────────────── - // Only fires when the client supplied an `expected` checksum. - // For BLAKE3 (the documented preferred choice) this is a string - // comparison against the hash assembly already produced. For - // MD5/SHA-256 we re-hash the assembled file on the blocking pool. - if let Some(expected) = expected_checksum { - let alg = alg.unwrap_or(ChecksumAlg::Blake3); - let computed = match Self::compute_assembled_hash(&assembled_path, alg, &hash).await { - Ok(c) => c, - Err(e) => { - let _ = tokio::fs::remove_file(&assembled_path).await; - return AppError::internal_error(format!( - "Failed to compute assembled checksum: {e}" - )) - .into_response(); - } + if let (Some(expected), Some(alg)) = (expected_checksum, alg) { + let computed = match alg { + ChecksumAlg::Blake3 => Some(ingested.hash.clone()), + _ => tee.as_ref().and_then(upload_ingest::finalize_checksum_tee), + }; + let Some(computed) = computed else { + upload_ingest::discard_ingested(dedup, &ingested).await; + return AppError::internal_error("Checksum tee produced no digest").into_response(); }; if !computed.eq_ignore_ascii_case(expected) { - let _ = tokio::fs::remove_file(&assembled_path).await; + upload_ingest::discard_ingested(dedup, &ingested).await; tracing::warn!( target: "audit", event = "chunked_upload.checksum_mismatch", @@ -449,36 +409,28 @@ impl ChunkedUploadHandler { } } - // ── MIME detection (magic bytes + extension fallback) ───── - let content_type = crate::common::mime_detect::refine_content_type_from_file( - &assembled_path, - &filename, - &content_type, - ) - .await; - - // Upload from assembled file on disk — zero extra RAM copies, hash pre-computed + // Register the file row against the ingested blob. + let size = ingested.size; match upload_service - .upload_file_from_path( - filename.clone(), - folder_id.clone(), - content_type, - &assembled_path, - Some(hash), + .upload_file_streaming( + parts.filename.clone(), + parts.folder_id.clone(), + ingested.content_type.clone(), + ingested.stored(), ) .await { Ok(file) => { - // Cleanup session + // Cleanup session (removes the chunk part files) let _ = chunked_service .finalize_upload(&upload_id, auth_user.id) .await; tracing::info!( "✅ CHUNKED UPLOAD COMPLETE: {} (ID: {}, {} bytes)", - filename, + parts.filename, file.id, - total_size + size ); ( @@ -486,14 +438,14 @@ impl ChunkedUploadHandler { Json(CompleteUploadResponse { file_id: file.id, filename: file.name, - size: total_size, + size, path: file.path, }), ) .into_response() } Err(e) => { - tracing::error!("Failed to create file from assembled upload: {:?}", e); + tracing::error!("Failed to create file from chunked upload: {:?}", e); AppError::internal_error(format!("Failed to create file: {}", e)).into_response() } } diff --git a/src/interfaces/api/handlers/dedup_handler.rs b/src/interfaces/api/handlers/dedup_handler.rs index 3defb869..3e2f29db 100644 --- a/src/interfaces/api/handlers/dedup_handler.rs +++ b/src/interfaces/api/handlers/dedup_handler.rs @@ -5,12 +5,11 @@ use axum::{ response::IntoResponse, }; use serde::Serialize; -use tokio::io::AsyncWriteExt; use utoipa::ToSchema; -use crate::application::ports::dedup_ports::DedupResultDto; use crate::common::di::AppState; use crate::interfaces::middleware::auth::AuthUser; +use crate::interfaces::upload_ingest; use std::sync::Arc; /// Global application state for dependency injection @@ -155,10 +154,10 @@ impl DedupHandler { /// Upload content with automatic deduplication (streaming). /// - /// Spools the upload to a temp file while computing the BLAKE3 hash - /// incrementally (hash-on-write). Memory usage is constant (~512 KB) - /// regardless of file size. Then delegates to `store_from_file` with - /// the pre-computed hash so the file is never re-read for hashing. + /// Streams the multipart field straight into the CDC chunk store — + /// chunking, BLAKE3 hashing and dedup checks happen while the bytes + /// arrive (no temp file, no re-read; peak RAM is bounded regardless + /// of file size). /// /// POST /api/dedup/upload pub(super) async fn upload_with_dedup_impl( @@ -177,64 +176,29 @@ impl DedupHandler { .content_type() .unwrap_or("application/octet-stream") .to_string(); + let filename = field.file_name().unwrap_or("unnamed").to_string(); - // ── Spool to temp file + BLAKE3 hash-on-write ──────── - let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp"); - let temp_path = temp_dir.join(format!("dedup-{}", uuid::Uuid::new_v4())); - - let mut total_size: u64 = 0; - let mut hasher = blake3::Hasher::new(); - let mut field = field; - - let spool_result: Result<(), String> = async { - let file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| format!("Failed to create temp file: {}", e))?; - - // 512 KB buffer — reduces write syscalls - let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); - - loop { - match field.chunk().await { - Ok(Some(chunk)) => { - total_size += chunk.len() as u64; - hasher.update(&chunk); - writer - .write_all(&chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; - } - Ok(None) => break, - Err(e) => { - return Err(format!( - "Connection lost during upload (received {} bytes): {}", - total_size, e - )); - } - } + // ── Stream into the CDC chunk store ────────────────── + let source = upload_ingest::multipart_field_stream(field); + let ingested = match upload_ingest::ingest_stream_to_cas( + source, + dedup, + &filename, + &content_type, + usize::MAX, + None, + ) + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::warn!("Dedup upload ingest failed: {}", e.message); + return e.into_response(); } + }; - writer - .flush() - .await - .map_err(|e| format!("Failed to flush temp file: {}", e))?; - Ok(()) - } - .await; - - if let Err(msg) = spool_result { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::warn!("Dedup upload spool failed: {}", msg); - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(format!(r#"{{"error": "{}"}}"#, msg))) - .unwrap() - .into_response(); - } - - if total_size == 0 { - let _ = tokio::fs::remove_file(&temp_path).await; + if ingested.size == 0 { + upload_ingest::discard_ingested(dedup, &ingested).await; return Response::builder() .status(StatusCode::BAD_REQUEST) .header(header::CONTENT_TYPE, "application/json") @@ -243,60 +207,33 @@ impl DedupHandler { .into_response(); } - let hash = hasher.finalize().to_hex().to_string(); + let metadata = dedup.get_blob_metadata(&ingested.hash).await; - // ── Store with deduplication (pre-computed hash) ────── - match dedup - .store_from_file(&temp_path, Some(content_type), Some(hash)) - .await - { - Ok(result) => { - let (is_new, bytes_saved) = match &result { - DedupResultDto::NewBlob { .. } => (true, 0), - DedupResultDto::ExistingBlob { saved_bytes, .. } => { - (false, *saved_bytes) - } - }; + let response = DedupUploadResponse { + is_new: ingested.is_new_blob, + hash: ingested.hash.clone(), + size: ingested.size, + bytes_saved: ingested.bytes_saved, + ref_count: metadata.map(|m| m.ref_count).unwrap_or(1), + }; - let metadata = dedup.get_blob_metadata(result.hash()).await; + tracing::info!( + "🔗 Dedup upload: hash={}, new={}, saved={}", + ingested.hash, + ingested.is_new_blob, + ingested.bytes_saved + ); - let response = DedupUploadResponse { - is_new, - hash: result.hash().to_string(), - size: result.size(), - bytes_saved, - ref_count: metadata.map(|m| m.ref_count).unwrap_or(1), - }; - - tracing::info!( - "🔗 Dedup upload: hash={}, new={}, saved={}", - result.hash(), - is_new, - bytes_saved - ); - - return Response::builder() - .status(if is_new { - StatusCode::CREATED - } else { - StatusCode::OK - }) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(serde_json::to_string(&response).unwrap())) - .unwrap() - .into_response(); - } - Err(e) => { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("Dedup upload failed: {}", e); - return Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(r#"{"error": "Upload failed"}"#)) - .unwrap() - .into_response(); - } - } + return Response::builder() + .status(if ingested.is_new_blob { + StatusCode::CREATED + } else { + StatusCode::OK + }) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&response).unwrap())) + .unwrap() + .into_response(); } } diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index d80cd494..903fd87e 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -21,6 +21,7 @@ use crate::common::di::AppState; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; use crate::interfaces::range_requests::not_modified_response; +use crate::interfaces::upload_ingest; use crate::{application::dtos::file_dto::FileDto, domain::services::authorization::Permission}; use std::sync::Arc; @@ -51,11 +52,12 @@ impl FileHandler { // UPLOAD // ═══════════════════════════════════════════════════════════════════════ - /// Streaming file upload — constant ~64 KB RAM regardless of file size. + /// Streaming file upload — bounded RAM regardless of file size. /// - /// **Hash-on-Write**: BLAKE3 is computed while spooling the multipart - /// body to the temp file. This eliminates the second sequential read - /// that dedup_service would otherwise need, cutting total I/O in half. + /// The multipart body is streamed straight into the CDC chunk store: + /// chunking, hashing and dedup checks happen while the bytes arrive. + /// No spool file, no re-read — chunks the store already has are never + /// written to disk at all. pub async fn upload_file( State(state): State, auth_user: AuthUser, @@ -71,7 +73,7 @@ impl FileHandler { /// [`Self::upload_file_with_thumbnails`]. /// /// Returns `(FileDto, blob_hash)` on success. The blob hash is the - /// BLAKE3 digest computed during the hash-on-write spool and is + /// BLAKE3 digest computed during the streaming ingest and is /// propagated without an extra database round-trip so that callers /// (e.g. thumbnail generation) can resolve the physical blob path /// immediately. @@ -158,120 +160,55 @@ impl FileHandler { } } - // ── Spool multipart field to temp file + hash-on-write ── - // .dedup_temp is created once by DedupService::initialize() at startup - let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp"); - let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4())); - - let mut total_size: u64 = 0; - let mut hasher = blake3::Hasher::new(); - let spool_result: Result<(), String> = async { - let file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| format!("Failed to create temp file: {}", e))?; - - // Pre-allocate if Content-Length is known (reduces fragmentation) - let hint = field - .headers() - .get(axum::http::header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - if let Some(len) = hint { - let _ = file.set_len(len).await; // best-effort - } - - // 512 KB buffer — 8× fewer write syscalls than 64 KB - let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); - let mut field = field; - // IMPORTANT: use explicit match instead of `while let Ok(Some(..))`. - // The old pattern silently swallowed Err (client disconnect) - // and accepted partially received data as a complete upload. - loop { - match field.chunk().await { - Ok(Some(chunk)) => { - total_size += chunk.len() as u64; - hasher.update(&chunk); - tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; - } - Ok(None) => break, // End of field — upload complete - Err(e) => { - return Err(format!( - "Connection lost during upload (received {} bytes): {}", - total_size, e - )); - } - } - } - tokio::io::AsyncWriteExt::flush(&mut writer) - .await - .map_err(|e| format!("Failed to flush temp file: {}", e))?; - Ok(()) - } - .await; - - if let Err(e) = spool_result { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("❌ UPLOAD SPOOL FAILED: {} - {}", filename, e); - return Err(Self::domain_error_response( - crate::common::errors::DomainError::internal_error("FileUpload", e), - )); - } - - // Empty file — use streaming path with the (empty) temp file - if total_size == 0 { - let hash = hasher.finalize().to_hex().to_string(); - let dto = upload_service - .upload_file_streaming( - filename, - folder_id, - content_type, - &temp_path, - 0, - Some(hash.clone()), - ) - .await - .map_err(Self::domain_error_response)?; - return Ok((dto, hash)); - } - - // Finalize hash - let hash = hasher.finalize().to_hex().to_string(); - - // ── MIME detection (magic bytes + extension fallback) ─ - let content_type = crate::common::mime_detect::refine_content_type_from_file( - &temp_path, + // ── Stream the field into the CDC chunk store ──────── + // Chunking (FastCDC) + hashing (BLAKE3) + dedup checks + + // MIME sniffing all happen while the bytes arrive; chunks + // the store already has never touch the disk. Size is + // capped globally by DefaultBodyLimit. + let dedup = &state.core.dedup_service; + let source = upload_ingest::multipart_field_stream(field); + let ingested = match upload_ingest::ingest_stream_to_cas( + source, + dedup, &filename, &content_type, + usize::MAX, + None, ) - .await; + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::error!("❌ UPLOAD INGEST FAILED: {} - {}", filename, e.message); + return Err(e.into_response()); + } + }; - // ── Quota enforcement ──────────────────────────────── + // ── Quota enforcement (exact size now known) ───────── if let Some(storage_svc) = state.storage_usage_service.as_ref() && let Err(err) = storage_svc - .check_storage_quota(auth_user.id, total_size) + .check_storage_quota(auth_user.id, ingested.size) .await { - let _ = tokio::fs::remove_file(&temp_path).await; + upload_ingest::discard_ingested(dedup, &ingested).await; tracing::warn!( "⛔ UPLOAD REJECTED (quota): user={}, file={}, size={}", auth_user.username, filename, - total_size + ingested.size ); return Err(Self::quota_error_response(err)); } - // ── Streaming upload (temp file → blob store, hash pre-computed) ─ + // ── Register the file row against the ingested blob ── + let hash = ingested.hash.clone(); + let size = ingested.size; match upload_service .upload_file_streaming( filename.clone(), folder_id, - content_type, - &temp_path, - total_size, - Some(hash.clone()), + ingested.content_type.clone(), + ingested.stored(), ) .await { @@ -279,13 +216,12 @@ impl FileHandler { tracing::info!( "✅ STREAMING UPLOAD: {} ({} bytes, ID: {})", filename, - total_size, + size, file.id ); return Ok((file, hash)); } Err(err) => { - let _ = tokio::fs::remove_file(&temp_path).await; tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err); return Err(Self::domain_error_response(err)); } diff --git a/src/interfaces/api/handlers/webdav_handler.rs b/src/interfaces/api/handlers/webdav_handler.rs index c0097eb3..4b8c2117 100644 --- a/src/interfaces/api/handlers/webdav_handler.rs +++ b/src/interfaces/api/handlers/webdav_handler.rs @@ -898,10 +898,11 @@ async fn handle_head( /** * Handles PUT requests to create or update files. * - * **Streaming implementation**: the request body is spooled to a temp file - * with incremental BLAKE3 hashing. Peak RAM usage is ~256 KB regardless - * of file size. The temp file is then atomically moved into blob storage - * via `update_file_streaming`. + * **Streaming implementation**: the request body is streamed straight into + * the CDC chunk store (FastCDC + BLAKE3 while the bytes arrive — no spool + * file, no re-read; peak RAM is bounded regardless of file size), then the + * file row is atomically swapped onto the ingested blob via + * `update_file_streaming`. * * @param state The application state containing service dependencies * @param path The requested resource path @@ -913,7 +914,7 @@ async fn handle_put( req: Request, path: String, ) -> Result, AppError> { - use crate::interfaces::upload_spool::spool_body_to_temp; + use crate::interfaces::upload_ingest; let user = extract_user(&req)?; @@ -972,32 +973,31 @@ async fn handle_put( .unwrap_or("application/octet-stream") .to_string(); - // ── Streaming spool: body → temp file + incremental hash ── - // Shared with the NextCloud-compat PUT handler; peak heap ~one frame - // regardless of file size. Honors `upload_temp_dir` to keep the spool - // off tmpfs/RAM. - let spooled = spool_body_to_temp( + // ── Streaming ingest: body → CDC chunk store ────────────── + // Shared with the NextCloud-compat PUT handler; chunking + hashing + + // dedup checks run while the body arrives — no spool file, no re-read. + let filename = crate::common::mime_detect::filename_from_path(&path).to_string(); + let ingested = upload_ingest::ingest_body_to_cas( req.into_body(), + &state.core.dedup_service, + &filename, + &content_type, max_upload, - state.core.config.storage.upload_temp_dir.clone(), ) .await?; - let temp_path = spooled.temp.path().to_path_buf(); - let total_bytes = spooled.size as usize; - let hash = spooled.hash; // ── Quota enforcement ──────────────────────────────────── if let Some(storage_svc) = state.storage_usage_service.as_ref() && let Err(err) = storage_svc - .check_storage_quota(user.id, total_bytes as u64) + .check_storage_quota(user.id, ingested.size) .await { - let _ = tokio::fs::remove_file(&temp_path).await; + upload_ingest::discard_ingested(&state.core.dedup_service, &ingested).await; tracing::warn!( "⛔ WEBDAV PUT REJECTED (quota): user={}, file={}, size={}", user.id, path, - total_bytes + ingested.size ); return Err(AppError::new( StatusCode::INSUFFICIENT_STORAGE, @@ -1006,21 +1006,12 @@ async fn handle_put( )); } - // ── Atomic store: temp file → dedup blob + DB metadata update ── + // ── Atomic store: swap the file row onto the ingested blob ── + let content_type = ingested.content_type.clone(); let result = file_upload_service - .update_file_streaming( - &path, - &temp_path, - total_bytes as u64, - &content_type, - Some(hash), - None, - ) + .update_file_streaming(&path, ingested.stored(), &content_type, None) .await; - // Clean up temp file (may already be moved by dedup, ignore error) - let _ = tokio::fs::remove_file(&temp_path).await; - match result { Ok(_file_dto) => Ok(Response::builder() .status(StatusCode::NO_CONTENT) diff --git a/src/interfaces/api/handlers/wopi_handler.rs b/src/interfaces/api/handlers/wopi_handler.rs index 80ed7b81..93e3320b 100644 --- a/src/interfaces/api/handlers/wopi_handler.rs +++ b/src/interfaces/api/handlers/wopi_handler.rs @@ -165,10 +165,6 @@ async fn put_file( State(state): State, req: Request, ) -> Response { - use http_body_util::BodyStream; - use tokio::io::AsyncWriteExt; - use tokio_stream::StreamExt; - let claims = match state .token_service .validate_token(&token_query.access_token) @@ -218,75 +214,32 @@ async fn put_file( Err(_) => return StatusCode::NOT_FOUND.into_response(), }; - // ── Streaming spool: body → temp file + incremental BLAKE3 ── - let temp_file = match tempfile::NamedTempFile::new() { - Ok(f) => f, - Err(e) => { - tracing::error!("WOPI PutFile: failed to create temp file: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - let temp_path = temp_file.path().to_path_buf(); - - let mut file_out = match tokio::fs::File::create(&temp_path).await { - Ok(f) => f, - Err(e) => { - tracing::error!("WOPI PutFile: failed to open temp file: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - + // ── Streaming ingest: body → CDC chunk store (no temp file) ── let content_type = file.mime_type.clone(); - let mut hasher = blake3::Hasher::new(); - let mut total_bytes: u64 = 0; - let mut stream = BodyStream::new(req.into_body()); - - while let Some(frame_result) = stream.next().await { - let frame = match frame_result { - Ok(f) => f, - Err(e) => { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: body read error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len() as u64; - hasher.update(chunk); - if let Err(e) = file_out.write_all(chunk).await { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: temp write error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } + let ingested = match crate::interfaces::upload_ingest::ingest_body_to_cas( + req.into_body(), + &state.app_state.core.dedup_service, + &file.name, + &content_type, + usize::MAX, + ) + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::error!("WOPI PutFile: ingest failed: {}", e.message); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - } - if let Err(e) = file_out.flush().await { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: flush error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - drop(file_out); + }; - let hash = hasher.finalize().to_hex().to_string(); - - // ── Atomic store: temp file → dedup blob + DB metadata update ── + // ── Atomic store: swap the file row onto the ingested blob ── let result = state .app_state .applications .file_upload_service - .update_file_streaming( - &file.path, - &temp_path, - total_bytes, - &content_type, - Some(hash), - None, - ) + .update_file_streaming(&file.path, ingested.stored(), &content_type, None) .await; - // Clean up temp file (may already be moved by dedup, ignore error) - let _ = tokio::fs::remove_file(&temp_path).await; - match result { Ok(_file_dto) => StatusCode::OK.into_response(), Err(e) => { diff --git a/src/interfaces/mod.rs b/src/interfaces/mod.rs index fef37883..f9c9f41f 100644 --- a/src/interfaces/mod.rs +++ b/src/interfaces/mod.rs @@ -3,7 +3,7 @@ pub mod errors; pub mod middleware; pub mod nextcloud; pub mod range_requests; -pub mod upload_spool; +pub mod upload_ingest; pub mod web; pub use api::create_api_routes; diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index 8bb6ca8f..16dd5b07 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -7,10 +7,12 @@ use std::sync::Arc; use crate::application::ports::file_ports::{FileRetrievalUseCase, FileUploadUseCase}; use crate::common::di::AppState; -use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; +use crate::common::mime_detect::filename_from_path; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; -use crate::interfaces::upload_spool::stream_body_to_path; +use crate::interfaces::upload_ingest::{ + discard_ingested, ingest_stream_to_cas, stream_body_to_path, stream_from_files, +}; /// Dispatch Nextcloud chunked upload WebDAV requests. /// @@ -239,18 +241,17 @@ async fn handle_assemble( let dest_subpath = extract_files_subpath(&destination, &user.username) .ok_or_else(|| AppError::bad_request("Invalid Destination URL"))?; - // Assemble chunks into a temp file with hash-on-write (BLAKE3 computed - // during the same read/write loop that copies chunks into the - // assembled file). The hash is passed downstream as `pre_computed_hash` - // so the dedup layer never re-reads the assembled file just to compute - // it — saves one full file-sized read pass per upload. - let (temp_path, size, blake3_hash) = nc + // Stream the chunk parts, in order, straight into the CDC chunk store — + // no assembled temp file is ever written. Chunking (FastCDC), BLAKE3 + // hashing, dedup checks and MIME sniffing (magic bytes off the first + // part) all happen in that single read pass. The parts stay on disk + // until the session cleanup below, so a failed completion is retryable. + let chunk_paths = nc .chunked_uploads - .assemble(&user.username, upload_id) + .ordered_chunk_paths(&user.username, upload_id) .await - .map_err(|e| AppError::internal_error(format!("Failed to assemble chunks: {}", e)))?; + .map_err(|e| AppError::internal_error(format!("Failed to list chunks: {}", e)))?; - // Write assembled file to storage via the upload service. let upload_service = &state.applications.file_upload_service; let file_service = &state.applications.file_retrieval_service; let folder_service = &state.applications.folder_service; @@ -261,35 +262,31 @@ async fn handle_assemble( dest_subpath.trim_matches('/') ); - // Detect content type via magic bytes + extension fallback. - let filename = filename_from_path(&dest_subpath); - let content_type = - refine_content_type_from_file(&temp_path, filename, "application/octet-stream").await; + let filename = filename_from_path(&dest_subpath).to_string(); + let ingested = ingest_stream_to_cas( + stream_from_files(chunk_paths), + &state.core.dedup_service, + &filename, + "application/octet-stream", + usize::MAX, + None, + ) + .await?; + let content_type = ingested.content_type.clone(); // Check if file exists (update vs create). let existing = file_service.get_file_by_path(&internal_path).await; let etag: Option = if existing.is_ok() { let dto = upload_service - .update_file_streaming( - &internal_path, - &temp_path, - size, - &content_type, - Some(blake3_hash.clone()), - oc_mtime, - ) + .update_file_streaming(&internal_path, ingested.stored(), &content_type, oc_mtime) .await .map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?; Some(dto.etag) } else { - // New-file branch: resolve the parent folder by path and pass the - // assembled file's path directly to `upload_file_from_path` so the - // bytes never get read back into RAM. Previously this branch did - // `tokio::fs::read(&temp_path)` — an extra full file-sized read - // pass AND a peak-RAM allocation equal to the upload size, which - // defeated the streaming model on large NC uploads. + // New-file branch: resolve the parent folder by path and register + // the file row against the already-ingested blob. let (parent_sub, filename) = match dest_subpath.rsplit_once('/') { Some((p, n)) => (p, n), None => ("", dest_subpath.as_str()), @@ -302,18 +299,23 @@ async fn handle_assemble( let parent_internal = parent_internal.trim_end_matches('/'); use crate::application::ports::folder_ports::FolderUseCase; - let parent_folder = folder_service - .get_folder_by_path(parent_internal) - .await - .map_err(|e| AppError::internal_error(format!("Parent folder lookup failed: {}", e)))?; + let parent_folder = match folder_service.get_folder_by_path(parent_internal).await { + Ok(folder) => folder, + Err(e) => { + discard_ingested(&state.core.dedup_service, &ingested).await; + return Err(AppError::internal_error(format!( + "Parent folder lookup failed: {}", + e + ))); + } + }; let dto = upload_service - .upload_file_from_path( + .upload_file_streaming( filename.to_string(), Some(parent_folder.id), content_type.to_string(), - &temp_path, - Some(blake3_hash), + ingested.stored(), ) .await .map_err(|e| AppError::internal_error(format!("Failed to create file: {}", e)))?; @@ -321,9 +323,6 @@ async fn handle_assemble( Some(dto.etag) }; - // Clean up temp file (session cleanup below removes the directory anyway). - let _ = tokio::fs::remove_file(&temp_path).await; - // Cleanup session. let _ = nc.chunked_uploads.cleanup(&user.username, upload_id).await; diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 8999fe04..e74aff48 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -22,12 +22,12 @@ use crate::application::ports::file_ports::{ use crate::application::ports::folder_ports::FolderUseCase; use crate::application::ports::trash_ports::TrashUseCase; use crate::common::di::AppState; -use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; +use crate::common::mime_detect::filename_from_path; use crate::interfaces::api::handlers::webdav_handler::PROPFIND_BATCH_SIZE; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; use crate::interfaces::range_requests::{not_modified_response, range_response}; -use crate::interfaces::upload_spool::spool_body_to_temp; +use crate::interfaces::upload_ingest::ingest_body_to_cas; /// Extension trait to map XML write errors to `String` concisely. trait XmlResultExt { @@ -575,46 +575,35 @@ async fn handle_put( // at 95 % loses everything. let max_upload = state.core.config.storage.direct_put_max_bytes; - // Stream the body to a temp file + incremental hash — never buffer the - // full upload in RAM. The old `body::to_bytes` path loaded the entire - // file (e.g. an 800 MB ISO) into anonymous memory before any dedup logic, - // OOMKilling the process even on dedup hits. Shared with the native - // WebDAV PUT handler; peak heap ~one frame regardless of file size. - let spooled = spool_body_to_temp( - req.into_body(), - max_upload, - state.core.config.storage.upload_temp_dir.clone(), - ) - .await?; - - // Detect real MIME type from the first bytes on disk (no full read). + // Stream the body straight into the CDC chunk store — never buffer the + // full upload in RAM and never spool it to disk. Chunking, hashing, + // dedup checks and MIME sniffing (magic bytes off the first frames) + // all run while the body arrives; chunks the store already has are + // never written at all. Shared with the native WebDAV PUT handler. + // // `filename` is owned so we don't hold a borrow of the `subpath` param // across the await (which would make the handler future non-Send). let filename = filename_from_path(subpath).to_string(); - let content_type = - refine_content_type_from_file(spooled.temp.path(), &filename, &claimed_type).await; + let ingested = ingest_body_to_cas( + req.into_body(), + &state.core.dedup_service, + &filename, + &claimed_type, + max_upload, + ) + .await?; + let content_type = ingested.content_type.clone(); // Distinguish create (201) vs update (204) for the response status. let existed = file_service.get_file_by_path(&internal_path).await.is_ok(); // Single streaming path — handles both update and create internally, - // passing the precomputed hash so the dedup fast path can short-circuit - // without re-reading the file. + // swapping the file row onto the already-ingested blob. let stored = upload_service - .update_file_streaming( - &internal_path, - spooled.temp.path(), - spooled.size, - &content_type, - Some(spooled.hash), - oc_mtime, - ) + .update_file_streaming(&internal_path, ingested.stored(), &content_type, oc_mtime) .await .map_err(|e| AppError::internal_error(format!("Failed to store file: {}", e)))?; - // dedup may have already moved the temp on a new-blob store; ignore error. - let _ = tokio::fs::remove_file(spooled.temp.path()).await; - let status = if existed { StatusCode::NO_CONTENT } else { diff --git a/src/interfaces/upload_ingest.rs b/src/interfaces/upload_ingest.rs new file mode 100644 index 00000000..c1ecf8a8 --- /dev/null +++ b/src/interfaces/upload_ingest.rs @@ -0,0 +1,588 @@ +//! Shared streaming upload ingestion: request body → CDC chunk store. +//! +//! Used by every upload surface (REST multipart, native WebDAV PUT, +//! NextCloud PUT, chunked-upload assembly, WOPI PutFile) so none of them +//! buffers the full body in RAM **or spools it to a temp file**. The bytes +//! flow straight into [`DedupService::store_from_stream`], which chunks +//! (FastCDC), hashes (BLAKE3) and dedup-checks them while they arrive — +//! each uploaded byte touches the disk at most once, and not at all when +//! the store already has its chunk. +//! +//! MIME refinement happens in-flight: when the claimed Content-Type is +//! generic, the first bytes are peeked off the stream for magic-byte +//! detection before being forwarded unchanged. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use axum::body::Body; +use bytes::Bytes; +use futures::stream::{self, Stream, StreamExt, TryStreamExt}; +use http_body_util::BodyStream; +// The `Digest` trait (re-exported by both `md5` and `sha2` from the +// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / +// `finalize` methods. Importing once via `sha2` covers both — +// otherwise every call site would need fully-qualified +// `::…` syntax. +use sha2::Digest as _; +use tokio::io::AsyncWriteExt; +use tokio_util::io::ReaderStream; + +use crate::application::ports::chunked_upload_ports::ChecksumAlg; +use crate::application::ports::file_ports::StoredBlob; +use crate::common::mime_detect::{MAGIC_BYTES_LEN, is_generic_mime, refine_content_type}; +use crate::infrastructure::services::dedup_service::DedupService; +use crate::interfaces::errors::AppError; + +/// Content stored in the chunk store by one upload ingest. +/// +/// The ingest holds ONE blob reference; pass [`IngestedBlob::stored`] to the +/// upload service (which takes ownership of the reference) or hand it back +/// via [`discard_ingested`] when the upload is rejected after the fact. +pub struct IngestedBlob { + /// BLAKE3 of the full content (the blob/manifest key). + pub hash: String, + /// Total bytes ingested. + pub size: u64, + /// Refined content type (claimed type or magic-byte detection). + pub content_type: String, + /// `false` when the exact content already existed (dedup hit). + pub is_new_blob: bool, + /// Bytes that did not need to be transferred to storage (dedup hit). + pub bytes_saved: u64, +} + +impl IngestedBlob { + /// The blob reference to hand to the upload service. + pub fn stored(&self) -> StoredBlob { + StoredBlob { + hash: self.hash.clone(), + size: self.size, + is_new_blob: self.is_new_blob, + } + } +} + +/// Hand back the blob reference taken by a successful ingest when the upload +/// is rejected after the fact (quota exceeded, checksum mismatch, …). +pub async fn discard_ingested(dedup: &DedupService, blob: &IngestedBlob) { + if let Err(e) = dedup.remove_reference(&blob.hash).await { + tracing::warn!( + "Failed to release blob reference of rejected upload {}: {e}", + &blob.hash[..blob.hash.len().min(12)] + ); + } +} + +/// Shared mutable tee for computing a client-requested checksum during the +/// ingest pass (REST chunked uploads) — no post-store re-read needed. +pub type ChecksumTee = Arc>>; + +/// Create a checksum tee for [`ingest_stream_to_cas`]. +pub fn checksum_tee(alg: ChecksumAlg) -> ChecksumTee { + Arc::new(StdMutex::new(Some(IncrementalHasher::new(alg)))) +} + +/// Finalize a checksum tee into its lowercase hex digest. +pub fn finalize_checksum_tee(tee: &ChecksumTee) -> Option { + tee.lock() + .ok() + .and_then(|mut h| h.take()) + .map(IncrementalHasher::finalize_hex) +} + +/// Out-of-band state observed by the stream adapters while the dedup engine +/// consumes the stream — lets the caller map an opaque engine error back to +/// the precise HTTP failure (413 vs 400). +struct IngestFlags { + too_large: AtomicBool, + source_error: StdMutex>, +} + +/// Stream a request body (or any byte stream) into the CDC chunk store. +/// +/// Single pass: size-cap enforcement, optional checksum tee, MIME sniffing +/// (first [`MAGIC_BYTES_LEN`] bytes, only when `claimed_type` is generic) +/// and the CDC chunk/hash/store pipeline all run while the bytes arrive. +/// Peak heap is bounded by the dedup engine (~9 MiB) regardless of size. +/// +/// On error nothing stays referenced — the engine compensates internally. +pub async fn ingest_stream_to_cas( + source: S, + dedup: &Arc, + filename: &str, + claimed_type: &str, + max_bytes: usize, + checksum: Option, +) -> Result +where + S: Stream> + Send, + E: std::fmt::Display, +{ + let flags = Arc::new(IngestFlags { + too_large: AtomicBool::new(false), + source_error: StdMutex::new(None), + }); + + // ── Adapter: cap + checksum tee + error capture ────────────── + let adapter_flags = flags.clone(); + let mut total: usize = 0; + let counted = source.map(move |item| match item { + Ok(bytes) => { + total += bytes.len(); + if total > max_bytes { + adapter_flags.too_large.store(true, Ordering::Relaxed); + return Err(std::io::Error::other("upload exceeds size cap")); + } + if let Some(tee) = &checksum + && let Ok(mut hasher) = tee.lock() + && let Some(hasher) = hasher.as_mut() + { + hasher.update(&bytes); + } + Ok(bytes) + } + Err(e) => { + let message = e.to_string(); + if let Ok(mut slot) = adapter_flags.source_error.lock() { + *slot = Some(message.clone()); + } + Err(std::io::Error::other(message)) + } + }); + // `fuse` is load-bearing: when the source is shorter than the MIME peek + // (< MAGIC_BYTES_LEN), the peek loop drains it to None and the `chain` + // below polls it once more — non-fused sources (e.g. `stream::unfold`, + // as used for multipart fields) panic on a post-None poll. + let mut counted = Box::pin(counted.fuse()); + + // ── In-flight MIME sniff (only when the claimed type is generic) ── + let mut head: Vec> = Vec::new(); + let content_type = if is_generic_mime(claimed_type) { + let mut head_len = 0usize; + while head_len < MAGIC_BYTES_LEN { + match counted.next().await { + Some(Ok(bytes)) => { + head_len += bytes.len(); + head.push(Ok(bytes)); + } + Some(Err(e)) => { + head.push(Err(e)); + break; + } + None => break, + } + } + let mut magic = Vec::with_capacity(head_len.min(MAGIC_BYTES_LEN)); + for item in head.iter().flatten() { + let take = (MAGIC_BYTES_LEN - magic.len()).min(item.len()); + magic.extend_from_slice(&item[..take]); + if magic.len() >= MAGIC_BYTES_LEN { + break; + } + } + refine_content_type(&magic, filename, claimed_type) + } else { + claimed_type.to_string() + }; + + // ── Store: peeked head + remainder, one continuous stream ──── + let full_stream = stream::iter(head).chain(counted); + let result = dedup + .store_from_stream(full_stream, Some(content_type.clone())) + .await; + + match result { + Ok(stored) => { + let is_new_blob = !stored.was_deduplicated(); + let bytes_saved = match &stored { + crate::application::ports::dedup_ports::DedupResultDto::ExistingBlob { + saved_bytes, + .. + } => *saved_bytes, + _ => 0, + }; + Ok(IngestedBlob { + hash: stored.hash().to_string(), + size: stored.size(), + content_type, + is_new_blob, + bytes_saved, + }) + } + Err(e) => { + if flags.too_large.load(Ordering::Relaxed) { + return Err(AppError::payload_too_large(format!( + "Upload body exceeds the direct-PUT cap ({max_bytes} bytes). \ + Use the chunked-upload protocol (REST: `/api/uploads/...`, \ + NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ + Chunked uploads are resumable on transient failure." + ))); + } + let source_error = flags.source_error.lock().ok().and_then(|s| s.clone()); + if let Some(message) = source_error { + return Err(AppError::bad_request(format!( + "Failed to read request body: {message}" + ))); + } + Err(AppError::from(e)) + } + } +} + +/// [`ingest_stream_to_cas`] for an HTTP request body. +pub async fn ingest_body_to_cas( + body: Body, + dedup: &Arc, + filename: &str, + claimed_type: &str, + max_bytes: usize, +) -> Result { + let source = BodyStream::new(body).filter_map(|item| async move { + match item { + Ok(frame) => frame.into_data().ok().map(Ok), + Err(e) => Some(Err(e)), + } + }); + ingest_stream_to_cas(source, dedup, filename, claimed_type, max_bytes, None).await +} + +/// Adapt a multipart field into a byte stream for [`ingest_stream_to_cas`]. +/// +/// Terminates after the first error — multipart fields are not resumable. +pub fn multipart_field_stream( + field: axum::extract::multipart::Field<'_>, +) -> impl Stream> + Send + '_ { + stream::unfold((field, false), |(mut field, done)| async move { + if done { + return None; + } + match field.chunk().await { + Ok(Some(bytes)) => Some((Ok(bytes), (field, false))), + Ok(None) => None, + Err(e) => Some((Err(e), (field, true))), + } + }) +} + +/// Concatenate already-uploaded chunk part files into one byte stream, in +/// the given order — feeds chunked-upload assembly into the CDC store +/// without ever materializing an assembled file on disk. +pub fn stream_from_files( + paths: Vec, +) -> impl Stream> + Send { + stream::iter(paths.into_iter().map(Ok::<_, std::io::Error>)) + .and_then(|path| async move { + tokio::fs::File::open(path) + .await + .map(|file| ReaderStream::with_capacity(file, 64 * 1024)) + }) + .try_flatten() +} + +/// Result of a streamed write to a caller-supplied path. +pub struct StreamedToPath { + /// Total bytes written. + pub bytes_written: u64, + /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` + /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. + pub checksum_hex: Option, + /// Algorithm used to compute `checksum_hex`. Echoed back so the + /// caller can include it in audit logs or response headers. + pub alg: Option, +} + +/// Stream an HTTP request body directly to a known destination file, +/// enforcing `max_bytes` as a hard size limit. +/// +/// Used by the chunked-upload PUT handlers — each chunk has a +/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` +/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the +/// REST surface), so there's no spool/move dance. Peak heap is ~one +/// HTTP frame regardless of chunk size or `max_bytes`. +/// +/// `checksum_alg` is the optional client-requested integrity check +/// (default `md5` per the legacy `Content-MD5` contract; `blake3` +/// available for forward-compat). When `Some`, the hash is computed +/// incrementally during streaming — no extra disk read for verification. +/// +/// On size overflow the partial file is removed before the function +/// returns, so a client retry against the same chunk name starts from +/// a clean slate. On any other I/O error the partial file is also +/// removed and the error surfaces — callers can assume the path is +/// either fully written or absent. +pub async fn stream_body_to_path( + body: Body, + path: &Path, + max_bytes: usize, + checksum_alg: Option, +) -> Result { + let mut file = tokio::fs::File::create(path) + .await + .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; + + let mut total_bytes: usize = 0; + let mut stream = BodyStream::new(body); + let mut hasher = checksum_alg.map(IncrementalHasher::new); + + while let Some(frame_result) = stream.next().await { + let frame = match frame_result { + Ok(f) => f, + Err(e) => { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::bad_request(format!( + "Failed to read request body: {e}" + ))); + } + }; + if let Some(chunk) = frame.data_ref() { + total_bytes += chunk.len(); + if total_bytes > max_bytes { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::payload_too_large(format!( + "Chunk exceeds maximum size of {max_bytes} bytes" + ))); + } + if let Some(h) = hasher.as_mut() { + h.update(chunk); + } + if let Err(e) = file.write_all(chunk).await { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::internal_error(format!( + "Failed to write chunk: {e}" + ))); + } + } + } + file.flush() + .await + .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; + drop(file); + + Ok(StreamedToPath { + bytes_written: total_bytes as u64, + checksum_hex: hasher.map(IncrementalHasher::finalize_hex), + alg: checksum_alg, + }) +} + +/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`] +/// and the [`ChecksumTee`] of chunked-upload completion. +/// Per-frame `update` is sub-millisecond for all three algorithms at the +/// 64 KB frame sizes axum's body stream produces, so we don't need +/// `spawn_blocking` (which the old buffered path used because it hashed +/// the full multi-MB chunk in one shot). +pub enum IncrementalHasher { + Md5(md5::Md5), + Sha256(sha2::Sha256), + // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 + // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the + // outlier keeps the enum size proportional to the common case + // rather than the worst case. + Blake3(Box), +} + +impl IncrementalHasher { + fn new(alg: ChecksumAlg) -> Self { + match alg { + ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), + ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), + ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), + } + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Self::Md5(h) => h.update(bytes), + Self::Sha256(h) => h.update(bytes), + Self::Blake3(h) => { + h.update(bytes); + } + } + } + + fn finalize_hex(self) -> String { + match self { + Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Blake3(h) => h.finalize().to_hex().to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + + #[tokio::test] + async fn stream_body_to_path_caps_oversized() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + // 5 MiB body, 4 MiB cap → must reject. + let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + assert!( + result.is_err(), + "expected PayloadTooLarge, got Ok(bytes_written={})", + result.ok().map(|r| r.bytes_written).unwrap_or(0) + ); + // Partial file must be removed on rejection. + assert!( + !path.exists(), + "rejected chunk file should be removed, but {} still exists", + path.display() + ); + } + + #[tokio::test] + async fn stream_body_to_path_accepts_under_cap() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + let outcome = result.expect("should succeed"); + assert_eq!(outcome.bytes_written, 1024 * 1024); + assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); + assert!(path.exists()); + } + + #[tokio::test] + async fn stream_body_to_path_caps_at_exact_boundary() { + // Edge case: body exactly equal to cap should succeed; cap+1 must fail. + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 100])); + let outcome = stream_body_to_path(body, &path, 100, None) + .await + .expect("100 bytes at 100-byte cap should succeed"); + assert_eq!(outcome.bytes_written, 100); + + let path2 = temp_dir.path().join("chunk2"); + let body = Body::from(Bytes::from(vec![1u8; 101])); + assert!( + stream_body_to_path(body, &path2, 100, None).await.is_err(), + "101 bytes at 100-byte cap must reject" + ); + assert!(!path2.exists()); + } + + #[tokio::test] + async fn ingest_rejects_oversized_before_touching_storage() { + // 1 KiB body against a 100-byte cap: the adapter must abort the + // stream before any flush, so the stub dedup service (which cannot + // reach PG) is never asked to settle a batch. + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from(vec![0u8; 1024]))]); + + let result = ingest_stream_to_cas( + source, + &dedup, + "file.bin", + "application/octet-stream", + 100, + None, + ) + .await; + + let err = result.err().expect("oversized body must be rejected"); + assert_eq!(err.status_code, axum::http::StatusCode::PAYLOAD_TOO_LARGE); + } + + #[tokio::test] + async fn ingest_surfaces_source_errors_as_bad_request() { + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::iter(vec![ + Ok::<_, std::io::Error>(Bytes::from_static(b"partial")), + Err(std::io::Error::other("connection reset by peer")), + ]); + + let result = + ingest_stream_to_cas(source, &dedup, "file.bin", "text/plain", usize::MAX, None).await; + + let err = result.err().expect("source error must surface"); + assert_eq!(err.status_code, axum::http::StatusCode::BAD_REQUEST); + assert!( + err.message.contains("connection reset by peer"), + "original cause must be preserved: {}", + err.message + ); + } + + /// Regression: a source shorter than the MIME peek (< MAGIC_BYTES_LEN) + /// is drained to None during sniffing and then polled once more by the + /// `chain` that re-attaches the peeked head. Non-fused sources — like + /// the `stream::unfold` used for multipart fields — panic on that + /// post-None poll ("Unfold must not be polled after it returned + /// `Poll::Ready(None)`") unless the ingest fuses the stream first. + /// The stub dedup service can't reach PG, so an orderly `Err` (not a + /// panic) proves the stream layer survived. + #[tokio::test] + async fn ingest_short_stream_with_generic_mime_does_not_repoll_source() { + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::unfold(false, |done| async move { + if done { + None + } else { + Some((Ok::<_, std::io::Error>(Bytes::from_static(b"tiny")), true)) + } + }); + + let result = ingest_stream_to_cas( + source, + &dedup, + "tiny.bin", + "application/octet-stream", + usize::MAX, + None, + ) + .await; + + assert!( + result.is_err(), + "stub DB must reject the store — but only AFTER the stream \ + layer survived the post-peek poll" + ); + } + + #[tokio::test] + async fn stream_from_files_concatenates_in_order() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let a = temp_dir.path().join("a"); + let b = temp_dir.path().join("b"); + tokio::fs::write(&a, b"Hello, ").await.unwrap(); + tokio::fs::write(&b, b"World!").await.unwrap(); + + let mut out = Vec::new(); + let s = stream_from_files(vec![a, b]); + futures::pin_mut!(s); + while let Some(chunk) = s.next().await { + out.extend_from_slice(&chunk.expect("read")); + } + assert_eq!(out, b"Hello, World!"); + } + + #[tokio::test] + async fn checksum_tee_roundtrip() { + let tee = checksum_tee(ChecksumAlg::Md5); + if let Ok(mut h) = tee.lock() + && let Some(h) = h.as_mut() + { + h.update(b"hello world"); + } + let hex = finalize_checksum_tee(&tee).expect("digest"); + assert_eq!(hex, "5eb63bbbe01eeed093cb22bb8f5acdc3"); + assert!( + finalize_checksum_tee(&tee).is_none(), + "second finalize returns None" + ); + } +} diff --git a/src/interfaces/upload_spool.rs b/src/interfaces/upload_spool.rs deleted file mode 100644 index 7ac2a56a..00000000 --- a/src/interfaces/upload_spool.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! Shared streaming upload spool: request body → temp file + incremental hash. -//! -//! Used by both the native WebDAV PUT handler and the NextCloud-compat PUT -//! handler so neither buffers the full request body in memory. Peak heap is -//! ~one HTTP frame regardless of file size; the body is written to a temp -//! file (off tmpfs when [`StorageConfig::upload_temp_dir`] is configured) and -//! BLAKE3-hashed on the fly so the dedup layer can short-circuit on a hit. - -use std::path::{Path, PathBuf}; - -use axum::body::Body; -use http_body_util::BodyStream; -// The `Digest` trait (re-exported by both `md5` and `sha2` from the -// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / -// `finalize` methods. Importing once via `sha2` covers both — -// otherwise every call site would need fully-qualified -// `::…` syntax. -use sha2::Digest as _; -use tempfile::NamedTempFile; -use tokio::io::AsyncWriteExt; -use tokio_stream::StreamExt; - -use crate::application::ports::chunked_upload_ports::ChecksumAlg; -use crate::common::temp::new_spool_temp_file; -use crate::interfaces::errors::AppError; - -/// Outcome of spooling a request body to disk. -pub struct SpooledBody { - /// The temp file holding the body. Kept alive by the caller (dropping it - /// removes the file unless the dedup layer already consumed/moved it). - pub temp: NamedTempFile, - /// Hex-encoded BLAKE3 of the full body — matches `DedupService::hash_file`, - /// so passing it as `pre_computed_hash` enables the dedup fast path. - pub hash: String, - /// Total bytes written. - pub size: u64, -} - -/// Stream an HTTP request body to a temp file, computing its BLAKE3 hash -/// incrementally and enforcing `max_upload` as a hard size limit. -/// -/// Peak heap is ~one frame — the body is never fully buffered in RAM. -/// -/// `temp_dir` is taken by value (not `&Path`) so the returned future captures -/// no borrowed lifetime — required for the handler future to stay `Send`. -pub async fn spool_body_to_temp( - body: Body, - max_upload: usize, - temp_dir: Option, -) -> Result { - let temp = new_spool_temp_file(temp_dir.as_deref()) - .map_err(|e| AppError::internal_error(format!("Failed to create temp file: {e}")))?; - let temp_path = temp.path().to_path_buf(); - - let mut file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| AppError::internal_error(format!("Failed to open temp file: {e}")))?; - - let mut hasher = blake3::Hasher::new(); - let mut total_bytes: usize = 0; - let mut stream = BodyStream::new(body); - - while let Some(frame_result) = stream.next().await { - let frame = frame_result - .map_err(|e| AppError::bad_request(format!("Failed to read request body: {e}")))?; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len(); - if total_bytes > max_upload { - // Abort early — stop reading, delete temp file. - drop(file); - let _ = tokio::fs::remove_file(&temp_path).await; - return Err(AppError::payload_too_large(format!( - "Upload body exceeds the direct-PUT cap ({max_upload} bytes). \ - Use the chunked-upload protocol (REST: `/api/uploads/...`, \ - NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ - Chunked uploads are resumable on transient failure." - ))); - } - hasher.update(chunk); - file.write_all(chunk).await.map_err(|e| { - AppError::internal_error(format!("Failed to write to temp file: {e}")) - })?; - } - } - file.flush() - .await - .map_err(|e| AppError::internal_error(format!("Failed to flush temp file: {e}")))?; - drop(file); - - let hash = hasher.finalize().to_hex().to_string(); - Ok(SpooledBody { - temp, - hash, - size: total_bytes as u64, - }) -} - -/// Result of a streamed write to a caller-supplied path. -pub struct StreamedToPath { - /// Total bytes written. - pub bytes_written: u64, - /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` - /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. - pub checksum_hex: Option, - /// Algorithm used to compute `checksum_hex`. Echoed back so the - /// caller can include it in audit logs or response headers. - pub alg: Option, -} - -/// Stream an HTTP request body directly to a known destination file, -/// enforcing `max_bytes` as a hard size limit. -/// -/// Used by the chunked-upload PUT handlers — each chunk has a -/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` -/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the -/// REST surface), so there's no spool/move dance. Peak heap is ~one -/// HTTP frame regardless of chunk size or `max_bytes`. -/// -/// `checksum_alg` is the optional client-requested integrity check -/// (default `md5` per the legacy `Content-MD5` contract; `blake3` -/// available for forward-compat). When `Some`, the hash is computed -/// incrementally during streaming — no extra disk read for verification. -/// -/// On size overflow the partial file is removed before the function -/// returns, so a client retry against the same chunk name starts from -/// a clean slate. On any other I/O error the partial file is also -/// removed and the error surfaces — callers can assume the path is -/// either fully written or absent. -pub async fn stream_body_to_path( - body: Body, - path: &Path, - max_bytes: usize, - checksum_alg: Option, -) -> Result { - let mut file = tokio::fs::File::create(path) - .await - .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; - - let mut total_bytes: usize = 0; - let mut stream = BodyStream::new(body); - let mut hasher = checksum_alg.map(IncrementalHasher::new); - - while let Some(frame_result) = stream.next().await { - let frame = match frame_result { - Ok(f) => f, - Err(e) => { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::bad_request(format!( - "Failed to read request body: {e}" - ))); - } - }; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len(); - if total_bytes > max_bytes { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::payload_too_large(format!( - "Chunk exceeds maximum size of {max_bytes} bytes" - ))); - } - if let Some(h) = hasher.as_mut() { - h.update(chunk); - } - if let Err(e) = file.write_all(chunk).await { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::internal_error(format!( - "Failed to write chunk: {e}" - ))); - } - } - } - file.flush() - .await - .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; - drop(file); - - Ok(StreamedToPath { - bytes_written: total_bytes as u64, - checksum_hex: hasher.map(IncrementalHasher::finalize_hex), - alg: checksum_alg, - }) -} - -/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`]. -/// Per-frame `update` is sub-millisecond for all three algorithms at the -/// 64 KB frame sizes axum's body stream produces, so we don't need -/// `spawn_blocking` (which the old buffered path used because it hashed -/// the full multi-MB chunk in one shot). -enum IncrementalHasher { - Md5(md5::Md5), - Sha256(sha2::Sha256), - // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 - // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the - // outlier keeps the enum size proportional to the common case - // rather than the worst case. - Blake3(Box), -} - -impl IncrementalHasher { - fn new(alg: ChecksumAlg) -> Self { - match alg { - ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), - ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), - ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), - } - } - - fn update(&mut self, bytes: &[u8]) { - match self { - Self::Md5(h) => h.update(bytes), - Self::Sha256(h) => h.update(bytes), - Self::Blake3(h) => { - h.update(bytes); - } - } - } - - fn finalize_hex(self) -> String { - match self { - Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), - Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), - Self::Blake3(h) => h.finalize().to_hex().to_string(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - - #[tokio::test] - async fn stream_body_to_path_caps_oversized() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - // 5 MiB body, 4 MiB cap → must reject. - let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); - let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; - assert!( - result.is_err(), - "expected PayloadTooLarge, got Ok(bytes_written={})", - result.ok().map(|r| r.bytes_written).unwrap_or(0) - ); - // Partial file must be removed on rejection. - assert!( - !path.exists(), - "rejected chunk file should be removed, but {} still exists", - path.display() - ); - } - - #[tokio::test] - async fn stream_body_to_path_accepts_under_cap() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB - let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; - let outcome = result.expect("should succeed"); - assert_eq!(outcome.bytes_written, 1024 * 1024); - assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); - assert!(path.exists()); - } - - #[tokio::test] - async fn stream_body_to_path_caps_at_exact_boundary() { - // Edge case: body exactly equal to cap should succeed; cap+1 must fail. - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - let body = Body::from(Bytes::from(vec![1u8; 100])); - let outcome = stream_body_to_path(body, &path, 100, None) - .await - .expect("100 bytes at 100-byte cap should succeed"); - assert_eq!(outcome.bytes_written, 100); - - let path2 = temp_dir.path().join("chunk2"); - let body = Body::from(Bytes::from(vec![1u8; 101])); - assert!( - stream_body_to_path(body, &path2, 100, None).await.is_err(), - "101 bytes at 100-byte cap must reject" - ); - assert!(!path2.exists()); - } -}