perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate

Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change
gated by a before/after in examples/bench_round2.rs — an AFTER that did
not beat its BEFORE was to be rolled back; none needed it):

- Range requests (REST/DAV/shares) answered from the moka content cache
  for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice.
  256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us).
- Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales
  with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster).
  Content-Length dropped (size unknown up front).
- NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM
  per-session counter (lazy rebuild on cold start). 1,000-chunk upload
  gate cost: 33.1s -> 0.09s cumulative.
- Delta download + commit-verify now use the CDC path's
  buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open
  latency 440ms -> 51ms; order preserved.
- CDC ingest settles batches on a spawned task (depth-1 pipeline) so
  the source stream keeps flowing during PG pin + backend writes;
  rollback ledger shared + lock-serialized so compensation stays exact
  on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s.
  OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch).
- Frontend: instant-upload BLAKE3 hashing moved off the main thread to
  a bounded Web Worker pool (File handles by reference); vitest gate
  asserts the pool beats sequential (first gate draft posting buffers
  was 2.6x slower and was rewritten — copies dominated).

Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544
integration tests green; 270 frontend tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w

src/infrastructure/services/dedup_service.rs

@@ -112,13 +112,33 @@ impl ChunkIngestOutcome {
 /// mid-stream — a client disconnect aborts the whole handler future — the
 /// guard spawns a rollback so pinned chunks don't leak references forever and
 /// written files become GC-collectible rows instead of invisible orphans.
-struct IngestGuard {
-    pool: Arc<PgPool>,
-    backend: Arc<dyn BlobStorageBackend>,
+/// Whether the ingest loop overlaps batch settling with source reading
+/// (default on). `OXICLOUD_INGEST_OVERLAP=0` restores the old inline
+/// behaviour — kept as a bench/ops escape hatch.
+fn ingest_overlap_enabled() -> bool {
+    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ENABLED.get_or_init(|| {
+        std::env::var("OXICLOUD_INGEST_OVERLAP").map_or(true, |v| v != "0" && v != "false")
+    })
+}
+
+/// Compensation ledger of one ingest session. Shared (`Arc<tokio::Mutex>`)
+/// between the ingest loop and the overlapped batch-settle task: the settler
+/// holds the lock for the whole batch and records progressively, so a
+/// rollback (explicit or Drop-spawned) that acquires the lock is guaranteed
+/// to observe every pin/write the in-flight settle made.
+#[derive(Default)]
+struct IngestState {
     /// Pre-existing chunks whose ref_count this session bumped (distinct).
     pinned: Vec<String>,
     /// Chunks written to the backend but not yet registered: (hash, size).
     written: Vec<(String, i64)>,
+}
+
+struct IngestGuard {
+    pool: Arc<PgPool>,
+    backend: Arc<dyn BlobStorageBackend>,
+    state: Arc<tokio::sync::Mutex<IngestState>>,
     armed: bool,
 }
 
@@ -127,8 +147,7 @@ impl IngestGuard {
         Self {
             pool,
             backend,
-            pinned: Vec::new(),
-            written: Vec::new(),
+            state: Arc::new(tokio::sync::Mutex::new(IngestState::default())),
             armed: true,
         }
     }
@@ -143,8 +162,15 @@ impl IngestGuard {
     /// spawned Drop path).
     async fn rollback(mut self) {
         self.armed = false;
-        let pinned = std::mem::take(&mut self.pinned);
-        let written = std::mem::take(&mut self.written);
+        // Lock acquisition serializes after any in-flight batch settle, so
+        // its pins/writes are visible here.
+        let (pinned, written) = {
+            let mut st = self.state.lock().await;
+            (
+                std::mem::take(&mut st.pinned),
+                std::mem::take(&mut st.written),
+            )
+        };
         Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await;
     }
 
@@ -211,24 +237,33 @@ impl IngestGuard {
 
 impl Drop for IngestGuard {
     fn drop(&mut self) {
-        if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) {
+        if !self.armed {
             return;
         }
-        let pinned = std::mem::take(&mut self.pinned);
-        let written = std::mem::take(&mut self.written);
+        // The rollback task locks the shared state first, so it naturally
+        // waits out an in-flight batch settle and observes its recordings.
+        let state = self.state.clone();
         match tokio::runtime::Handle::try_current() {
             Ok(handle) => {
                 let pool = self.pool.clone();
                 let backend = self.backend.clone();
                 handle.spawn(async move {
+                    let (pinned, written) = {
+                        let mut st = state.lock().await;
+                        (
+                            std::mem::take(&mut st.pinned),
+                            std::mem::take(&mut st.written),
+                        )
+                    };
+                    if pinned.is_empty() && written.is_empty() {
+                        return;
+                    }
                     Self::run_rollback(pool, backend, pinned, written).await;
                 });
             }
             Err(_) => tracing::warn!(
-                "Ingest guard dropped outside a runtime: {} pins / {} written chunks \
+                "Ingest guard dropped outside a runtime: any pins / written chunks \
                  stay leaked until the next GC sweep",
-                pinned.len(),
-                written.len()
             ),
         }
     }
@@ -628,6 +663,13 @@ impl DedupService {
         .map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}")))
     }
 
+    /// Read-ahead depth the backend recommends for multi-chunk drains
+    /// (1 local, 8 for request-latency-bound object stores) — see
+    /// `BlobStorageBackend::read_prefetch` and benches/BLOB-PREFETCH.md.
+    pub fn read_prefetch(&self) -> usize {
+        self.backend.read_prefetch()
+    }
+
     /// Stream one chunk's raw bytes from the backend. The caller is
     /// responsible for entitlement (see [`claimable_chunks`]).
     pub async fn chunk_stream(
@@ -876,8 +918,28 @@ impl DedupService {
         let mut hasher = blake3::Hasher::new();
         let mut head: Vec<u8> = Vec::with_capacity(sniff_len.min(16 * 1024));
 
-        for (hash, declared_size) in chunks {
-            let mut stream = self.backend.get_blob_stream(hash).await?;
+        // Overlap the NEXT chunk's open with the current chunk's hash+drain
+        // — the same `buffered(read_prefetch)` combinator as the download
+        // path (benches/BLOB-PREFETCH.md measured +7-12 % on local disk;
+        // request-latency-bound object stores gain far more). Hashing stays
+        // strictly in manifest order: `buffered` yields in input order.
+        let prefetch = self.backend.read_prefetch().max(1);
+        let backend = self.backend.clone();
+        let mut opened = futures::stream::iter(chunks.iter().cloned())
+            .map(move |(hash, declared_size)| {
+                let backend = backend.clone();
+                async move {
+                    backend
+                        .get_blob_stream(&hash)
+                        .await
+                        .map(|s| (hash, declared_size, s))
+                }
+            })
+            .buffered(prefetch);
+
+        while let Some(next) = opened.next().await {
+            let (hash, declared_size, mut stream) = next?;
+            let (hash, declared_size) = (&hash, &declared_size);
             let mut actual: u64 = 0;
             while let Some(part) = stream.next().await {
                 let part = part.map_err(|e| {
@@ -954,7 +1016,7 @@ impl DedupService {
     where
         S: Stream<Item = Result<Bytes, std::io::Error>> + Send,
     {
-        let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone());
+        let guard = IngestGuard::new(self.pool.clone(), self.backend.clone());
 
         let reader = StreamReader::new(Box::pin(source));
         let mut chunker = fastcdc::v2020::AsyncStreamCDC::new(
@@ -973,11 +1035,35 @@ impl DedupService {
         let mut session_seen: HashSet<String> = HashSet::new();
         let mut pending: Vec<(String, Bytes)> = Vec::new();
         let mut pending_bytes: usize = 0;
+        // Depth-1 settle pipeline: batch N settles on a spawned task while
+        // the loop keeps reading/chunking/hashing batch N+1 from the source
+        // — the inline shape froze the reader (and the client's socket) for
+        // every settle (benches/INGEST-OVERLAP.md). The task records into
+        // the guard's shared state under its lock, so rollback stays exact
+        // even if this future is dropped mid-settle.
+        let mut in_flight: Option<tokio::task::JoinHandle<Result<(), DomainError>>> = None;
+
+        /// Await the previous batch's settle, mapping panics/aborts to a
+        /// domain error so both are compensated identically.
+        async fn join_settle(
+            handle: tokio::task::JoinHandle<Result<(), DomainError>>,
+        ) -> Result<(), DomainError> {
+            match handle.await {
+                Ok(res) => res,
+                Err(e) => Err(DomainError::internal_error(
+                    "Dedup",
+                    format!("Chunk settle task failed: {e}"),
+                )),
+            }
+        }
 
         while let Some(item) = chunk_stream.next().await {
             let chunk = match item {
                 Ok(chunk) => chunk,
                 Err(e) => {
+                    if let Some(handle) = in_flight.take() {
+                        let _ = join_settle(handle).await;
+                    }
                     guard.rollback().await;
                     return Err(DomainError::internal_error(
                         "Dedup",
@@ -1000,7 +1086,26 @@ impl DedupService {
                 pending.push((hash, Bytes::from(data)));
                 if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES
                 {
-                    if let Err(e) = self.flush_pending(&mut guard, &mut pending).await {
+                    if let Some(handle) = in_flight.take()
+                        && let Err(e) = join_settle(handle).await
+                    {
+                        guard.rollback().await;
+                        return Err(e);
+                    }
+                    let batch = std::mem::take(&mut pending);
+                    let handle = tokio::spawn(Self::settle_batch(
+                        self.pool.clone(),
+                        self.backend.clone(),
+                        guard.state.clone(),
+                        batch,
+                    ));
+                    // Bench/ops escape hatch: OXICLOUD_INGEST_OVERLAP=0
+                    // reproduces the old inline-settle behaviour (await the
+                    // batch before reading on) — used by
+                    // benches/INGEST-OVERLAP.md for an in-binary A/B.
+                    if ingest_overlap_enabled() {
+                        in_flight = Some(handle);
+                    } else if let Err(e) = join_settle(handle).await {
                         guard.rollback().await;
                         return Err(e);
                     }
@@ -1009,7 +1114,20 @@ impl DedupService {
             }
         }
 
-        if let Err(e) = self.flush_pending(&mut guard, &mut pending).await {
+        if let Some(handle) = in_flight.take()
+            && let Err(e) = join_settle(handle).await
+        {
+            guard.rollback().await;
+            return Err(e);
+        }
+        if let Err(e) = Self::settle_batch(
+            self.pool.clone(),
+            self.backend.clone(),
+            guard.state.clone(),
+            std::mem::take(&mut pending),
+        )
+        .await
+        {
             guard.rollback().await;
             return Err(e);
         }
@@ -1018,10 +1136,15 @@ impl DedupService {
         // One batched fsync sweep (no-op for remote backends, durable on
         // PUT), then one batched INSERT. A crash before the INSERT leaves
         // only unreferenced files; never a row pointing at unsynced bytes.
-        if !guard.written.is_empty() {
-            let new_hashes: Vec<String> = guard.written.iter().map(|(h, _)| h.clone()).collect();
-            let new_sizes: Vec<i64> = guard.written.iter().map(|(_, s)| *s).collect();
-
+        // No settle is in flight past this point — the lock is uncontended.
+        let (new_hashes, new_sizes): (Vec<String>, Vec<i64>) = {
+            let st = guard.state.lock().await;
+            (
+                st.written.iter().map(|(h, _)| h.clone()).collect(),
+                st.written.iter().map(|(_, s)| *s).collect(),
+            )
+        };
+        if !new_hashes.is_empty() {
             if let Err(e) = self.backend.sync_blobs(&new_hashes).await {
                 guard.rollback().await;
                 return Err(e);
@@ -1047,7 +1170,7 @@ impl DedupService {
             }
         }
 
-        let newly_written = guard.written.len();
+        let newly_written = new_hashes.len();
         guard.disarm();
 
         Ok(ChunkIngestOutcome {
@@ -1061,18 +1184,23 @@ impl DedupService {
 
     /// Settle one batch of distinct in-RAM chunks against PG + the backend.
     ///
-    /// Successfully pinned hashes and written chunks are recorded on the
-    /// guard as they happen, so a failure mid-batch leaves nothing
-    /// untracked for rollback.
-    async fn flush_pending(
-        &self,
-        guard: &mut IngestGuard,
-        pending: &mut Vec<(String, Bytes)>,
+    /// Static (no `&self`) so the ingest loop can run it on a spawned task
+    /// and keep consuming the source stream while the batch settles — the
+    /// inline shape stalled the reader for the whole settle every 8 MiB
+    /// (benches/INGEST-OVERLAP.md). The shared-state lock is held for the
+    /// entire batch: pinned hashes and written chunks are recorded
+    /// progressively under it, so a failure (or a rollback racing this
+    /// settle) leaves nothing untracked.
+    async fn settle_batch(
+        pool: Arc<PgPool>,
+        backend: Arc<dyn BlobStorageBackend>,
+        state: Arc<tokio::sync::Mutex<IngestState>>,
+        batch: Vec<(String, Bytes)>,
     ) -> Result<(), DomainError> {
-        if pending.is_empty() {
+        if batch.is_empty() {
             return Ok(());
         }
-        let batch = std::mem::take(pending);
+        let mut guard = state.lock().await;
         let hashes: Vec<String> = batch.iter().map(|(h, _)| h.clone()).collect();
 
         // Pin-or-classify in one statement: rows that exist take this
@@ -1084,7 +1212,7 @@ impl DedupService {
               RETURNING hash",
         )
         .bind(&hashes)
-        .fetch_all(self.pool.as_ref())
+        .fetch_all(pool.as_ref())
         .await
         .map_err(|e| {
             DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}"))
@@ -1106,7 +1234,6 @@ impl DedupService {
 
         // Unsynced writes — durability comes from the single end-of-stream
         // sweep, before any PG row references these chunks.
-        let backend = self.backend.clone();
         let results: Vec<Result<(String, i64), DomainError>> = stream::iter(to_write)
             .map(|(hash, data)| {
                 let backend = backend.clone();

src/infrastructure/services/nextcloud_chunked_upload_service.rs

@@ -1,24 +1,84 @@
 use std::path::PathBuf;
+use std::time::Duration;
 use tokio::fs;
 
 use crate::common::errors::{DomainError, Result};
 
+/// In-RAM running byte counter per upload session (`user/upload_id` →
+/// bytes accepted so far). The per-chunk quota gate used to recompute
+/// this by listing the whole session directory and stat-ing every chunk
+/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload
+/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the
+/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds
+/// from the directory listing, so crash-correctness is unchanged
+/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the
+/// TTL reaps counters for sessions the client abandoned.
+fn build_session_bytes_cache() -> moka::sync::Cache<String, u64> {
+    moka::sync::Cache::builder()
+        .max_capacity(100_000)
+        .time_to_idle(Duration::from_secs(24 * 3600))
+        .build()
+}
+
 #[derive(Clone)]
 pub struct NextcloudChunkedUploadService {
     pub base_dir: PathBuf,
+    /// See [`build_session_bytes_cache`]. Cloning the service shares the
+    /// counter (moka `Cache` clones are handles to the same store).
+    session_bytes: moka::sync::Cache<String, u64>,
 }
 
 impl NextcloudChunkedUploadService {
     pub fn new(base_dir: PathBuf) -> Self {
-        Self { base_dir }
+        Self {
+            base_dir,
+            session_bytes: build_session_bytes_cache(),
+        }
     }
 
     pub fn new_stub() -> Self {
         Self {
             base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
+            session_bytes: build_session_bytes_cache(),
         }
     }
 
+    fn bytes_key(user: &str, upload_id: &str) -> String {
+        format!("{user}/{upload_id}")
+    }
+
+    /// Session bytes accepted so far, if the counter is warm.
+    /// `None` = rebuild from the directory listing and call
+    /// [`Self::set_session_bytes`].
+    pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option<u64> {
+        self.session_bytes.get(&Self::bytes_key(user, upload_id))
+    }
+
+    /// Seed / overwrite the session counter (post-rebuild or on MKCOL).
+    pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) {
+        self.session_bytes
+            .insert(Self::bytes_key(user, upload_id), bytes);
+    }
+
+    /// Add an accepted chunk's bytes to the counter (no-op when cold —
+    /// the next gate rebuilds from disk). Two racing PUTs on one session
+    /// could drop an increment; the counter is a gate hint, and the
+    /// MOVE-time quota check stays authoritative.
+    pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) {
+        let key = Self::bytes_key(user, upload_id);
+        if let Some(current) = self.session_bytes.get(&key) {
+            self.session_bytes
+                .insert(key, current.saturating_add(delta));
+        }
+    }
+
+    /// Drop the counter (session cleanup, or a chunk overwrite made the
+    /// running total untrustworthy — rebuilt lazily on next use).
+    pub fn forget_session_bytes(&self, user: &str, upload_id: &str) {
+        self.session_bytes
+            .invalidate(&Self::bytes_key(user, upload_id));
+    }
+
     /// Validate that a path component contains no traversal characters.
     fn validate_path_component(name: &str, label: &str) -> Result<()> {
         if name.is_empty()
@@ -48,6 +108,7 @@ impl NextcloudChunkedUploadService {
         fs::create_dir_all(&session_dir)
             .await
             .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
+        self.set_session_bytes(user, upload_id, 0);
         Ok(())
     }
 
@@ -97,9 +158,17 @@ impl NextcloudChunkedUploadService {
         data: &[u8],
     ) -> Result<()> {
         let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
+        let overwrite = fs::metadata(&chunk_path).await.is_ok();
         fs::write(&chunk_path, data)
             .await
-            .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))
+            .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
+        if overwrite {
+            // Retried chunk — running total is stale; rebuild lazily.
+            self.forget_session_bytes(user, upload_id);
+        } else {
+            self.bump_session_bytes(user, upload_id, data.len() as u64);
+        }
+        Ok(())
     }
 
     /// List the session's chunk files in assembly (numeric) order.
@@ -146,6 +215,7 @@ impl NextcloudChunkedUploadService {
                 .await
                 .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
         }
+        self.forget_session_bytes(user, upload_id);
         Ok(())
     }
 

src/infrastructure/services/zip_service.rs

@@ -44,8 +44,9 @@ impl From<ZipError> for DomainError {
     }
 }
 
-/// Type alias for the fully-async ZIP writer backed by a buffered tokio file.
-type AsyncZipWriter = ZipFileWriter<Compat<BufWriter<tokio::fs::File>>>;
+/// Fully-async ZIP writer over any buffered tokio sink (temp file for the
+/// legacy path, one half of a `tokio::io::duplex` for the streaming path).
+type AsyncZipWriter<W> = ZipFileWriter<Compat<BufWriter<W>>>;
 
 /// One planned archive entry, in final ZIP order.
 enum ZipPlanEntry {
@@ -119,6 +120,110 @@ impl ZipService {
         folder_id: &str,
         folder_name: &str,
     ) -> Result<NamedTempFile> {
+        let plan = self.plan_archive(folder_id, folder_name).await?;
+
+        // ── Open the temp file + ZIP writer ──────────────────────────────
+        let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
+        let tokio_file = tokio::fs::File::create(temp.path())
+            .await
+            .map_err(ZipError::IoError)?;
+
+        let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
+        let _prefetcher = tokio::spawn(Self::prefetch_files(
+            self.file_service.clone(),
+            Self::planned_file_ids(&plan),
+            tx,
+        ));
+        Self::write_archive(tokio_file, &plan, &mut rx).await?;
+
+        Ok(temp)
+    }
+
+    /// Streaming variant: the archive bytes are produced on a spawned task
+    /// and yielded as they are written — the client's first byte arrives
+    /// after the first entry starts, not after the whole archive has been
+    /// built (the temp-file variant's time-to-first-byte grows with folder
+    /// size; benches/ZIP-STREAM.md). The plan phase still runs inline so
+    /// planning errors surface as proper HTTP errors; a blob-read error
+    /// mid-archive can only truncate the stream (no central directory →
+    /// clients detect the corrupt archive), which is the standard tradeoff
+    /// for streamed ZIPs.
+    pub async fn create_folder_zip_stream(
+        &self,
+        folder_id: &str,
+        folder_name: &str,
+    ) -> Result<impl futures::Stream<Item = std::io::Result<bytes::Bytes>> + Send + use<>> {
+        let plan = self.plan_archive(folder_id, folder_name).await?;
+
+        let (writer, reader) = tokio::io::duplex(256 * 1024);
+        let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
+        let _prefetcher = tokio::spawn(Self::prefetch_files(
+            self.file_service.clone(),
+            Self::planned_file_ids(&plan),
+            tx,
+        ));
+        tokio::spawn(async move {
+            if let Err(e) = Self::write_archive(writer, &plan, &mut rx).await {
+                // Dropping the writer EOFs the reader early — the truncated
+                // archive has no central directory, so clients flag it.
+                warn!("Streaming ZIP aborted mid-archive: {e}");
+            }
+        });
+
+        Ok(tokio_util::io::ReaderStream::new(reader))
+    }
+
+    /// File ids of the plan, in archive order (the prefetcher's read list).
+    fn planned_file_ids(plan: &[ZipPlanEntry]) -> Vec<String> {
+        plan.iter()
+            .filter_map(|entry| match entry {
+                ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
+                ZipPlanEntry::Dir(_) => None,
+            })
+            .collect()
+    }
+
+    /// Write every planned entry through a buffered ZIP writer over `sink`,
+    /// then finalize (central directory + flush). Shared by the temp-file
+    /// and streaming variants.
+    async fn write_archive<W: tokio::io::AsyncWrite + Unpin>(
+        sink: W,
+        plan: &[ZipPlanEntry],
+        rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
+    ) -> Result<()> {
+        let buf_writer = BufWriter::with_capacity(256 * 1024, sink);
+        let mut zip = ZipFileWriter::with_tokio(buf_writer);
+
+        for entry in plan {
+            match entry {
+                ZipPlanEntry::Dir(zip_dir) => {
+                    let dir_entry =
+                        ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
+                    match zip.write_entry_whole(dir_entry, &[]).await {
+                        Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
+                        Err(e) => {
+                            warn!("Could not add folder entry (may already exist): {}", e);
+                        }
+                    }
+                }
+                ZipPlanEntry::File {
+                    zip_path,
+                    compression,
+                    ..
+                } => {
+                    Self::write_prefetched_file(&mut zip, zip_path, *compression, rx).await?;
+                }
+            }
+        }
+
+        let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
+        compat_writer.close().await.map_err(ZipError::IoError)?;
+        Ok(())
+    }
+
+    /// Resolve the folder, fetch its subtree (2 bulk queries) and lay out
+    /// the archive entries in final ZIP order.
+    async fn plan_archive(&self, folder_id: &str, folder_name: &str) -> Result<Vec<ZipPlanEntry>> {
         info!(
             "Creating ZIP for folder: {} (ID: {})",
             folder_name, folder_id
@@ -200,61 +305,7 @@ impl ZipService {
             }
         }
 
-        // ── 5. Open the temp file + ZIP writer ───────────────────────────
-        let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
-        let tokio_file = tokio::fs::File::create(temp.path())
-            .await
-            .map_err(ZipError::IoError)?;
-        let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
-        let mut zip = ZipFileWriter::with_tokio(buf_writer);
-
-        // ── 6. Write entries: 2-stage pipeline ───────────────────────────
-        // The prefetch task reads blob streams for the planned files, in
-        // order, ahead of the writer — the next file's blob-store latency
-        // overlaps the current file's deflate. If the writer bails out,
-        // dropping the receiver makes the prefetcher's next send fail and
-        // it stops on its own.
-        let file_ids: Vec<String> = plan
-            .iter()
-            .filter_map(|entry| match entry {
-                ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
-                ZipPlanEntry::Dir(_) => None,
-            })
-            .collect();
-        let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
-        let _prefetcher = tokio::spawn(Self::prefetch_files(
-            self.file_service.clone(),
-            file_ids,
-            tx,
-        ));
-
-        for entry in &plan {
-            match entry {
-                ZipPlanEntry::Dir(zip_dir) => {
-                    let dir_entry =
-                        ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
-                    match zip.write_entry_whole(dir_entry, &[]).await {
-                        Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
-                        Err(e) => {
-                            warn!("Could not add folder entry (may already exist): {}", e);
-                        }
-                    }
-                }
-                ZipPlanEntry::File {
-                    zip_path,
-                    compression,
-                    ..
-                } => {
-                    Self::write_prefetched_file(&mut zip, zip_path, *compression, &mut rx).await?;
-                }
-            }
-        }
-
-        // ── 7. Finalize ──────────────────────────────────────────────────
-        let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
-        compat_writer.close().await.map_err(ZipError::IoError)?;
-
-        Ok(temp)
+        Ok(plan)
     }
 
     /// Prefetch stage: streams each planned file's content from the blob
@@ -302,8 +353,8 @@ impl ZipService {
     /// (`Stored` for already-compressed media, `Deflate` otherwise — see
     /// `entry_compression`). Peak memory stays bounded by the channel,
     /// independent of individual file sizes.
-    async fn write_prefetched_file(
-        zip: &mut AsyncZipWriter,
+    async fn write_prefetched_file<W: tokio::io::AsyncWrite + Unpin>(
+        zip: &mut AsyncZipWriter<W>,
         zip_path: &str,
         compression: Compression,
         rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,