perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate

Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change
gated by a before/after in examples/bench_round2.rs — an AFTER that did
not beat its BEFORE was to be rolled back; none needed it):

- Range requests (REST/DAV/shares) answered from the moka content cache
  for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice.
  256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us).
- Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales
  with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster).
  Content-Length dropped (size unknown up front).
- NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM
  per-session counter (lazy rebuild on cold start). 1,000-chunk upload
  gate cost: 33.1s -> 0.09s cumulative.
- Delta download + commit-verify now use the CDC path's
  buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open
  latency 440ms -> 51ms; order preserved.
- CDC ingest settles batches on a spawned task (depth-1 pipeline) so
  the source stream keeps flowing during PG pin + backend writes;
  rollback ledger shared + lock-serialized so compensation stays exact
  on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s.
  OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch).
- Frontend: instant-upload BLAKE3 hashing moved off the main thread to
  a bounded Web Worker pool (File handles by reference); vitest gate
  asserts the pool beats sequential (first gate draft posting buffers
  was 2.6x slower and was rewritten — copies dominated).

Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544
integration tests green; 270 frontend tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w
This commit is contained in:
Claude
2026-07-16 16:50:07 +00:00
parent aba89c4f5d
commit 82ee7da0d2
18 changed files with 1245 additions and 195 deletions
@@ -1,24 +1,84 @@
use std::path::PathBuf;
use std::time::Duration;
use tokio::fs;
use crate::common::errors::{DomainError, Result};
/// In-RAM running byte counter per upload session (`user/upload_id` →
/// bytes accepted so far). The per-chunk quota gate used to recompute
/// this by listing the whole session directory and stat-ing every chunk
/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload
/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the
/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds
/// from the directory listing, so crash-correctness is unchanged
/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the
/// TTL reaps counters for sessions the client abandoned.
fn build_session_bytes_cache() -> moka::sync::Cache<String, u64> {
moka::sync::Cache::builder()
.max_capacity(100_000)
.time_to_idle(Duration::from_secs(24 * 3600))
.build()
}
#[derive(Clone)]
pub struct NextcloudChunkedUploadService {
pub base_dir: PathBuf,
/// See [`build_session_bytes_cache`]. Cloning the service shares the
/// counter (moka `Cache` clones are handles to the same store).
session_bytes: moka::sync::Cache<String, u64>,
}
impl NextcloudChunkedUploadService {
pub fn new(base_dir: PathBuf) -> Self {
Self { base_dir }
Self {
base_dir,
session_bytes: build_session_bytes_cache(),
}
}
pub fn new_stub() -> Self {
Self {
base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
session_bytes: build_session_bytes_cache(),
}
}
fn bytes_key(user: &str, upload_id: &str) -> String {
format!("{user}/{upload_id}")
}
/// Session bytes accepted so far, if the counter is warm.
/// `None` = rebuild from the directory listing and call
/// [`Self::set_session_bytes`].
pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option<u64> {
self.session_bytes.get(&Self::bytes_key(user, upload_id))
}
/// Seed / overwrite the session counter (post-rebuild or on MKCOL).
pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) {
self.session_bytes
.insert(Self::bytes_key(user, upload_id), bytes);
}
/// Add an accepted chunk's bytes to the counter (no-op when cold —
/// the next gate rebuilds from disk). Two racing PUTs on one session
/// could drop an increment; the counter is a gate hint, and the
/// MOVE-time quota check stays authoritative.
pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) {
let key = Self::bytes_key(user, upload_id);
if let Some(current) = self.session_bytes.get(&key) {
self.session_bytes
.insert(key, current.saturating_add(delta));
}
}
/// Drop the counter (session cleanup, or a chunk overwrite made the
/// running total untrustworthy — rebuilt lazily on next use).
pub fn forget_session_bytes(&self, user: &str, upload_id: &str) {
self.session_bytes
.invalidate(&Self::bytes_key(user, upload_id));
}
/// Validate that a path component contains no traversal characters.
fn validate_path_component(name: &str, label: &str) -> Result<()> {
if name.is_empty()
@@ -48,6 +108,7 @@ impl NextcloudChunkedUploadService {
fs::create_dir_all(&session_dir)
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
self.set_session_bytes(user, upload_id, 0);
Ok(())
}
@@ -97,9 +158,17 @@ impl NextcloudChunkedUploadService {
data: &[u8],
) -> Result<()> {
let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
let overwrite = fs::metadata(&chunk_path).await.is_ok();
fs::write(&chunk_path, data)
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
if overwrite {
// Retried chunk — running total is stale; rebuild lazily.
self.forget_session_bytes(user, upload_id);
} else {
self.bump_session_bytes(user, upload_id, data.len() as u64);
}
Ok(())
}
/// List the session's chunk files in assembly (numeric) order.
@@ -146,6 +215,7 @@ impl NextcloudChunkedUploadService {
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
}
self.forget_session_bytes(user, upload_id);
Ok(())
}