fix(upload): stream WebDAV/NextCloud PUT to disk to prevent OOM on large files

Large uploads (e.g. ~800 MB ISOs) could OOMKill the process, even on
dedup hits, due to three separate full-file-in-memory paths:

- NextCloud PUT (/remote.php/dav) buffered the entire body in RAM via
  body::to_bytes before any dedup logic, then re-wrote and re-hashed it.
  Now streams the body to a temp file with incremental BLAKE3 and goes
  through update_file_streaming (shared spool helper with the native
  WebDAV PUT handler); peak heap is ~one HTTP frame regardless of size.

- DedupService::store_chunks materialized every new chunk's data in a Vec
  before uploading. Now reads each new chunk by positioned I/O
  (read_exact_at, off the runtime via spawn_blocking) just before its
  upload; peak heap bounded to ~CHUNK_UPLOAD_CONCURRENCY x CDC_MAX_CHUNK.

- The upload spool used the OS temp dir, often tmpfs/RAM in containers
  where its page-cache counts against the cgroup memory limit. Add
  OXICLOUD_UPLOAD_TMPDIR to point the spool at real disk.

Also collapse a pre-existing clippy collapsible_else_if in carddav_handler.

Refs #404

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DioCrafts
2026-06-07 01:03:52 +02:00
parent c5e0800336
commit 061306cc84
12 changed files with 315 additions and 153 deletions
@@ -1,4 +1,4 @@
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::application::dtos::file_dto::FileDto;
@@ -55,6 +55,9 @@ pub struct FileUploadService {
content_cache: Option<Arc<FileContentCache>>,
/// Single lifecycle dispatcher — fires on_file_created / on_file_updated.
file_lifecycle_hook: Option<Arc<dyn FileLifecycleHook>>,
/// Directory for spool temp files (`&[u8]` upload variants). When `Some`,
/// keeps spools off tmpfs/RAM so they don't count against the cgroup limit.
upload_temp_dir: Option<PathBuf>,
}
impl FileUploadService {
@@ -66,6 +69,7 @@ impl FileUploadService {
storage_usage_service: None,
content_cache: None,
file_lifecycle_hook: None,
upload_temp_dir: None,
}
}
@@ -80,9 +84,16 @@ impl FileUploadService {
storage_usage_service: None,
content_cache: None,
file_lifecycle_hook: None,
upload_temp_dir: None,
}
}
/// Configures the spool directory for the `&[u8]` upload variants.
pub fn with_upload_temp_dir(mut self, dir: Option<PathBuf>) -> Self {
self.upload_temp_dir = dir;
self
}
/// Configures the content cache for invalidation on file updates.
pub fn with_content_cache(mut self, cache: Arc<FileContentCache>) -> Self {
self.content_cache = Some(cache);
@@ -106,6 +117,11 @@ impl FileUploadService {
// ── private helpers ──────────────────────────────────────────
/// Create a spool temp file, honoring the configured upload temp dir.
fn new_temp(&self) -> std::io::Result<tempfile::NamedTempFile> {
crate::common::temp::new_spool_temp_file(self.upload_temp_dir.as_deref())
}
/// Optionally update storage usage after a successful upload.
fn maybe_update_storage_usage(&self, file: &FileDto) {
if let Some(storage_service) = &self.storage_usage_service {
@@ -220,7 +236,8 @@ impl FileUploadUseCase for FileUploadService {
};
// Spool to temp file + hash
let temp = tempfile::NamedTempFile::new()
let temp = self
.new_temp()
.map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?;
tokio::fs::write(temp.path(), content)
.await
@@ -260,7 +277,8 @@ impl FileUploadUseCase for FileUploadService {
modified_at: Option<i64>,
) -> Result<FileDto, DomainError> {
// Spool to temp file + hash
let temp = tempfile::NamedTempFile::new()
let temp = self
.new_temp()
.map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?;
tokio::fs::write(temp.path(), content)
.await