fix(upload): stream WebDAV/NextCloud PUT to disk to prevent OOM on large files
Large uploads (e.g. ~800 MB ISOs) could OOMKill the process, even on dedup hits, due to three separate full-file-in-memory paths: - NextCloud PUT (/remote.php/dav) buffered the entire body in RAM via body::to_bytes before any dedup logic, then re-wrote and re-hashed it. Now streams the body to a temp file with incremental BLAKE3 and goes through update_file_streaming (shared spool helper with the native WebDAV PUT handler); peak heap is ~one HTTP frame regardless of size. - DedupService::store_chunks materialized every new chunk's data in a Vec before uploading. Now reads each new chunk by positioned I/O (read_exact_at, off the runtime via spawn_blocking) just before its upload; peak heap bounded to ~CHUNK_UPLOAD_CONCURRENCY x CDC_MAX_CHUNK. - The upload spool used the OS temp dir, often tmpfs/RAM in containers where its page-cache counts against the cgroup memory limit. Add OXICLOUD_UPLOAD_TMPDIR to point the spool at real disk. Also collapse a pre-existing clippy collapsible_else_if in carddav_handler. Refs #404 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -20,9 +20,10 @@ use crate::application::ports::file_ports::{
|
||||
use crate::application::ports::folder_ports::FolderUseCase;
|
||||
use crate::application::ports::trash_ports::TrashUseCase;
|
||||
use crate::common::di::AppState;
|
||||
use crate::common::mime_detect::{filename_from_path, refine_content_type};
|
||||
use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file};
|
||||
use crate::interfaces::errors::AppError;
|
||||
use crate::interfaces::middleware::auth::{AuthUser, CurrentUser};
|
||||
use crate::interfaces::upload_spool::spool_body_to_temp;
|
||||
|
||||
/// Extension trait to map XML write errors to `String` concisely.
|
||||
trait XmlResultExt<T> {
|
||||
@@ -531,51 +532,59 @@ async fn handle_put(
|
||||
.and_then(|v| v.parse::<i64>().ok());
|
||||
|
||||
let max_upload = state.core.config.storage.max_upload_size;
|
||||
let body_bytes = body::to_bytes(req.into_body(), max_upload)
|
||||
|
||||
// Stream the body to a temp file + incremental hash — never buffer the
|
||||
// full upload in RAM. The old `body::to_bytes` path loaded the entire
|
||||
// file (e.g. an 800 MB ISO) into anonymous memory before any dedup logic,
|
||||
// OOMKilling the process even on dedup hits. Shared with the native
|
||||
// WebDAV PUT handler; peak heap ~one frame regardless of file size.
|
||||
let spooled = spool_body_to_temp(
|
||||
req.into_body(),
|
||||
max_upload,
|
||||
state.core.config.storage.upload_temp_dir.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Detect real MIME type from the first bytes on disk (no full read).
|
||||
// `filename` is owned so we don't hold a borrow of the `subpath` param
|
||||
// across the await (which would make the handler future non-Send).
|
||||
let filename = filename_from_path(subpath).to_string();
|
||||
let content_type =
|
||||
refine_content_type_from_file(spooled.temp.path(), &filename, &claimed_type).await;
|
||||
|
||||
// Distinguish create (201) vs update (204) for the response status.
|
||||
let existed = file_service.get_file_by_path(&internal_path).await.is_ok();
|
||||
|
||||
// Single streaming path — handles both update and create internally,
|
||||
// passing the precomputed hash so the dedup fast path can short-circuit
|
||||
// without re-reading the file.
|
||||
let stored = upload_service
|
||||
.update_file_streaming(
|
||||
&internal_path,
|
||||
spooled.temp.path(),
|
||||
spooled.size,
|
||||
&content_type,
|
||||
Some(spooled.hash),
|
||||
oc_mtime,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| AppError::bad_request(format!("Failed to read body: {}", e)))?;
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to store file: {}", e)))?;
|
||||
|
||||
// Detect real MIME type via magic bytes + extension, falling back to client header.
|
||||
let filename = filename_from_path(subpath);
|
||||
let content_type = refine_content_type(&body_bytes, filename, &claimed_type);
|
||||
// dedup may have already moved the temp on a new-blob store; ignore error.
|
||||
let _ = tokio::fs::remove_file(spooled.temp.path()).await;
|
||||
|
||||
// Check if the file already exists (update vs create).
|
||||
let existing = file_service.get_file_by_path(&internal_path).await;
|
||||
|
||||
if existing.is_ok() {
|
||||
// Update existing file — returns FileDto with fresh content-hash etag.
|
||||
let updated = upload_service
|
||||
.update_file(&internal_path, &body_bytes, &content_type, oc_mtime)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?;
|
||||
|
||||
return Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.header(header::ETAG, format!("\"{}\"", updated.etag))
|
||||
.header("oc-etag", format!("\"{}\"", updated.etag))
|
||||
.body(Body::empty())
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
// Create new file — split subpath into parent dir and filename.
|
||||
let (parent_subpath, filename) = match subpath.rsplit_once('/') {
|
||||
Some((parent, name)) => (parent, name),
|
||||
None => ("", subpath),
|
||||
let status = if existed {
|
||||
StatusCode::NO_CONTENT
|
||||
} else {
|
||||
StatusCode::CREATED
|
||||
};
|
||||
|
||||
let parent_internal = nc_to_internal_path(&user.username, parent_subpath)?;
|
||||
|
||||
let file_dto = upload_service
|
||||
.create_file(&parent_internal, filename, &body_bytes, &content_type)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to create file: {}", e)))?;
|
||||
|
||||
let builder = Response::builder()
|
||||
.status(StatusCode::CREATED)
|
||||
.header(header::ETAG, format!("\"{}\"", file_dto.etag))
|
||||
.header("oc-etag", format!("\"{}\"", file_dto.etag));
|
||||
|
||||
Ok(builder.body(Body::empty()).unwrap())
|
||||
Ok(Response::builder()
|
||||
.status(status)
|
||||
.header(header::ETAG, format!("\"{}\"", stored.etag))
|
||||
.header("oc-etag", format!("\"{}\"", stored.etag))
|
||||
.body(Body::empty())
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
// ──────────────────── MKCOL ────────────────────
|
||||
|
||||
Reference in New Issue
Block a user