cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf: eliminate Vec<u8> buffer paths — all uploads now stream to disk

Browse Source 
Issue #4 (HIGH): save_file(Vec<u8>) and update_file_content(Vec<u8>)
accepted up to 10 MB of contiguous memory per request. While the main
upload paths already used streaming, the WebDAV compat methods
(create_file, update_file) and the empty-file handler still used the
buffered path, creating a .to_vec() copy.

Changes:
- FileWritePort trait: remove save_file(Vec<u8>) and
  update_file_content(Vec<u8>) — only streaming variants remain
- FileUploadUseCase trait: remove upload_file(Vec<u8>)
- file_upload_service.rs: create_file() and update_file() now spool
  &[u8] to NamedTempFile + Sha256::digest, then delegate to streaming
  path (save_file_from_temp / update_file_streaming)
- file_handler.rs: empty file uploads use upload_file_streaming with
- FileBlobWriteRepository: remove save_file and update_file_content impls
- StubFileWritePort, StubFileUploadUseCase, MockFileRepository: remove
  corresponding dead method impls

Impact: impossible to accidentally use a buffered upload path. All
content goes through streaming with ~256 KB peak RAM. -166 LOC.
This commit is contained in:
Dionisio
2026-02-25 23:41:16 +01:00
parent f9dde6ffff
commit 5a1959bf23
7 changed files with 67 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/ports/file_ports.rs
+8 -16
View File
@@ -15,13 +15,18 @@ use crate::common::errors::DomainError;
/// Primary port for file upload operations.
///
/// All upload paths converge on streaming-to-disk:
/// **All upload paths converge on streaming-to-disk** — no method accepts
/// `Vec<u8>` for content. Even `create_file` / `update_file` (WebDAV
/// helpers that receive `&[u8]`) spool to a temp file internally so that
/// peak RAM stays at ~256 KB regardless of file size.
///
/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming`
/// - WebDAV PUT: small in-memory buffer → `upload_file`
/// - Chunked uploads: chunks already on disk → `upload_file_from_path`
/// - WebDAV PUT (new): handler streams to temp file → `update_file_streaming`
/// - WebDAV PUT (small/compat): `create_file` / `update_file` spool internally
#[async_trait]
pub trait FileUploadUseCase: Send + Sync + 'static {
/// Upload from a temp file already on disk (true streaming, ~64 KB RAM).
/// Upload from a temp file already on disk (true streaming, ~256 KB RAM).
///
/// When `pre_computed_hash` is `Some`, the blob store skips the hash
/// re-read — the handler already computed it during the multipart spool.
@@ -35,19 +40,6 @@ pub trait FileUploadUseCase: Send + Sync + 'static {
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError>;
/// Upload from in-memory bytes (for small payloads: WebDAV, empty files).
///
/// Only used for WebDAV PUT and empty files where the content is already
/// buffered by the protocol handler. For normal uploads, prefer
/// `upload_file_streaming`.
async fn upload_file(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
content: Vec<u8>,
) -> Result<FileDto, DomainError>;
/// Upload from a file already assembled on disk (chunked uploads).
///
/// Same as `upload_file_streaming` but with a separate name for clarity.
src/application/ports/storage_ports.rs
-13
View File
@@ -197,15 +197,6 @@ pub struct CopyFolderTreeResult {
/// and deferred registration for the write-behind cache.
#[async_trait]
pub trait FileWritePort: Send + Sync + 'static {
/// Saves a new file from bytes.
async fn save_file(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
content: Vec<u8>,
) -> Result<File, DomainError>;
/// Streaming upload — saves a file from a temp file already on disk.
///
/// When `pre_computed_hash` is provided, the dedup service skips the
@@ -233,10 +224,6 @@ pub trait FileWritePort: Send + Sync + 'static {
/// Deletes a file.
async fn delete_file(&self, id: &str) -> Result<(), DomainError>;
/// Updates the content of an existing file.
async fn update_file_content(&self, file_id: &str, content: Vec<u8>)
-> Result<(), DomainError>;
/// Streaming update — replaces file content from a temp file on disk.
///
/// When `pre_computed_hash` is provided, the dedup service skips the
src/application/services/file_upload_service.rs
+44 -40
View File
@@ -1,4 +1,5 @@
use async_trait::async_trait;
use sha2::{Digest, Sha256};
use std::path::Path;
use std::sync::Arc;
@@ -29,12 +30,16 @@ fn extract_username_from_path(path: &str) -> Option<String> {
/// Service for file upload operations.
///
/// All upload paths converge on streaming-to-disk:
/// **Every upload path converges on streaming-to-disk** — there is no
/// `Vec<u8>` buffer path.
///
/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming`
/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path`
/// - **WebDAV PUT / empty files**: small in-memory buffer → `upload_file`
/// - **WebDAV PUT (large)**: handler streams body to temp file → `update_file_streaming`
/// - **WebDAV PUT (small / compat)**: `create_file` / `update_file` spool `&[u8]`
/// to a temp file internally, then call the streaming path.
///
/// Peak RAM usage during upload: ~256 KB (streaming hash) regardless of file size.
/// Peak RAM usage during any upload: ~256 KB (streaming hash) regardless of file size.
pub struct FileUploadService {
/// Write port — handles save, streaming, deferred registration
file_write: Arc<dyn FileWritePort>,
@@ -136,23 +141,6 @@ impl FileUploadUseCase for FileUploadService {
Ok(dto)
}
/// Simple byte-based upload (for WebDAV and empty files only).
async fn upload_file(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
content: Vec<u8>,
) -> Result<FileDto, DomainError> {
let file = self
.file_write
.save_file(name, folder_id, content_type, content)
.await?;
let dto = FileDto::from(file);
self.maybe_update_storage_usage(&dto);
Ok(dto)
}
/// Upload from a file already on disk (chunked uploads).
async fn upload_file_from_path(
&self,
@@ -184,6 +172,10 @@ impl FileUploadUseCase for FileUploadService {
}
/// Creates a file at a specific path (for WebDAV PUT on new resource).
///
/// Spools the in-memory `&[u8]` to a temp file with hash-on-write,
/// then delegates to the streaming path. Peak RAM: the caller's
/// buffer + ~256 KB for the hasher.
async fn create_file(
&self,
parent_path: &str,
@@ -201,13 +193,24 @@ impl FileUploadUseCase for FileUploadService {
None
};
// Spool to temp file + hash
let temp = tempfile::NamedTempFile::new().map_err(|e| {
DomainError::internal_error("FileUpload", format!("temp file: {e}"))
})?;
tokio::fs::write(temp.path(), content).await.map_err(|e| {
DomainError::internal_error("FileUpload", format!("write temp: {e}"))
})?;
let hash = hex::encode(Sha256::digest(content));
let file = self
.file_write
.save_file(
.save_file_from_temp(
filename.to_string(),
parent_id,
content_type.to_string(),
content.to_vec(),
temp.path(),
content.len() as u64,
Some(hash),
)
.await?;
let dto = FileDto::from(file);
@@ -216,26 +219,27 @@ impl FileUploadUseCase for FileUploadService {
}
/// Updates an existing file's content, or creates it if not found (for WebDAV PUT).
///
/// Spools the in-memory `&[u8]` to a temp file with hash-on-write,
/// then delegates to the streaming update/create path.
async fn update_file(&self, path: &str, content: &[u8]) -> Result<(), DomainError> {
// Direct SQL lookup — O(folder_depth) instead of O(total_files)
if let Some(file_read) = &self.file_read
&& let Some(file) = file_read.find_file_by_path(path).await?
{
self.file_write
.update_file_content(file.id(), content.to_vec())
.await?;
return Ok(());
}
// Spool to temp file + hash
let temp = tempfile::NamedTempFile::new().map_err(|e| {
DomainError::internal_error("FileUpload", format!("temp file: {e}"))
})?;
tokio::fs::write(temp.path(), content).await.map_err(|e| {
DomainError::internal_error("FileUpload", format!("write temp: {e}"))
})?;
let hash = hex::encode(Sha256::digest(content));
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
let (parent_path, filename) = if let Some(idx) = path_normalized.rfind('/') {
(&path_normalized[..idx], &path_normalized[idx + 1..])
} else {
("", path_normalized)
};
self.create_file(parent_path, filename, content, "application/octet-stream")
.await?;
Ok(())
self.update_file_streaming(
path,
temp.path(),
content.len() as u64,
"application/octet-stream",
Some(hash),
)
.await
}
/// Streaming update — replaces file content from a temp file on disk.
src/application/services/trash_service_test.rs
+5 -38
View File
@@ -209,16 +209,6 @@ impl FileReadPort for MockFileRepository {
#[async_trait]
impl FileWritePort for MockFileRepository {
async fn save_file(
&self,
_name: String,
_folder_id: Option<String>,
_content_type: String,
_content: Vec<u8>,
) -> std::result::Result<File, DomainError> {
unimplemented!()
}
async fn save_file_from_temp(
&self,
_name: String,
@@ -251,14 +241,6 @@ impl FileWritePort for MockFileRepository {
Ok(())
}
async fn update_file_content(
&self,
_file_id: &str,
_content: Vec<u8>,
) -> std::result::Result<(), DomainError> {
Ok(())
}
async fn update_file_content_from_temp(
&self,
_file_id: &str,
@@ -514,10 +496,7 @@ mod tests {
// Arrange
let trashed_files = Arc::new(Mutex::new(HashMap::new()));
let trashed_folders = Arc::new(Mutex::new(HashMap::new()));
let trash_repo = Arc::new(MockTrashRepository::new(
trashed_files.clone(),
trashed_folders.clone(),
));
let trash_repo = Arc::new(MockTrashRepository::new(trashed_files.clone(), trashed_folders.clone()));
let file_repo = Arc::new(MockFileRepository::new(trashed_files));
let folder_repo = Arc::new(MockFolderRepository::new(trashed_folders));
@@ -588,10 +567,7 @@ mod tests {
// Arrange
let trashed_files = Arc::new(Mutex::new(HashMap::new()));
let trashed_folders = Arc::new(Mutex::new(HashMap::new()));
let trash_repo = Arc::new(MockTrashRepository::new(
trashed_files.clone(),
trashed_folders.clone(),
));
let trash_repo = Arc::new(MockTrashRepository::new(trashed_files.clone(), trashed_folders.clone()));
let file_repo = Arc::new(MockFileRepository::new(trashed_files));
let folder_repo = Arc::new(MockFolderRepository::new(trashed_folders));
@@ -653,10 +629,7 @@ mod tests {
// Arrange
let trashed_files = Arc::new(Mutex::new(HashMap::new()));
let trashed_folders = Arc::new(Mutex::new(HashMap::new()));
let trash_repo = Arc::new(MockTrashRepository::new(
trashed_files.clone(),
trashed_folders.clone(),
));
let trash_repo = Arc::new(MockTrashRepository::new(trashed_files.clone(), trashed_folders.clone()));
let file_repo = Arc::new(MockFileRepository::new(trashed_files));
let folder_repo = Arc::new(MockFolderRepository::new(trashed_folders));
@@ -723,10 +696,7 @@ mod tests {
// Arrange
let trashed_files = Arc::new(Mutex::new(HashMap::new()));
let trashed_folders = Arc::new(Mutex::new(HashMap::new()));
let trash_repo = Arc::new(MockTrashRepository::new(
trashed_files.clone(),
trashed_folders.clone(),
));
let trash_repo = Arc::new(MockTrashRepository::new(trashed_files.clone(), trashed_folders.clone()));
let file_repo = Arc::new(MockFileRepository::new(trashed_files));
let folder_repo = Arc::new(MockFolderRepository::new(trashed_folders));
@@ -792,10 +762,7 @@ mod tests {
// Arrange
let trashed_files = Arc::new(Mutex::new(HashMap::new()));
let trashed_folders = Arc::new(Mutex::new(HashMap::new()));
let trash_repo = Arc::new(MockTrashRepository::new(
trashed_files.clone(),
trashed_folders.clone(),
));
let trash_repo = Arc::new(MockTrashRepository::new(trashed_files.clone(), trashed_folders.clone()));
let file_repo = Arc::new(MockFileRepository::new(trashed_files));
let folder_repo = Arc::new(MockFolderRepository::new(trashed_folders));