cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

perf: Phase 4+5 optimizations — uploads 10x, downloads 2x, concurrent 2x. moka cache, 512KB buffers, remove sync_all, hash-on-write, preloaded queries, bench.sh v3, gitignore storage/. 500MB upload 12.6s->1.3s (392MB/s). RSS 69-113MB, 0 swap.

Browse Source
This commit is contained in:
Dionisio
2026-02-15 17:53:25 +01:00
parent fac0b5e77b
commit 1ed20f425f
57 changed files with 1700 additions and 1184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/services/file_upload_service.rs
+62 -193
View File
@@ -1,22 +1,13 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use std::pin::Pin;
use std::path::Path;
use std::sync::Arc;
use crate::application::dtos::file_dto::FileDto;
use crate::application::ports::cache_ports::WriteBehindCachePort;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::file_ports::{FileUploadUseCase, UploadStrategy};
use crate::application::ports::file_ports::FileUploadUseCase;
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::common::errors::DomainError;
use tracing::{debug, info, warn};
/// Threshold for using streaming upload (files >= 1MB use streaming)
const STREAMING_UPLOAD_THRESHOLD: usize = 1024 * 1024;
/// Threshold for write-behind cache (files < 256KB get instant response)
const WRITE_BEHIND_THRESHOLD: usize = 256 * 1024;
/// Helper function to extract username from folder path string.
/// e.g. "My Folder - user1/subfolder/file.txt" → "user1"
fn extract_username_from_path(path: &str) -> Option<String> {
@@ -27,7 +18,6 @@ fn extract_username_from_path(path: &str) -> Option<String> {
if parts.len() <= 1 {
return None;
}
// Take only the first segment (username), not any subfolders
let remainder = parts[1].trim();
let username = remainder.split('/').next().unwrap_or(remainder);
let username = username.trim();
@@ -37,58 +27,35 @@ fn extract_username_from_path(path: &str) -> Option<String> {
Some(username.to_string())
}
/// Service for file upload operations
/// Service for file upload operations.
///
/// Encapsulates the three-tier upload strategy:
/// 1. **Write-Behind** (<256 KB): store in RAM, respond instantly, flush async.
/// 2. **Buffered** (256 KB – 1 MB): collect bytes, write, respond.
/// 3. **Streaming** (≥1 MB): pipe chunks to disk with constant memory.
/// All upload paths converge on streaming-to-disk:
/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming`
/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path`
/// - **WebDAV PUT / empty files**: small in-memory buffer → `upload_file`
///
/// Also runs deduplication so duplicate content is never stored twice.
/// Peak RAM usage during upload: ~256 KB (streaming hash) regardless of file size.
pub struct FileUploadService {
/// Write port — handles save, streaming, deferred registration
file_write: Arc<dyn FileWritePort>,
/// Read port — needed for WebDAV create_file / update_file
file_read: Option<Arc<dyn FileReadPort>>,
/// Optional write-behind cache for instant uploads
write_behind: Option<Arc<dyn WriteBehindCachePort>>,
/// Optional dedup service for content-addressable storage
dedup: Option<Arc<dyn DedupPort>>,
/// Optional storage usage tracking
storage_usage_service:
Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>,
}
impl FileUploadService {
/// Backward-compatible constructor (no write-behind, no dedup).
/// Constructor with write port only (minimal).
pub fn new(file_repository: Arc<dyn FileWritePort>) -> Self {
Self {
file_write: file_repository,
file_read: None,
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
/// Full constructor with all infrastructure ports.
pub fn new_full(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
write_behind: Arc<dyn WriteBehindCachePort>,
dedup: Arc<dyn DedupPort>,
) -> Self {
Self {
file_write,
file_read: Some(file_read),
write_behind: Some(write_behind),
dedup: Some(dedup),
storage_usage_service: None,
}
}
/// Constructor for blob-storage model: write + read ports only.
/// Dedup is handled at the repository layer — no write-behind needed.
/// Constructor for blob-storage model: write + read ports.
pub fn new_with_read(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
@@ -96,8 +63,6 @@ impl FileUploadService {
Self {
file_write,
file_read: Some(file_read),
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
@@ -113,37 +78,9 @@ impl FileUploadService {
// ── private helpers ──────────────────────────────────────────
/// Run dedup tracking (non-fatal on failure).
async fn run_dedup(&self, data: &[u8], content_type: &str) {
let Some(dedup) = &self.dedup else { return };
match dedup
.store_bytes(data, Some(content_type.to_string()))
.await
{
Ok(result) => {
if result.was_deduplicated() {
info!(
"🔗 DEDUP: content already exists (hash: {}, saved {} bytes)",
&result.hash()[..12],
result.size()
);
} else {
info!(
"💾 DEDUP: new content stored (hash: {})",
&result.hash()[..12]
);
}
}
Err(e) => {
warn!("⚠️ DEDUP: Failed to store in blob store: {}", e);
}
}
}
/// Optionally update storage usage after a successful upload.
fn maybe_update_storage_usage(&self, file: &FileDto) {
if let Some(storage_service) = &self.storage_usage_service {
// Extract username from the file's own path (contains folder structure)
let file_path = file.path.clone();
if let Some(username) = extract_username_from_path(&file_path) {
let service_clone = Arc::clone(storage_service);
@@ -166,7 +103,33 @@ impl FileUploadService {
#[async_trait]
impl FileUploadUseCase for FileUploadService {
/// Simple byte-based upload (backward compatible).
/// Streaming upload from a temp file on disk.
///
/// Peak RAM: ~256 KB (hash calculation) regardless of file size.
/// The temp file is consumed (moved/deleted) by the blob store.
async fn upload_file_streaming(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
temp_path: &Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
let file = self
.file_write
.save_file_from_temp(name.clone(), folder_id, content_type, temp_path, size, pre_computed_hash)
.await?;
let dto = FileDto::from(file);
info!(
"📡 STREAMING UPLOAD: {} ({} bytes, ID: {})",
name, size, dto.id
);
self.maybe_update_storage_usage(&dto);
Ok(dto)
}
/// Simple byte-based upload (for WebDAV and empty files only).
async fn upload_file(
&self,
name: String,
@@ -183,105 +146,27 @@ impl FileUploadUseCase for FileUploadService {
Ok(dto)
}
/// Smart three-tier upload with write-behind cache and dedup.
async fn smart_upload(
/// Upload from a file already on disk (chunked uploads).
async fn upload_file_from_path(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
chunks: Vec<Bytes>,
total_size: usize,
) -> Result<(FileDto, UploadStrategy), DomainError> {
use futures::stream;
file_path: &Path,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
let size = tokio::fs::metadata(file_path)
.await
.map_err(|e| {
DomainError::internal_error(
"FileUpload",
format!("Failed to read file metadata: {}", e),
)
})?
.len();
// ─── Dedup (runs for all tiers) ──────────────────────
{
let dedup_data: Vec<u8> = {
let mut combined = Vec::with_capacity(total_size);
for chunk in &chunks {
combined.extend_from_slice(chunk);
}
combined
};
self.run_dedup(&dedup_data, &content_type).await;
}
// ─── TIER 1: Write-Behind (<256 KB) ──────────────────
if total_size < WRITE_BEHIND_THRESHOLD
&& let Some(wb) = &self.write_behind
&& wb.is_eligible_size(total_size)
{
let data: Bytes = if chunks.len() == 1 {
chunks.into_iter().next().unwrap()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined.into()
};
let (file, target_path) = self
.file_write
.register_file_deferred(name.clone(), folder_id, content_type, total_size as u64)
.await?;
let dto = FileDto::from(file);
if let Err(e) = wb.put_pending(dto.id.clone(), data, target_path).await {
return Err(DomainError::internal_error(
"file",
format!("Write-behind cache failed: {}", e),
));
}
info!(
"⚡ WRITE-BEHIND UPLOAD: {} (ID: {}, ~0ms latency)",
name, dto.id
);
self.maybe_update_storage_usage(&dto);
return Ok((dto, UploadStrategy::WriteBehind));
}
// ─── TIER 2: Streaming (≥1 MB) ──────────────────────
if total_size >= STREAMING_UPLOAD_THRESHOLD {
let chunk_stream = stream::iter(chunks.into_iter().map(Ok::<_, std::io::Error>));
let pinned_stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
Box::pin(chunk_stream);
let file = self
.file_write
.save_file_from_stream(name.clone(), folder_id, content_type, pinned_stream)
.await?;
let dto = FileDto::from(file);
info!(
"✅ STREAMING UPLOAD: {} ({} MB, ID: {})",
name,
total_size / (1024 * 1024),
dto.id
);
self.maybe_update_storage_usage(&dto);
return Ok((dto, UploadStrategy::Streaming));
}
// ─── TIER 3: Buffered (256 KB – 1 MB) ───────────────
let data = if chunks.len() == 1 {
chunks.into_iter().next().unwrap().to_vec()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined
};
let file = self
.file_write
.save_file(name.clone(), folder_id, content_type, data)
.await?;
let dto = FileDto::from(file);
info!("✅ BUFFERED UPLOAD: {} (ID: {})", name, dto.id);
self.maybe_update_storage_usage(&dto);
Ok((dto, UploadStrategy::Buffered))
self.upload_file_streaming(name, folder_id, content_type, file_path, size, pre_computed_hash)
.await
}
/// Creates a file at a specific path (for WebDAV PUT on new resource).
@@ -292,13 +177,9 @@ impl FileUploadUseCase for FileUploadService {
content: &[u8],
content_type: &str,
) -> Result<FileDto, DomainError> {
// Resolve parent folder ID from path
let parent_id = if !parent_path.is_empty() {
if let Some(file_read) = &self.file_read {
match file_read.get_parent_folder_id(parent_path).await {
Ok(id) => Some(id),
Err(_) => None, // If parent doesn't exist, use root
}
file_read.get_parent_folder_id(parent_path).await.ok()
} else {
None
}
@@ -322,28 +203,16 @@ impl FileUploadUseCase for FileUploadService {
/// Updates an existing file's content, or creates it if not found (for WebDAV PUT).
async fn update_file(&self, path: &str, content: &[u8]) -> Result<(), DomainError> {
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
// Try to find the existing file by path
if let Some(file_read) = &self.file_read {
let all_files = file_read.list_files(None).await?;
for file in &all_files {
let dto = FileDto::from(file.clone());
let dto_path = dto.path.trim_start_matches('/').trim_end_matches('/');
if dto_path == path_normalized
|| dto_path.ends_with(&format!("/{}", path_normalized))
|| path_normalized.ends_with(&format!("/{}", dto_path))
{
// Found it — update in place
self.file_write
.update_file_content(file.id(), content.to_vec())
.await?;
return Ok(());
}
// Direct SQL lookup — O(folder_depth) instead of O(total_files)
if let Some(file_read) = &self.file_read
&& let Some(file) = file_read.find_file_by_path(path).await? {
self.file_write
.update_file_content(file.id(), content.to_vec())
.await?;
return Ok(());
}
}
// File not found — create it
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
let (parent_path, filename) = if let Some(idx) = path_normalized.rfind('/') {
(&path_normalized[..idx], &path_normalized[idx + 1..])
} else {