perf: Phase 4+5 optimizations — uploads 10x, downloads 2x, concurrent 2x. moka cache, 512KB buffers, remove sync_all, hash-on-write, preloaded queries, bench.sh v3, gitignore storage/. 500MB upload 12.6s->1.3s (392MB/s). RSS 69-113MB, 0 swap.

This commit is contained in:
Dionisio
2026-02-15 17:53:25 +01:00
parent fac0b5e77b
commit 1ed20f425f
57 changed files with 1700 additions and 1184 deletions
+26 -30
View File
@@ -78,22 +78,20 @@ impl CalDavAdapter {
s if s == "filter" || s.ends_with(":filter") => in_filter = true,
s if s == "time-range" || s.ends_with(":time-range") => {
// Parse time-range attributes
for attr in e.attributes() {
if let Ok(attr) = attr {
let attr_name =
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
let attr_value = attr.unescape_value().unwrap_or_default();
for attr in e.attributes().flatten() {
let attr_name =
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
let attr_value = attr.unescape_value().unwrap_or_default();
if attr_name == "start" {
// Parse ISO date format with Z for UTC
start_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
} else if attr_name == "end" {
end_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
}
if attr_name == "start" {
// Parse ISO date format with Z for UTC
start_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
} else if attr_name == "end" {
end_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
}
}
}
@@ -151,22 +149,20 @@ impl CalDavAdapter {
props.push(QualifiedName::new(namespace, prop_name));
} else if name_str == "time-range" || name_str.ends_with(":time-range") {
// Parse time-range attributes
for attr in e.attributes() {
if let Ok(attr) = attr {
let attr_name =
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
let attr_value = attr.unescape_value().unwrap_or_default();
for attr in e.attributes().flatten() {
let attr_name =
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
let attr_value = attr.unescape_value().unwrap_or_default();
if attr_name == "start" {
// Parse ISO date format with Z for UTC
start_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
} else if attr_name == "end" {
end_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
}
if attr_name == "start" {
// Parse ISO date format with Z for UTC
start_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
} else if attr_name == "end" {
end_time = DateTime::parse_from_rfc3339(&attr_value)
.ok()
.map(|dt| dt.with_timezone(&Utc));
}
}
}
+6 -3
View File
@@ -61,11 +61,14 @@ impl QualifiedName {
}
}
pub fn to_string(&self) -> String {
}
impl std::fmt::Display for QualifiedName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.namespace.is_empty() {
self.name.clone()
write!(f, "{}", self.name)
} else {
format!("{{{}}}{}", self.namespace, self.name)
write!(f, "{{{}}}{}", self.namespace, self.name)
}
}
}
+1 -5
View File
@@ -73,11 +73,7 @@ impl PaginationRequestDto {
}
// Ensure the page size is between 10 and 500
if page_size < 10 {
page_size = 10;
} else if page_size > 500 {
page_size = 500;
}
page_size = page_size.clamp(10, 500);
Self { page, page_size }
}
@@ -84,11 +84,13 @@ pub trait ChunkedUploadPort: Send + Sync + 'static {
/// Assemble all chunks into the final file.
///
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size)`.
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`.
/// The hash is computed during assembly (hash-on-write), eliminating a
/// second sequential read of the assembled file.
async fn complete_upload(
&self,
upload_id: &str,
) -> Result<(PathBuf, String, Option<String>, String, u64), DomainError>;
) -> Result<(PathBuf, String, Option<String>, String, u64, String), DomainError>;
/// Finalize upload: clean up the session and temporary files.
async fn finalize_upload(&self, upload_id: &str) -> Result<(), DomainError>;
+30
View File
@@ -7,8 +7,10 @@
use crate::common::errors::DomainError;
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use serde::Serialize;
use std::path::{Path, PathBuf};
use std::pin::Pin;
/// Metadata of a stored blob in the dedup system.
#[derive(Debug, Clone, Serialize)]
@@ -103,10 +105,15 @@ pub trait DedupPort: Send + Sync + 'static {
) -> Result<DedupResultDto, DomainError>;
/// Store content with deduplication (streaming from file).
///
/// If `pre_computed_hash` is provided (e.g. hash-on-write from the handler),
/// the file will NOT be re-read to calculate the hash — saving one full
/// sequential read of the file.
async fn store_from_file(
&self,
source_path: &Path,
content_type: Option<String>,
pre_computed_hash: Option<String>,
) -> Result<DedupResultDto, DomainError>;
/// Check if a blob with the given hash exists.
@@ -121,6 +128,29 @@ pub trait DedupPort: Send + Sync + 'static {
/// Read blob content as `Bytes`.
async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError>;
/// Stream blob content in chunks (64 KB default) — constant memory usage.
///
/// Unlike `read_blob()`, this never loads the entire file into RAM.
async fn read_blob_stream(
&self,
hash: &str,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>;
/// Stream a byte range of a blob — only reads the requested portion.
///
/// Uses seek + take so a 1 MB range on a 1 GB file only reads 1 MB from disk.
async fn read_blob_range_stream(
&self,
hash: &str,
start: u64,
end: Option<u64>,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>;
/// Get the size of a blob without reading its content.
///
/// Used by HEAD requests to return Content-Length without loading the file.
async fn blob_size(&self, hash: &str) -> Result<u64, DomainError>;
/// Add a reference to a blob (increment ref_count).
async fn add_reference(&self, hash: &str) -> Result<(), DomainError>;
+47 -20
View File
@@ -1,6 +1,7 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
@@ -11,21 +12,33 @@ use crate::common::errors::DomainError;
// Upload port
// ─────────────────────────────────────────────────────
/// Strategy chosen by the upload service based on file size.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UploadStrategy {
/// Instant (<256KB): write-behind cache, ~0ms latency
WriteBehind,
/// Buffered (256KB–1MB): full bytes in memory then write
Buffered,
/// Streaming (≥1MB): pipe chunks directly to disk
Streaming,
}
/// Primary port for file upload operations
/// Primary port for file upload operations.
///
/// All upload paths converge on streaming-to-disk:
/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming`
/// - WebDAV PUT: small in-memory buffer → `upload_file`
/// - Chunked uploads: chunks already on disk → `upload_file_from_path`
#[async_trait]
pub trait FileUploadUseCase: Send + Sync + 'static {
/// Uploads a new file from bytes
/// Upload from a temp file already on disk (true streaming, ~64 KB RAM).
///
/// When `pre_computed_hash` is `Some`, the blob store skips the hash
/// re-read — the handler already computed it during the multipart spool.
async fn upload_file_streaming(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
temp_path: &Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError>;
/// Upload from in-memory bytes (for small payloads: WebDAV, empty files).
///
/// Only used for WebDAV PUT and empty files where the content is already
/// buffered by the protocol handler. For normal uploads, prefer
/// `upload_file_streaming`.
async fn upload_file(
&self,
name: String,
@@ -34,18 +47,17 @@ pub trait FileUploadUseCase: Send + Sync + 'static {
content: Vec<u8>,
) -> Result<FileDto, DomainError>;
/// Smart upload: picks the best strategy (write-behind / buffered / streaming)
/// and handles dedup automatically.
/// Upload from a file already assembled on disk (chunked uploads).
///
/// Returns `(FileDto, UploadStrategy)` so the handler can log the chosen tier.
async fn smart_upload(
/// Same as `upload_file_streaming` but with a separate name for clarity.
async fn upload_file_from_path(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
chunks: Vec<Bytes>,
total_size: usize,
) -> Result<(FileDto, UploadStrategy), DomainError>;
file_path: &Path,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError>;
/// Creates a new file at the specified path (for WebDAV)
async fn create_file(
@@ -115,6 +127,21 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static {
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError>;
/// Like `get_file_optimized` but accepts an already-fetched `FileDto`,
/// avoiding a redundant metadata query when the handler already has it.
async fn get_file_optimized_preloaded(
&self,
id: &str,
file_dto: FileDto,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
// Default: ignore pre-fetched meta, re-fetch everything.
let _ = file_dto;
self.get_file_optimized(id, accept_webp, prefer_original)
.await
}
/// Range-based streaming for HTTP Range Requests (video seek, resumable DL).
async fn get_file_range_stream(
&self,
+28 -3
View File
@@ -56,6 +56,26 @@ pub trait FileReadPort: Send + Sync + 'static {
/// Gets the parent folder ID from a path (WebDAV).
async fn get_parent_folder_id(&self, path: &str) -> Result<String, DomainError>;
/// Find a file by its logical path (folder_name/.../file_name).
///
/// The default implementation falls back to `list_files(None)` + linear
/// scan (O(N)). Repositories should override with a direct SQL query.
async fn find_file_by_path(&self, path: &str) -> Result<Option<File>, DomainError> {
let path = path.trim_start_matches('/').trim_end_matches('/');
let all_files = self.list_files(None).await?;
for file in all_files {
let file_path = file.path_string();
let file_path = file_path.trim_start_matches('/').trim_end_matches('/');
if file_path == path
|| file_path.ends_with(&format!("/{}", path))
|| path.ends_with(&format!("/{}", file_path))
{
return Ok(Some(file));
}
}
Ok(None)
}
}
// ─────────────────────────────────────────────────────
@@ -77,13 +97,18 @@ pub trait FileWritePort: Send + Sync + 'static {
content: Vec<u8>,
) -> Result<File, DomainError>;
/// Streaming upload — writes chunks to disk without accumulating in RAM.
async fn save_file_from_stream(
/// Streaming upload — saves a file from a temp file already on disk.
///
/// When `pre_computed_hash` is provided, the dedup service skips the
/// hash re-read — zero extra I/O beyond the initial spool.
async fn save_file_from_temp(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
temp_path: &std::path::Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<File, DomainError>;
/// Moves a file to another folder.
@@ -168,7 +168,7 @@ impl AuthApplicationService {
/// Returns whether OIDC is configured and enabled
pub fn oidc_enabled(&self) -> bool {
let state = self.oidc.read().unwrap();
state.service.is_some() && state.config.as_ref().map_or(false, |c| c.enabled)
state.service.is_some() && state.config.as_ref().is_some_and(|c| c.enabled)
}
/// Returns whether password login is disabled (OIDC-only mode)
@@ -177,7 +177,7 @@ impl AuthApplicationService {
state
.config
.as_ref()
.map_or(false, |c| c.disable_password_login)
.is_some_and(|c| c.disable_password_login)
}
/// Returns a clone of the OIDC config if available
+14 -14
View File
@@ -114,13 +114,13 @@ impl ContactService {
let lines: Vec<&str> = vcard_data.lines().collect();
for i in 0..lines.len() {
let line = lines[i].trim();
for line in &lines {
let line = line.trim();
if line.starts_with("FN:") {
contact.set_full_name(Some(line[3..].to_string()));
} else if line.starts_with("N:") {
let parts: Vec<&str> = line[2..].split(';').collect();
if let Some(stripped) = line.strip_prefix("FN:") {
contact.set_full_name(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("N:") {
let parts: Vec<&str> = stripped.split(';').collect();
if parts.len() >= 2 {
contact.set_last_name(Some(parts[0].to_string()));
contact.set_first_name(Some(parts[1].to_string()));
@@ -163,14 +163,14 @@ impl ContactService {
is_primary: contact.phone_is_empty(), // First one is primary
});
}
} else if line.starts_with("ORG:") {
contact.set_organization(Some(line[4..].to_string()));
} else if line.starts_with("TITLE:") {
contact.set_title(Some(line[6..].to_string()));
} else if line.starts_with("NOTE:") {
contact.set_notes(Some(line[5..].to_string()));
} else if line.starts_with("UID:") {
contact.set_uid(line[4..].to_string());
} else if let Some(stripped) = line.strip_prefix("ORG:") {
contact.set_organization(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("TITLE:") {
contact.set_title(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("NOTE:") {
contact.set_notes(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("UID:") {
contact.set_uid(stripped.to_string());
}
}
@@ -13,8 +13,6 @@ use tracing::{debug, info, warn};
/// Threshold below which files are served from RAM cache (10 MB).
const CACHE_THRESHOLD: u64 = 10 * 1024 * 1024;
/// Threshold above which mmap is used instead of streaming (100 MB).
const MMAP_THRESHOLD: u64 = 100 * 1024 * 1024;
/// Service for file retrieval operations
///
@@ -103,63 +101,16 @@ impl FileRetrievalService {
_ => None,
}
}
}
#[async_trait]
impl FileRetrievalUseCase for FileRetrievalService {
async fn get_file(&self, id: &str) -> Result<FileDto, DomainError> {
let file = self.file_read.get_file(id).await?;
Ok(FileDto::from(file))
}
async fn get_file_by_path(&self, path: &str) -> Result<FileDto, DomainError> {
// Normalize the path (remove leading/trailing slashes)
let path = path.trim_start_matches('/').trim_end_matches('/');
// List all files and find the one with matching path
let all_files = self.list_files(None).await?;
for file in all_files {
let file_path = file.path.trim_start_matches('/').trim_end_matches('/');
if file_path == path
|| file_path.ends_with(&format!("/{}", path))
|| path.ends_with(&format!("/{}", file_path))
{
return Ok(file);
}
}
Err(DomainError::not_found(
"File",
format!("not found at path: {}", path),
))
}
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<FileDto>, DomainError> {
let files = self.file_read.list_files(folder_id).await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
self.file_read.get_file_content(id).await
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.file_read.get_file_stream(id).await
}
/// Multi-tier optimized download.
async fn get_file_optimized(
/// Core multi-tier download logic shared by `get_file_optimized` and
/// `get_file_optimized_preloaded`.
async fn optimized_inner(
&self,
id: &str,
dto: FileDto,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
let file = self.file_read.get_file(id).await?;
let dto = FileDto::from(file);
let mime_type = dto.mime_type.clone();
let file_size = dto.size;
let file_name = dto.name.clone();
@@ -274,27 +225,9 @@ impl FileRetrievalUseCase for FileRetrievalService {
));
}
// ── Tier 2: MMAP (10–100 MB) ────────────────────────
if file_size < MMAP_THRESHOLD {
info!(
"🗺️ TIER 2 MMAP: {} ({} MB)",
file_name,
file_size / (1024 * 1024)
);
match self.file_read.get_file_mmap(id).await {
Ok(mmap_content) => {
return Ok((dto, OptimizedFileContent::Mmap(mmap_content)));
}
Err(e) => {
warn!("MMAP failed, falling back to streaming: {}", e);
// fall through to streaming
}
}
}
// ── Tier 3: Streaming (≥100 MB) ─────────────────────
// ── Tier 2 + 3: Streaming (≥10 MB) ──────────────────
info!(
"📡 TIER 3 STREAMING: {} ({} MB)",
"📡 TIER 2 STREAMING: {} ({} MB)",
file_name,
file_size / (1024 * 1024)
);
@@ -314,6 +247,67 @@ impl FileRetrievalUseCase for FileRetrievalService {
}
}
}
}
#[async_trait]
impl FileRetrievalUseCase for FileRetrievalService {
async fn get_file(&self, id: &str) -> Result<FileDto, DomainError> {
let file = self.file_read.get_file(id).await?;
Ok(FileDto::from(file))
}
async fn get_file_by_path(&self, path: &str) -> Result<FileDto, DomainError> {
// Direct SQL lookup — O(folder_depth) queries instead of O(total_files)
if let Some(file) = self.file_read.find_file_by_path(path).await? {
return Ok(FileDto::from(file));
}
Err(DomainError::not_found(
"File",
format!("not found at path: {}", path),
))
}
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<FileDto>, DomainError> {
let files = self.file_read.list_files(folder_id).await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
self.file_read.get_file_content(id).await
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.file_read.get_file_stream(id).await
}
/// Multi-tier optimized download.
async fn get_file_optimized(
&self,
id: &str,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
let file = self.file_read.get_file(id).await?;
let dto = FileDto::from(file);
self.optimized_inner(id, dto, accept_webp, prefer_original)
.await
}
/// Like `get_file_optimized` but skips the metadata re-fetch.
async fn get_file_optimized_preloaded(
&self,
id: &str,
file_dto: FileDto,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
self.optimized_inner(id, file_dto, accept_webp, prefer_original)
.await
}
/// Range-based streaming for HTTP Range Requests.
async fn get_file_range_stream(
+62 -193
View File
@@ -1,22 +1,13 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use std::pin::Pin;
use std::path::Path;
use std::sync::Arc;
use crate::application::dtos::file_dto::FileDto;
use crate::application::ports::cache_ports::WriteBehindCachePort;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::file_ports::{FileUploadUseCase, UploadStrategy};
use crate::application::ports::file_ports::FileUploadUseCase;
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::common::errors::DomainError;
use tracing::{debug, info, warn};
/// Threshold for using streaming upload (files >= 1MB use streaming)
const STREAMING_UPLOAD_THRESHOLD: usize = 1024 * 1024;
/// Threshold for write-behind cache (files < 256KB get instant response)
const WRITE_BEHIND_THRESHOLD: usize = 256 * 1024;
/// Helper function to extract username from folder path string.
/// e.g. "My Folder - user1/subfolder/file.txt" → "user1"
fn extract_username_from_path(path: &str) -> Option<String> {
@@ -27,7 +18,6 @@ fn extract_username_from_path(path: &str) -> Option<String> {
if parts.len() <= 1 {
return None;
}
// Take only the first segment (username), not any subfolders
let remainder = parts[1].trim();
let username = remainder.split('/').next().unwrap_or(remainder);
let username = username.trim();
@@ -37,58 +27,35 @@ fn extract_username_from_path(path: &str) -> Option<String> {
Some(username.to_string())
}
/// Service for file upload operations
/// Service for file upload operations.
///
/// Encapsulates the three-tier upload strategy:
/// 1. **Write-Behind** (<256 KB): store in RAM, respond instantly, flush async.
/// 2. **Buffered** (256 KB – 1 MB): collect bytes, write, respond.
/// 3. **Streaming** (≥1 MB): pipe chunks to disk with constant memory.
/// All upload paths converge on streaming-to-disk:
/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming`
/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path`
/// - **WebDAV PUT / empty files**: small in-memory buffer → `upload_file`
///
/// Also runs deduplication so duplicate content is never stored twice.
/// Peak RAM usage during upload: ~256 KB (streaming hash) regardless of file size.
pub struct FileUploadService {
/// Write port — handles save, streaming, deferred registration
file_write: Arc<dyn FileWritePort>,
/// Read port — needed for WebDAV create_file / update_file
file_read: Option<Arc<dyn FileReadPort>>,
/// Optional write-behind cache for instant uploads
write_behind: Option<Arc<dyn WriteBehindCachePort>>,
/// Optional dedup service for content-addressable storage
dedup: Option<Arc<dyn DedupPort>>,
/// Optional storage usage tracking
storage_usage_service:
Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>,
}
impl FileUploadService {
/// Backward-compatible constructor (no write-behind, no dedup).
/// Constructor with write port only (minimal).
pub fn new(file_repository: Arc<dyn FileWritePort>) -> Self {
Self {
file_write: file_repository,
file_read: None,
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
/// Full constructor with all infrastructure ports.
pub fn new_full(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
write_behind: Arc<dyn WriteBehindCachePort>,
dedup: Arc<dyn DedupPort>,
) -> Self {
Self {
file_write,
file_read: Some(file_read),
write_behind: Some(write_behind),
dedup: Some(dedup),
storage_usage_service: None,
}
}
/// Constructor for blob-storage model: write + read ports only.
/// Dedup is handled at the repository layer — no write-behind needed.
/// Constructor for blob-storage model: write + read ports.
pub fn new_with_read(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
@@ -96,8 +63,6 @@ impl FileUploadService {
Self {
file_write,
file_read: Some(file_read),
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
@@ -113,37 +78,9 @@ impl FileUploadService {
// ── private helpers ──────────────────────────────────────────
/// Run dedup tracking (non-fatal on failure).
async fn run_dedup(&self, data: &[u8], content_type: &str) {
let Some(dedup) = &self.dedup else { return };
match dedup
.store_bytes(data, Some(content_type.to_string()))
.await
{
Ok(result) => {
if result.was_deduplicated() {
info!(
"🔗 DEDUP: content already exists (hash: {}, saved {} bytes)",
&result.hash()[..12],
result.size()
);
} else {
info!(
"💾 DEDUP: new content stored (hash: {})",
&result.hash()[..12]
);
}
}
Err(e) => {
warn!("⚠️ DEDUP: Failed to store in blob store: {}", e);
}
}
}
/// Optionally update storage usage after a successful upload.
fn maybe_update_storage_usage(&self, file: &FileDto) {
if let Some(storage_service) = &self.storage_usage_service {
// Extract username from the file's own path (contains folder structure)
let file_path = file.path.clone();
if let Some(username) = extract_username_from_path(&file_path) {
let service_clone = Arc::clone(storage_service);
@@ -166,7 +103,33 @@ impl FileUploadService {
#[async_trait]
impl FileUploadUseCase for FileUploadService {
/// Simple byte-based upload (backward compatible).
/// Streaming upload from a temp file on disk.
///
/// Peak RAM: ~256 KB (hash calculation) regardless of file size.
/// The temp file is consumed (moved/deleted) by the blob store.
async fn upload_file_streaming(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
temp_path: &Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
let file = self
.file_write
.save_file_from_temp(name.clone(), folder_id, content_type, temp_path, size, pre_computed_hash)
.await?;
let dto = FileDto::from(file);
info!(
"📡 STREAMING UPLOAD: {} ({} bytes, ID: {})",
name, size, dto.id
);
self.maybe_update_storage_usage(&dto);
Ok(dto)
}
/// Simple byte-based upload (for WebDAV and empty files only).
async fn upload_file(
&self,
name: String,
@@ -183,105 +146,27 @@ impl FileUploadUseCase for FileUploadService {
Ok(dto)
}
/// Smart three-tier upload with write-behind cache and dedup.
async fn smart_upload(
/// Upload from a file already on disk (chunked uploads).
async fn upload_file_from_path(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
chunks: Vec<Bytes>,
total_size: usize,
) -> Result<(FileDto, UploadStrategy), DomainError> {
use futures::stream;
file_path: &Path,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
let size = tokio::fs::metadata(file_path)
.await
.map_err(|e| {
DomainError::internal_error(
"FileUpload",
format!("Failed to read file metadata: {}", e),
)
})?
.len();
// ─── Dedup (runs for all tiers) ──────────────────────
{
let dedup_data: Vec<u8> = {
let mut combined = Vec::with_capacity(total_size);
for chunk in &chunks {
combined.extend_from_slice(chunk);
}
combined
};
self.run_dedup(&dedup_data, &content_type).await;
}
// ─── TIER 1: Write-Behind (<256 KB) ──────────────────
if total_size < WRITE_BEHIND_THRESHOLD
&& let Some(wb) = &self.write_behind
&& wb.is_eligible_size(total_size)
{
let data: Bytes = if chunks.len() == 1 {
chunks.into_iter().next().unwrap()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined.into()
};
let (file, target_path) = self
.file_write
.register_file_deferred(name.clone(), folder_id, content_type, total_size as u64)
.await?;
let dto = FileDto::from(file);
if let Err(e) = wb.put_pending(dto.id.clone(), data, target_path).await {
return Err(DomainError::internal_error(
"file",
format!("Write-behind cache failed: {}", e),
));
}
info!(
"⚡ WRITE-BEHIND UPLOAD: {} (ID: {}, ~0ms latency)",
name, dto.id
);
self.maybe_update_storage_usage(&dto);
return Ok((dto, UploadStrategy::WriteBehind));
}
// ─── TIER 2: Streaming (≥1 MB) ──────────────────────
if total_size >= STREAMING_UPLOAD_THRESHOLD {
let chunk_stream = stream::iter(chunks.into_iter().map(Ok::<_, std::io::Error>));
let pinned_stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
Box::pin(chunk_stream);
let file = self
.file_write
.save_file_from_stream(name.clone(), folder_id, content_type, pinned_stream)
.await?;
let dto = FileDto::from(file);
info!(
"✅ STREAMING UPLOAD: {} ({} MB, ID: {})",
name,
total_size / (1024 * 1024),
dto.id
);
self.maybe_update_storage_usage(&dto);
return Ok((dto, UploadStrategy::Streaming));
}
// ─── TIER 3: Buffered (256 KB – 1 MB) ───────────────
let data = if chunks.len() == 1 {
chunks.into_iter().next().unwrap().to_vec()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined
};
let file = self
.file_write
.save_file(name.clone(), folder_id, content_type, data)
.await?;
let dto = FileDto::from(file);
info!("✅ BUFFERED UPLOAD: {} (ID: {})", name, dto.id);
self.maybe_update_storage_usage(&dto);
Ok((dto, UploadStrategy::Buffered))
self.upload_file_streaming(name, folder_id, content_type, file_path, size, pre_computed_hash)
.await
}
/// Creates a file at a specific path (for WebDAV PUT on new resource).
@@ -292,13 +177,9 @@ impl FileUploadUseCase for FileUploadService {
content: &[u8],
content_type: &str,
) -> Result<FileDto, DomainError> {
// Resolve parent folder ID from path
let parent_id = if !parent_path.is_empty() {
if let Some(file_read) = &self.file_read {
match file_read.get_parent_folder_id(parent_path).await {
Ok(id) => Some(id),
Err(_) => None, // If parent doesn't exist, use root
}
file_read.get_parent_folder_id(parent_path).await.ok()
} else {
None
}
@@ -322,28 +203,16 @@ impl FileUploadUseCase for FileUploadService {
/// Updates an existing file's content, or creates it if not found (for WebDAV PUT).
async fn update_file(&self, path: &str, content: &[u8]) -> Result<(), DomainError> {
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
// Try to find the existing file by path
if let Some(file_read) = &self.file_read {
let all_files = file_read.list_files(None).await?;
for file in &all_files {
let dto = FileDto::from(file.clone());
let dto_path = dto.path.trim_start_matches('/').trim_end_matches('/');
if dto_path == path_normalized
|| dto_path.ends_with(&format!("/{}", path_normalized))
|| path_normalized.ends_with(&format!("/{}", dto_path))
{
// Found it — update in place
self.file_write
.update_file_content(file.id(), content.to_vec())
.await?;
return Ok(());
}
// Direct SQL lookup — O(folder_depth) instead of O(total_files)
if let Some(file_read) = &self.file_read
&& let Some(file) = file_read.find_file_by_path(path).await? {
self.file_write
.update_file_content(file.id(), content.to_vec())
.await?;
return Ok(());
}
}
// File not found — create it
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
let (parent_path, filename) = if let Some(idx) = path_normalized.rfind('/') {
(&path_normalized[..idx], &path_normalized[idx + 1..])
} else {
@@ -43,7 +43,7 @@ impl I18nApplicationService {
/// Get a translation for a key and locale
pub async fn translate(&self, key: &str, locale: Option<Locale>) -> I18nResult<String> {
let locale = locale.unwrap_or(Locale::default());
let locale = locale.unwrap_or_default();
self.i18n_service.translate(key, locale).await
}
+1 -1
View File
@@ -19,7 +19,7 @@ impl RecentService {
pub fn new(repo: Arc<dyn RecentItemsRepositoryPort>, max_recent_items: i32) -> Self {
Self {
repo,
max_recent_items: max_recent_items.max(1).min(100),
max_recent_items: max_recent_items.clamp(1, 100),
}
}
}
+36 -24
View File
@@ -193,14 +193,14 @@ impl FileWritePort for MockFileRepository {
unimplemented!()
}
async fn save_file_from_stream(
async fn save_file_from_temp(
&self,
_name: String,
_folder_id: Option<String>,
_content_type: String,
_stream: std::pin::Pin<
Box<dyn Stream<Item = std::result::Result<Bytes, std::io::Error>> + Send>,
>,
_temp_path: &std::path::Path,
_size: u64,
_pre_computed_hash: Option<String>,
) -> std::result::Result<File, DomainError> {
unimplemented!()
}
@@ -243,6 +243,14 @@ impl FileWritePort for MockFileRepository {
unimplemented!()
}
async fn copy_file(
&self,
_file_id: &str,
_target_folder_id: Option<String>,
) -> std::result::Result<File, DomainError> {
unimplemented!()
}
async fn move_to_trash(&self, id: &str) -> std::result::Result<(), DomainError> {
let mut files = self.files.lock().unwrap();
let mut trashed = self.trashed_files.lock().unwrap();
@@ -608,17 +616,19 @@ mod tests {
);
// Verify the file is restored in file repository
let files = file_repo.files.lock().unwrap();
let trashed_files = file_repo.trashed_files.lock().unwrap();
{
let files = file_repo.files.lock().unwrap();
let trashed_files = file_repo.trashed_files.lock().unwrap();
assert!(
files.get(file_id).is_some(),
"File should be back in main storage"
);
assert!(
trashed_files.get(file_id).is_none(),
"File should no longer be in trash storage"
);
assert!(
files.get(file_id).is_some(),
"File should be back in main storage"
);
assert!(
trashed_files.get(file_id).is_none(),
"File should no longer be in trash storage"
);
}
// Verify the trash item is removed
let trash_items = trash_repo.get_trash_items(&user_uuid).await.unwrap();
@@ -670,17 +680,19 @@ mod tests {
);
// Verify the file is permanently deleted
let files = file_repo.files.lock().unwrap();
let trashed_files = file_repo.trashed_files.lock().unwrap();
{
let files = file_repo.files.lock().unwrap();
let trashed_files = file_repo.trashed_files.lock().unwrap();
assert!(
files.get(file_id).is_none(),
"File should not be in main storage"
);
assert!(
trashed_files.get(file_id).is_none(),
"File should not be in trash storage"
);
assert!(
files.get(file_id).is_none(),
"File should not be in main storage"
);
assert!(
trashed_files.get(file_id).is_none(),
"File should not be in trash storage"
);
}
// Verify the trash item is removed
let trash_items = trash_repo.get_trash_items(&user_uuid).await.unwrap();