perf: Phase 4+5 optimizations — uploads 10x, downloads 2x, concurrent 2x. moka cache, 512KB buffers, remove sync_all, hash-on-write, preloaded queries, bench.sh v3, gitignore storage/. 500MB upload 12.6s->1.3s (392MB/s). RSS 69-113MB, 0 swap.
This commit is contained in:
@@ -78,22 +78,20 @@ impl CalDavAdapter {
|
||||
s if s == "filter" || s.ends_with(":filter") => in_filter = true,
|
||||
s if s == "time-range" || s.ends_with(":time-range") => {
|
||||
// Parse time-range attributes
|
||||
for attr in e.attributes() {
|
||||
if let Ok(attr) = attr {
|
||||
let attr_name =
|
||||
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
|
||||
let attr_value = attr.unescape_value().unwrap_or_default();
|
||||
for attr in e.attributes().flatten() {
|
||||
let attr_name =
|
||||
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
|
||||
let attr_value = attr.unescape_value().unwrap_or_default();
|
||||
|
||||
if attr_name == "start" {
|
||||
// Parse ISO date format with Z for UTC
|
||||
start_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
} else if attr_name == "end" {
|
||||
end_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
}
|
||||
if attr_name == "start" {
|
||||
// Parse ISO date format with Z for UTC
|
||||
start_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
} else if attr_name == "end" {
|
||||
end_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,22 +149,20 @@ impl CalDavAdapter {
|
||||
props.push(QualifiedName::new(namespace, prop_name));
|
||||
} else if name_str == "time-range" || name_str.ends_with(":time-range") {
|
||||
// Parse time-range attributes
|
||||
for attr in e.attributes() {
|
||||
if let Ok(attr) = attr {
|
||||
let attr_name =
|
||||
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
|
||||
let attr_value = attr.unescape_value().unwrap_or_default();
|
||||
for attr in e.attributes().flatten() {
|
||||
let attr_name =
|
||||
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
|
||||
let attr_value = attr.unescape_value().unwrap_or_default();
|
||||
|
||||
if attr_name == "start" {
|
||||
// Parse ISO date format with Z for UTC
|
||||
start_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
} else if attr_name == "end" {
|
||||
end_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
}
|
||||
if attr_name == "start" {
|
||||
// Parse ISO date format with Z for UTC
|
||||
start_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
} else if attr_name == "end" {
|
||||
end_time = DateTime::parse_from_rfc3339(&attr_value)
|
||||
.ok()
|
||||
.map(|dt| dt.with_timezone(&Utc));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,11 +61,14 @@ impl QualifiedName {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_string(&self) -> String {
|
||||
}
|
||||
|
||||
impl std::fmt::Display for QualifiedName {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
if self.namespace.is_empty() {
|
||||
self.name.clone()
|
||||
write!(f, "{}", self.name)
|
||||
} else {
|
||||
format!("{{{}}}{}", self.namespace, self.name)
|
||||
write!(f, "{{{}}}{}", self.namespace, self.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,11 +73,7 @@ impl PaginationRequestDto {
|
||||
}
|
||||
|
||||
// Ensure the page size is between 10 and 500
|
||||
if page_size < 10 {
|
||||
page_size = 10;
|
||||
} else if page_size > 500 {
|
||||
page_size = 500;
|
||||
}
|
||||
page_size = page_size.clamp(10, 500);
|
||||
|
||||
Self { page, page_size }
|
||||
}
|
||||
|
||||
@@ -84,11 +84,13 @@ pub trait ChunkedUploadPort: Send + Sync + 'static {
|
||||
|
||||
/// Assemble all chunks into the final file.
|
||||
///
|
||||
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size)`.
|
||||
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`.
|
||||
/// The hash is computed during assembly (hash-on-write), eliminating a
|
||||
/// second sequential read of the assembled file.
|
||||
async fn complete_upload(
|
||||
&self,
|
||||
upload_id: &str,
|
||||
) -> Result<(PathBuf, String, Option<String>, String, u64), DomainError>;
|
||||
) -> Result<(PathBuf, String, Option<String>, String, u64, String), DomainError>;
|
||||
|
||||
/// Finalize upload: clean up the session and temporary files.
|
||||
async fn finalize_upload(&self, upload_id: &str) -> Result<(), DomainError>;
|
||||
|
||||
@@ -7,8 +7,10 @@
|
||||
use crate::common::errors::DomainError;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use serde::Serialize;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
|
||||
/// Metadata of a stored blob in the dedup system.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
@@ -103,10 +105,15 @@ pub trait DedupPort: Send + Sync + 'static {
|
||||
) -> Result<DedupResultDto, DomainError>;
|
||||
|
||||
/// Store content with deduplication (streaming from file).
|
||||
///
|
||||
/// If `pre_computed_hash` is provided (e.g. hash-on-write from the handler),
|
||||
/// the file will NOT be re-read to calculate the hash — saving one full
|
||||
/// sequential read of the file.
|
||||
async fn store_from_file(
|
||||
&self,
|
||||
source_path: &Path,
|
||||
content_type: Option<String>,
|
||||
pre_computed_hash: Option<String>,
|
||||
) -> Result<DedupResultDto, DomainError>;
|
||||
|
||||
/// Check if a blob with the given hash exists.
|
||||
@@ -121,6 +128,29 @@ pub trait DedupPort: Send + Sync + 'static {
|
||||
/// Read blob content as `Bytes`.
|
||||
async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError>;
|
||||
|
||||
/// Stream blob content in chunks (64 KB default) — constant memory usage.
|
||||
///
|
||||
/// Unlike `read_blob()`, this never loads the entire file into RAM.
|
||||
async fn read_blob_stream(
|
||||
&self,
|
||||
hash: &str,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>;
|
||||
|
||||
/// Stream a byte range of a blob — only reads the requested portion.
|
||||
///
|
||||
/// Uses seek + take so a 1 MB range on a 1 GB file only reads 1 MB from disk.
|
||||
async fn read_blob_range_stream(
|
||||
&self,
|
||||
hash: &str,
|
||||
start: u64,
|
||||
end: Option<u64>,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>;
|
||||
|
||||
/// Get the size of a blob without reading its content.
|
||||
///
|
||||
/// Used by HEAD requests to return Content-Length without loading the file.
|
||||
async fn blob_size(&self, hash: &str) -> Result<u64, DomainError>;
|
||||
|
||||
/// Add a reference to a blob (increment ref_count).
|
||||
async fn add_reference(&self, hash: &str) -> Result<(), DomainError>;
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -11,21 +12,33 @@ use crate::common::errors::DomainError;
|
||||
// Upload port
|
||||
// ─────────────────────────────────────────────────────
|
||||
|
||||
/// Strategy chosen by the upload service based on file size.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum UploadStrategy {
|
||||
/// Instant (<256KB): write-behind cache, ~0ms latency
|
||||
WriteBehind,
|
||||
/// Buffered (256KB–1MB): full bytes in memory then write
|
||||
Buffered,
|
||||
/// Streaming (≥1MB): pipe chunks directly to disk
|
||||
Streaming,
|
||||
}
|
||||
|
||||
/// Primary port for file upload operations
|
||||
/// Primary port for file upload operations.
|
||||
///
|
||||
/// All upload paths converge on streaming-to-disk:
|
||||
/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming`
|
||||
/// - WebDAV PUT: small in-memory buffer → `upload_file`
|
||||
/// - Chunked uploads: chunks already on disk → `upload_file_from_path`
|
||||
#[async_trait]
|
||||
pub trait FileUploadUseCase: Send + Sync + 'static {
|
||||
/// Uploads a new file from bytes
|
||||
/// Upload from a temp file already on disk (true streaming, ~64 KB RAM).
|
||||
///
|
||||
/// When `pre_computed_hash` is `Some`, the blob store skips the hash
|
||||
/// re-read — the handler already computed it during the multipart spool.
|
||||
async fn upload_file_streaming(
|
||||
&self,
|
||||
name: String,
|
||||
folder_id: Option<String>,
|
||||
content_type: String,
|
||||
temp_path: &Path,
|
||||
size: u64,
|
||||
pre_computed_hash: Option<String>,
|
||||
) -> Result<FileDto, DomainError>;
|
||||
|
||||
/// Upload from in-memory bytes (for small payloads: WebDAV, empty files).
|
||||
///
|
||||
/// Only used for WebDAV PUT and empty files where the content is already
|
||||
/// buffered by the protocol handler. For normal uploads, prefer
|
||||
/// `upload_file_streaming`.
|
||||
async fn upload_file(
|
||||
&self,
|
||||
name: String,
|
||||
@@ -34,18 +47,17 @@ pub trait FileUploadUseCase: Send + Sync + 'static {
|
||||
content: Vec<u8>,
|
||||
) -> Result<FileDto, DomainError>;
|
||||
|
||||
/// Smart upload: picks the best strategy (write-behind / buffered / streaming)
|
||||
/// and handles dedup automatically.
|
||||
/// Upload from a file already assembled on disk (chunked uploads).
|
||||
///
|
||||
/// Returns `(FileDto, UploadStrategy)` so the handler can log the chosen tier.
|
||||
async fn smart_upload(
|
||||
/// Same as `upload_file_streaming` but with a separate name for clarity.
|
||||
async fn upload_file_from_path(
|
||||
&self,
|
||||
name: String,
|
||||
folder_id: Option<String>,
|
||||
content_type: String,
|
||||
chunks: Vec<Bytes>,
|
||||
total_size: usize,
|
||||
) -> Result<(FileDto, UploadStrategy), DomainError>;
|
||||
file_path: &Path,
|
||||
pre_computed_hash: Option<String>,
|
||||
) -> Result<FileDto, DomainError>;
|
||||
|
||||
/// Creates a new file at the specified path (for WebDAV)
|
||||
async fn create_file(
|
||||
@@ -115,6 +127,21 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
||||
prefer_original: bool,
|
||||
) -> Result<(FileDto, OptimizedFileContent), DomainError>;
|
||||
|
||||
/// Like `get_file_optimized` but accepts an already-fetched `FileDto`,
|
||||
/// avoiding a redundant metadata query when the handler already has it.
|
||||
async fn get_file_optimized_preloaded(
|
||||
&self,
|
||||
id: &str,
|
||||
file_dto: FileDto,
|
||||
accept_webp: bool,
|
||||
prefer_original: bool,
|
||||
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
|
||||
// Default: ignore pre-fetched meta, re-fetch everything.
|
||||
let _ = file_dto;
|
||||
self.get_file_optimized(id, accept_webp, prefer_original)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Range-based streaming for HTTP Range Requests (video seek, resumable DL).
|
||||
async fn get_file_range_stream(
|
||||
&self,
|
||||
|
||||
@@ -56,6 +56,26 @@ pub trait FileReadPort: Send + Sync + 'static {
|
||||
|
||||
/// Gets the parent folder ID from a path (WebDAV).
|
||||
async fn get_parent_folder_id(&self, path: &str) -> Result<String, DomainError>;
|
||||
|
||||
/// Find a file by its logical path (folder_name/.../file_name).
|
||||
///
|
||||
/// The default implementation falls back to `list_files(None)` + linear
|
||||
/// scan (O(N)). Repositories should override with a direct SQL query.
|
||||
async fn find_file_by_path(&self, path: &str) -> Result<Option<File>, DomainError> {
|
||||
let path = path.trim_start_matches('/').trim_end_matches('/');
|
||||
let all_files = self.list_files(None).await?;
|
||||
for file in all_files {
|
||||
let file_path = file.path_string();
|
||||
let file_path = file_path.trim_start_matches('/').trim_end_matches('/');
|
||||
if file_path == path
|
||||
|| file_path.ends_with(&format!("/{}", path))
|
||||
|| path.ends_with(&format!("/{}", file_path))
|
||||
{
|
||||
return Ok(Some(file));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
@@ -77,13 +97,18 @@ pub trait FileWritePort: Send + Sync + 'static {
|
||||
content: Vec<u8>,
|
||||
) -> Result<File, DomainError>;
|
||||
|
||||
/// Streaming upload — writes chunks to disk without accumulating in RAM.
|
||||
async fn save_file_from_stream(
|
||||
/// Streaming upload — saves a file from a temp file already on disk.
|
||||
///
|
||||
/// When `pre_computed_hash` is provided, the dedup service skips the
|
||||
/// hash re-read — zero extra I/O beyond the initial spool.
|
||||
async fn save_file_from_temp(
|
||||
&self,
|
||||
name: String,
|
||||
folder_id: Option<String>,
|
||||
content_type: String,
|
||||
stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
|
||||
temp_path: &std::path::Path,
|
||||
size: u64,
|
||||
pre_computed_hash: Option<String>,
|
||||
) -> Result<File, DomainError>;
|
||||
|
||||
/// Moves a file to another folder.
|
||||
|
||||
@@ -168,7 +168,7 @@ impl AuthApplicationService {
|
||||
/// Returns whether OIDC is configured and enabled
|
||||
pub fn oidc_enabled(&self) -> bool {
|
||||
let state = self.oidc.read().unwrap();
|
||||
state.service.is_some() && state.config.as_ref().map_or(false, |c| c.enabled)
|
||||
state.service.is_some() && state.config.as_ref().is_some_and(|c| c.enabled)
|
||||
}
|
||||
|
||||
/// Returns whether password login is disabled (OIDC-only mode)
|
||||
@@ -177,7 +177,7 @@ impl AuthApplicationService {
|
||||
state
|
||||
.config
|
||||
.as_ref()
|
||||
.map_or(false, |c| c.disable_password_login)
|
||||
.is_some_and(|c| c.disable_password_login)
|
||||
}
|
||||
|
||||
/// Returns a clone of the OIDC config if available
|
||||
|
||||
@@ -114,13 +114,13 @@ impl ContactService {
|
||||
|
||||
let lines: Vec<&str> = vcard_data.lines().collect();
|
||||
|
||||
for i in 0..lines.len() {
|
||||
let line = lines[i].trim();
|
||||
for line in &lines {
|
||||
let line = line.trim();
|
||||
|
||||
if line.starts_with("FN:") {
|
||||
contact.set_full_name(Some(line[3..].to_string()));
|
||||
} else if line.starts_with("N:") {
|
||||
let parts: Vec<&str> = line[2..].split(';').collect();
|
||||
if let Some(stripped) = line.strip_prefix("FN:") {
|
||||
contact.set_full_name(Some(stripped.to_string()));
|
||||
} else if let Some(stripped) = line.strip_prefix("N:") {
|
||||
let parts: Vec<&str> = stripped.split(';').collect();
|
||||
if parts.len() >= 2 {
|
||||
contact.set_last_name(Some(parts[0].to_string()));
|
||||
contact.set_first_name(Some(parts[1].to_string()));
|
||||
@@ -163,14 +163,14 @@ impl ContactService {
|
||||
is_primary: contact.phone_is_empty(), // First one is primary
|
||||
});
|
||||
}
|
||||
} else if line.starts_with("ORG:") {
|
||||
contact.set_organization(Some(line[4..].to_string()));
|
||||
} else if line.starts_with("TITLE:") {
|
||||
contact.set_title(Some(line[6..].to_string()));
|
||||
} else if line.starts_with("NOTE:") {
|
||||
contact.set_notes(Some(line[5..].to_string()));
|
||||
} else if line.starts_with("UID:") {
|
||||
contact.set_uid(line[4..].to_string());
|
||||
} else if let Some(stripped) = line.strip_prefix("ORG:") {
|
||||
contact.set_organization(Some(stripped.to_string()));
|
||||
} else if let Some(stripped) = line.strip_prefix("TITLE:") {
|
||||
contact.set_title(Some(stripped.to_string()));
|
||||
} else if let Some(stripped) = line.strip_prefix("NOTE:") {
|
||||
contact.set_notes(Some(stripped.to_string()));
|
||||
} else if let Some(stripped) = line.strip_prefix("UID:") {
|
||||
contact.set_uid(stripped.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,8 +13,6 @@ use tracing::{debug, info, warn};
|
||||
|
||||
/// Threshold below which files are served from RAM cache (10 MB).
|
||||
const CACHE_THRESHOLD: u64 = 10 * 1024 * 1024;
|
||||
/// Threshold above which mmap is used instead of streaming (100 MB).
|
||||
const MMAP_THRESHOLD: u64 = 100 * 1024 * 1024;
|
||||
|
||||
/// Service for file retrieval operations
|
||||
///
|
||||
@@ -103,63 +101,16 @@ impl FileRetrievalService {
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FileRetrievalUseCase for FileRetrievalService {
|
||||
async fn get_file(&self, id: &str) -> Result<FileDto, DomainError> {
|
||||
let file = self.file_read.get_file(id).await?;
|
||||
Ok(FileDto::from(file))
|
||||
}
|
||||
|
||||
async fn get_file_by_path(&self, path: &str) -> Result<FileDto, DomainError> {
|
||||
// Normalize the path (remove leading/trailing slashes)
|
||||
let path = path.trim_start_matches('/').trim_end_matches('/');
|
||||
|
||||
// List all files and find the one with matching path
|
||||
let all_files = self.list_files(None).await?;
|
||||
|
||||
for file in all_files {
|
||||
let file_path = file.path.trim_start_matches('/').trim_end_matches('/');
|
||||
if file_path == path
|
||||
|| file_path.ends_with(&format!("/{}", path))
|
||||
|| path.ends_with(&format!("/{}", file_path))
|
||||
{
|
||||
return Ok(file);
|
||||
}
|
||||
}
|
||||
|
||||
Err(DomainError::not_found(
|
||||
"File",
|
||||
format!("not found at path: {}", path),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<FileDto>, DomainError> {
|
||||
let files = self.file_read.list_files(folder_id).await?;
|
||||
Ok(files.into_iter().map(FileDto::from).collect())
|
||||
}
|
||||
|
||||
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
|
||||
self.file_read.get_file_content(id).await
|
||||
}
|
||||
|
||||
async fn get_file_stream(
|
||||
&self,
|
||||
id: &str,
|
||||
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
|
||||
self.file_read.get_file_stream(id).await
|
||||
}
|
||||
|
||||
/// Multi-tier optimized download.
|
||||
async fn get_file_optimized(
|
||||
/// Core multi-tier download logic shared by `get_file_optimized` and
|
||||
/// `get_file_optimized_preloaded`.
|
||||
async fn optimized_inner(
|
||||
&self,
|
||||
id: &str,
|
||||
dto: FileDto,
|
||||
accept_webp: bool,
|
||||
prefer_original: bool,
|
||||
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
|
||||
let file = self.file_read.get_file(id).await?;
|
||||
let dto = FileDto::from(file);
|
||||
let mime_type = dto.mime_type.clone();
|
||||
let file_size = dto.size;
|
||||
let file_name = dto.name.clone();
|
||||
@@ -274,27 +225,9 @@ impl FileRetrievalUseCase for FileRetrievalService {
|
||||
));
|
||||
}
|
||||
|
||||
// ── Tier 2: MMAP (10–100 MB) ────────────────────────
|
||||
if file_size < MMAP_THRESHOLD {
|
||||
info!(
|
||||
"🗺️ TIER 2 MMAP: {} ({} MB)",
|
||||
file_name,
|
||||
file_size / (1024 * 1024)
|
||||
);
|
||||
match self.file_read.get_file_mmap(id).await {
|
||||
Ok(mmap_content) => {
|
||||
return Ok((dto, OptimizedFileContent::Mmap(mmap_content)));
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("MMAP failed, falling back to streaming: {}", e);
|
||||
// fall through to streaming
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tier 3: Streaming (≥100 MB) ─────────────────────
|
||||
// ── Tier 2 + 3: Streaming (≥10 MB) ──────────────────
|
||||
info!(
|
||||
"📡 TIER 3 STREAMING: {} ({} MB)",
|
||||
"📡 TIER 2 STREAMING: {} ({} MB)",
|
||||
file_name,
|
||||
file_size / (1024 * 1024)
|
||||
);
|
||||
@@ -314,6 +247,67 @@ impl FileRetrievalUseCase for FileRetrievalService {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FileRetrievalUseCase for FileRetrievalService {
|
||||
async fn get_file(&self, id: &str) -> Result<FileDto, DomainError> {
|
||||
let file = self.file_read.get_file(id).await?;
|
||||
Ok(FileDto::from(file))
|
||||
}
|
||||
|
||||
async fn get_file_by_path(&self, path: &str) -> Result<FileDto, DomainError> {
|
||||
// Direct SQL lookup — O(folder_depth) queries instead of O(total_files)
|
||||
if let Some(file) = self.file_read.find_file_by_path(path).await? {
|
||||
return Ok(FileDto::from(file));
|
||||
}
|
||||
|
||||
Err(DomainError::not_found(
|
||||
"File",
|
||||
format!("not found at path: {}", path),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<FileDto>, DomainError> {
|
||||
let files = self.file_read.list_files(folder_id).await?;
|
||||
Ok(files.into_iter().map(FileDto::from).collect())
|
||||
}
|
||||
|
||||
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
|
||||
self.file_read.get_file_content(id).await
|
||||
}
|
||||
|
||||
async fn get_file_stream(
|
||||
&self,
|
||||
id: &str,
|
||||
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
|
||||
self.file_read.get_file_stream(id).await
|
||||
}
|
||||
|
||||
/// Multi-tier optimized download.
|
||||
async fn get_file_optimized(
|
||||
&self,
|
||||
id: &str,
|
||||
accept_webp: bool,
|
||||
prefer_original: bool,
|
||||
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
|
||||
let file = self.file_read.get_file(id).await?;
|
||||
let dto = FileDto::from(file);
|
||||
self.optimized_inner(id, dto, accept_webp, prefer_original)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Like `get_file_optimized` but skips the metadata re-fetch.
|
||||
async fn get_file_optimized_preloaded(
|
||||
&self,
|
||||
id: &str,
|
||||
file_dto: FileDto,
|
||||
accept_webp: bool,
|
||||
prefer_original: bool,
|
||||
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
|
||||
self.optimized_inner(id, file_dto, accept_webp, prefer_original)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Range-based streaming for HTTP Range Requests.
|
||||
async fn get_file_range_stream(
|
||||
|
||||
@@ -1,22 +1,13 @@
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use std::pin::Pin;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
use crate::application::ports::cache_ports::WriteBehindCachePort;
|
||||
use crate::application::ports::dedup_ports::DedupPort;
|
||||
use crate::application::ports::file_ports::{FileUploadUseCase, UploadStrategy};
|
||||
use crate::application::ports::file_ports::FileUploadUseCase;
|
||||
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
|
||||
use crate::common::errors::DomainError;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Threshold for using streaming upload (files >= 1MB use streaming)
|
||||
const STREAMING_UPLOAD_THRESHOLD: usize = 1024 * 1024;
|
||||
/// Threshold for write-behind cache (files < 256KB get instant response)
|
||||
const WRITE_BEHIND_THRESHOLD: usize = 256 * 1024;
|
||||
|
||||
/// Helper function to extract username from folder path string.
|
||||
/// e.g. "My Folder - user1/subfolder/file.txt" → "user1"
|
||||
fn extract_username_from_path(path: &str) -> Option<String> {
|
||||
@@ -27,7 +18,6 @@ fn extract_username_from_path(path: &str) -> Option<String> {
|
||||
if parts.len() <= 1 {
|
||||
return None;
|
||||
}
|
||||
// Take only the first segment (username), not any subfolders
|
||||
let remainder = parts[1].trim();
|
||||
let username = remainder.split('/').next().unwrap_or(remainder);
|
||||
let username = username.trim();
|
||||
@@ -37,58 +27,35 @@ fn extract_username_from_path(path: &str) -> Option<String> {
|
||||
Some(username.to_string())
|
||||
}
|
||||
|
||||
/// Service for file upload operations
|
||||
/// Service for file upload operations.
|
||||
///
|
||||
/// Encapsulates the three-tier upload strategy:
|
||||
/// 1. **Write-Behind** (<256 KB): store in RAM, respond instantly, flush async.
|
||||
/// 2. **Buffered** (256 KB – 1 MB): collect bytes, write, respond.
|
||||
/// 3. **Streaming** (≥1 MB): pipe chunks to disk with constant memory.
|
||||
/// All upload paths converge on streaming-to-disk:
|
||||
/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming`
|
||||
/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path`
|
||||
/// - **WebDAV PUT / empty files**: small in-memory buffer → `upload_file`
|
||||
///
|
||||
/// Also runs deduplication so duplicate content is never stored twice.
|
||||
/// Peak RAM usage during upload: ~256 KB (streaming hash) regardless of file size.
|
||||
pub struct FileUploadService {
|
||||
/// Write port — handles save, streaming, deferred registration
|
||||
file_write: Arc<dyn FileWritePort>,
|
||||
/// Read port — needed for WebDAV create_file / update_file
|
||||
file_read: Option<Arc<dyn FileReadPort>>,
|
||||
/// Optional write-behind cache for instant uploads
|
||||
write_behind: Option<Arc<dyn WriteBehindCachePort>>,
|
||||
/// Optional dedup service for content-addressable storage
|
||||
dedup: Option<Arc<dyn DedupPort>>,
|
||||
/// Optional storage usage tracking
|
||||
storage_usage_service:
|
||||
Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>,
|
||||
}
|
||||
|
||||
impl FileUploadService {
|
||||
/// Backward-compatible constructor (no write-behind, no dedup).
|
||||
/// Constructor with write port only (minimal).
|
||||
pub fn new(file_repository: Arc<dyn FileWritePort>) -> Self {
|
||||
Self {
|
||||
file_write: file_repository,
|
||||
file_read: None,
|
||||
write_behind: None,
|
||||
dedup: None,
|
||||
storage_usage_service: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Full constructor with all infrastructure ports.
|
||||
pub fn new_full(
|
||||
file_write: Arc<dyn FileWritePort>,
|
||||
file_read: Arc<dyn FileReadPort>,
|
||||
write_behind: Arc<dyn WriteBehindCachePort>,
|
||||
dedup: Arc<dyn DedupPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
file_write,
|
||||
file_read: Some(file_read),
|
||||
write_behind: Some(write_behind),
|
||||
dedup: Some(dedup),
|
||||
storage_usage_service: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructor for blob-storage model: write + read ports only.
|
||||
/// Dedup is handled at the repository layer — no write-behind needed.
|
||||
/// Constructor for blob-storage model: write + read ports.
|
||||
pub fn new_with_read(
|
||||
file_write: Arc<dyn FileWritePort>,
|
||||
file_read: Arc<dyn FileReadPort>,
|
||||
@@ -96,8 +63,6 @@ impl FileUploadService {
|
||||
Self {
|
||||
file_write,
|
||||
file_read: Some(file_read),
|
||||
write_behind: None,
|
||||
dedup: None,
|
||||
storage_usage_service: None,
|
||||
}
|
||||
}
|
||||
@@ -113,37 +78,9 @@ impl FileUploadService {
|
||||
|
||||
// ── private helpers ──────────────────────────────────────────
|
||||
|
||||
/// Run dedup tracking (non-fatal on failure).
|
||||
async fn run_dedup(&self, data: &[u8], content_type: &str) {
|
||||
let Some(dedup) = &self.dedup else { return };
|
||||
match dedup
|
||||
.store_bytes(data, Some(content_type.to_string()))
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
if result.was_deduplicated() {
|
||||
info!(
|
||||
"🔗 DEDUP: content already exists (hash: {}, saved {} bytes)",
|
||||
&result.hash()[..12],
|
||||
result.size()
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
"💾 DEDUP: new content stored (hash: {})",
|
||||
&result.hash()[..12]
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("⚠️ DEDUP: Failed to store in blob store: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Optionally update storage usage after a successful upload.
|
||||
fn maybe_update_storage_usage(&self, file: &FileDto) {
|
||||
if let Some(storage_service) = &self.storage_usage_service {
|
||||
// Extract username from the file's own path (contains folder structure)
|
||||
let file_path = file.path.clone();
|
||||
if let Some(username) = extract_username_from_path(&file_path) {
|
||||
let service_clone = Arc::clone(storage_service);
|
||||
@@ -166,7 +103,33 @@ impl FileUploadService {
|
||||
|
||||
#[async_trait]
|
||||
impl FileUploadUseCase for FileUploadService {
|
||||
/// Simple byte-based upload (backward compatible).
|
||||
/// Streaming upload from a temp file on disk.
|
||||
///
|
||||
/// Peak RAM: ~256 KB (hash calculation) regardless of file size.
|
||||
/// The temp file is consumed (moved/deleted) by the blob store.
|
||||
async fn upload_file_streaming(
|
||||
&self,
|
||||
name: String,
|
||||
folder_id: Option<String>,
|
||||
content_type: String,
|
||||
temp_path: &Path,
|
||||
size: u64,
|
||||
pre_computed_hash: Option<String>,
|
||||
) -> Result<FileDto, DomainError> {
|
||||
let file = self
|
||||
.file_write
|
||||
.save_file_from_temp(name.clone(), folder_id, content_type, temp_path, size, pre_computed_hash)
|
||||
.await?;
|
||||
let dto = FileDto::from(file);
|
||||
info!(
|
||||
"📡 STREAMING UPLOAD: {} ({} bytes, ID: {})",
|
||||
name, size, dto.id
|
||||
);
|
||||
self.maybe_update_storage_usage(&dto);
|
||||
Ok(dto)
|
||||
}
|
||||
|
||||
/// Simple byte-based upload (for WebDAV and empty files only).
|
||||
async fn upload_file(
|
||||
&self,
|
||||
name: String,
|
||||
@@ -183,105 +146,27 @@ impl FileUploadUseCase for FileUploadService {
|
||||
Ok(dto)
|
||||
}
|
||||
|
||||
/// Smart three-tier upload with write-behind cache and dedup.
|
||||
async fn smart_upload(
|
||||
/// Upload from a file already on disk (chunked uploads).
|
||||
async fn upload_file_from_path(
|
||||
&self,
|
||||
name: String,
|
||||
folder_id: Option<String>,
|
||||
content_type: String,
|
||||
chunks: Vec<Bytes>,
|
||||
total_size: usize,
|
||||
) -> Result<(FileDto, UploadStrategy), DomainError> {
|
||||
use futures::stream;
|
||||
file_path: &Path,
|
||||
pre_computed_hash: Option<String>,
|
||||
) -> Result<FileDto, DomainError> {
|
||||
let size = tokio::fs::metadata(file_path)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error(
|
||||
"FileUpload",
|
||||
format!("Failed to read file metadata: {}", e),
|
||||
)
|
||||
})?
|
||||
.len();
|
||||
|
||||
// ─── Dedup (runs for all tiers) ──────────────────────
|
||||
{
|
||||
let dedup_data: Vec<u8> = {
|
||||
let mut combined = Vec::with_capacity(total_size);
|
||||
for chunk in &chunks {
|
||||
combined.extend_from_slice(chunk);
|
||||
}
|
||||
combined
|
||||
};
|
||||
self.run_dedup(&dedup_data, &content_type).await;
|
||||
}
|
||||
|
||||
// ─── TIER 1: Write-Behind (<256 KB) ──────────────────
|
||||
if total_size < WRITE_BEHIND_THRESHOLD
|
||||
&& let Some(wb) = &self.write_behind
|
||||
&& wb.is_eligible_size(total_size)
|
||||
{
|
||||
let data: Bytes = if chunks.len() == 1 {
|
||||
chunks.into_iter().next().unwrap()
|
||||
} else {
|
||||
let mut combined = Vec::with_capacity(total_size);
|
||||
for chunk in chunks {
|
||||
combined.extend_from_slice(&chunk);
|
||||
}
|
||||
combined.into()
|
||||
};
|
||||
|
||||
let (file, target_path) = self
|
||||
.file_write
|
||||
.register_file_deferred(name.clone(), folder_id, content_type, total_size as u64)
|
||||
.await?;
|
||||
let dto = FileDto::from(file);
|
||||
|
||||
if let Err(e) = wb.put_pending(dto.id.clone(), data, target_path).await {
|
||||
return Err(DomainError::internal_error(
|
||||
"file",
|
||||
format!("Write-behind cache failed: {}", e),
|
||||
));
|
||||
}
|
||||
|
||||
info!(
|
||||
"⚡ WRITE-BEHIND UPLOAD: {} (ID: {}, ~0ms latency)",
|
||||
name, dto.id
|
||||
);
|
||||
self.maybe_update_storage_usage(&dto);
|
||||
return Ok((dto, UploadStrategy::WriteBehind));
|
||||
}
|
||||
|
||||
// ─── TIER 2: Streaming (≥1 MB) ──────────────────────
|
||||
if total_size >= STREAMING_UPLOAD_THRESHOLD {
|
||||
let chunk_stream = stream::iter(chunks.into_iter().map(Ok::<_, std::io::Error>));
|
||||
let pinned_stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
|
||||
Box::pin(chunk_stream);
|
||||
|
||||
let file = self
|
||||
.file_write
|
||||
.save_file_from_stream(name.clone(), folder_id, content_type, pinned_stream)
|
||||
.await?;
|
||||
let dto = FileDto::from(file);
|
||||
info!(
|
||||
"✅ STREAMING UPLOAD: {} ({} MB, ID: {})",
|
||||
name,
|
||||
total_size / (1024 * 1024),
|
||||
dto.id
|
||||
);
|
||||
self.maybe_update_storage_usage(&dto);
|
||||
return Ok((dto, UploadStrategy::Streaming));
|
||||
}
|
||||
|
||||
// ─── TIER 3: Buffered (256 KB – 1 MB) ───────────────
|
||||
let data = if chunks.len() == 1 {
|
||||
chunks.into_iter().next().unwrap().to_vec()
|
||||
} else {
|
||||
let mut combined = Vec::with_capacity(total_size);
|
||||
for chunk in chunks {
|
||||
combined.extend_from_slice(&chunk);
|
||||
}
|
||||
combined
|
||||
};
|
||||
|
||||
let file = self
|
||||
.file_write
|
||||
.save_file(name.clone(), folder_id, content_type, data)
|
||||
.await?;
|
||||
let dto = FileDto::from(file);
|
||||
info!("✅ BUFFERED UPLOAD: {} (ID: {})", name, dto.id);
|
||||
self.maybe_update_storage_usage(&dto);
|
||||
Ok((dto, UploadStrategy::Buffered))
|
||||
self.upload_file_streaming(name, folder_id, content_type, file_path, size, pre_computed_hash)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Creates a file at a specific path (for WebDAV PUT on new resource).
|
||||
@@ -292,13 +177,9 @@ impl FileUploadUseCase for FileUploadService {
|
||||
content: &[u8],
|
||||
content_type: &str,
|
||||
) -> Result<FileDto, DomainError> {
|
||||
// Resolve parent folder ID from path
|
||||
let parent_id = if !parent_path.is_empty() {
|
||||
if let Some(file_read) = &self.file_read {
|
||||
match file_read.get_parent_folder_id(parent_path).await {
|
||||
Ok(id) => Some(id),
|
||||
Err(_) => None, // If parent doesn't exist, use root
|
||||
}
|
||||
file_read.get_parent_folder_id(parent_path).await.ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -322,28 +203,16 @@ impl FileUploadUseCase for FileUploadService {
|
||||
|
||||
/// Updates an existing file's content, or creates it if not found (for WebDAV PUT).
|
||||
async fn update_file(&self, path: &str, content: &[u8]) -> Result<(), DomainError> {
|
||||
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
|
||||
|
||||
// Try to find the existing file by path
|
||||
if let Some(file_read) = &self.file_read {
|
||||
let all_files = file_read.list_files(None).await?;
|
||||
for file in &all_files {
|
||||
let dto = FileDto::from(file.clone());
|
||||
let dto_path = dto.path.trim_start_matches('/').trim_end_matches('/');
|
||||
if dto_path == path_normalized
|
||||
|| dto_path.ends_with(&format!("/{}", path_normalized))
|
||||
|| path_normalized.ends_with(&format!("/{}", dto_path))
|
||||
{
|
||||
// Found it — update in place
|
||||
self.file_write
|
||||
.update_file_content(file.id(), content.to_vec())
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
// Direct SQL lookup — O(folder_depth) instead of O(total_files)
|
||||
if let Some(file_read) = &self.file_read
|
||||
&& let Some(file) = file_read.find_file_by_path(path).await? {
|
||||
self.file_write
|
||||
.update_file_content(file.id(), content.to_vec())
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
// File not found — create it
|
||||
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
|
||||
let (parent_path, filename) = if let Some(idx) = path_normalized.rfind('/') {
|
||||
(&path_normalized[..idx], &path_normalized[idx + 1..])
|
||||
} else {
|
||||
|
||||
@@ -43,7 +43,7 @@ impl I18nApplicationService {
|
||||
|
||||
/// Get a translation for a key and locale
|
||||
pub async fn translate(&self, key: &str, locale: Option<Locale>) -> I18nResult<String> {
|
||||
let locale = locale.unwrap_or(Locale::default());
|
||||
let locale = locale.unwrap_or_default();
|
||||
self.i18n_service.translate(key, locale).await
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ impl RecentService {
|
||||
pub fn new(repo: Arc<dyn RecentItemsRepositoryPort>, max_recent_items: i32) -> Self {
|
||||
Self {
|
||||
repo,
|
||||
max_recent_items: max_recent_items.max(1).min(100),
|
||||
max_recent_items: max_recent_items.clamp(1, 100),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,14 +193,14 @@ impl FileWritePort for MockFileRepository {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
async fn save_file_from_stream(
|
||||
async fn save_file_from_temp(
|
||||
&self,
|
||||
_name: String,
|
||||
_folder_id: Option<String>,
|
||||
_content_type: String,
|
||||
_stream: std::pin::Pin<
|
||||
Box<dyn Stream<Item = std::result::Result<Bytes, std::io::Error>> + Send>,
|
||||
>,
|
||||
_temp_path: &std::path::Path,
|
||||
_size: u64,
|
||||
_pre_computed_hash: Option<String>,
|
||||
) -> std::result::Result<File, DomainError> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -243,6 +243,14 @@ impl FileWritePort for MockFileRepository {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
async fn copy_file(
|
||||
&self,
|
||||
_file_id: &str,
|
||||
_target_folder_id: Option<String>,
|
||||
) -> std::result::Result<File, DomainError> {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
async fn move_to_trash(&self, id: &str) -> std::result::Result<(), DomainError> {
|
||||
let mut files = self.files.lock().unwrap();
|
||||
let mut trashed = self.trashed_files.lock().unwrap();
|
||||
@@ -608,17 +616,19 @@ mod tests {
|
||||
);
|
||||
|
||||
// Verify the file is restored in file repository
|
||||
let files = file_repo.files.lock().unwrap();
|
||||
let trashed_files = file_repo.trashed_files.lock().unwrap();
|
||||
{
|
||||
let files = file_repo.files.lock().unwrap();
|
||||
let trashed_files = file_repo.trashed_files.lock().unwrap();
|
||||
|
||||
assert!(
|
||||
files.get(file_id).is_some(),
|
||||
"File should be back in main storage"
|
||||
);
|
||||
assert!(
|
||||
trashed_files.get(file_id).is_none(),
|
||||
"File should no longer be in trash storage"
|
||||
);
|
||||
assert!(
|
||||
files.get(file_id).is_some(),
|
||||
"File should be back in main storage"
|
||||
);
|
||||
assert!(
|
||||
trashed_files.get(file_id).is_none(),
|
||||
"File should no longer be in trash storage"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify the trash item is removed
|
||||
let trash_items = trash_repo.get_trash_items(&user_uuid).await.unwrap();
|
||||
@@ -670,17 +680,19 @@ mod tests {
|
||||
);
|
||||
|
||||
// Verify the file is permanently deleted
|
||||
let files = file_repo.files.lock().unwrap();
|
||||
let trashed_files = file_repo.trashed_files.lock().unwrap();
|
||||
{
|
||||
let files = file_repo.files.lock().unwrap();
|
||||
let trashed_files = file_repo.trashed_files.lock().unwrap();
|
||||
|
||||
assert!(
|
||||
files.get(file_id).is_none(),
|
||||
"File should not be in main storage"
|
||||
);
|
||||
assert!(
|
||||
trashed_files.get(file_id).is_none(),
|
||||
"File should not be in trash storage"
|
||||
);
|
||||
assert!(
|
||||
files.get(file_id).is_none(),
|
||||
"File should not be in main storage"
|
||||
);
|
||||
assert!(
|
||||
trashed_files.get(file_id).is_none(),
|
||||
"File should not be in trash storage"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify the trash item is removed
|
||||
let trash_items = trash_repo.get_trash_items(&user_uuid).await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user