cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf: implement findings #2, #3, #17 from architecture audit

Browse Source 
- Finding #2: Replace Mutex<HashMap> with moka::sync::Cache in file_blob_read_repository (10K cap, 30s TTI)
- Finding #3: Add chunked upload persistence with session.json + progress.bin bitmask for crash recovery
- Finding #17: Remove manual gzip compression, delegate entirely to tower-http CompressionLayer
- Remove dead code: StubCompressionPort, GzipCompressionService re-export, duplicate response structs
- All 114 tests passing
This commit is contained in:
Diocrafts
2026-02-22 14:12:53 +01:00
parent 0975f1f7ab
commit fd9e509648
8 changed files with 805 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -38,7 +38,7 @@ hyper = { version = "1.8.1", features = ["full"] }
quick-xml = "0.39.0"
dotenvy = "0.15.7"
lru = "0.16.3"
moka = { version = "0.12", features = ["future"] }
moka = { version = "0.12", features = ["future", "sync"] }
http-range-header = "0.4"
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
md5 = "0.8.0"
src/common/di.rs
+6 -13
View File
@@ -7,7 +7,7 @@ use crate::application::services::auth_application_service::AuthApplicationServi
use crate::application::ports::cache_ports::ContentCachePort;
use crate::application::ports::chunked_upload_ports::ChunkedUploadPort;
use crate::application::ports::compression_ports::CompressionPort;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::favorites_ports::FavoritesUseCase;
use crate::application::ports::file_ports::{
@@ -46,7 +46,7 @@ use crate::infrastructure::services::path_service::PathService;
use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService;
use crate::common::stubs::{
StubCompressionPort, StubDedupPort, StubFileManagementUseCase, StubFileReadPort,
StubDedupPort, StubFileManagementUseCase, StubFileReadPort,
StubFileRetrievalUseCase, StubFileUploadUseCase, StubFileUseCaseFactory, StubFileWritePort,
StubFolderStoragePort, StubFolderUseCase, StubI18nService, StubSearchUseCase, StubZipPort,
};
@@ -124,7 +124,8 @@ impl AppServiceFactory {
let chunked_upload_service = Arc::new(
crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new(
chunked_temp_dir,
),
)
.await,
);
// Image transcoding service for automatic WebP conversion
@@ -146,13 +147,8 @@ impl AppServiceFactory {
);
dedup_service.initialize().await?;
// Compression service (gzip)
let compression_service: Arc<dyn CompressionPort> = Arc::new(
crate::infrastructure::services::compression_service::GzipCompressionService::new(),
);
tracing::info!(
"Core services initialized: path service, file content cache, thumbnails, chunked upload, image transcode, dedup (PRIMARY blob storage), compression"
"Core services initialized: path service, file content cache, thumbnails, chunked upload, image transcode, dedup (PRIMARY blob storage)"
);
Ok(CoreServices {
@@ -162,7 +158,6 @@ impl AppServiceFactory {
chunked_upload_service,
image_transcode_service,
dedup_service,
compression_service,
zip_service: Arc::new(StubZipPort), // Placeholder - replaced after app services init
config: self.config.clone(),
})
@@ -688,7 +683,6 @@ pub struct CoreServices {
pub chunked_upload_service: Arc<dyn ChunkedUploadPort>,
pub image_transcode_service: Arc<dyn ImageTranscodePort>,
pub dedup_service: Arc<dyn DedupPort>,
pub compression_service: Arc<dyn CompressionPort>,
pub zip_service: Arc<dyn ZipPort>,
pub config: AppConfig,
}
@@ -800,7 +794,7 @@ impl Default for AppState {
// Create dummy chunked upload service
let dummy_chunked_upload_service: Arc<dyn ChunkedUploadPort> = Arc::new(
crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new(
crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new_stub(
std::path::PathBuf::from("./storage/.uploads"),
),
);
@@ -825,7 +819,6 @@ impl Default for AppState {
chunked_upload_service: dummy_chunked_upload_service,
image_transcode_service: dummy_image_transcode_service,
dedup_service: dummy_dedup_service,
compression_service: Arc::new(StubCompressionPort) as Arc<dyn CompressionPort>,
zip_service: Arc::new(StubZipPort) as Arc<dyn ZipPort>,
config: config.clone(),
};
src/common/stubs.rs
-26
View File
@@ -22,7 +22,6 @@ use crate::application::dtos::pagination::{PaginatedResponseDto, PaginationReque
use crate::application::dtos::search_dto::{
SearchCriteriaDto, SearchResultsDto, SearchSuggestionsDto,
};
use crate::application::ports::compression_ports::{CompressionLevel, CompressionPort};
use crate::application::ports::file_ports::{
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, FileUseCaseFactory,
OptimizedFileContent,
@@ -59,31 +58,6 @@ impl ZipPort for StubZipPort {
}
}
// ---------------------------------------------------------------------------
// CompressionPort
// ---------------------------------------------------------------------------
pub struct StubCompressionPort;
#[async_trait]
impl CompressionPort for StubCompressionPort {
async fn compress_data(
&self,
_data: &[u8],
_level: CompressionLevel,
) -> Result<Vec<u8>, DomainError> {
Ok(Vec::new())
}
async fn decompress_data(&self, _compressed_data: &[u8]) -> Result<Vec<u8>, DomainError> {
Ok(Vec::new())
}
fn should_compress(&self, _mime_type: &str, _size: u64) -> bool {
false
}
}
// ---------------------------------------------------------------------------
// FileReadPort
// ---------------------------------------------------------------------------
src/infrastructure/repositories/pg/file_blob_read_repository.rs
+175 -11
View File
@@ -10,9 +10,10 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use moka::sync::Cache;
use sqlx::PgPool;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use crate::application::dtos::search_dto::SearchCriteriaDto;
use crate::application::ports::dedup_ports::DedupPort;
@@ -25,10 +26,11 @@ use crate::domain::services::path_service::StoragePath;
pub struct FileBlobReadRepository {
pool: Arc<PgPool>,
dedup: Arc<dyn DedupPort>,
/// Lightweight cache: file_id → blob_hash.
/// Lock-free cache: file_id → blob_hash.
/// Populated by `get_file()`, consumed by `resolve_blob_hash()`.
/// Avoids an extra SQL round-trip on the hot download path.
hash_cache: std::sync::Mutex<HashMap<String, String>>,
/// Uses moka with TTI eviction to prevent unbounded growth.
hash_cache: Cache<String, String>,
}
impl FileBlobReadRepository {
@@ -40,7 +42,10 @@ impl FileBlobReadRepository {
Self {
pool,
dedup,
hash_cache: std::sync::Mutex::new(HashMap::new()),
hash_cache: Cache::builder()
.max_capacity(10_000)
.time_to_idle(Duration::from_secs(30))
.build(),
}
}
@@ -80,10 +85,11 @@ impl FileBlobReadRepository {
}
/// Resolve the blob hash for a file (internal helper).
/// Checks the in-memory cache first (populated by `get_file`).
/// Checks the lock-free moka cache first (populated by `get_file`).
async fn resolve_blob_hash(&self, file_id: &str) -> Result<String, DomainError> {
// Fast path: already cached from a prior get_file call
if let Some(hash) = self.hash_cache.lock().unwrap().remove(file_id) {
// Fast path: cached from a prior get_file call (lock-free read)
if let Some(hash) = self.hash_cache.get(file_id) {
self.hash_cache.invalidate(file_id);
return Ok(hash);
}
// Slow path: DB round-trip
@@ -136,10 +142,7 @@ impl FileReadPort for FileBlobReadRepository {
// Cache blob_hash so the subsequent get_file_stream / get_file_content
// call doesn't need a separate DB round-trip.
self.hash_cache
.lock()
.unwrap()
.insert(id.to_string(), row.8.clone());
self.hash_cache.insert(id.to_string(), row.8.clone());
Self::row_to_file(
row.0, row.1, row.2, row.3, row.4, row.5, row.6, row.7, row.9,
@@ -629,3 +632,164 @@ impl FileReadPort for FileBlobReadRepository {
Ok(count)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::common::stubs::StubDedupPort;
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
/// Helper: build a `FileBlobReadRepository` without a real PgPool.
/// Only the moka `hash_cache` is exercised — no SQL is executed.
fn make_repo() -> FileBlobReadRepository {
let _folder_repo = Arc::new(FolderDbRepository::new_stub());
// StubDedupPort satisfies the trait but is never called in cache-only tests
let dedup: Arc<dyn DedupPort> = Arc::new(StubDedupPort);
// PgPool is required by the struct but we won't hit any SQL in these tests.
// We create a repo with a stub pool placeholder — only hash_cache is tested.
FileBlobReadRepository {
pool: Arc::new(
// Use an intentionally invalid URL; tests never reach PG.
sqlx::pool::PoolOptions::<sqlx::Postgres>::new()
.max_connections(1)
.connect_lazy("postgres://invalid:5432/none")
.unwrap(),
),
dedup,
hash_cache: Cache::builder()
.max_capacity(10_000)
.time_to_idle(Duration::from_secs(30))
.build(),
}
}
#[tokio::test]
async fn test_cache_insert_and_consume() {
let repo = make_repo();
// Insert a hash
repo.hash_cache
.insert("file-1".to_string(), "abc123".to_string());
// First read should return the cached value
let cached = repo.hash_cache.get("file-1");
assert_eq!(cached.as_deref(), Some("abc123"));
// Simulate the one-shot consume pattern used in resolve_blob_hash
repo.hash_cache.invalidate("file-1");
assert!(
repo.hash_cache.get("file-1").is_none(),
"Entry must be gone after invalidation"
);
}
#[tokio::test]
async fn test_cache_miss_returns_none() {
let repo = make_repo();
assert!(
repo.hash_cache.get("nonexistent").is_none(),
"Cache miss must return None"
);
}
#[tokio::test]
async fn test_cache_multiple_files_independent() {
let repo = make_repo();
repo.hash_cache
.insert("file-a".to_string(), "hash-a".to_string());
repo.hash_cache
.insert("file-b".to_string(), "hash-b".to_string());
// Consuming file-a should not affect file-b
assert_eq!(repo.hash_cache.get("file-a").as_deref(), Some("hash-a"));
repo.hash_cache.invalidate("file-a");
assert!(repo.hash_cache.get("file-a").is_none());
assert_eq!(
repo.hash_cache.get("file-b").as_deref(),
Some("hash-b"),
"Independent entries must not interfere"
);
}
#[tokio::test]
async fn test_cache_overwrite_updates_value() {
let repo = make_repo();
repo.hash_cache
.insert("file-1".to_string(), "old-hash".to_string());
repo.hash_cache
.insert("file-1".to_string(), "new-hash".to_string());
assert_eq!(
repo.hash_cache.get("file-1").as_deref(),
Some("new-hash"),
"Last insert wins"
);
}
#[tokio::test]
async fn test_cache_capacity_eviction() {
// Build a tiny cache to verify eviction behaviour
let repo = FileBlobReadRepository {
pool: Arc::new(
sqlx::pool::PoolOptions::<sqlx::Postgres>::new()
.max_connections(1)
.connect_lazy("postgres://invalid:5432/none")
.unwrap(),
),
dedup: Arc::new(StubDedupPort),
hash_cache: Cache::builder()
.max_capacity(2) // only 2 entries
.build(),
};
repo.hash_cache
.insert("a".to_string(), "ha".to_string());
repo.hash_cache
.insert("b".to_string(), "hb".to_string());
repo.hash_cache
.insert("c".to_string(), "hc".to_string());
// Force moka to run pending eviction tasks
repo.hash_cache.run_pending_tasks();
// At most 2 entries should survive
let alive = ["a", "b", "c"]
.iter()
.filter(|k| repo.hash_cache.get(**k).is_some())
.count();
assert!(
alive <= 2,
"Cache must evict when capacity is exceeded (alive: {alive})"
);
}
#[tokio::test]
async fn test_cache_concurrent_access() {
use std::sync::Arc;
use std::thread;
let repo = Arc::new(make_repo());
let mut handles = vec![];
// Spawn 50 threads doing inserts + reads simultaneously
for i in 0..50 {
let repo = Arc::clone(&repo);
handles.push(thread::spawn(move || {
let key = format!("file-{i}");
let hash = format!("hash-{i}");
repo.hash_cache.insert(key.clone(), hash.clone());
// Read back — should be our value or already evicted, never panic
let _ = repo.hash_cache.get(&key);
repo.hash_cache.invalidate(&key);
}));
}
for h in handles {
h.join().expect("Thread must not panic — no poison possible with moka");
}
}
}
src/infrastructure/services/chunked_upload_service.rs
+601 -123
View File
File diff suppressed because it is too large Load Diff
src/interfaces/api/handlers/file_handler.rs
+9 -53
View File
@@ -10,7 +10,6 @@ use http_range_header::parse_range_header;
use serde::Deserialize;
use std::collections::HashMap;
use crate::application::ports::compression_ports::{CompressionLevel, CompressionPort};
use crate::application::ports::file_ports::OptimizedFileContent;
use crate::common::di::AppState;
use crate::interfaces::middleware::auth::{AuthUser, OptionalUserId};
@@ -464,11 +463,7 @@ impl FileHandler {
&mime_type,
&disposition,
&etag,
file_dto.size,
&params,
&*state.core.compression_service,
)
.await
.into_response(),
OptimizedFileContent::Mmap(mmap_data) => Response::builder()
.status(StatusCode::OK)
@@ -862,71 +857,32 @@ impl FileHandler {
.unwrap()
}
/// Build response for cached/small files with optional compression.
async fn build_cached_response(
/// Build response for cached/small files.
///
/// Compression is handled uniformly by `CompressionLayer` (tower-http)
/// which negotiates `Accept-Encoding` and applies gzip/brotli in streaming
/// mode. No manual compression is done here to avoid double-encoding.
fn build_cached_response(
content: Bytes,
mime_type: &str,
disposition: &str,
etag: &str,
file_size: u64,
params: &HashMap<String, String>,
compression_service: &dyn CompressionPort,
) -> Response<Body> {
let compression_param = params.get("compress").map(|v| v.as_str());
let force_compress = compression_param == Some("true") || compression_param == Some("1");
let force_no_compress =
compression_param == Some("false") || compression_param == Some("0");
let should_compress = if force_no_compress {
false
} else if force_compress {
true
} else {
compression_service.should_compress(mime_type, file_size)
};
let compression_level = match params.get("compression_level").map(|v| v.as_str()) {
Some("fast") => CompressionLevel::Fast,
Some("best") => CompressionLevel::Best,
_ => CompressionLevel::Default,
};
let builder = Response::builder()
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, mime_type)
.header(header::CONTENT_DISPOSITION, disposition)
.header(header::ETAG, etag)
.header(
header::CACHE_CONTROL,
"private, max-age=3600, must-revalidate",
)
.header(header::VARY, "Accept-Encoding");
if should_compress {
match compression_service
.compress_data(&content, compression_level)
.await
{
Ok(compressed) => builder
.header(header::CONTENT_TYPE, mime_type)
.header(header::CONTENT_ENCODING, "gzip")
.header(header::CONTENT_LENGTH, compressed.len())
.body(Body::from(compressed))
.unwrap(),
Err(_) => builder
.header(header::CONTENT_TYPE, mime_type)
.header(header::CONTENT_LENGTH, content.len())
.body(Body::from(content))
.unwrap(),
}
} else {
builder
.header(header::CONTENT_TYPE, mime_type)
.header(header::VARY, "Accept-Encoding")
.header(header::CONTENT_LENGTH, content.len())
.body(Body::from(content))
.unwrap()
}
}
}
/// Payload for moving a file
#[derive(Debug, Deserialize)]
src/interfaces/api/routes.rs
+3 -1
View File
@@ -358,7 +358,9 @@ pub fn create_api_routes(app_state: &AppState) -> Router<AppState> {
let admin_router = admin_handler::admin_routes().with_state(app_state.clone());
router = router.nest("/admin", admin_router);
// Compression for API responses (JSON) — excludes file downloads
// Transparent compression (gzip + brotli) for all API responses.
// tower-http negotiates via Accept-Encoding and skips already-compressed
// content types automatically. No manual compression in handlers.
router
.layer(CompressionLayer::new().br(true).gzip(true))
.layer(TraceLayer::new_for_http())
src/lib.rs
-1
View File
@@ -9,5 +9,4 @@ pub mod interfaces;
pub use application::services::folder_service::FolderService;
pub use application::services::i18n_application_service::I18nApplicationService;
pub use domain::services::path_service::StoragePath;
pub use infrastructure::services::compression_service::GzipCompressionService;
pub use infrastructure::services::path_service::PathService;