cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(#113): 100% blob storage model — PostgreSQL metadata + DedupService blobs

Browse Source 
BREAKING CHANGE: Storage model completely rewritten. All file/folder
metadata now lives in PostgreSQL (storage schema). File content stored
as content-addressable blobs via DedupService. Filesystem directories
are no longer used for user storage.

New components:
- storage.folders / storage.files / storage.trash_items (PG schema)
- FolderDbRepository: virtual folders backed by PG
- FileBlobReadRepository: file reads via PG metadata + dedup blobs
- FileBlobWriteRepository: file writes via PG metadata + dedup blobs
- TrashDbRepository: soft-delete trash using is_trashed flags

Removed legacy FS components (~5500 lines deleted):
- FolderFsRepository, FileFsReadRepository, FileFsWriteRepository
- CompositeFileRepository, ParallelFileProcessor
- IdMappingService, IdMappingOptimizer, FileMetadataCache
- BufferPool, FileSystemUtils, RepositoryErrors
- TrashFsRepository, FolderFsRepositoryTrash

DI rewired: build_app_state() now requires PgPool (no FS fallback).
FileUploadService.new_with_read() and FileRetrievalService.new_with_cache()
constructors added for blob model (no write-behind needed).

Closes #113
This commit is contained in:
Dionisio
2026-02-14 17:54:25 +01:00
parent f25987e553
commit 3c7c16f07e
27 changed files with 1883 additions and 7429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/schema.sql
+75
View File
@@ -303,3 +303,78 @@ COMMENT ON TABLE carddav.contacts IS 'Contacts stored with vCard data for round-
COMMENT ON TABLE carddav.address_book_shares IS 'Address book sharing permissions between users';
COMMENT ON TABLE carddav.contact_groups IS 'Contact groups within address books';
COMMENT ON TABLE carddav.group_memberships IS 'Many-to-many relationship between contacts and groups';
-- ============================================================
-- 4. STORAGE SCHEMA — 100% Blob Storage Model
-- ============================================================
-- All file/folder metadata lives here. Actual file content is stored
-- as content-addressable blobs on the filesystem via DedupService
-- (.blobs/{prefix}/{hash}.blob). No physical directories are created
-- for user folders — they are virtual records in this schema.
-- ============================================================
CREATE SCHEMA IF NOT EXISTS storage;
-- Virtual folders (replaces physical directories on disk)
CREATE TABLE IF NOT EXISTS storage.folders (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
parent_id UUID REFERENCES storage.folders(id) ON DELETE CASCADE,
user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
is_trashed BOOLEAN NOT NULL DEFAULT FALSE,
trashed_at TIMESTAMP WITH TIME ZONE,
original_parent_id UUID,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- A user cannot have two non-trashed folders with the same name in the same parent
CREATE UNIQUE INDEX IF NOT EXISTS idx_folders_unique_name
ON storage.folders(parent_id, name, user_id) WHERE NOT is_trashed AND parent_id IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_folders_unique_name_root
ON storage.folders(name, user_id) WHERE NOT is_trashed AND parent_id IS NULL;
CREATE INDEX IF NOT EXISTS idx_folders_user_id ON storage.folders(user_id);
CREATE INDEX IF NOT EXISTS idx_folders_parent_id ON storage.folders(parent_id);
CREATE INDEX IF NOT EXISTS idx_folders_trashed ON storage.folders(user_id, is_trashed);
-- Files as references to content-addressable blobs
CREATE TABLE IF NOT EXISTS storage.files (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
folder_id UUID REFERENCES storage.folders(id) ON DELETE SET NULL,
user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE,
blob_hash VARCHAR(64) NOT NULL,
size BIGINT NOT NULL DEFAULT 0,
mime_type TEXT NOT NULL DEFAULT 'application/octet-stream',
is_trashed BOOLEAN NOT NULL DEFAULT FALSE,
trashed_at TIMESTAMP WITH TIME ZONE,
original_folder_id UUID,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- A user cannot have two non-trashed files with the same name in the same folder
CREATE UNIQUE INDEX IF NOT EXISTS idx_files_unique_name_in_folder
ON storage.files(folder_id, name, user_id) WHERE NOT is_trashed AND folder_id IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_files_unique_name_at_root
ON storage.files(name, user_id) WHERE NOT is_trashed AND folder_id IS NULL;
CREATE INDEX IF NOT EXISTS idx_files_user_id ON storage.files(user_id);
CREATE INDEX IF NOT EXISTS idx_files_folder_id ON storage.files(folder_id);
CREATE INDEX IF NOT EXISTS idx_files_blob_hash ON storage.files(blob_hash);
CREATE INDEX IF NOT EXISTS idx_files_trashed ON storage.files(user_id, is_trashed);
CREATE INDEX IF NOT EXISTS idx_files_name_search ON storage.files(user_id, name text_pattern_ops);
-- Trash view combining trashed files and folders for the TrashRepository
CREATE OR REPLACE VIEW storage.trash_items AS
SELECT id, name, 'file' AS item_type, user_id, trashed_at,
original_folder_id AS original_parent_id, created_at
FROM storage.files WHERE is_trashed = TRUE
UNION ALL
SELECT id, name, 'folder' AS item_type, user_id, trashed_at,
original_parent_id, created_at
FROM storage.folders WHERE is_trashed = TRUE;
COMMENT ON TABLE storage.folders IS 'Virtual folder hierarchy — no physical directories on disk';
COMMENT ON TABLE storage.files IS 'File metadata pointing to content-addressable blobs';
COMMENT ON VIEW storage.trash_items IS 'Unified view of all trashed files and folders';
src/application/services/file_retrieval_service.rs
+15
View File
@@ -56,6 +56,21 @@ impl FileRetrievalService {
}
}
/// Constructor for blob-storage model: read + content cache + transcode.
/// No write-behind needed — dedup handled at the repository layer.
pub fn new_with_cache(
file_read: Arc<dyn FileReadPort>,
content_cache: Arc<dyn ContentCachePort>,
transcode: Arc<dyn ImageTranscodePort>,
) -> Self {
Self {
file_read,
write_behind: None,
content_cache: Some(content_cache),
transcode: Some(transcode),
}
}
// ── private helpers ──────────────────────────────────────────
/// Try to transcode image content to WebP and return transcoded variant.
src/application/services/file_upload_service.rs
+15
View File
@@ -87,6 +87,21 @@ impl FileUploadService {
}
}
/// Constructor for blob-storage model: write + read ports only.
/// Dedup is handled at the repository layer — no write-behind needed.
pub fn new_with_read(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
) -> Self {
Self {
file_write,
file_read: Some(file_read),
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
/// Configures the storage usage service
pub fn with_storage_usage_service(
mut self,
src/common/di.rs
+184 -370
View File
@@ -1,61 +1,50 @@
use sqlx::PgPool;
use std::path::PathBuf;
use std::sync::Arc;
use sqlx::PgPool;
use crate::application::services::admin_settings_service::AdminSettingsService;
use crate::application::services::auth_application_service::AuthApplicationService;
use crate::application::services::admin_settings_service::AdminSettingsService;
use crate::application::ports::cache_ports::{ContentCachePort, WriteBehindCachePort};
use crate::application::ports::chunked_upload_ports::ChunkedUploadPort;
use crate::application::ports::compression_ports::CompressionPort;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::favorites_ports::FavoritesUseCase;
use crate::application::ports::file_ports::{
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, FileUseCaseFactory,
use crate::infrastructure::services::path_service::PathService;
use crate::infrastructure::repositories::share_fs_repository::ShareFsRepository;
use crate::infrastructure::repositories::pg::{
FolderDbRepository, FileBlobReadRepository, FileBlobWriteRepository, TrashDbRepository,
};
use crate::application::ports::inbound::{FolderUseCase, SearchUseCase};
use crate::application::ports::outbound::FolderStoragePort;
use crate::application::ports::recent_ports::RecentItemsUseCase;
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::application::ports::thumbnail_ports::ThumbnailPort;
use crate::application::ports::transcode_ports::ImageTranscodePort;
use crate::application::ports::trash_ports::TrashUseCase;
use crate::application::ports::zip_ports::ZipPort;
use crate::application::services::favorites_service::FavoritesService;
use crate::infrastructure::services::file_system_i18n_service::FileSystemI18nService;
use crate::infrastructure::services::file_content_cache::{FileContentCache, FileContentCacheConfig};
use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService;
use crate::application::services::folder_service::FolderService;
use crate::application::services::i18n_application_service::I18nApplicationService;
use crate::application::services::recent_service::RecentService;
use crate::application::services::trash_service::TrashService;
use crate::application::services::search_service::SearchService;
use crate::application::services::share_service::ShareService;
use crate::application::services::storage_mediator::{FileSystemStorageMediator, StorageMediator};
use crate::application::services::trash_service::TrashService;
use crate::application::services::{
AppFileUseCaseFactory, FileManagementService, FileRetrievalService, FileUploadService,
};
use crate::common::config::AppConfig;
use crate::application::services::favorites_service::FavoritesService;
use crate::application::services::recent_service::RecentService;
use crate::application::ports::trash_ports::TrashUseCase;
use crate::application::ports::inbound::{FolderUseCase, SearchUseCase};
use crate::application::ports::outbound::FolderStoragePort;
use crate::application::ports::favorites_ports::FavoritesUseCase;
use crate::application::ports::recent_ports::RecentItemsUseCase;
use crate::application::ports::file_ports::{FileUploadUseCase, FileRetrievalUseCase, FileManagementUseCase, FileUseCaseFactory};
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::application::services::{FileUploadService, FileRetrievalService, FileManagementService, AppFileUseCaseFactory};
use crate::common::errors::DomainError;
use crate::domain::services::i18n_service::I18nService;
use crate::infrastructure::repositories::folder_fs_repository::FolderFsRepository;
use crate::infrastructure::repositories::parallel_file_processor::ParallelFileProcessor;
use crate::infrastructure::repositories::share_fs_repository::ShareFsRepository;
use crate::infrastructure::repositories::trash_fs_repository::TrashFsRepository;
use crate::infrastructure::repositories::{FileFsReadRepository, FileFsWriteRepository};
use crate::infrastructure::services::buffer_pool::BufferPool;
use crate::infrastructure::services::file_content_cache::{
FileContentCache, FileContentCacheConfig,
};
use crate::infrastructure::services::file_metadata_cache::FileMetadataCache;
use crate::infrastructure::services::file_system_i18n_service::FileSystemI18nService;
use crate::infrastructure::services::id_mapping_optimizer::IdMappingOptimizer;
use crate::infrastructure::services::id_mapping_service::IdMappingService;
use crate::infrastructure::services::path_service::PathService;
use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService;
use crate::common::config::AppConfig;
use crate::application::ports::cache_ports::ContentCachePort;
use crate::application::ports::thumbnail_ports::ThumbnailPort;
use crate::application::ports::transcode_ports::ImageTranscodePort;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::chunked_upload_ports::ChunkedUploadPort;
use crate::application::ports::compression_ports::CompressionPort;
use crate::application::ports::zip_ports::ZipPort;
use crate::common::stubs::{
StubCompressionPort, StubFileManagementUseCase, StubFileReadPort, StubFileRetrievalUseCase,
StubFileUploadUseCase, StubFileUseCaseFactory, StubFileWritePort, StubFolderStoragePort,
StubFolderUseCase, StubI18nService, StubIdMappingService, StubSearchUseCase,
StubStorageMediator, StubZipPort,
StubZipPort, StubCompressionPort,
StubFileReadPort, StubFileWritePort, StubFolderStoragePort,
StubI18nService, StubFolderUseCase, StubFileUploadUseCase,
StubFileRetrievalUseCase, StubFileManagementUseCase, StubFileUseCaseFactory,
StubSearchUseCase,
};
/// Factory for the different application components
@@ -99,7 +88,7 @@ impl AppServiceFactory {
/// Initializes the core system services
pub async fn create_core_services(&self) -> Result<CoreServices, DomainError> {
// Path service
// Path service (still needed for blob storage root + thumbnails)
let path_service = Arc::new(PathService::new(self.storage_path.clone()));
// File content cache for ultra-fast file serving (hot files in RAM)
@@ -110,43 +99,21 @@ impl AppServiceFactory {
}));
tracing::info!("FileContentCache initialized: max 10MB/file, 512MB total, 10k entries");
// ID mapping service for folders
let folder_id_mapping_path = self.storage_path.join("folder_ids.json");
let folder_id_mapping_service =
Arc::new(IdMappingService::new(folder_id_mapping_path).await?);
// ID mapping service for files
let file_id_mapping_path = self.storage_path.join("file_ids.json");
let file_id_mapping_service = Arc::new(IdMappingService::new(file_id_mapping_path).await?);
// Optimizer with batch processing and caching
let id_mapping_optimizer =
Arc::new(IdMappingOptimizer::new(folder_id_mapping_service.clone()));
// Start optimizer cleanup task
IdMappingOptimizer::start_cleanup_task(id_mapping_optimizer.clone());
// Thumbnail service for thumbnail generation
let thumbnail_service = Arc::new(
crate::infrastructure::services::thumbnail_service::ThumbnailService::new(
&self.storage_path,
5000, // max 5000 thumbnails in cache
100 * 1024 * 1024, // max 100MB cache
),
)
);
// Initialize thumbnail directories
thumbnail_service.initialize().await?;
// Write-behind cache for instant uploads of small files
let write_behind_cache =
crate::infrastructure::services::write_behind_cache::WriteBehindCache::new();
// Chunked upload service for large files (>10MB)
let chunked_temp_dir = std::path::PathBuf::from(&self.storage_path).join(".uploads");
let chunked_upload_service = Arc::new(
crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new(
chunked_temp_dir,
),
crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new(chunked_temp_dir)
);
// Image transcoding service for automatic WebP conversion
@@ -155,37 +122,27 @@ impl AppServiceFactory {
&self.storage_path,
2000, // max 2000 transcoded images in cache
50 * 1024 * 1024, // max 50MB in-memory cache
),
)
);
image_transcode_service.initialize().await?;
// Deduplication service for removing duplicate files
// Deduplication service — PRIMARY blob storage engine
let dedup_service = Arc::new(
crate::infrastructure::services::dedup_service::DedupService::new(&self.storage_path),
crate::infrastructure::services::dedup_service::DedupService::new(&self.storage_path)
);
dedup_service.initialize().await?;
// Compression service (gzip)
let compression_service: Arc<dyn CompressionPort> = Arc::new(
crate::infrastructure::services::compression_service::GzipCompressionService::new(),
crate::infrastructure::services::compression_service::GzipCompressionService::new()
);
tracing::info!(
"Core services initialized: path service, cache manager, file content cache, ID mapping, thumbnails, write-behind cache, chunked upload, image transcode, dedup, compression"
);
// NOTE: zip_service requires ApplicationServices (FileRetrievalUseCase, FolderUseCase) which are
// created later. It will be set via AppState::with_zip_service() after application services are ready.
// For now we use a placeholder that will be replaced.
tracing::info!("Core services initialized: path service, file content cache, thumbnails, chunked upload, image transcode, dedup (PRIMARY blob storage), compression");
Ok(CoreServices {
path_service,
file_content_cache,
id_mapping_service: folder_id_mapping_service,
file_id_mapping_service,
id_mapping_optimizer,
thumbnail_service,
write_behind_cache,
chunked_upload_service,
image_transcode_service,
dedup_service,
@@ -195,92 +152,50 @@ impl AppServiceFactory {
})
}
/// Initializes the repository services
pub fn create_repository_services(&self, core: &CoreServices) -> RepositoryServices {
// Storage mediator - uses stub initially, will be replaced after folder repo is ready
let storage_mediator_stub: Arc<dyn StorageMediator> =
Arc::new(FileSystemStorageMediator::new_stub());
/// Initializes the repository services (blob-storage model).
///
/// Requires a PgPool since all metadata lives in PostgreSQL.
pub fn create_repository_services(&self, core: &CoreServices, db_pool: &Arc<PgPool>) -> RepositoryServices {
// Folder repository — PostgreSQL-backed virtual folders
let folder_repo_concrete = Arc::new(FolderDbRepository::new(db_pool.clone()));
let folder_repository: Arc<dyn FolderStoragePort> = folder_repo_concrete.clone();
// Folder repository — implements FolderStoragePort directly
let folder_repository = Arc::new(FolderFsRepository::new(
self.storage_path.clone(),
storage_mediator_stub.clone(),
core.id_mapping_service.clone(),
core.path_service.clone(),
// File repositories — PostgreSQL metadata + blob content via DedupService
let file_read_repository: Arc<dyn FileReadPort> = Arc::new(FileBlobReadRepository::new(
db_pool.clone(),
core.dedup_service.clone(),
folder_repo_concrete.clone(),
));
// Now create the real storage mediator with the folder repo (as FolderStoragePort)
let storage_mediator: Arc<dyn StorageMediator> = Arc::new(FileSystemStorageMediator::new(
folder_repository.clone() as Arc<dyn FolderStoragePort>,
core.path_service.clone(),
core.id_mapping_optimizer.clone(),
));
// Metadata cache
let metadata_cache = Arc::new(FileMetadataCache::default_with_config(core.config.clone()));
// Start metadata cache cleanup task
let cache_clone = metadata_cache.clone();
tokio::spawn(async move {
FileMetadataCache::start_cleanup_task(cache_clone).await;
});
// Buffer pool for memory optimization
let buffer_pool = BufferPool::new(256 * 1024, 50, 120); // 256KB buffers, 50 max, 2 min TTL
BufferPool::start_cleaner(buffer_pool.clone());
// Parallel file processor
let parallel_processor = Arc::new(ParallelFileProcessor::new_with_buffer_pool(
core.config.clone(),
buffer_pool.clone(),
));
// Separate file repositories for reading and writing
let file_read_repository = Arc::new(FileFsReadRepository::new(
self.storage_path.clone(),
storage_mediator.clone(),
core.file_id_mapping_service.clone(),
core.path_service.clone(),
metadata_cache.clone(),
core.config.clone(),
Some(parallel_processor.clone()),
));
let file_write_repository = Arc::new(FileFsWriteRepository::new(
self.storage_path.clone(),
storage_mediator.clone(),
core.file_id_mapping_service.clone(),
core.path_service.clone(),
metadata_cache.clone(),
core.config.clone(),
Some(parallel_processor.clone()),
let file_write_repository: Arc<dyn FileWritePort> = Arc::new(FileBlobWriteRepository::new(
db_pool.clone(),
core.dedup_service.clone(),
folder_repo_concrete.clone(),
));
// I18n repository
let i18n_repository = Arc::new(FileSystemI18nService::new(self.locales_path.clone()));
let i18n_repository = Arc::new(FileSystemI18nService::new(
self.locales_path.clone()
));
// Trash repository
// Trash repository — reads soft-delete flags from storage.files/folders
let trash_repository = if core.config.features.enable_trash {
Some(Arc::new(TrashFsRepository::new(
self.storage_path.as_path(),
core.id_mapping_service.clone(),
))
as Arc<
dyn crate::domain::repositories::trash_repository::TrashRepository,
>)
Some(Arc::new(TrashDbRepository::new(
db_pool.clone(),
core.config.storage.trash_retention_days,
)) as Arc<dyn crate::domain::repositories::trash_repository::TrashRepository>)
} else {
None
};
tracing::info!("Repository services initialized with parallel processing and buffer pool");
tracing::info!("Repository services initialized with 100% blob storage model (PG metadata + DedupService blobs)");
RepositoryServices {
folder_repository,
folder_repo_concrete,
file_read_repository,
file_write_repository,
i18n_repository,
storage_mediator,
metadata_cache,
trash_repository,
}
}
@@ -293,19 +208,19 @@ impl AppServiceFactory {
trash_service: Option<Arc<dyn TrashUseCase>>,
) -> ApplicationServices {
// Main services
let folder_service = Arc::new(FolderService::new(repos.folder_repository.clone()));
// Refactored services with all infrastructure ports
let file_upload_service = Arc::new(FileUploadService::new_full(
repos.file_write_repository.clone(),
repos.file_read_repository.clone(),
core.write_behind_cache.clone(),
core.dedup_service.clone(),
let folder_service = Arc::new(FolderService::new(
repos.folder_repository.clone()
));
let file_retrieval_service = Arc::new(FileRetrievalService::new_full(
// Refactored services with all infrastructure ports
// In blob model, dedup is handled by the repository — no separate write-behind needed
let file_upload_service = Arc::new(FileUploadService::new_with_read(
repos.file_write_repository.clone(),
repos.file_read_repository.clone(),
));
let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache(
repos.file_read_repository.clone(),
core.write_behind_cache.clone(),
core.file_content_cache.clone(),
core.image_transcode_service.clone(),
));
@@ -320,10 +235,12 @@ impl AppServiceFactory {
let file_use_case_factory = Arc::new(AppFileUseCaseFactory::new(
repos.file_read_repository.clone(),
repos.file_write_repository.clone(),
repos.file_write_repository.clone()
));
let i18n_service = Arc::new(I18nApplicationService::new(repos.i18n_repository.clone()));
let i18n_service = Arc::new(I18nApplicationService::new(
repos.i18n_repository.clone()
));
// Search service with cache
let search_service: Option<Arc<dyn SearchUseCase>> = Some(Arc::new(SearchService::new(
@@ -397,7 +314,9 @@ impl AppServiceFactory {
return None;
}
let share_repository = Arc::new(ShareFsRepository::new(Arc::new(self.config.clone())));
let share_repository = Arc::new(ShareFsRepository::new(
Arc::new(self.config.clone())
));
// Build a password hasher for share password verification
let password_hasher: Arc<dyn crate::application::ports::auth_ports::PasswordHasherPort> =
@@ -416,9 +335,12 @@ impl AppServiceFactory {
}
/// Creates the favorites service (requires database)
pub fn create_favorites_service(&self, db_pool: &Arc<PgPool>) -> Arc<dyn FavoritesUseCase> {
pub fn create_favorites_service(
&self,
db_pool: &Arc<PgPool>,
) -> Arc<dyn FavoritesUseCase> {
let repo = Arc::new(
crate::infrastructure::repositories::pg::FavoritesPgRepository::new(db_pool.clone()),
crate::infrastructure::repositories::pg::FavoritesPgRepository::new(db_pool.clone())
);
let service = Arc::new(FavoritesService::new(repo));
tracing::info!("Favorites service initialized");
@@ -426,12 +348,16 @@ impl AppServiceFactory {
}
/// Creates the recent items service (requires database)
pub fn create_recent_service(&self, db_pool: &Arc<PgPool>) -> Arc<dyn RecentItemsUseCase> {
pub fn create_recent_service(
&self,
db_pool: &Arc<PgPool>,
) -> Arc<dyn RecentItemsUseCase> {
let repo = Arc::new(
crate::infrastructure::repositories::pg::RecentItemsPgRepository::new(db_pool.clone()),
crate::infrastructure::repositories::pg::RecentItemsPgRepository::new(db_pool.clone())
);
let service = Arc::new(RecentService::new(
repo, 50, // Maximum recent items per user
repo,
50 // Maximum recent items per user
));
tracing::info!("Recent items service initialized");
service
@@ -459,17 +385,6 @@ impl AppServiceFactory {
tracing::info!("Translations preloaded");
}
/// Preloads directories into cache
pub async fn preload_cache(&self, metadata_cache: &FileMetadataCache) {
tracing::info!("Preloading common directories to warm up cache...");
if let Ok(count) = metadata_cache
.preload_directory(&self.storage_path, true, 1)
.await
{
tracing::info!("Preloaded {} directory entries into cache", count);
}
}
/// Creates the storage usage service (requires database)
pub fn create_storage_usage_service(
&self,
@@ -477,13 +392,13 @@ impl AppServiceFactory {
db_pool: &Arc<PgPool>,
) -> Arc<dyn crate::application::ports::storage_ports::StorageUsagePort> {
let user_repository = Arc::new(
crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone()),
crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone())
);
let service = Arc::new(
crate::application::services::storage_usage_service::StorageUsageService::new(
repos.file_read_repository.clone(),
user_repository,
),
)
);
tracing::info!("Storage usage service initialized");
service
@@ -496,11 +411,16 @@ impl AppServiceFactory {
&self,
db_pool: Option<Arc<PgPool>>,
) -> Result<AppState, DomainError> {
// Database is REQUIRED in 100% blob storage model
let pool = db_pool.clone().ok_or_else(|| {
DomainError::internal_error("Database", "PostgreSQL database is required for blob storage model")
})?;
// 1. Core services
let core = self.create_core_services().await?;
// 2. Repository services
let repos = self.create_repository_services(&core);
// 2. Repository services (requires PgPool for all metadata)
let repos = self.create_repository_services(&core, &pool);
// 3. Trash service (needed before application services)
let trash_service = self.create_trash_service(&repos).await;
@@ -512,24 +432,22 @@ impl AppServiceFactory {
let share_service = self.create_share_service(&repos);
apps.share_service = share_service.clone();
// 6. Database-dependent services
let mut favorites_service: Option<Arc<dyn FavoritesUseCase>> = None;
let mut recent_service: Option<Arc<dyn RecentItemsUseCase>> = None;
let mut storage_usage_service: Option<
Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>,
> = None;
// 6. Database-dependent services (PgPool always available in blob model)
let favorites_service: Option<Arc<dyn FavoritesUseCase>>;
let recent_service: Option<Arc<dyn RecentItemsUseCase>>;
let storage_usage_service: Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>;
let mut auth_services: Option<crate::common::di::AuthServices> = None;
if let Some(ref pool) = db_pool {
let favs = self.create_favorites_service(pool);
{
let favs = self.create_favorites_service(&pool);
favorites_service = Some(favs.clone());
apps.favorites_service = Some(favs);
let recent = self.create_recent_service(pool);
let recent = self.create_recent_service(&pool);
recent_service = Some(recent.clone());
apps.recent_service = Some(recent);
storage_usage_service = Some(self.create_storage_usage_service(&repos, pool));
storage_usage_service = Some(self.create_storage_usage_service(&repos, &pool));
// Auth services
if self.config.features.enable_auth {
@@ -537,9 +455,7 @@ impl AppServiceFactory {
&self.config,
pool.clone(),
Some(apps.folder_service_concrete.clone()),
)
.await
{
).await {
Ok(services) => {
tracing::info!("Authentication services initialized successfully");
auth_services = Some(services);
@@ -554,20 +470,17 @@ impl AppServiceFactory {
// 7. Preload translations
self.preload_translations(&apps.i18n_service).await;
// 8. Preload cache
self.preload_cache(&repos.metadata_cache).await;
// 9. Build the ZipService with real application services
// 8. Build the ZipService with real application services
let zip_service: Arc<dyn crate::application::ports::zip_ports::ZipPort> = Arc::new(
crate::infrastructure::services::zip_service::ZipService::new(
apps.file_retrieval_service.clone(),
apps.folder_service.clone(),
),
)
);
let mut core = core;
core.zip_service = zip_service;
// 10. Assemble final AppState
// 9. Assemble final AppState
let mut app_state = AppState {
core,
repositories: repos,
@@ -587,10 +500,10 @@ impl AppServiceFactory {
contact_use_case: None,
};
// 10b. Wire admin settings service when auth + DB are available
if let (Some(auth_svc), Some(pool)) = (&app_state.auth_service, &db_pool) {
// 9b. Wire admin settings service when auth is available
if let Some(auth_svc) = &app_state.auth_service {
let settings_repo = Arc::new(
crate::infrastructure::repositories::pg::SettingsPgRepository::new(pool.clone()),
crate::infrastructure::repositories::pg::SettingsPgRepository::new(pool.clone())
);
let server_base_url = self.config.base_url();
@@ -606,50 +519,34 @@ impl AppServiceFactory {
// Hot-reload OIDC from DB settings if configured
match admin_svc.load_effective_oidc_config().await {
Ok(eff)
if eff.enabled
&& !eff.issuer_url.is_empty()
&& !eff.client_id.is_empty()
&& !eff.client_secret.is_empty() =>
Ok(eff) if eff.enabled && !eff.issuer_url.is_empty()
&& !eff.client_id.is_empty() && !eff.client_secret.is_empty() =>
{
let oidc_svc = Arc::new(
crate::infrastructure::services::oidc_service::OidcService::new(
eff.clone(),
),
crate::infrastructure::services::oidc_service::OidcService::new(eff.clone())
);
auth_svc.auth_application_service.reload_oidc(oidc_svc, eff);
tracing::info!("OIDC config loaded from admin settings (database)");
}
Ok(_) => {
tracing::info!(
"No active OIDC config in admin settings — using env vars or defaults"
);
tracing::info!("No active OIDC config in admin settings — using env vars or defaults");
}
Err(e) => {
tracing::warn!(
"Failed to load OIDC settings from database (table may not exist yet): {}",
e
);
tracing::warn!("Failed to load OIDC settings from database (table may not exist yet): {}", e);
}
}
app_state.admin_settings_service = Some(admin_svc);
}
// 11. Wire CalDAV/CardDAV services when database is available
if let Some(ref pool) = db_pool {
// 10. Wire CalDAV/CardDAV services
{
// CalDAV
let calendar_repo: Arc<
dyn crate::domain::repositories::calendar_repository::CalendarRepository,
> = Arc::new(
crate::infrastructure::repositories::pg::CalendarPgRepository::new(pool.clone()),
let calendar_repo: Arc<dyn crate::domain::repositories::calendar_repository::CalendarRepository> = Arc::new(
crate::infrastructure::repositories::pg::CalendarPgRepository::new(pool.clone())
);
let event_repo: Arc<
dyn crate::domain::repositories::calendar_event_repository::CalendarEventRepository,
> = Arc::new(
crate::infrastructure::repositories::pg::CalendarEventPgRepository::new(
pool.clone(),
),
let event_repo: Arc<dyn crate::domain::repositories::calendar_event_repository::CalendarEventRepository> = Arc::new(
crate::infrastructure::repositories::pg::CalendarEventPgRepository::new(pool.clone())
);
let calendar_storage = Arc::new(
crate::infrastructure::adapters::calendar_storage_adapter::CalendarStorageAdapter::new(
@@ -658,32 +555,19 @@ impl AppServiceFactory {
)
);
let calendar_service = Arc::new(
crate::application::services::calendar_service::CalendarService::new(
calendar_storage,
),
);
app_state.calendar_use_case = Some(
calendar_service
as Arc<dyn crate::application::ports::calendar_ports::CalendarUseCase>,
crate::application::services::calendar_service::CalendarService::new(calendar_storage)
);
app_state.calendar_use_case = Some(calendar_service as Arc<dyn crate::application::ports::calendar_ports::CalendarUseCase>);
// CardDAV
let address_book_repo: Arc<
dyn crate::domain::repositories::address_book_repository::AddressBookRepository,
> = Arc::new(
crate::infrastructure::repositories::pg::AddressBookPgRepository::new(pool.clone()),
let address_book_repo: Arc<dyn crate::domain::repositories::address_book_repository::AddressBookRepository> = Arc::new(
crate::infrastructure::repositories::pg::AddressBookPgRepository::new(pool.clone())
);
let contact_repo: Arc<
dyn crate::domain::repositories::contact_repository::ContactRepository,
> = Arc::new(
crate::infrastructure::repositories::pg::ContactPgRepository::new(pool.clone()),
let contact_repo: Arc<dyn crate::domain::repositories::contact_repository::ContactRepository> = Arc::new(
crate::infrastructure::repositories::pg::ContactPgRepository::new(pool.clone())
);
let group_repo: Arc<
dyn crate::domain::repositories::contact_repository::ContactGroupRepository,
> = Arc::new(
crate::infrastructure::repositories::pg::ContactGroupPgRepository::new(
pool.clone(),
),
let group_repo: Arc<dyn crate::domain::repositories::contact_repository::ContactGroupRepository> = Arc::new(
crate::infrastructure::repositories::pg::ContactGroupPgRepository::new(pool.clone())
);
let contact_storage = Arc::new(
crate::infrastructure::adapters::contact_storage_adapter::ContactStorageAdapter::new(
@@ -692,12 +576,8 @@ impl AppServiceFactory {
group_repo,
)
);
app_state.addressbook_use_case = Some(contact_storage.clone()
as Arc<dyn crate::application::ports::carddav_ports::AddressBookUseCase>);
app_state.contact_use_case = Some(
contact_storage
as Arc<dyn crate::application::ports::carddav_ports::ContactUseCase>,
);
app_state.addressbook_use_case = Some(contact_storage.clone() as Arc<dyn crate::application::ports::carddav_ports::AddressBookUseCase>);
app_state.contact_use_case = Some(contact_storage as Arc<dyn crate::application::ports::carddav_ports::ContactUseCase>);
tracing::info!("CalDAV and CardDAV services initialized with PostgreSQL repositories");
}
@@ -711,11 +591,7 @@ impl AppServiceFactory {
pub struct CoreServices {
pub path_service: Arc<PathService>,
pub file_content_cache: Arc<dyn ContentCachePort>,
pub id_mapping_service: Arc<dyn crate::application::ports::outbound::IdMappingPort>,
pub file_id_mapping_service: Arc<IdMappingService>,
pub id_mapping_optimizer: Arc<IdMappingOptimizer>,
pub thumbnail_service: Arc<dyn ThumbnailPort>,
pub write_behind_cache: Arc<dyn WriteBehindCachePort>,
pub chunked_upload_service: Arc<dyn ChunkedUploadPort>,
pub image_transcode_service: Arc<dyn ImageTranscodePort>,
pub dedup_service: Arc<dyn DedupPort>,
@@ -728,13 +604,11 @@ pub struct CoreServices {
#[derive(Clone)]
pub struct RepositoryServices {
pub folder_repository: Arc<dyn FolderStoragePort>,
pub folder_repo_concrete: Arc<FolderDbRepository>,
pub file_read_repository: Arc<dyn FileReadPort>,
pub file_write_repository: Arc<dyn FileWritePort>,
pub i18n_repository: Arc<dyn I18nService>,
pub storage_mediator: Arc<dyn StorageMediator>,
pub metadata_cache: Arc<FileMetadataCache>,
pub trash_repository:
Option<Arc<dyn crate::domain::repositories::trash_repository::TrashRepository>>,
pub trash_repository: Option<Arc<dyn crate::domain::repositories::trash_repository::TrashRepository>>,
}
/// Container for application services
@@ -776,50 +650,32 @@ pub struct AppState {
pub share_service: Option<Arc<dyn crate::application::ports::share_ports::ShareUseCase>>,
pub favorites_service: Option<Arc<dyn FavoritesUseCase>>,
pub recent_service: Option<Arc<dyn RecentItemsUseCase>>,
pub storage_usage_service:
Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>,
pub storage_usage_service: Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>,
pub calendar_service: Option<Arc<dyn crate::application::ports::storage_ports::StorageUseCase>>,
pub contact_service: Option<Arc<dyn crate::application::ports::storage_ports::StorageUseCase>>,
pub calendar_use_case:
Option<Arc<dyn crate::application::ports::calendar_ports::CalendarUseCase>>,
pub addressbook_use_case:
Option<Arc<dyn crate::application::ports::carddav_ports::AddressBookUseCase>>,
pub calendar_use_case: Option<Arc<dyn crate::application::ports::calendar_ports::CalendarUseCase>>,
pub addressbook_use_case: Option<Arc<dyn crate::application::ports::carddav_ports::AddressBookUseCase>>,
pub contact_use_case: Option<Arc<dyn crate::application::ports::carddav_ports::ContactUseCase>>,
}
impl Default for AppState {
fn default() -> Self {
// Minimal stub version for auth middleware and route construction.
// Real services are wired in main.rs via AppServiceFactory or manual init.
// Real services are wired in main.rs via AppServiceFactory.
let config = crate::common::config::AppConfig::default();
let path_service = Arc::new(
crate::infrastructure::services::path_service::PathService::new(
std::path::PathBuf::from("./storage"),
),
std::path::PathBuf::from("./storage")
)
);
// Create service instances from the stubs module
let id_mapping_service = Arc::new(StubIdMappingService)
as Arc<dyn crate::application::ports::outbound::IdMappingPort>;
let storage_mediator = Arc::new(StubStorageMediator)
as Arc<dyn crate::application::services::storage_mediator::StorageMediator>;
let i18n_repository = Arc::new(StubI18nService)
as Arc<dyn crate::domain::services::i18n_service::I18nService>;
let folder_service = Arc::new(StubFolderUseCase)
as Arc<dyn crate::application::ports::inbound::FolderUseCase>;
let file_upload_service = Arc::new(StubFileUploadUseCase)
as Arc<dyn crate::application::ports::file_ports::FileUploadUseCase>;
let file_retrieval_service = Arc::new(StubFileRetrievalUseCase)
as Arc<dyn crate::application::ports::file_ports::FileRetrievalUseCase>;
let file_management_service = Arc::new(StubFileManagementUseCase)
as Arc<dyn crate::application::ports::file_ports::FileManagementUseCase>;
let file_use_case_factory = Arc::new(StubFileUseCaseFactory)
as Arc<dyn crate::application::ports::file_ports::FileUseCaseFactory>;
// Create dummy ID mapping service for files
let dummy_file_id_mapping = Arc::new(IdMappingService::dummy());
let dummy_id_optimizer = Arc::new(IdMappingOptimizer::new(dummy_file_id_mapping.clone()));
let i18n_repository = Arc::new(StubI18nService) as Arc<dyn crate::domain::services::i18n_service::I18nService>;
let folder_service = Arc::new(StubFolderUseCase) as Arc<dyn crate::application::ports::inbound::FolderUseCase>;
let file_upload_service = Arc::new(StubFileUploadUseCase) as Arc<dyn crate::application::ports::file_ports::FileUploadUseCase>;
let file_retrieval_service = Arc::new(StubFileRetrievalUseCase) as Arc<dyn crate::application::ports::file_ports::FileRetrievalUseCase>;
let file_management_service = Arc::new(StubFileManagementUseCase) as Arc<dyn crate::application::ports::file_ports::FileManagementUseCase>;
let file_use_case_factory = Arc::new(StubFileUseCaseFactory) as Arc<dyn crate::application::ports::file_ports::FileUseCaseFactory>;
// Create file content cache for stub
let file_content_cache = Arc::new(FileContentCache::new(FileContentCacheConfig::default()));
@@ -830,18 +686,14 @@ impl Default for AppState {
&std::path::PathBuf::from("./storage"),
100,
10 * 1024 * 1024,
),
)
);
// Create dummy write-behind cache
let dummy_write_behind_cache: Arc<dyn WriteBehindCachePort> =
crate::infrastructure::services::write_behind_cache::WriteBehindCache::new();
// Create dummy chunked upload service
let dummy_chunked_upload_service: Arc<dyn ChunkedUploadPort> = Arc::new(
crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new(
std::path::PathBuf::from("./storage/.uploads"),
),
std::path::PathBuf::from("./storage/.uploads")
)
);
// Create dummy image transcode service
@@ -850,25 +702,21 @@ impl Default for AppState {
&std::path::PathBuf::from("./storage"),
100,
10 * 1024 * 1024,
),
)
);
// Create dummy dedup service
let dummy_dedup_service: Arc<dyn DedupPort> = Arc::new(
crate::infrastructure::services::dedup_service::DedupService::new(
&std::path::PathBuf::from("./storage"),
),
&std::path::PathBuf::from("./storage")
)
);
// Core services using stubs
let core_services = CoreServices {
path_service: path_service.clone(),
file_content_cache,
id_mapping_service: id_mapping_service.clone(),
file_id_mapping_service: dummy_file_id_mapping,
id_mapping_optimizer: dummy_id_optimizer,
thumbnail_service: dummy_thumbnail_service,
write_behind_cache: dummy_write_behind_cache,
chunked_upload_service: dummy_chunked_upload_service,
image_transcode_service: dummy_image_transcode_service,
dedup_service: dummy_dedup_service,
@@ -877,33 +725,27 @@ impl Default for AppState {
config: config.clone(),
};
// Dummy metadata cache
let dummy_metadata_cache = Arc::new(FileMetadataCache::default_with_config(config.clone()));
// Stub folder repo concrete — uses a dummy PgPool-less constructor
// In practice, Default is only used for routing stubs; real app always goes through build_app_state
let dummy_folder_repo_concrete = Arc::new(FolderDbRepository::new_stub());
// Repository services using stubs
let repository_services = RepositoryServices {
folder_repository: Arc::new(StubFolderStoragePort)
as Arc<dyn crate::application::ports::outbound::FolderStoragePort>,
file_read_repository: Arc::new(StubFileReadPort)
as Arc<dyn crate::application::ports::storage_ports::FileReadPort>,
file_write_repository: Arc::new(StubFileWritePort)
as Arc<dyn crate::application::ports::storage_ports::FileWritePort>,
folder_repository: Arc::new(StubFolderStoragePort) as Arc<dyn crate::application::ports::outbound::FolderStoragePort>,
folder_repo_concrete: dummy_folder_repo_concrete,
file_read_repository: Arc::new(StubFileReadPort) as Arc<dyn crate::application::ports::storage_ports::FileReadPort>,
file_write_repository: Arc::new(StubFileWritePort) as Arc<dyn crate::application::ports::storage_ports::FileWritePort>,
i18n_repository,
storage_mediator: storage_mediator.clone(),
metadata_cache: dummy_metadata_cache,
trash_repository: None,
};
// Dummy concrete services for compatibility
let dummy_folder_storage = Arc::new(StubFolderStoragePort)
as Arc<dyn crate::application::ports::outbound::FolderStoragePort>;
let dummy_folder_storage = Arc::new(StubFolderStoragePort) as Arc<dyn crate::application::ports::outbound::FolderStoragePort>;
let folder_service_concrete = Arc::new(FolderService::new(dummy_folder_storage));
// Dummy I18nApplicationService
let dummy_i18n_app_service =
crate::application::services::i18n_application_service::I18nApplicationService::new(
Arc::new(StubI18nService)
as Arc<dyn crate::domain::services::i18n_service::I18nService>,
let dummy_i18n_app_service = crate::application::services::i18n_application_service::I18nApplicationService::new(
Arc::new(StubI18nService) as Arc<dyn crate::domain::services::i18n_service::I18nService>
);
// Application services using stubs
@@ -916,8 +758,7 @@ impl Default for AppState {
file_use_case_factory,
i18n_service: Arc::new(dummy_i18n_app_service),
trash_service: None,
search_service: Some(Arc::new(StubSearchUseCase)
as Arc<dyn crate::application::ports::inbound::SearchUseCase>),
search_service: Some(Arc::new(StubSearchUseCase) as Arc<dyn crate::application::ports::inbound::SearchUseCase>),
share_service: None,
favorites_service: None,
recent_service: None,
@@ -982,15 +823,11 @@ impl AppState {
/// This keeps `routes.rs` free of any `crate::infrastructure` references.
pub fn for_routing(
folder_service: Arc<FolderService>,
file_retrieval_service: Arc<
dyn crate::application::ports::file_ports::FileRetrievalUseCase,
>,
file_retrieval_service: Arc<dyn crate::application::ports::file_ports::FileRetrievalUseCase>,
file_upload_service: Arc<dyn FileUploadUseCase>,
file_management_service: Arc<dyn FileManagementUseCase>,
folder_use_case: Arc<dyn crate::application::ports::inbound::FolderUseCase>,
i18n_service: Option<
Arc<crate::application::services::i18n_application_service::I18nApplicationService>,
>,
i18n_service: Option<Arc<crate::application::services::i18n_application_service::I18nApplicationService>>,
trash_service: Option<Arc<dyn TrashUseCase>>,
search_service: Option<Arc<dyn crate::application::ports::inbound::SearchUseCase>>,
share_service: Option<Arc<dyn crate::application::ports::share_ports::ShareUseCase>>,
@@ -1025,11 +862,9 @@ impl AppState {
// Create real ZipService with the actual file/folder services
state.core.zip_service = Arc::new(
crate::infrastructure::services::zip_service::ZipService::new(
file_retrieval_service
as Arc<dyn crate::application::ports::file_ports::FileRetrievalUseCase>,
folder_service.clone()
as Arc<dyn crate::application::ports::inbound::FolderUseCase>,
),
file_retrieval_service as Arc<dyn crate::application::ports::file_ports::FileRetrievalUseCase>,
folder_service.clone() as Arc<dyn crate::application::ports::inbound::FolderUseCase>,
)
);
state
@@ -1045,10 +880,7 @@ impl AppState {
self
}
pub fn with_share_service(
mut self,
share_service: Arc<dyn crate::application::ports::share_ports::ShareUseCase>,
) -> Self {
pub fn with_share_service(mut self, share_service: Arc<dyn crate::application::ports::share_ports::ShareUseCase>) -> Self {
self.share_service = Some(share_service);
self
}
@@ -1063,50 +895,32 @@ impl AppState {
self
}
pub fn with_storage_usage_service(
mut self,
storage_usage_service: Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>,
) -> Self {
pub fn with_storage_usage_service(mut self, storage_usage_service: Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>) -> Self {
self.storage_usage_service = Some(storage_usage_service);
self
}
pub fn with_calendar_service(
mut self,
calendar_service: Arc<dyn crate::application::ports::storage_ports::StorageUseCase>,
) -> Self {
pub fn with_calendar_service(mut self, calendar_service: Arc<dyn crate::application::ports::storage_ports::StorageUseCase>) -> Self {
self.calendar_service = Some(calendar_service);
self
}
pub fn with_contact_service(
mut self,
contact_service: Arc<dyn crate::application::ports::storage_ports::StorageUseCase>,
) -> Self {
pub fn with_contact_service(mut self, contact_service: Arc<dyn crate::application::ports::storage_ports::StorageUseCase>) -> Self {
self.contact_service = Some(contact_service);
self
}
pub fn with_calendar_use_case(
mut self,
calendar_use_case: Arc<dyn crate::application::ports::calendar_ports::CalendarUseCase>,
) -> Self {
pub fn with_calendar_use_case(mut self, calendar_use_case: Arc<dyn crate::application::ports::calendar_ports::CalendarUseCase>) -> Self {
self.calendar_use_case = Some(calendar_use_case);
self
}
pub fn with_addressbook_use_case(
mut self,
addressbook_use_case: Arc<dyn crate::application::ports::carddav_ports::AddressBookUseCase>,
) -> Self {
pub fn with_addressbook_use_case(mut self, addressbook_use_case: Arc<dyn crate::application::ports::carddav_ports::AddressBookUseCase>) -> Self {
self.addressbook_use_case = Some(addressbook_use_case);
self
}
pub fn with_contact_use_case(
mut self,
contact_use_case: Arc<dyn crate::application::ports::carddav_ports::ContactUseCase>,
) -> Self {
pub fn with_contact_use_case(mut self, contact_use_case: Arc<dyn crate::application::ports::carddav_ports::ContactUseCase>) -> Self {
self.contact_use_case = Some(contact_use_case);
self
}
src/common/stubs.rs
-123
View File
@@ -25,10 +25,8 @@ use crate::application::ports::file_ports::{
OptimizedFileContent, UploadStrategy,
};
use crate::application::ports::inbound::{FolderUseCase, SearchUseCase};
use crate::application::ports::outbound::IdMappingPort;
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::application::ports::zip_ports::ZipPort;
use crate::application::services::storage_mediator::{StorageMediator, StorageMediatorError};
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::entities::folder::Folder;
@@ -83,127 +81,6 @@ impl CompressionPort for StubCompressionPort {
}
}
// ---------------------------------------------------------------------------
// IdMappingPort
// ---------------------------------------------------------------------------
pub struct StubIdMappingService;
#[async_trait]
impl IdMappingPort for StubIdMappingService {
async fn get_or_create_id(&self, _path: &StoragePath) -> Result<String, DomainError> {
Ok("dummy-id".to_string())
}
async fn get_path_by_id(&self, _id: &str) -> Result<StoragePath, DomainError> {
Ok(StoragePath::from_string("/"))
}
async fn update_path(&self, _id: &str, _new_path: &StoragePath) -> Result<(), DomainError> {
Ok(())
}
async fn remove_id(&self, _id: &str) -> Result<(), DomainError> {
Ok(())
}
async fn save_changes(&self) -> Result<(), DomainError> {
Ok(())
}
}
// ---------------------------------------------------------------------------
// StorageMediator
// ---------------------------------------------------------------------------
pub struct StubStorageMediator;
#[async_trait]
impl StorageMediator for StubStorageMediator {
async fn get_folder_path(&self, _folder_id: &str) -> Result<PathBuf, StorageMediatorError> {
Ok(PathBuf::from("/tmp"))
}
async fn get_folder_storage_path(
&self,
_folder_id: &str,
) -> Result<StoragePath, StorageMediatorError> {
Ok(StoragePath::root())
}
async fn get_folder(&self, _folder_id: &str) -> Result<Folder, StorageMediatorError> {
Err(StorageMediatorError::NotFound(
"Stub not implemented".to_string(),
))
}
async fn file_exists_at_path(&self, _path: &Path) -> Result<bool, StorageMediatorError> {
Ok(false)
}
async fn file_exists_at_storage_path(
&self,
_storage_path: &StoragePath,
) -> Result<bool, StorageMediatorError> {
Ok(false)
}
async fn folder_exists_at_path(&self, _path: &Path) -> Result<bool, StorageMediatorError> {
Ok(false)
}
async fn folder_exists_at_storage_path(
&self,
_storage_path: &StoragePath,
) -> Result<bool, StorageMediatorError> {
Ok(false)
}
fn resolve_path(&self, _relative_path: &Path) -> PathBuf {
PathBuf::from("/tmp")
}
fn resolve_storage_path(&self, _storage_path: &StoragePath) -> PathBuf {
PathBuf::from("/tmp")
}
async fn ensure_directory(&self, _path: &Path) -> Result<(), StorageMediatorError> {
Ok(())
}
async fn ensure_storage_directory(
&self,
_storage_path: &StoragePath,
) -> Result<(), StorageMediatorError> {
Ok(())
}
}
// ---------------------------------------------------------------------------
// IdMappingPort
// ---------------------------------------------------------------------------
pub struct StubIdMappingPort;
#[async_trait]
impl IdMappingPort for StubIdMappingPort {
async fn get_or_create_id(&self, _path: &StoragePath) -> Result<String, DomainError> {
Ok("stub-id".to_string())
}
async fn get_path_by_id(&self, _id: &str) -> Result<StoragePath, DomainError> {
Ok(StoragePath::from_string("/"))
}
async fn update_path(&self, _id: &str, _new_path: &StoragePath) -> Result<(), DomainError> {
Ok(())
}
async fn remove_id(&self, _id: &str) -> Result<(), DomainError> {
Ok(())
}
async fn save_changes(&self) -> Result<(), DomainError> {
Ok(())
}
}
// ---------------------------------------------------------------------------
// FileReadPort
// ---------------------------------------------------------------------------
src/infrastructure/repositories/composite_file_repository.rs
-157
View File
@@ -1,157 +0,0 @@
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::services::path_service::StoragePath;
/// Composite that wraps `Arc<dyn FileReadPort>` + `Arc<dyn FileWritePort>`
/// and delegates each method to the corresponding port.
///
/// Thanks to the blanket impl `impl<T: FileReadPort + FileWritePort> FileStoragePort for T {}`
/// this type automatically gets `FileStoragePort`.
pub struct CompositeFileRepository {
read: Arc<dyn FileReadPort>,
write: Arc<dyn FileWritePort>,
}
impl CompositeFileRepository {
pub fn new(read: Arc<dyn FileReadPort>, write: Arc<dyn FileWritePort>) -> Self {
Self { read, write }
}
}
// ─────────────────────────────────────────────────────
// FileReadPort — delegate to self.read
// ─────────────────────────────────────────────────────
#[async_trait]
impl FileReadPort for CompositeFileRepository {
async fn get_file(&self, id: &str) -> Result<File, DomainError> {
self.read.get_file(id).await
}
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<File>, DomainError> {
self.read.list_files(folder_id).await
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
self.read.get_file_content(id).await
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.read.get_file_stream(id).await
}
async fn get_file_range_stream(
&self,
id: &str,
start: u64,
end: Option<u64>,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.read.get_file_range_stream(id, start, end).await
}
async fn get_file_mmap(&self, id: &str) -> Result<Bytes, DomainError> {
self.read.get_file_mmap(id).await
}
async fn get_file_path(&self, id: &str) -> Result<StoragePath, DomainError> {
self.read.get_file_path(id).await
}
async fn get_parent_folder_id(&self, path: &str) -> Result<String, DomainError> {
self.read.get_parent_folder_id(path).await
}
}
// ─────────────────────────────────────────────────────
// FileWritePort — delegate to self.write
// ─────────────────────────────────────────────────────
#[async_trait]
impl FileWritePort for CompositeFileRepository {
async fn save_file(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
content: Vec<u8>,
) -> Result<File, DomainError> {
self.write
.save_file(name, folder_id, content_type, content)
.await
}
async fn save_file_from_stream(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
) -> Result<File, DomainError> {
self.write
.save_file_from_stream(name, folder_id, content_type, stream)
.await
}
async fn move_file(
&self,
file_id: &str,
target_folder_id: Option<String>,
) -> Result<File, DomainError> {
self.write.move_file(file_id, target_folder_id).await
}
async fn rename_file(&self, file_id: &str, new_name: &str) -> Result<File, DomainError> {
self.write.rename_file(file_id, new_name).await
}
async fn delete_file(&self, id: &str) -> Result<(), DomainError> {
self.write.delete_file(id).await
}
async fn update_file_content(
&self,
file_id: &str,
content: Vec<u8>,
) -> Result<(), DomainError> {
self.write.update_file_content(file_id, content).await
}
async fn register_file_deferred(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
size: u64,
) -> Result<(File, PathBuf), DomainError> {
self.write
.register_file_deferred(name, folder_id, content_type, size)
.await
}
async fn move_to_trash(&self, file_id: &str) -> Result<(), DomainError> {
self.write.move_to_trash(file_id).await
}
async fn restore_from_trash(
&self,
file_id: &str,
original_path: &str,
) -> Result<(), DomainError> {
self.write.restore_from_trash(file_id, original_path).await
}
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
self.write.delete_file_permanently(file_id).await
}
}
src/infrastructure/repositories/file_fs_read_repository.rs
-473
View File
@@ -1,473 +0,0 @@
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use bytes::Bytes;
use futures::{Stream, StreamExt};
use mime_guess::from_path;
use tokio::fs::File as TokioFile;
use tokio::task;
use tokio::{fs, time};
use tokio_util::codec::{BytesCodec, FramedRead};
use crate::application::ports::cache_ports::MetadataCachePort;
use crate::application::ports::storage_ports::FileReadPort;
use crate::application::services::storage_mediator::StorageMediator;
use crate::common::config::AppConfig;
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::services::path_service::StoragePath;
use crate::infrastructure::repositories::parallel_file_processor::ParallelFileProcessor;
use crate::infrastructure::repositories::repository_errors::{
FileRepositoryError, FileRepositoryResult,
};
use crate::infrastructure::services::path_service::PathService;
/// Repository implementation for file **read** operations.
///
/// Implements `FileReadPort`:
/// get_file, list_files, get_file_content, get_file_stream,
/// get_file_range_stream, get_file_mmap, get_file_path, get_parent_folder_id.
pub struct FileFsReadRepository {
root_path: PathBuf,
storage_mediator: Arc<dyn StorageMediator>,
id_mapping_service: Arc<dyn crate::application::ports::outbound::IdMappingPort>,
path_service: Arc<PathService>,
metadata_cache: Arc<dyn MetadataCachePort>,
config: AppConfig,
parallel_processor: Option<Arc<ParallelFileProcessor>>,
}
impl FileFsReadRepository {
/// Full constructor with all infrastructure dependencies.
pub fn new(
root_path: PathBuf,
storage_mediator: Arc<dyn StorageMediator>,
id_mapping_service: Arc<dyn crate::application::ports::outbound::IdMappingPort>,
path_service: Arc<PathService>,
metadata_cache: Arc<dyn MetadataCachePort>,
config: AppConfig,
parallel_processor: Option<Arc<ParallelFileProcessor>>,
) -> Self {
Self {
root_path,
storage_mediator,
id_mapping_service,
path_service,
metadata_cache,
config,
parallel_processor,
}
}
/// Stub for testing (does not perform real I/O).
pub fn default_stub() -> Self {
Self {
root_path: PathBuf::from("./storage"),
storage_mediator: Arc::new(
crate::application::services::storage_mediator::FileSystemStorageMediator::new_stub(
),
),
id_mapping_service: Arc::new(crate::common::stubs::StubIdMappingPort),
path_service: Arc::new(PathService::new(PathBuf::from("./storage"))),
metadata_cache: Arc::new(
crate::infrastructure::services::file_metadata_cache::FileMetadataCache::default(),
) as Arc<dyn MetadataCachePort>,
config: AppConfig::default(),
parallel_processor: None,
}
}
// ─── internal helpers ─────────────────────────────────────
fn resolve_storage_path(&self, storage_path: &StoragePath) -> PathBuf {
self.path_service.resolve_path(storage_path)
}
async fn get_file_metadata_raw(
&self,
abs_path: &PathBuf,
) -> FileRepositoryResult<(u64, u64, u64)> {
// Cache first
if let Some(cached) = self.metadata_cache.get_metadata(abs_path).await
&& let (Some(s), Some(c), Some(m)) =
(cached.size, cached.created_at, cached.modified_at)
{
return Ok((s, c, m));
}
let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(abs_path))
.await
.map_err(|_| {
FileRepositoryError::StorageError(format!(
"Timeout metadata: {}",
abs_path.display()
))
})?
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
let size = metadata.len();
let created_at = metadata
.created()
.map(|t| {
t.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.unwrap_or(0);
let modified_at = metadata
.modified()
.map(|t| {
t.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.unwrap_or(0);
let _ = self.metadata_cache.refresh_metadata(abs_path).await;
Ok((size, created_at, modified_at))
}
async fn get_file_by_id(&self, id: &str) -> FileRepositoryResult<File> {
let storage_path = self
.id_mapping_service
.get_path_by_id(id)
.await
.map_err(|e| FileRepositoryError::Other(e.to_string()))?;
let abs_path = self.resolve_storage_path(&storage_path);
if !abs_path.exists() || !abs_path.is_file() {
return Err(FileRepositoryError::NotFound(format!(
"File {} not found at {}",
id,
storage_path.to_string()
)));
}
let (size, created_at, modified_at) = self.get_file_metadata_raw(&abs_path).await?;
let name = storage_path
.file_name()
.ok_or_else(|| FileRepositoryError::InvalidPath(storage_path.to_string()))?;
let mime_type = from_path(&abs_path).first_or_octet_stream().to_string();
File::with_timestamps(
id.to_string(),
name,
storage_path,
size,
mime_type,
None,
created_at,
modified_at,
)
.map_err(|e| FileRepositoryError::Other(e.to_string()))
}
}
impl Clone for FileFsReadRepository {
fn clone(&self) -> Self {
Self {
root_path: self.root_path.clone(),
storage_mediator: self.storage_mediator.clone(),
id_mapping_service: self.id_mapping_service.clone(),
path_service: self.path_service.clone(),
metadata_cache: self.metadata_cache.clone(),
config: self.config.clone(),
parallel_processor: self.parallel_processor.clone(),
}
}
}
#[async_trait]
impl FileReadPort for FileFsReadRepository {
async fn get_file(&self, id: &str) -> Result<File, DomainError> {
self.get_file_by_id(id).await.map_err(|e| match e {
FileRepositoryError::NotFound(msg) => DomainError::not_found("File", msg),
FileRepositoryError::StorageError(msg) => DomainError::internal_error("File", msg),
other => DomainError::internal_error("File", other.to_string()),
})
}
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<File>, DomainError> {
let folder_storage_path = match folder_id {
Some(id) => match self.storage_mediator.get_folder_path(id).await {
Ok(path) => {
let lossy = path.to_string_lossy().to_string();
let folder_name = path.file_name().and_then(|f| f.to_str()).unwrap_or(&lossy);
StoragePath::from_string(folder_name)
}
Err(_) => return Ok(Vec::new()),
},
None => StoragePath::root(),
};
let abs_folder_path = self.path_service.resolve_path(&folder_storage_path);
if !abs_folder_path.exists() || !abs_folder_path.is_dir() {
return Ok(Vec::new());
}
let mut files_result = Vec::new();
let mut entries = fs::read_dir(&abs_folder_path)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
while let Some(entry) = entries
.next_entry()
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?
{
let path = entry.path();
if !path.is_file() {
continue;
}
let file_name = entry.file_name().to_string_lossy().to_string();
if file_name.starts_with('.')
|| file_name == "folder_ids.json"
|| file_name == "file_ids.json"
{
continue;
}
let metadata = match fs::metadata(&path).await {
Ok(m) => m,
Err(_) => continue,
};
let file_storage_path = folder_storage_path.join(&file_name);
let id = match self
.id_mapping_service
.get_or_create_id(&file_storage_path)
.await
{
Ok(id) => id,
Err(_) => continue,
};
let size = metadata.len();
let created_at = metadata
.created()
.map(|t| {
t.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.unwrap_or(0);
let modified_at = metadata
.modified()
.map(|t| {
t.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.unwrap_or(0);
let mime_type = from_path(&path).first_or_octet_stream().to_string();
match File::with_timestamps(
id,
file_name,
file_storage_path,
size,
mime_type,
folder_id.map(String::from),
created_at,
modified_at,
) {
Ok(file) => files_result.push(file),
Err(_) => continue,
}
}
// Persist any new ID mappings
let _ = self.id_mapping_service.save_changes().await;
Ok(files_result)
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
let file = self
.get_file_by_id(id)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let abs_path = self.resolve_storage_path(file.storage_path());
let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs_path))
.await
.map_err(|_| {
DomainError::internal_error(
"File",
format!("Timeout metadata: {}", abs_path.display()),
)
})?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let file_size = metadata.len();
if !self.config.resources.can_load_in_memory(file_size) {
return Err(DomainError::internal_error(
"File",
format!(
"File too large for memory: {} MB",
file_size / (1024 * 1024)
),
));
}
// Parallel read for very large files
if self
.config
.resources
.needs_parallel_processing(file_size, &self.config.concurrency)
{
let content = if let Some(processor) = &self.parallel_processor {
processor.read_file_parallel(&abs_path).await
} else {
let processor = ParallelFileProcessor::new(self.config.clone());
processor.read_file_parallel(&abs_path).await
};
return content.map_err(|e| DomainError::internal_error("File", e.to_string()));
}
// spawn_blocking for large-ish files
if self.config.resources.is_large_file(file_size) {
let abs_clone = abs_path.clone();
let chunk_size = self.config.resources.chunk_size_bytes;
let content = task::spawn_blocking(move || -> std::io::Result<Vec<u8>> {
use std::io::{BufReader, Read};
let file = std::fs::File::open(&abs_clone)?;
let mut reader = BufReader::with_capacity(chunk_size, file);
let mut buf = Vec::with_capacity(file_size as usize);
reader.read_to_end(&mut buf)?;
Ok(buf)
})
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
return Ok(content);
}
// Small files — async read
time::timeout(self.config.timeouts.file_timeout(), fs::read(&abs_path))
.await
.map_err(|_| {
DomainError::internal_error(
"File",
format!("Timeout reading: {}", abs_path.display()),
)
})?
.map_err(|e| DomainError::internal_error("File", e.to_string()))
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
let file = self
.get_file_by_id(id)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let abs_path = self.resolve_storage_path(file.storage_path());
let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs_path))
.await
.map_err(|_| DomainError::internal_error("File", "Timeout getting metadata"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let file_size = metadata.len();
let is_large = self.config.resources.is_large_file(file_size);
let fh = time::timeout(
self.config.timeouts.file_timeout(),
TokioFile::open(&abs_path),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout opening file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let chunk_size = if is_large {
self.config.resources.chunk_size_bytes
} else {
4096
};
let codec = BytesCodec::new();
let stream =
FramedRead::with_capacity(fh, codec, chunk_size).map(|r| r.map(|bm| bm.freeze()));
Ok(Box::new(stream))
}
async fn get_file_range_stream(
&self,
id: &str,
start: u64,
end: Option<u64>,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
use tokio::io::AsyncSeekExt;
let file = self
.get_file_by_id(id)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let abs_path = self.resolve_storage_path(file.storage_path());
let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs_path))
.await
.map_err(|_| DomainError::internal_error("File", "Timeout"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let file_size = metadata.len();
if start >= file_size {
return Err(DomainError::internal_error(
"File",
format!("Range start {} beyond file size {}", start, file_size),
));
}
let actual_end = end.map(|e| e.min(file_size - 1)).unwrap_or(file_size - 1);
let range_length = actual_end - start + 1;
let mut fh = time::timeout(
self.config.timeouts.file_timeout(),
TokioFile::open(&abs_path),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout opening file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
fh.seek(std::io::SeekFrom::Start(start))
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let chunk_size = if range_length > 1024 * 1024 {
self.config.resources.chunk_size_bytes
} else {
8192
};
use tokio::io::AsyncReadExt;
let limited = fh.take(range_length);
let codec = BytesCodec::new();
let stream =
FramedRead::with_capacity(limited, codec, chunk_size).map(|r| r.map(|bm| bm.freeze()));
Ok(Box::new(stream))
}
async fn get_file_mmap(&self, id: &str) -> Result<Bytes, DomainError> {
use memmap2::Mmap;
let file = self
.get_file_by_id(id)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let abs_path = self.resolve_storage_path(file.storage_path());
let path_clone = abs_path.clone();
task::spawn_blocking(move || -> Result<Bytes, DomainError> {
let fh = std::fs::File::open(&path_clone)
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let mmap = unsafe { Mmap::map(&fh) }
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
Ok(Bytes::copy_from_slice(&mmap[..]))
})
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?
}
async fn get_file_path(&self, id: &str) -> Result<StoragePath, DomainError> {
self.id_mapping_service.get_path_by_id(id).await
}
async fn get_parent_folder_id(&self, path: &str) -> Result<String, DomainError> {
let storage_path = StoragePath::from_string(path);
match storage_path.parent() {
Some(parent) if !parent.is_empty() => {
self.id_mapping_service.get_or_create_id(&parent).await
}
_ => Ok("root".to_string()),
}
}
}
src/infrastructure/repositories/file_fs_write_repository.rs
-800
View File
@@ -1,800 +0,0 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::{Stream, StreamExt};
use mime_guess::from_path;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::fs::File as TokioFile;
use tokio::io::AsyncWriteExt;
use tokio::task;
use tokio::{fs, time};
use crate::application::ports::cache_ports::MetadataCachePort;
use crate::application::ports::storage_ports::FileWritePort;
use crate::application::services::storage_mediator::StorageMediator;
use crate::common::config::AppConfig;
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::services::path_service::StoragePath;
use crate::infrastructure::repositories::parallel_file_processor::ParallelFileProcessor;
use crate::infrastructure::repositories::repository_errors::{
FileRepositoryError, FileRepositoryResult,
};
use crate::infrastructure::services::file_system_utils::FileSystemUtils;
use crate::infrastructure::services::path_service::PathService;
/// Repository implementation for file **write** operations.
///
/// Implements `FileWritePort`:
/// save_file, save_file_from_stream, move_file, delete_file,
/// update_file_content, register_file_deferred.
pub struct FileFsWriteRepository {
root_path: PathBuf,
storage_mediator: Arc<dyn StorageMediator>,
id_mapping_service: Arc<dyn crate::application::ports::outbound::IdMappingPort>,
path_service: Arc<PathService>,
metadata_cache: Arc<dyn MetadataCachePort>,
config: AppConfig,
parallel_processor: Option<Arc<ParallelFileProcessor>>,
}
impl FileFsWriteRepository {
/// Full constructor with all dependencies.
pub fn new(
root_path: PathBuf,
storage_mediator: Arc<dyn StorageMediator>,
id_mapping_service: Arc<dyn crate::application::ports::outbound::IdMappingPort>,
path_service: Arc<PathService>,
metadata_cache: Arc<dyn MetadataCachePort>,
config: AppConfig,
parallel_processor: Option<Arc<ParallelFileProcessor>>,
) -> Self {
Self {
root_path,
storage_mediator,
id_mapping_service,
path_service,
metadata_cache,
config,
parallel_processor,
}
}
/// Stub for testing (does not perform real I/O).
pub fn default_stub() -> Self {
Self {
root_path: PathBuf::from("./storage"),
storage_mediator: Arc::new(
crate::application::services::storage_mediator::FileSystemStorageMediator::new_stub(
),
),
id_mapping_service: Arc::new(crate::common::stubs::StubIdMappingPort),
path_service: Arc::new(PathService::new(PathBuf::from("./storage"))),
metadata_cache: Arc::new(
crate::infrastructure::services::file_metadata_cache::FileMetadataCache::default(),
) as Arc<dyn MetadataCachePort>,
config: AppConfig::default(),
parallel_processor: None,
}
}
// ─── helpers ─────────────────────────────────────────────
fn resolve_storage_path(&self, storage_path: &StoragePath) -> PathBuf {
self.path_service.resolve_path(storage_path)
}
async fn ensure_parent_directory(&self, abs_path: &PathBuf) -> FileRepositoryResult<()> {
if let Some(parent) = abs_path.parent() {
time::timeout(
self.config.timeouts.dir_timeout(),
FileSystemUtils::create_dir_with_sync(parent),
)
.await
.map_err(|_| {
FileRepositoryError::StorageError(format!(
"Timeout creating dir: {}",
parent.display()
))
})?
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
}
Ok(())
}
async fn file_exists_at_storage_path(
&self,
storage_path: &StoragePath,
) -> FileRepositoryResult<bool> {
let abs = self.resolve_storage_path(storage_path);
if let Some(is_file) = self.metadata_cache.is_file(&abs).await {
return Ok(is_file);
}
match time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs)).await {
Ok(Ok(m)) => {
let _ = self.metadata_cache.refresh_metadata(&abs).await;
Ok(m.is_file())
}
Ok(Err(_)) => Ok(false),
Err(_) => Err(FileRepositoryError::StorageError(format!(
"Timeout: {}",
abs.display()
))),
}
}
async fn get_file_metadata_raw(
&self,
abs_path: &PathBuf,
) -> FileRepositoryResult<(u64, u64, u64)> {
if let Some(cached) = self.metadata_cache.get_metadata(abs_path).await
&& let (Some(s), Some(c), Some(m)) =
(cached.size, cached.created_at, cached.modified_at)
{
return Ok((s, c, m));
}
let meta = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(abs_path))
.await
.map_err(|_| {
FileRepositoryError::StorageError(format!("Timeout: {}", abs_path.display()))
})?
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
let s = meta.len();
let c = meta
.created()
.map(|t| {
t.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.unwrap_or(0);
let m = meta
.modified()
.map(|t| {
t.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.unwrap_or(0);
let _ = self.metadata_cache.refresh_metadata(abs_path).await;
Ok((s, c, m))
}
/// Resolve folder to StoragePath
async fn resolve_folder_path(&self, folder_id: &Option<String>) -> StoragePath {
match folder_id {
Some(id) => match self.storage_mediator.get_folder_path(id).await {
Ok(path) => {
let lossy = path.to_string_lossy().to_string();
let folder_name = path.file_name().and_then(|f| f.to_str()).unwrap_or(&lossy);
StoragePath::from_string(folder_name)
}
Err(_) => StoragePath::root(),
},
None => StoragePath::root(),
}
}
/// Generate unique file path avoiding name collisions.
async fn unique_file_path(
&self,
folder_path: &StoragePath,
name: &str,
) -> FileRepositoryResult<(StoragePath, String)> {
let mut file_path = folder_path.join(name);
let mut actual_name = name.to_string();
let mut counter = 1;
while self.file_exists_at_storage_path(&file_path).await? {
let (stem, ext) = if let Some(dot) = name.rfind('.') {
(name[..dot].to_string(), name[dot..].to_string())
} else {
(name.to_string(), String::new())
};
actual_name = format!("{}_{}{}", stem, counter, ext);
file_path = folder_path.join(&actual_name);
counter += 1;
}
Ok((file_path, actual_name))
}
async fn delete_file_non_blocking(&self, abs_path: PathBuf) -> FileRepositoryResult<()> {
let file_size = match fs::metadata(&abs_path).await {
Ok(m) => m.len(),
Err(_) => 0,
};
if self.config.resources.is_large_file(file_size) {
task::spawn_blocking(move || {
let _ = std::fs::remove_file(&abs_path);
})
.await
.map_err(|e| FileRepositoryError::Other(e.to_string()))?;
} else {
time::timeout(
self.config.timeouts.file_timeout(),
fs::remove_file(&abs_path),
)
.await
.map_err(|_| FileRepositoryError::StorageError("Timeout deleting file".into()))?
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
}
Ok(())
}
/// Persist ID mapping with retry + verification.
async fn persist_id_mapping(&self, id: &str, expected_path: &str) -> FileRepositoryResult<()> {
for attempt in 1..=3 {
match self.id_mapping_service.save_changes().await {
Ok(_) => {
if let Ok(verified) = self.id_mapping_service.get_path_by_id(id).await
&& verified.to_string() == expected_path
{
return Ok(());
}
if attempt == 3 {
return Err(FileRepositoryError::Other(
"Failed to verify ID mapping after 3 attempts".into(),
));
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
Err(e) if attempt < 3 => {
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
tracing::warn!("ID mapping save retry {}: {}", attempt, e);
}
Err(e) => {
return Err(FileRepositoryError::Other(format!(
"Save ID mapping failed: {}",
e
)));
}
}
}
Ok(())
}
}
impl Clone for FileFsWriteRepository {
fn clone(&self) -> Self {
Self {
root_path: self.root_path.clone(),
storage_mediator: self.storage_mediator.clone(),
id_mapping_service: self.id_mapping_service.clone(),
path_service: self.path_service.clone(),
metadata_cache: self.metadata_cache.clone(),
config: self.config.clone(),
parallel_processor: self.parallel_processor.clone(),
}
}
}
fn map_repo_err(e: FileRepositoryError) -> DomainError {
match e {
FileRepositoryError::NotFound(m) => DomainError::not_found("File", m),
FileRepositoryError::AlreadyExists(m) => DomainError::already_exists("File", m),
FileRepositoryError::StorageError(m) => DomainError::internal_error("File", m),
other => DomainError::internal_error("File", other.to_string()),
}
}
#[async_trait]
impl FileWritePort for FileFsWriteRepository {
async fn save_file(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
content: Vec<u8>,
) -> Result<File, DomainError> {
let folder_path = self.resolve_folder_path(&folder_id).await;
let (file_storage_path, actual_name) = self
.unique_file_path(&folder_path, &name)
.await
.map_err(map_repo_err)?;
let abs_path = self.resolve_storage_path(&file_storage_path);
self.ensure_parent_directory(&abs_path)
.await
.map_err(map_repo_err)?;
let content_size = content.len() as u64;
// Write strategy based on file size
if self
.config
.resources
.needs_parallel_processing(content_size, &self.config.concurrency)
{
if let Some(proc) = &self.parallel_processor {
proc.write_file_parallel(&abs_path, &content)
.await
.map_err(map_repo_err)?;
} else {
let proc = ParallelFileProcessor::new(self.config.clone());
proc.write_file_parallel(&abs_path, &content)
.await
.map_err(map_repo_err)?;
}
} else if content_size > self.config.resources.large_file_threshold_mb * 1024 * 1024 {
let mut fh = time::timeout(
self.config.timeouts.file_timeout(),
TokioFile::create(&abs_path),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout creating file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let chunk_size = self.config.resources.chunk_size_bytes;
for chunk in content.chunks(chunk_size) {
fh.write_all(chunk)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
}
fh.flush()
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
} else {
let mut fh = time::timeout(
self.config.timeouts.file_timeout(),
TokioFile::create(&abs_path),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout creating file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
fh.write_all(&content)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
fh.flush()
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
}
let (size, created_at, modified_at) = self
.get_file_metadata_raw(&abs_path)
.await
.map_err(map_repo_err)?;
let mime = if content_type.is_empty() {
from_path(&abs_path).first_or_octet_stream().to_string()
} else {
content_type
};
let id = self
.id_mapping_service
.get_or_create_id(&file_storage_path)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let path_string = file_storage_path.to_string();
let file = File::with_timestamps(
id.clone(),
actual_name,
file_storage_path,
size,
mime,
folder_id,
created_at,
modified_at,
)
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
self.persist_id_mapping(&id, &path_string)
.await
.map_err(map_repo_err)?;
if let Some(parent) = abs_path.parent() {
self.metadata_cache.invalidate_directory(parent).await;
}
Ok(file)
}
async fn save_file_from_stream(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
mut stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
) -> Result<File, DomainError> {
let folder_path = self.resolve_folder_path(&folder_id).await;
let (file_storage_path, actual_name) = self
.unique_file_path(&folder_path, &name)
.await
.map_err(map_repo_err)?;
let abs_path = self.resolve_storage_path(&file_storage_path);
self.ensure_parent_directory(&abs_path)
.await
.map_err(map_repo_err)?;
let temp_path = abs_path.with_extension("tmp.upload");
let mut fh = time::timeout(
self.config.timeouts.file_timeout(),
TokioFile::create(&temp_path),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout creating temp file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let mut total_bytes: u64 = 0;
while let Some(chunk_result) = stream.next().await {
let chunk =
chunk_result.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
fh.write_all(&chunk)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
total_bytes += chunk.len() as u64;
}
fh.flush()
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
fh.sync_all()
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
drop(fh);
// Atomic rename
fs::rename(&temp_path, &abs_path)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let (size, created_at, modified_at) = self
.get_file_metadata_raw(&abs_path)
.await
.map_err(map_repo_err)?;
let mime = if content_type.is_empty() {
from_path(&abs_path).first_or_octet_stream().to_string()
} else {
content_type
};
let id = self
.id_mapping_service
.get_or_create_id(&file_storage_path)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let path_string = file_storage_path.to_string();
let log_name = actual_name.clone();
let file = File::with_timestamps(
id.clone(),
actual_name,
file_storage_path,
size,
mime,
folder_id,
created_at,
modified_at,
)
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
self.persist_id_mapping(&id, &path_string)
.await
.map_err(map_repo_err)?;
if let Some(parent) = abs_path.parent() {
self.metadata_cache.invalidate_directory(parent).await;
}
tracing::info!(
"✅ STREAMING UPLOAD COMPLETE: {} ({} bytes)",
log_name,
total_bytes
);
Ok(file)
}
async fn move_file(
&self,
file_id: &str,
target_folder_id: Option<String>,
) -> Result<File, DomainError> {
// Get original file
let original_path = self.id_mapping_service.get_path_by_id(file_id).await?;
let old_abs = self.resolve_storage_path(&original_path);
if !old_abs.exists() || !old_abs.is_file() {
return Err(DomainError::not_found("File", file_id.to_string()));
}
let (size, created_at, modified_at) = self
.get_file_metadata_raw(&old_abs)
.await
.map_err(map_repo_err)?;
let name = original_path
.file_name()
.ok_or_else(|| DomainError::internal_error("File", "Invalid path"))?;
let mime = from_path(&old_abs).first_or_octet_stream().to_string();
// Build target path
let target_folder_path = self.resolve_folder_path(&target_folder_id).await;
let new_storage_path = target_folder_path.join(&name);
if self
.file_exists_at_storage_path(&new_storage_path)
.await
.map_err(map_repo_err)?
{
return Err(DomainError::already_exists(
"File",
format!("File already exists at {}", new_storage_path.to_string()),
));
}
let new_abs = self.resolve_storage_path(&new_storage_path);
self.ensure_parent_directory(&new_abs)
.await
.map_err(map_repo_err)?;
// Rename
time::timeout(
self.config.timeouts.file_timeout(),
FileSystemUtils::rename_with_sync(&old_abs, &new_abs),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout moving file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
// Update mapping
self.id_mapping_service
.update_path(file_id, &new_storage_path)
.await?;
let _ = self.id_mapping_service.save_changes().await;
File::with_timestamps(
file_id.to_string(),
name,
new_storage_path,
size,
mime,
target_folder_id,
created_at,
modified_at,
)
.map_err(|e| DomainError::internal_error("File", e.to_string()))
}
async fn rename_file(&self, file_id: &str, new_name: &str) -> Result<File, DomainError> {
// 1. Get current file info
let original_path = self.id_mapping_service.get_path_by_id(file_id).await?;
let old_abs = self.resolve_storage_path(&original_path);
if !old_abs.exists() || !old_abs.is_file() {
return Err(DomainError::not_found("File", file_id.to_string()));
}
let (size, created_at, modified_at) = self
.get_file_metadata_raw(&old_abs)
.await
.map_err(map_repo_err)?;
// 2. Build new path (same parent directory, different filename)
let parent = original_path
.parent()
.unwrap_or_else(|| StoragePath::new(vec![]));
let new_storage_path = parent.join(new_name);
if self
.file_exists_at_storage_path(&new_storage_path)
.await
.map_err(map_repo_err)?
{
return Err(DomainError::already_exists(
"File",
format!("File already exists: {}", new_name),
));
}
let new_abs = self.resolve_storage_path(&new_storage_path);
let mime = from_path(&new_abs).first_or_octet_stream().to_string();
// 3. Rename on disk
time::timeout(
self.config.timeouts.file_timeout(),
FileSystemUtils::rename_with_sync(&old_abs, &new_abs),
)
.await
.map_err(|_| DomainError::internal_error("File", "Timeout renaming file"))?
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
// 4. Update id→path mapping
self.id_mapping_service
.update_path(file_id, &new_storage_path)
.await?;
let _ = self.id_mapping_service.save_changes().await;
File::with_timestamps(
file_id.to_string(),
new_name.to_string(),
new_storage_path,
size,
mime,
None,
created_at,
modified_at,
)
.map_err(|e| DomainError::internal_error("File", e.to_string()))
}
async fn delete_file(&self, id: &str) -> Result<(), DomainError> {
let storage_path = self.id_mapping_service.get_path_by_id(id).await?;
let abs_path = self.resolve_storage_path(&storage_path);
self.metadata_cache.invalidate(&abs_path).await;
if let Some(parent) = abs_path.parent() {
self.metadata_cache.invalidate_directory(parent).await;
}
self.delete_file_non_blocking(abs_path)
.await
.map_err(map_repo_err)?;
// Clean up the ID mapping so we don't leave orphaned entries
if let Err(e) = self.id_mapping_service.remove_id(id).await {
tracing::warn!("Failed to remove ID mapping for deleted file {}: {}", id, e);
}
let _ = self.id_mapping_service.save_changes().await;
Ok(())
}
async fn update_file_content(
&self,
file_id: &str,
content: Vec<u8>,
) -> Result<(), DomainError> {
let storage_path = self.id_mapping_service.get_path_by_id(file_id).await?;
let physical_path = self.resolve_storage_path(&storage_path);
FileSystemUtils::atomic_write(&physical_path, &content)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
// Refresh cache
let _ = self.metadata_cache.refresh_metadata(&physical_path).await;
Ok(())
}
async fn register_file_deferred(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
size: u64,
) -> Result<(File, PathBuf), DomainError> {
let folder_path = self.resolve_folder_path(&folder_id).await;
let (file_storage_path, actual_name) = self
.unique_file_path(&folder_path, &name)
.await
.map_err(map_repo_err)?;
let abs_path = self.resolve_storage_path(&file_storage_path);
self.ensure_parent_directory(&abs_path)
.await
.map_err(map_repo_err)?;
let mime = if content_type.is_empty() {
from_path(&abs_path).first_or_octet_stream().to_string()
} else {
content_type
};
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let id = self
.id_mapping_service
.get_or_create_id(&file_storage_path)
.await
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
let _ = self.id_mapping_service.save_changes().await;
let file = File::with_timestamps(
id.clone(),
actual_name,
file_storage_path,
size,
mime,
folder_id,
now,
now,
)
.map_err(|e| DomainError::internal_error("File", e.to_string()))?;
tracing::debug!("⚡ Registered deferred file: {} -> {:?}", id, abs_path);
Ok((file, abs_path))
}
async fn move_to_trash(&self, file_id: &str) -> Result<(), DomainError> {
// Get the file's current path
let storage_path = self.id_mapping_service.get_path_by_id(file_id).await?;
let abs_path = self.resolve_storage_path(&storage_path);
if !abs_path.exists() || !abs_path.is_file() {
return Err(DomainError::not_found("File", file_id.to_string()));
}
// Create trash directory
let trash_dir = self.root_path.join(".trash").join("files");
fs::create_dir_all(&trash_dir).await.map_err(|e| {
DomainError::internal_error("File", format!("Failed to create trash dir: {}", e))
})?;
// Move file to trash
let trash_path = trash_dir.join(file_id);
fs::rename(&abs_path, &trash_path).await.map_err(|e| {
DomainError::internal_error("File", format!("Failed to move file to trash: {}", e))
})?;
// Update mapping to trash location
let trash_storage_path = StoragePath::from_string(&format!(".trash/files/{}", file_id));
self.id_mapping_service
.update_path(file_id, &trash_storage_path)
.await?;
let _ = self.id_mapping_service.save_changes().await;
// Invalidate cache
self.metadata_cache.invalidate(&abs_path).await;
if let Some(parent) = abs_path.parent() {
self.metadata_cache.invalidate_directory(parent).await;
}
tracing::debug!(
"File moved to trash: {} -> {}",
file_id,
trash_path.display()
);
Ok(())
}
async fn restore_from_trash(
&self,
file_id: &str,
original_path: &str,
) -> Result<(), DomainError> {
// Get current path (should be in trash)
let current_storage_path = self.id_mapping_service.get_path_by_id(file_id).await?;
let current_abs_path = self.resolve_storage_path(&current_storage_path);
if !current_abs_path.exists() {
return Err(DomainError::not_found(
"File",
format!("File {} not found in trash", file_id),
));
}
// Ensure parent directory exists for original location
let original_storage_path = StoragePath::from_string(original_path);
let original_abs_path = self.resolve_storage_path(&original_storage_path);
if let Some(parent) = original_abs_path.parent() {
fs::create_dir_all(parent).await.map_err(|e| {
DomainError::internal_error("File", format!("Failed to create parent dir: {}", e))
})?;
}
// Move file back to original location
fs::rename(&current_abs_path, &original_abs_path)
.await
.map_err(|e| {
DomainError::internal_error("File", format!("Failed to restore file: {}", e))
})?;
// Update mapping back to original path
self.id_mapping_service
.update_path(file_id, &original_storage_path)
.await?;
let _ = self.id_mapping_service.save_changes().await;
tracing::debug!(
"File restored from trash: {} -> {}",
file_id,
original_abs_path.display()
);
Ok(())
}
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
// Get current path (could be in trash or original location)
let storage_path = self.id_mapping_service.get_path_by_id(file_id).await?;
let abs_path = self.resolve_storage_path(&storage_path);
// Delete the physical file if it exists
if abs_path.exists() {
self.delete_file_non_blocking(abs_path.clone())
.await
.map_err(map_repo_err)?;
}
// Remove ID mapping
self.id_mapping_service.remove_id(file_id).await?;
let _ = self.id_mapping_service.save_changes().await;
// Invalidate cache
self.metadata_cache.invalidate(&abs_path).await;
tracing::debug!("File permanently deleted: {}", file_id);
Ok(())
}
}
src/infrastructure/repositories/folder_fs_repository.rs
-1010
View File
File diff suppressed because it is too large Load Diff
src/infrastructure/repositories/folder_fs_repository_trash.rs
-192
View File
@@ -1,192 +0,0 @@
use std::path::PathBuf;
use tokio::fs;
use tracing::{debug, error};
use crate::infrastructure::repositories::folder_fs_repository::FolderFsRepository;
use crate::infrastructure::repositories::repository_errors::FolderRepositoryResult;
// This file contains the implementation of trash-related methods
// for the FolderFsRepository folder repository
// Implementation of trash methods for the folder repository
impl FolderFsRepository {
// Gets the full path to the trash directory
fn get_trash_dir(&self) -> PathBuf {
self.get_root_path().join(".trash").join("folders")
}
// Creates a unique path in the trash for the folder
async fn create_trash_folder_path(&self, folder_id: &str) -> FolderRepositoryResult<PathBuf> {
let trash_dir = self.get_trash_dir();
// Ensure the trash directory exists
if !trash_dir.exists() {
fs::create_dir_all(&trash_dir)
.await
.map_err(|e| FolderRepositoryError::StorageError(e.to_string()))?;
}
// Create a unique path for the folder in the trash
Ok(trash_dir.join(folder_id))
}
}
// Implementation of public FolderRepository trait methods related to trash
// Implementation of internal methods for trash functionality
// These will be enabled when the trash feature is re-enabled
impl FolderFsRepository {
/// Helper method that will be used for trash functionality
pub(crate) async fn _trash_move_to_trash(&self, folder_id: &str) -> FolderRepositoryResult<()> {
debug!("Moving folder to trash: {}", folder_id);
// Get the physical path of the folder
let folder_path = match self.get_mapped_folder_path(folder_id).await {
Ok(path) => path,
Err(e) => {
error!("Error getting folder path {}: {:?}", folder_id, e);
return Err(e);
}
};
let folder_path_buf = PathBuf::from(folder_path.to_string());
// Verify the folder exists
if !folder_path_buf.exists() {
return Err(FolderRepositoryError::NotFound(format!(
"Folder not found: {}",
folder_id
)));
}
// Create directory in the trash
let trash_folder_path = self.create_trash_folder_path(folder_id).await?;
// Physically move the folder to the trash
match fs::rename(&folder_path_buf, &trash_folder_path).await {
Ok(_) => {
debug!(
"Folder moved to trash: {} -> {}",
folder_path_buf.display(),
trash_folder_path.display()
);
// Update the mapping to the new path in the trash
if let Err(e) = self
.update_mapped_folder_path(folder_id, &trash_folder_path)
.await
{
error!("Error updating folder mapping in trash: {}", e);
return Err(e);
}
Ok(())
}
Err(e) => {
error!("Error moving folder to trash: {}", e);
Err(FolderRepositoryError::StorageError(e.to_string()))
}
}
}
/// Restores a folder from the trash to its original location
pub(crate) async fn _trash_restore_from_trash(
&self,
folder_id: &str,
original_path: &str,
) -> FolderRepositoryResult<()> {
debug!("Restoring folder {} to {}", folder_id, original_path);
// Get the current path in the trash
let current_path = match self.get_mapped_folder_path(folder_id).await {
Ok(path) => PathBuf::from(path),
Err(e) => {
error!("Error getting current folder path {}: {:?}", folder_id, e);
return Err(e);
}
};
// Convert the original path to PathBuf
let original_path_buf = PathBuf::from(original_path);
// Ensure the destination parent directory exists
if let Some(parent) = original_path_buf.parent()
&& !parent.exists()
{
fs::create_dir_all(parent).await.map_err(|e| {
error!("Error creating parent directory for restoration: {}", e);
FolderRepositoryError::StorageError(e.to_string())
})?;
}
// Move the folder from the trash to its original location
match fs::rename(&current_path, &original_path_buf).await {
Ok(_) => {
debug!(
"Folder restored: {} -> {}",
current_path.display(),
original_path_buf.display()
);
// Update the mapping to the original path
if let Err(e) = self
.update_mapped_folder_path(folder_id, &original_path_buf)
.await
{
error!("Error updating restored folder mapping: {}", e);
return Err(e);
}
Ok(())
}
Err(e) => {
error!("Error restoring folder: {}", e);
Err(FolderRepositoryError::StorageError(e.to_string()))
}
}
}
/// Permanently deletes a folder (used by the trash)
pub(crate) async fn _trash_delete_folder_permanently(
&self,
folder_id: &str,
) -> FolderRepositoryResult<()> {
debug!("Permanently deleting folder: {}", folder_id);
// Similar to delete_folder but without additional validations
let folder_path = match self.get_mapped_folder_path(folder_id).await {
Ok(path) => PathBuf::from(path),
Err(e) => {
error!("Error getting folder path {}: {:?}", folder_id, e);
return Err(e);
}
};
// Delete the folder recursively
if folder_path.exists() {
match fs::remove_dir_all(&folder_path).await {
Ok(_) => {
debug!("Folder permanently deleted: {}", folder_path.display());
}
Err(e) => {
error!("Error permanently deleting folder: {}", e);
// Don't report error if the folder no longer exists
if e.kind() != std::io::ErrorKind::NotFound {
return Err(FolderRepositoryError::StorageError(e.to_string()));
}
}
}
}
// Remove the mapping
if let Err(e) = self.remove_mapped_folder_id(folder_id).await {
error!("Error removing folder mapping: {}", e);
return Err(e);
}
debug!("Folder permanently deleted successfully: {}", folder_id);
Ok(())
}
}
// Re-exports needed by the compiler
use crate::infrastructure::repositories::repository_errors::FolderRepositoryError;
src/infrastructure/repositories/mod.rs
+5 -16
View File
@@ -1,21 +1,10 @@
pub mod folder_fs_repository;
pub mod parallel_file_processor;
pub mod repository_errors;
// Repositorios CQRS (Read/Write) + composite
pub mod composite_file_repository;
pub mod file_fs_read_repository;
pub mod file_fs_write_repository;
pub mod folder_fs_repository_trash;
pub mod share_fs_repository;
pub mod trash_fs_repository;
// Repositorios PostgreSQL
// Repositorios PostgreSQL (blob-storage model)
pub mod pg;
// Re-exportar para facilitar acceso
pub use composite_file_repository::CompositeFileRepository;
pub use file_fs_read_repository::FileFsReadRepository;
pub use file_fs_write_repository::FileFsWriteRepository;
pub use pg::{SessionPgRepository, UserPgRepository};
pub use pg::{
FileBlobReadRepository, FileBlobWriteRepository, FolderDbRepository,
SessionPgRepository, TrashDbRepository, UserPgRepository,
};
src/infrastructure/repositories/parallel_file_processor.rs
-571
View File
@@ -1,571 +0,0 @@
use bytes::{Bytes, BytesMut};
use futures::future::join_all;
use std::io::{self, SeekFrom};
use std::path::PathBuf;
use std::sync::Arc;
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::sync::{Mutex, Semaphore};
use tokio::task;
use tracing::{debug, error, info};
use crate::common::config::AppConfig;
use crate::infrastructure::repositories::repository_errors::FileRepositoryError;
use crate::infrastructure::services::buffer_pool::BufferPool;
/// Structure for the byte range to process
#[derive(Debug, Clone, Copy)]
pub struct ChunkRange {
/// Chunk index
pub index: usize,
/// Start position in bytes
pub start: u64,
/// Chunk size in bytes
pub size: usize,
}
/// Specific buffer pooling for BytesMut
pub struct BytesBufferPool {
buffers: Mutex<Vec<BytesMut>>,
buffer_size: usize,
max_buffers: usize,
}
impl BytesBufferPool {
pub fn new(buffer_size: usize, max_buffers: usize) -> Self {
Self {
buffers: Mutex::new(Vec::with_capacity(max_buffers)),
buffer_size,
max_buffers,
}
}
/// Get a buffer from the pool or create a new one
pub async fn get_buffer(&self) -> BytesMut {
let mut buffers = self.buffers.lock().await;
if let Some(mut buffer) = buffers.pop() {
// Reuse existing buffer
buffer.clear(); // Keep capacity, clear content
buffer
} else {
// Create new buffer if the pool is empty
BytesMut::with_capacity(self.buffer_size)
}
}
/// Return a buffer to the pool for reuse
pub async fn return_buffer(&self, mut buffer: BytesMut) {
// Reset the buffer for reuse
buffer.clear();
let mut buffers = self.buffers.lock().await;
// Only keep up to max_buffers
if buffers.len() < self.max_buffers {
buffers.push(buffer);
}
// If we already have enough buffers, this one will be discarded
}
}
/// Parallel file processor for IO-intensive operations
pub struct ParallelFileProcessor {
/// Application configuration
config: AppConfig,
/// Semaphore to limit global concurrency
concurrency_limiter: Arc<Semaphore>,
/// Buffer pool to optimize memory
buffer_pool: Option<Arc<BufferPool>>,
/// BytesMut buffer pool for zero-copy operations
bytes_pool: Arc<BytesBufferPool>,
}
impl ParallelFileProcessor {
/// Creates a new processor instance
pub fn new(config: AppConfig) -> Self {
let concurrency_limiter = Arc::new(Semaphore::new(config.concurrency.max_concurrent_io));
// Create BytesMut pool for efficient operations
let chunk_size = config.resources.chunk_size_bytes;
let max_chunks = config.concurrency.max_parallel_chunks;
let bytes_pool = Arc::new(BytesBufferPool::new(chunk_size, max_chunks * 2));
Self {
config,
concurrency_limiter,
buffer_pool: None,
bytes_pool,
}
}
/// Creates a new processor instance with a buffer pool
pub fn new_with_buffer_pool(config: AppConfig, buffer_pool: Arc<BufferPool>) -> Self {
let concurrency_limiter = Arc::new(Semaphore::new(config.concurrency.max_concurrent_io));
// Create BytesMut pool for efficient operations
let chunk_size = config.resources.chunk_size_bytes;
let max_chunks = config.concurrency.max_parallel_chunks;
let bytes_pool = Arc::new(BytesBufferPool::new(chunk_size, max_chunks * 2));
Self {
config,
concurrency_limiter,
buffer_pool: Some(buffer_pool),
bytes_pool,
}
}
/// Divides a file into chunks for parallel processing
pub fn calculate_chunks(&self, file_size: u64) -> Vec<ChunkRange> {
// Determine if the file needs parallel processing
let needs_parallel = self
.config
.resources
.needs_parallel_processing(file_size, &self.config.concurrency);
if !needs_parallel {
// For small files, use a single chunk
return vec![ChunkRange {
index: 0,
start: 0,
size: file_size as usize,
}];
}
// Calculate optimal number of chunks
let chunk_count = self
.config
.resources
.calculate_optimal_chunks(file_size, &self.config.concurrency);
// Calculate size of each chunk
let chunk_size = self
.config
.resources
.calculate_chunk_size(file_size, chunk_count);
// Create chunk ranges
let mut chunks = Vec::with_capacity(chunk_count);
let mut start = 0;
for i in 0..chunk_count {
let current_chunk_size = if i == chunk_count - 1 {
// Last chunk might be smaller
(file_size - start) as usize
} else {
chunk_size
};
chunks.push(ChunkRange {
index: i,
start,
size: current_chunk_size,
});
start += current_chunk_size as u64;
}
debug!(
"File size: {} bytes, divided into {} chunks of ~{} bytes each",
file_size,
chunks.len(),
chunk_size
);
chunks
}
/// Reads a file in parallel and returns the complete content
/// Optimized implementation using BytesMut to reduce memory copies
pub async fn read_file_parallel(
&self,
file_path: &PathBuf,
) -> Result<Vec<u8>, FileRepositoryError> {
// Get file size
let metadata = tokio::fs::metadata(file_path)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
let file_size = metadata.len();
// Check if the file is too large for memory
if !self.config.resources.can_load_in_memory(file_size) {
return Err(FileRepositoryError::Other(format!(
"File too large to load in memory: {} MB (max: {} MB)",
file_size / (1024 * 1024),
self.config.resources.max_in_memory_file_size_mb
)));
}
// Calculate chunks
let chunks = self.calculate_chunks(file_size);
if chunks.len() == 1 {
// For a single chunk, use simple reading with buffer pool if available
info!(
"Reading file with size {}MB as a single chunk",
file_size / (1024 * 1024)
);
if let Some(pool) = &self.buffer_pool {
// Use buffer from the pool for efficient reading
debug!("Using buffer pool for single chunk read");
let mut buffer = pool.get_buffer().await;
// If the buffer is too small, revert to standard implementation
if buffer.capacity() < file_size as usize {
debug!(
"Buffer from pool too small ({}), using standard read",
buffer.capacity()
);
let content = tokio::fs::read(file_path)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
return Ok(content);
}
// Use memory buffer from the pool
let mut file = File::open(file_path)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
let read_size = file
.read(buffer.as_mut_slice())
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
buffer.set_used(read_size);
// Convert to Vec<u8>
let content = buffer.into_vec();
return Ok(content);
} else {
// Standard implementation without pool
let content = tokio::fs::read(file_path)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
return Ok(content);
}
}
// For multiple chunks, use parallel reading
info!(
"Reading file with size {}MB in {} parallel chunks using BytesMut",
file_size / (1024 * 1024),
chunks.len()
);
// Create final result buffer (pre-allocated)
let mut result = BytesMut::with_capacity(file_size as usize);
result.resize(file_size as usize, 0);
let result_mutex = Arc::new(Mutex::new(result));
// Create tasks for each chunk
let mut tasks = Vec::with_capacity(chunks.len());
// Open file once and share it
let file = Arc::new(
File::open(file_path)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?,
);
// Reference to BytesMut pool
let bytes_pool = self.bytes_pool.clone();
// Process chunks in parallel
for chunk in chunks {
let file_clone = file.clone();
let result_clone = result_mutex.clone();
let semaphore_clone = self.concurrency_limiter.clone();
let bytes_pool_clone = bytes_pool.clone();
// Spawn task for this chunk - no need to copy the original data
let task = task::spawn(async move {
// Acquire semaphore permit
let _permit = semaphore_clone.acquire().await.unwrap();
// Get a reusable buffer from the BytesMut pool
let mut chunk_buffer = bytes_pool_clone.get_buffer().await;
// Ensure it has sufficient capacity
if chunk_buffer.capacity() < chunk.size {
chunk_buffer = BytesMut::with_capacity(chunk.size);
}
// Resize to the exact size needed
chunk_buffer.resize(chunk.size, 0);
// Create a duplicate file descriptor for independent use
let mut file_handle = file_clone.try_clone().await?;
// Position and read directly into the BytesMut
file_handle.seek(SeekFrom::Start(chunk.start)).await?;
let bytes_read = file_handle
.read_exact(&mut chunk_buffer[..chunk.size])
.await?;
if bytes_read != chunk.size {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
format!(
"Expected to read {} bytes but got {}",
chunk.size, bytes_read
),
));
}
// Write to final result
let mut result_lock = result_clone.lock().await;
let start_pos = chunk.start as usize;
let end_pos = start_pos + chunk.size;
// Use copy_from_slice to copy from BytesMut to result buffer
result_lock[start_pos..end_pos].copy_from_slice(&chunk_buffer[..chunk.size]);
// Return the buffer to the pool for reuse
bytes_pool_clone.return_buffer(chunk_buffer).await;
// Log progress
debug!(
"Chunk {} processed: {} bytes from offset {}",
chunk.index, chunk.size, chunk.start
);
Ok::<_, io::Error>(())
});
tasks.push(task);
}
// Wait for all tasks to complete
let results = join_all(tasks).await;
// Check for errors
for (i, task_result) in results.into_iter().enumerate() {
match task_result {
Ok(Ok(())) => {}
Ok(Err(e)) => {
error!("Error in chunk {}: {}", i, e);
return Err(FileRepositoryError::StorageError(e.to_string()));
}
Err(e) => {
error!("Task error in chunk {}: {}", i, e);
return Err(FileRepositoryError::Other(format!("Task error: {}", e)));
}
}
}
// Get the final result and convert to Vec<u8>
let result_buffer = result_mutex.lock().await;
let result_vec = result_buffer.to_vec();
info!(
"Successfully read file of {}MB in parallel with optimized BytesMut",
file_size / (1024 * 1024)
);
Ok(result_vec)
}
/// Writes a file in parallel from a buffer
/// Optimized implementation using BytesMut/Bytes to reduce memory copies
pub async fn write_file_parallel(
&self,
file_path: &PathBuf,
content: &[u8],
) -> Result<(), FileRepositoryError> {
let file_size = content.len() as u64;
// Calculate chunks
let chunks = self.calculate_chunks(file_size);
if chunks.len() == 1 {
// For a single chunk, use simple writing
info!(
"Writing file with size {}MB as a single chunk",
file_size / (1024 * 1024)
);
// Standard implementation (buffer pooling offers no advantages for simple writing)
tokio::fs::write(file_path, content)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
return Ok(());
}
// For multiple chunks, use parallel writing
info!(
"Writing file with size {}MB in {} parallel chunks using Bytes",
file_size / (1024 * 1024),
chunks.len()
);
// Create file (we don't use Mutex to reduce contention)
let file = File::create(file_path)
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
// Convert content to Bytes (single copy step)
let content_bytes = Bytes::copy_from_slice(content);
// Create tasks for each chunk
let mut tasks = Vec::with_capacity(chunks.len());
// Process chunks in parallel
for chunk in chunks {
let file_clone = file
.try_clone()
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
let semaphore_clone = self.concurrency_limiter.clone();
// Create Bytes slice (doesn't copy data, only references)
let start_idx = chunk.start as usize;
let end_idx = start_idx + chunk.size;
let chunk_data = content_bytes.slice(start_idx..end_idx);
// Create and launch task
let task = task::spawn(async move {
// Acquire semaphore permit
let _permit = semaphore_clone.acquire().await.unwrap();
// Position and write
let mut file_handle = file_clone;
file_handle.seek(SeekFrom::Start(chunk.start)).await?;
file_handle.write_all(&chunk_data).await?;
// Log progress
debug!(
"Chunk {} written: {} bytes at offset {}",
chunk.index, chunk.size, chunk.start
);
Ok::<_, io::Error>(())
});
tasks.push(task);
}
// Wait for all tasks to complete
let results = join_all(tasks).await;
// Check for errors
for (i, task_result) in results.into_iter().enumerate() {
match task_result {
Ok(Ok(())) => {}
Ok(Err(e)) => {
error!("Error in chunk {}: {}", i, e);
return Err(FileRepositoryError::StorageError(e.to_string()));
}
Err(e) => {
error!("Task error in chunk {}: {}", i, e);
return Err(FileRepositoryError::Other(format!("Task error: {}", e)));
}
}
}
// Ensure everything has been written correctly
let mut file_handle = file;
file_handle
.flush()
.await
.map_err(|e| FileRepositoryError::StorageError(e.to_string()))?;
info!(
"Successfully wrote file of {}MB in parallel with optimized Bytes",
file_size / (1024 * 1024)
);
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use bytes::BufMut;
use tempfile::tempdir;
#[tokio::test]
async fn test_parallel_read_write() {
// Create configuration with low threshold for testing
let mut config = AppConfig::default();
config.concurrency.min_size_for_parallel_chunks_mb = 1; // 1MB for testing
config.concurrency.max_parallel_chunks = 4;
let processor = ParallelFileProcessor::new(config);
// Create temporary directory
let temp_dir = tempdir().unwrap();
let file_path = temp_dir.path().join("test_file.bin");
// Create test data (2MB)
let size = 2 * 1024 * 1024;
let mut test_data = Vec::with_capacity(size);
for i in 0..size {
test_data.push((i % 256) as u8);
}
// Write file in parallel
processor
.write_file_parallel(&file_path, &test_data)
.await
.unwrap();
// Read file in parallel
let read_data = processor.read_file_parallel(&file_path).await.unwrap();
// Verify that the data is identical
assert_eq!(test_data.len(), read_data.len());
assert_eq!(test_data, read_data);
}
#[tokio::test]
async fn test_bytesmut_pool() {
// Create pool
let pool = BytesBufferPool::new(1024, 5);
// Get buffer
let mut buffer1 = pool.get_buffer().await;
buffer1.put_slice(b"test data");
assert_eq!(&buffer1[..9], b"test data");
// Return buffer to the pool
pool.return_buffer(buffer1).await;
// Get another buffer (should be the same one)
let buffer2 = pool.get_buffer().await;
assert_eq!(buffer2.capacity(), 1024);
// The buffer should be empty (cleared)
assert_eq!(buffer2.len(), 0);
}
#[test]
fn test_chunk_calculation() {
// Create test configuration
let mut config = AppConfig::default();
config.concurrency.min_size_for_parallel_chunks_mb = 100; // 100MB
config.concurrency.max_parallel_chunks = 4;
config.concurrency.parallel_chunk_size_bytes = 50 * 1024 * 1024; // 50MB
let processor = ParallelFileProcessor::new(config);
// Small file (10MB)
let small_file_size = 10 * 1024 * 1024;
let chunks = processor.calculate_chunks(small_file_size);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].size as u64, small_file_size);
// Large file (300MB)
let large_file_size = 300 * 1024 * 1024;
let chunks = processor.calculate_chunks(large_file_size);
assert_eq!(chunks.len(), 4); // Limited to max_parallel_chunks
// Verify that all chunks add up to the total size
let total_size: u64 = chunks.iter().map(|c| c.size as u64).sum();
assert_eq!(total_size, large_file_size);
}
}
src/infrastructure/repositories/pg/file_blob_read_repository.rs
+271
View File
@@ -0,0 +1,271 @@
//! PostgreSQL + Blob-backed file read repository.
//!
//! Implements `FileReadPort` using:
//! - `storage.files` table for metadata lookups
//! - `DedupPort` for reading content-addressable blobs from the filesystem
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use sqlx::PgPool;
use std::sync::Arc;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::storage_ports::FileReadPort;
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::repositories::folder_repository::FolderRepository;
use crate::domain::services::path_service::StoragePath;
use super::folder_db_repository::FolderDbRepository;
/// File read repository backed by PostgreSQL metadata + blob storage.
pub struct FileBlobReadRepository {
pool: Arc<PgPool>,
dedup: Arc<dyn DedupPort>,
folder_repo: Arc<FolderDbRepository>,
}
impl FileBlobReadRepository {
pub fn new(
pool: Arc<PgPool>,
dedup: Arc<dyn DedupPort>,
folder_repo: Arc<FolderDbRepository>,
) -> Self {
Self {
pool,
dedup,
folder_repo,
}
}
/// Build a virtual StoragePath for a file.
async fn build_file_path(
&self,
folder_id: Option<&str>,
file_name: &str,
) -> Result<StoragePath, DomainError> {
if let Some(fid) = folder_id {
let folder_path = self.folder_repo.get_folder_path(fid).await?;
Ok(folder_path.join(file_name))
} else {
Ok(StoragePath::from_string(file_name))
}
}
/// Convert a database row into a `File` domain entity.
async fn row_to_file(
&self,
id: String,
name: String,
folder_id: Option<String>,
size: i64,
mime_type: String,
created_at: i64,
modified_at: i64,
) -> Result<File, DomainError> {
let storage_path = self
.build_file_path(folder_id.as_deref(), &name)
.await?;
File::with_timestamps(
id,
name,
storage_path,
size as u64,
mime_type,
folder_id,
created_at as u64,
modified_at as u64,
)
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("entity: {e}")))
}
/// Get the blob hash for a file.
async fn get_blob_hash(&self, file_id: &str) -> Result<String, DomainError> {
sqlx::query_scalar::<_, String>(
"SELECT blob_hash FROM storage.files WHERE id = $1::uuid AND NOT is_trashed",
)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("hash lookup: {e}")))?
.ok_or_else(|| DomainError::not_found("File", file_id))
}
}
#[async_trait]
impl FileReadPort for FileBlobReadRepository {
async fn get_file(&self, id: &str) -> Result<File, DomainError> {
let row = sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64)>(
r#"
SELECT id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.files
WHERE id = $1::uuid AND NOT is_trashed
"#,
)
.bind(id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("get: {e}")))?
.ok_or_else(|| DomainError::not_found("File", id))?;
self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6)
.await
}
async fn list_files(
&self,
folder_id: Option<&str>,
) -> Result<Vec<File>, DomainError> {
let rows: Vec<(String, String, Option<String>, i64, String, i64, i64)> =
if let Some(fid) = folder_id {
sqlx::query_as(
r#"
SELECT id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.files
WHERE folder_id = $1::uuid AND NOT is_trashed
ORDER BY name
"#,
)
.bind(fid)
.fetch_all(self.pool.as_ref())
.await
} else {
sqlx::query_as(
r#"
SELECT id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.files
WHERE folder_id IS NULL AND NOT is_trashed
ORDER BY name
"#,
)
.fetch_all(self.pool.as_ref())
.await
}
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("list: {e}")))?;
let mut files = Vec::with_capacity(rows.len());
for (id, name, fid, size, mime, ca, ma) in rows {
files.push(self.row_to_file(id, name, fid, size, mime, ca, ma).await?);
}
Ok(files)
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
let blob_hash = self.get_blob_hash(id).await?;
self.dedup.read_blob(&blob_hash).await
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
// Read blob as bytes and wrap in a single-chunk stream.
// For very large files, a true streaming implementation from the
// blob file would be better, but DedupPort API currently returns bytes.
let blob_hash = self.get_blob_hash(id).await?;
let content = self.dedup.read_blob_bytes(&blob_hash).await?;
let stream = futures::stream::once(async move { Ok(content) });
Ok(Box::new(stream))
}
async fn get_file_range_stream(
&self,
id: &str,
start: u64,
end: Option<u64>,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
let blob_hash = self.get_blob_hash(id).await?;
let content = self.dedup.read_blob_bytes(&blob_hash).await?;
let start = start as usize;
let end = end.map_or(content.len(), |e| e as usize).min(content.len());
if start >= content.len() {
return Ok(Box::new(futures::stream::empty()));
}
let slice = content.slice(start..end);
let stream = futures::stream::once(async move { Ok(slice) });
Ok(Box::new(stream))
}
async fn get_file_mmap(&self, id: &str) -> Result<Bytes, DomainError> {
let blob_hash = self.get_blob_hash(id).await?;
self.dedup.read_blob_bytes(&blob_hash).await
}
async fn get_file_path(&self, id: &str) -> Result<StoragePath, DomainError> {
let row = sqlx::query_as::<_, (String, Option<String>)>(
r#"
SELECT name, folder_id::text
FROM storage.files
WHERE id = $1::uuid AND NOT is_trashed
"#,
)
.bind(id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("path: {e}")))?
.ok_or_else(|| DomainError::not_found("File", id))?;
self.build_file_path(row.1.as_deref(), &row.0).await
}
async fn get_parent_folder_id(&self, path: &str) -> Result<String, DomainError> {
// Walk the path to find the parent folder, searching by folder names
let path = path.trim_start_matches('/').trim_end_matches('/');
let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
if segments.is_empty() {
return Err(DomainError::not_found("Folder", "empty path"));
}
// For path "a/b/c/file.txt", the parent folder path is "a/b/c"
// But we don't know which part is folders vs filename.
// Walk segments trying to find matching folders.
let mut current_parent: Option<String> = None;
for segment in &segments {
let row = if let Some(ref pid) = current_parent {
sqlx::query_as::<_, (String,)>(
r#"
SELECT id::text FROM storage.folders
WHERE name = $1 AND parent_id = $2::uuid AND NOT is_trashed
"#,
)
.bind(segment)
.bind(pid)
.fetch_optional(self.pool.as_ref())
.await
} else {
sqlx::query_as::<_, (String,)>(
r#"
SELECT id::text FROM storage.folders
WHERE name = $1 AND parent_id IS NULL AND NOT is_trashed
"#,
)
.bind(segment)
.fetch_optional(self.pool.as_ref())
.await
}
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("path walk: {e}")))?;
match row {
Some(r) => current_parent = Some(r.0),
None => break, // This segment is not a folder → it's the filename
}
}
current_parent.ok_or_else(|| {
DomainError::not_found("Folder", format!("parent for path: {path}"))
})
}
}
src/infrastructure/repositories/pg/file_blob_write_repository.rs
+435
View File
@@ -0,0 +1,435 @@
//! PostgreSQL + Blob-backed file write repository.
//!
//! Implements `FileWritePort` using:
//! - `storage.files` table for metadata
//! - `DedupPort` for content-addressable blob storage on the filesystem
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use sqlx::PgPool;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::storage_ports::FileWritePort;
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::repositories::folder_repository::FolderRepository;
use crate::domain::services::path_service::StoragePath;
use super::folder_db_repository::FolderDbRepository;
/// File write repository backed by PostgreSQL metadata + blob storage.
pub struct FileBlobWriteRepository {
pool: Arc<PgPool>,
dedup: Arc<dyn DedupPort>,
folder_repo: Arc<FolderDbRepository>,
}
impl FileBlobWriteRepository {
pub fn new(
pool: Arc<PgPool>,
dedup: Arc<dyn DedupPort>,
folder_repo: Arc<FolderDbRepository>,
) -> Self {
Self {
pool,
dedup,
folder_repo,
}
}
/// Build a virtual StoragePath for a file from its DB metadata.
async fn build_file_path(
&self,
folder_id: Option<&str>,
file_name: &str,
) -> Result<StoragePath, DomainError> {
if let Some(fid) = folder_id {
let folder_path = self.folder_repo.get_folder_path(fid).await?;
Ok(folder_path.join(file_name))
} else {
Ok(StoragePath::from_string(file_name))
}
}
/// Convert a database row into a `File` domain entity.
async fn row_to_file(
&self,
id: String,
name: String,
folder_id: Option<String>,
size: i64,
mime_type: String,
created_at: i64,
modified_at: i64,
) -> Result<File, DomainError> {
let storage_path = self
.build_file_path(folder_id.as_deref(), &name)
.await?;
File::with_timestamps(
id,
name,
storage_path,
size as u64,
mime_type,
folder_id,
created_at as u64,
modified_at as u64,
)
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("entity: {e}")))
}
/// Derive user_id from the parent folder, or error if folder_id is None.
async fn resolve_user_id(&self, folder_id: Option<&str>) -> Result<String, DomainError> {
match folder_id {
Some(fid) => self.folder_repo.get_folder_user_id(fid).await,
None => Err(DomainError::internal_error(
"FileBlobWrite",
"folder_id is required to determine file owner",
)),
}
}
}
#[async_trait]
impl FileWritePort for FileBlobWriteRepository {
async fn save_file(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
content: Vec<u8>,
) -> Result<File, DomainError> {
let user_id = self.resolve_user_id(folder_id.as_deref()).await?;
let size = content.len() as i64;
// Store content in blob store
let dedup_result = self
.dedup
.store_bytes(&content, Some(content_type.clone()))
.await?;
let blob_hash = dedup_result.hash().to_string();
// Insert file metadata
let row = sqlx::query_as::<_, (String, i64, i64)>(
r#"
INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type)
VALUES ($1, $2::uuid, $3, $4, $5, $6)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(&name)
.bind(&folder_id)
.bind(&user_id)
.bind(&blob_hash)
.bind(size)
.bind(&content_type)
.fetch_one(self.pool.as_ref())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"File",
format!("{name} already exists in folder"),
);
}
}
DomainError::internal_error("FileBlobWrite", format!("insert: {e}"))
})?;
tracing::info!(
"💾 BLOB WRITE: {} ({} bytes, hash: {})",
name,
size,
&blob_hash[..12]
);
self.row_to_file(
row.0,
name,
folder_id,
size,
content_type,
row.1,
row.2,
)
.await
}
async fn save_file_from_stream(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
) -> Result<File, DomainError> {
use futures::StreamExt;
// Collect stream into bytes (blobs are content-addressed, need full content for hash)
let mut content = Vec::new();
let mut stream = stream;
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| {
DomainError::internal_error("FileBlobWrite", format!("stream read: {e}"))
})?;
content.extend_from_slice(&chunk);
}
self.save_file(name, folder_id, content_type, content).await
}
async fn move_file(
&self,
file_id: &str,
target_folder_id: Option<String>,
) -> Result<File, DomainError> {
// If moving to a different folder, get the new user_id (must be same user)
let row = sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64)>(
r#"
UPDATE storage.files
SET folder_id = $1::uuid, updated_at = NOW()
WHERE id = $2::uuid AND NOT is_trashed
RETURNING id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(&target_folder_id)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("move: {e}")))?
.ok_or_else(|| DomainError::not_found("File", file_id))?;
self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6)
.await
}
async fn rename_file(
&self,
file_id: &str,
new_name: &str,
) -> Result<File, DomainError> {
let row = sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64)>(
r#"
UPDATE storage.files
SET name = $1, updated_at = NOW()
WHERE id = $2::uuid AND NOT is_trashed
RETURNING id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(new_name)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"File",
format!("{new_name} already exists"),
);
}
}
DomainError::internal_error("FileBlobWrite", format!("rename: {e}"))
})?
.ok_or_else(|| DomainError::not_found("File", file_id))?;
self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6)
.await
}
async fn delete_file(&self, id: &str) -> Result<(), DomainError> {
// Get blob_hash before deleting so we can decrement ref
let hash = sqlx::query_scalar::<_, String>(
"SELECT blob_hash FROM storage.files WHERE id = $1::uuid",
)
.bind(id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("hash lookup: {e}")))?;
let result = sqlx::query("DELETE FROM storage.files WHERE id = $1::uuid")
.bind(id)
.execute(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("delete: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("File", id));
}
// Decrement blob reference
if let Some(h) = hash {
if let Err(e) = self.dedup.remove_reference(&h).await {
tracing::warn!("Failed to decrement blob ref for {}: {}", &h[..12], e);
}
}
Ok(())
}
async fn update_file_content(
&self,
file_id: &str,
content: Vec<u8>,
) -> Result<(), DomainError> {
// Get old blob hash to decrement ref
let old_hash = sqlx::query_scalar::<_, String>(
"SELECT blob_hash FROM storage.files WHERE id = $1::uuid",
)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("old hash: {e}")))?
.ok_or_else(|| DomainError::not_found("File", file_id))?;
// Store new content
let new_size = content.len() as i64;
let dedup_result = self.dedup.store_bytes(&content, None).await?;
let new_hash = dedup_result.hash().to_string();
// Update file metadata
sqlx::query(
r#"
UPDATE storage.files
SET blob_hash = $1, size = $2, updated_at = NOW()
WHERE id = $3::uuid
"#,
)
.bind(&new_hash)
.bind(new_size)
.bind(file_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("update: {e}")))?;
// Decrement old blob ref (only if hash changed)
if old_hash != new_hash {
if let Err(e) = self.dedup.remove_reference(&old_hash).await {
tracing::warn!(
"Failed to decrement old blob ref {}: {}",
&old_hash[..12],
e
);
}
}
Ok(())
}
async fn register_file_deferred(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
size: u64,
) -> Result<(File, PathBuf), DomainError> {
let user_id = self.resolve_user_id(folder_id.as_deref()).await?;
// For deferred registration we use a placeholder hash.
// The write-behind cache will call update_file_content later.
let placeholder_hash = "0000000000000000000000000000000000000000000000000000000000000000";
let row = sqlx::query_as::<_, (String, i64, i64)>(
r#"
INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type)
VALUES ($1, $2::uuid, $3, $4, $5, $6)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(&name)
.bind(&folder_id)
.bind(&user_id)
.bind(placeholder_hash)
.bind(size as i64)
.bind(&content_type)
.fetch_one(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("deferred: {e}")))?;
let file = self
.row_to_file(
row.0.clone(),
name,
folder_id,
size as i64,
content_type,
row.1,
row.2,
)
.await?;
// The target_path is not meaningful for blob storage (content goes to .blobs/)
// but the WriteBehindCache API requires it. We return a synthetic path.
let target_path = PathBuf::from(format!(".pending/{}", row.0));
Ok((file, target_path))
}
// ── Trash operations ──
async fn move_to_trash(&self, file_id: &str) -> Result<(), DomainError> {
let result = sqlx::query(
r#"
UPDATE storage.files
SET is_trashed = TRUE,
trashed_at = NOW(),
original_folder_id = folder_id,
updated_at = NOW()
WHERE id = $1::uuid AND NOT is_trashed
"#,
)
.bind(file_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("trash: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("File", file_id));
}
Ok(())
}
async fn restore_from_trash(
&self,
file_id: &str,
_original_path: &str,
) -> Result<(), DomainError> {
let result = sqlx::query(
r#"
UPDATE storage.files
SET is_trashed = FALSE,
trashed_at = NULL,
folder_id = COALESCE(original_folder_id, folder_id),
original_folder_id = NULL,
updated_at = NOW()
WHERE id = $1::uuid AND is_trashed
"#,
)
.bind(file_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("restore: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("File", file_id));
}
Ok(())
}
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
// Same as delete_file — removes from DB and decrements blob ref
self.delete_file(file_id).await
}
}
src/infrastructure/repositories/pg/folder_db_repository.rs
+603
View File
@@ -0,0 +1,603 @@
//! PostgreSQL-backed folder repository.
//!
//! Implements `FolderRepository` (and thus `FolderStoragePort`) using the
//! `storage.folders` table. Folders are purely virtual — no physical
//! directories are created on the filesystem.
use async_trait::async_trait;
use sqlx::PgPool;
use std::sync::Arc;
use crate::common::errors::DomainError;
use crate::domain::entities::folder::Folder;
use crate::domain::repositories::folder_repository::FolderRepository;
use crate::domain::services::path_service::StoragePath;
/// PostgreSQL-backed folder repository.
///
/// All folder metadata lives in the `storage.folders` table. The physical
/// filesystem is never touched for folder operations.
pub struct FolderDbRepository {
pool: Option<Arc<PgPool>>,
}
impl FolderDbRepository {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool: Some(pool) }
}
/// Creates a stub instance for `AppState::default()`.
/// This is never called in production — only used for route scaffolding.
pub fn new_stub() -> Self {
Self { pool: None }
}
/// Get the pool, panicking if stub.
fn pool(&self) -> &PgPool {
self.pool.as_deref().expect("FolderDbRepository: pool not available (stub instance)")
}
// ── helpers ──────────────────────────────────────────────────
/// Build the full virtual path for a folder by walking up the `parent_id` chain.
async fn build_folder_path(&self, folder_id: &str) -> Result<StoragePath, DomainError> {
// CTE-based recursive query to build path segments
let _rows = sqlx::query_as::<_, (String,)>(
r#"
WITH RECURSIVE ancestors AS (
SELECT id, name, parent_id
FROM storage.folders
WHERE id = $1::uuid
UNION ALL
SELECT f.id, f.name, f.parent_id
FROM storage.folders f
JOIN ancestors a ON f.id = a.parent_id
)
SELECT name FROM ancestors ORDER BY name
"#,
)
.bind(folder_id)
.fetch_all(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("path query: {e}")))?;
// Actually we need a proper ordering. Let me rewrite with depth tracking.
// Re-query with depth.
let rows = sqlx::query_as::<_, (String, i32)>(
r#"
WITH RECURSIVE ancestors AS (
SELECT id, name, parent_id, 0 AS depth
FROM storage.folders
WHERE id = $1::uuid
UNION ALL
SELECT f.id, f.name, f.parent_id, a.depth + 1
FROM storage.folders f
JOIN ancestors a ON f.id = a.parent_id
)
SELECT name, depth FROM ancestors ORDER BY depth DESC
"#,
)
.bind(folder_id)
.fetch_all(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("path query: {e}")))?;
let path_parts: Vec<&str> = rows.iter().map(|(name, _)| name.as_str()).collect();
let path_str = path_parts.join("/");
Ok(StoragePath::from_string(&path_str))
}
/// Convert a database row into a `Folder` domain entity.
async fn row_to_folder(
&self,
id: String,
name: String,
parent_id: Option<String>,
created_at: i64,
modified_at: i64,
) -> Result<Folder, DomainError> {
let storage_path = self.build_folder_path(&id).await?;
Folder::with_timestamps(
id,
name,
storage_path,
parent_id,
created_at as u64,
modified_at as u64,
)
.map_err(|e| DomainError::internal_error("FolderDb", format!("entity: {e}")))
}
}
#[async_trait]
impl FolderRepository for FolderDbRepository {
async fn create_folder(
&self,
name: String,
parent_id: Option<String>,
) -> Result<Folder, DomainError> {
// Derive user_id from parent folder. Root-level folders require the
// caller to have set up the home folder beforehand (done during user
// registration).
let user_id: String = if let Some(ref pid) = parent_id {
sqlx::query_scalar::<_, String>(
"SELECT user_id FROM storage.folders WHERE id = $1::uuid",
)
.bind(pid)
.fetch_optional(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("parent lookup: {e}")))?
.ok_or_else(|| DomainError::not_found("Folder", pid))?
} else {
return Err(DomainError::internal_error(
"FolderDb",
"Cannot create root folder without user_id — use create_home_folder instead",
));
};
let row = sqlx::query_as::<_, (String, i64, i64)>(
r#"
INSERT INTO storage.folders (name, parent_id, user_id)
VALUES ($1, $2::uuid, $3)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(&name)
.bind(&parent_id)
.bind(&user_id)
.fetch_one(self.pool())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"Folder",
format!("{name} already exists in parent"),
);
}
}
DomainError::internal_error("FolderDb", format!("insert: {e}"))
})?;
self.row_to_folder(row.0, name, parent_id, row.1, row.2)
.await
}
async fn get_folder(&self, id: &str) -> Result<Folder, DomainError> {
let row = sqlx::query_as::<_, (String, String, Option<String>, i64, i64)>(
r#"
SELECT id::text, name, parent_id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.folders
WHERE id = $1::uuid AND NOT is_trashed
"#,
)
.bind(id)
.fetch_optional(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("get: {e}")))?
.ok_or_else(|| DomainError::not_found("Folder", id))?;
self.row_to_folder(row.0, row.1, row.2, row.3, row.4)
.await
}
async fn get_folder_by_path(
&self,
storage_path: &StoragePath,
) -> Result<Folder, DomainError> {
// Walk the path segments to find the folder.
let path_str = storage_path.to_string();
let segments: Vec<&str> = path_str.split('/').filter(|s| !s.is_empty()).collect();
if segments.is_empty() {
return Err(DomainError::not_found("Folder", "empty path"));
}
let mut current_parent: Option<String> = None;
let mut current_id = String::new();
for segment in &segments {
let row = if let Some(ref pid) = current_parent {
sqlx::query_as::<_, (String,)>(
r#"
SELECT id::text FROM storage.folders
WHERE name = $1 AND parent_id = $2::uuid AND NOT is_trashed
"#,
)
.bind(segment)
.bind(pid)
.fetch_optional(self.pool())
.await
} else {
sqlx::query_as::<_, (String,)>(
r#"
SELECT id::text FROM storage.folders
WHERE name = $1 AND parent_id IS NULL AND NOT is_trashed
"#,
)
.bind(segment)
.fetch_optional(self.pool())
.await
}
.map_err(|e| DomainError::internal_error("FolderDb", format!("path walk: {e}")))?
.ok_or_else(|| {
DomainError::not_found("Folder", format!("segment '{segment}' in path"))
})?;
current_id = row.0;
current_parent = Some(current_id.clone());
}
self.get_folder(&current_id).await
}
async fn list_folders(
&self,
parent_id: Option<&str>,
) -> Result<Vec<Folder>, DomainError> {
let rows: Vec<(String, String, Option<String>, i64, i64)> = if let Some(pid) = parent_id {
sqlx::query_as(
r#"
SELECT id::text, name, parent_id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.folders
WHERE parent_id = $1::uuid AND NOT is_trashed
ORDER BY name
"#,
)
.bind(pid)
.fetch_all(self.pool())
.await
} else {
sqlx::query_as(
r#"
SELECT id::text, name, parent_id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.folders
WHERE parent_id IS NULL AND NOT is_trashed
ORDER BY name
"#,
)
.fetch_all(self.pool())
.await
}
.map_err(|e| DomainError::internal_error("FolderDb", format!("list: {e}")))?;
let mut folders = Vec::with_capacity(rows.len());
for (id, name, pid, ca, ma) in rows {
folders.push(self.row_to_folder(id, name, pid, ca, ma).await?);
}
Ok(folders)
}
async fn list_folders_paginated(
&self,
parent_id: Option<&str>,
offset: usize,
limit: usize,
include_total: bool,
) -> Result<(Vec<Folder>, Option<usize>), DomainError> {
let total = if include_total {
let count: i64 = if let Some(pid) = parent_id {
sqlx::query_scalar(
"SELECT COUNT(*) FROM storage.folders WHERE parent_id = $1::uuid AND NOT is_trashed",
)
.bind(pid)
.fetch_one(self.pool())
.await
} else {
sqlx::query_scalar(
"SELECT COUNT(*) FROM storage.folders WHERE parent_id IS NULL AND NOT is_trashed",
)
.fetch_one(self.pool())
.await
}
.map_err(|e| DomainError::internal_error("FolderDb", format!("count: {e}")))?;
Some(count as usize)
} else {
None
};
let rows: Vec<(String, String, Option<String>, i64, i64)> = if let Some(pid) = parent_id {
sqlx::query_as(
r#"
SELECT id::text, name, parent_id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.folders
WHERE parent_id = $1::uuid AND NOT is_trashed
ORDER BY name
LIMIT $2 OFFSET $3
"#,
)
.bind(pid)
.bind(limit as i64)
.bind(offset as i64)
.fetch_all(self.pool())
.await
} else {
sqlx::query_as(
r#"
SELECT id::text, name, parent_id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.folders
WHERE parent_id IS NULL AND NOT is_trashed
ORDER BY name
LIMIT $1 OFFSET $2
"#,
)
.bind(limit as i64)
.bind(offset as i64)
.fetch_all(self.pool())
.await
}
.map_err(|e| DomainError::internal_error("FolderDb", format!("paginate: {e}")))?;
let mut folders = Vec::with_capacity(rows.len());
for (id, name, pid, ca, ma) in rows {
folders.push(self.row_to_folder(id, name, pid, ca, ma).await?);
}
Ok((folders, total))
}
async fn rename_folder(
&self,
id: &str,
new_name: String,
) -> Result<Folder, DomainError> {
sqlx::query(
r#"
UPDATE storage.folders
SET name = $1, updated_at = NOW()
WHERE id = $2::uuid AND NOT is_trashed
"#,
)
.bind(&new_name)
.bind(id)
.execute(self.pool())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"Folder",
format!("{new_name} already exists"),
);
}
}
DomainError::internal_error("FolderDb", format!("rename: {e}"))
})?;
self.get_folder(id).await
}
async fn move_folder(
&self,
id: &str,
new_parent_id: Option<&str>,
) -> Result<Folder, DomainError> {
sqlx::query(
r#"
UPDATE storage.folders
SET parent_id = $1::uuid, updated_at = NOW()
WHERE id = $2::uuid AND NOT is_trashed
"#,
)
.bind(new_parent_id)
.bind(id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("move: {e}")))?;
self.get_folder(id).await
}
async fn delete_folder(&self, id: &str) -> Result<(), DomainError> {
// Hard delete folder and all descendants (CASCADE handles children)
let result = sqlx::query("DELETE FROM storage.folders WHERE id = $1::uuid")
.bind(id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("delete: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("Folder", id));
}
Ok(())
}
async fn folder_exists(&self, storage_path: &StoragePath) -> Result<bool, DomainError> {
// Try to find by walking the path
match self.get_folder_by_path(storage_path).await {
Ok(_) => Ok(true),
Err(e) if e.to_string().contains("not found") => Ok(false),
Err(e) => Err(e),
}
}
async fn get_folder_path(&self, id: &str) -> Result<StoragePath, DomainError> {
self.build_folder_path(id).await
}
// ── Trash operations ──
async fn move_to_trash(&self, folder_id: &str) -> Result<(), DomainError> {
// Soft-delete: set is_trashed = true and remember original parent
let result = sqlx::query(
r#"
UPDATE storage.folders
SET is_trashed = TRUE,
trashed_at = NOW(),
original_parent_id = parent_id,
updated_at = NOW()
WHERE id = $1::uuid AND NOT is_trashed
"#,
)
.bind(folder_id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("trash: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("Folder", folder_id));
}
// Also trash all files inside the folder (recursively)
sqlx::query(
r#"
WITH RECURSIVE descendants AS (
SELECT id FROM storage.folders WHERE id = $1::uuid
UNION ALL
SELECT f.id FROM storage.folders f JOIN descendants d ON f.parent_id = d.id
)
UPDATE storage.files
SET is_trashed = TRUE, trashed_at = NOW(), original_folder_id = folder_id
WHERE folder_id IN (SELECT id FROM descendants) AND NOT is_trashed
"#,
)
.bind(folder_id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("trash files: {e}")))?;
Ok(())
}
async fn restore_from_trash(
&self,
folder_id: &str,
_original_path: &str,
) -> Result<(), DomainError> {
// Restore: set is_trashed = false, restore parent_id from original_parent_id
let result = sqlx::query(
r#"
UPDATE storage.folders
SET is_trashed = FALSE,
trashed_at = NULL,
parent_id = COALESCE(original_parent_id, parent_id),
original_parent_id = NULL,
updated_at = NOW()
WHERE id = $1::uuid AND is_trashed
"#,
)
.bind(folder_id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("restore: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("Folder", folder_id));
}
// Also restore files that were trashed with this folder
sqlx::query(
r#"
WITH RECURSIVE descendants AS (
SELECT id FROM storage.folders WHERE id = $1::uuid
UNION ALL
SELECT f.id FROM storage.folders f JOIN descendants d ON f.parent_id = d.id
)
UPDATE storage.files
SET is_trashed = FALSE,
trashed_at = NULL,
folder_id = COALESCE(original_folder_id, folder_id),
original_folder_id = NULL
WHERE folder_id IN (SELECT id FROM descendants) AND is_trashed
"#,
)
.bind(folder_id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("restore files: {e}")))?;
Ok(())
}
async fn delete_folder_permanently(&self, folder_id: &str) -> Result<(), DomainError> {
// Permanently delete — CASCADE handles children
let result = sqlx::query("DELETE FROM storage.folders WHERE id = $1::uuid")
.bind(folder_id)
.execute(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("perm delete: {e}")))?;
if result.rows_affected() == 0 {
return Err(DomainError::not_found("Folder", folder_id));
}
Ok(())
}
}
// ── Extra helpers for blob-storage bootstrap ──
impl FolderDbRepository {
/// Creates a root-level home folder for a user.
/// This is called during user registration.
pub async fn create_home_folder(
&self,
user_id: &str,
name: &str,
) -> Result<Folder, DomainError> {
let row = sqlx::query_as::<_, (String, i64, i64)>(
r#"
INSERT INTO storage.folders (name, parent_id, user_id)
VALUES ($1, NULL, $2)
ON CONFLICT DO NOTHING
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(name)
.bind(user_id)
.fetch_optional(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("home folder: {e}")))?;
match row {
Some((id, ca, ma)) => {
self.row_to_folder(id, name.to_string(), None, ca, ma).await
}
None => {
// Already exists — fetch it
let existing = sqlx::query_as::<_, (String, i64, i64)>(
r#"
SELECT id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.folders
WHERE name = $1 AND user_id = $2 AND parent_id IS NULL
"#,
)
.bind(name)
.bind(user_id)
.fetch_one(self.pool())
.await
.map_err(|e| {
DomainError::internal_error("FolderDb", format!("home fetch: {e}"))
})?;
self.row_to_folder(existing.0, name.to_string(), None, existing.1, existing.2)
.await
}
}
}
/// Returns user_id for a given folder. Used by file repositories.
pub async fn get_folder_user_id(&self, folder_id: &str) -> Result<String, DomainError> {
sqlx::query_scalar::<_, String>(
"SELECT user_id FROM storage.folders WHERE id = $1::uuid",
)
.bind(folder_id)
.fetch_optional(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("user_id lookup: {e}")))?
.ok_or_else(|| DomainError::not_found("Folder", folder_id))
}
}
src/infrastructure/repositories/pg/mod.rs
+10
View File
@@ -11,6 +11,12 @@ mod settings_pg_repository;
mod transaction_utils;
mod user_pg_repository;
// ── Blob-storage repositories ──
pub mod file_blob_read_repository;
pub mod file_blob_write_repository;
pub mod folder_db_repository;
pub mod trash_db_repository;
pub use address_book_pg_repository::AddressBookPgRepository;
pub use calendar_event_pg_repository::CalendarEventPgRepository;
pub use calendar_pg_repository::CalendarPgRepository;
@@ -18,7 +24,11 @@ pub use contact_group_pg_repository::ContactGroupPgRepository;
pub use contact_persistence_dto::*;
pub use contact_pg_repository::ContactPgRepository;
pub use favorites_pg_repository::FavoritesPgRepository;
pub use file_blob_read_repository::FileBlobReadRepository;
pub use file_blob_write_repository::FileBlobWriteRepository;
pub use folder_db_repository::FolderDbRepository;
pub use recent_items_pg_repository::RecentItemsPgRepository;
pub use session_pg_repository::SessionPgRepository;
pub use settings_pg_repository::SettingsPgRepository;
pub use trash_db_repository::TrashDbRepository;
pub use user_pg_repository::UserPgRepository;
src/infrastructure/repositories/pg/trash_db_repository.rs
+191
View File
@@ -0,0 +1,191 @@
//! PostgreSQL-backed trash repository.
//!
//! Implements `TrashRepository` using soft-delete columns in `storage.files`
//! and `storage.folders`. There is no separate trash table — trashed items
//! are files/folders with `is_trashed = TRUE`.
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use sqlx::PgPool;
use std::sync::Arc;
use uuid::Uuid;
use crate::common::errors::{DomainError, Result};
use crate::domain::entities::trashed_item::{TrashedItem, TrashedItemType};
use crate::domain::repositories::trash_repository::TrashRepository;
/// Default retention period (days) used when computing deletion_date.
const _DEFAULT_RETENTION_DAYS: i64 = 30;
/// PostgreSQL-backed trash repository using soft-delete flags.
pub struct TrashDbRepository {
pool: Arc<PgPool>,
retention_days: i64,
}
impl TrashDbRepository {
pub fn new(pool: Arc<PgPool>, retention_days: u32) -> Self {
Self {
pool,
retention_days: retention_days as i64,
}
}
/// Convert a trash_items view row into a TrashedItem entity.
fn row_to_trashed_item(
&self,
id: Uuid,
name: String,
item_type: String,
user_id: String,
trashed_at: Option<DateTime<Utc>>,
) -> TrashedItem {
let trashed_at = trashed_at.unwrap_or_else(Utc::now);
let deletion_date = trashed_at + chrono::Duration::days(self.retention_days);
let item_type_enum = match item_type.as_str() {
"folder" => TrashedItemType::Folder,
_ => TrashedItemType::File,
};
let user_uuid = Uuid::parse_str(&user_id).unwrap_or_else(|_| Uuid::nil());
// In the soft-delete model, the trash entry ID is the same as the
// original item ID since there is no separate trash table.
TrashedItem::from_raw(
id, // trash entry id (same as original)
id, // original item id
user_uuid, // owner
item_type_enum,
name.clone(),
String::new(), // original_path — not stored separately in soft-delete model
trashed_at,
deletion_date,
)
}
}
#[async_trait]
impl TrashRepository for TrashDbRepository {
async fn add_to_trash(&self, _item: &TrashedItem) -> Result<()> {
// No-op: the actual flagging is done by FileWritePort::move_to_trash
// or FolderRepository::move_to_trash. This method exists for interface
// compatibility with the TrashService.
Ok(())
}
async fn get_trash_items(&self, user_id: &Uuid) -> Result<Vec<TrashedItem>> {
let rows = sqlx::query_as::<_, (Uuid, String, String, String, Option<DateTime<Utc>>)>(
r#"
SELECT id, name, item_type, user_id, trashed_at
FROM storage.trash_items
WHERE user_id = $1
ORDER BY trashed_at DESC
"#,
)
.bind(user_id.to_string())
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("TrashDb", format!("list: {e}"))
})?;
Ok(rows
.into_iter()
.map(|(id, name, item_type, uid, trashed_at)| {
self.row_to_trashed_item(id, name, item_type, uid, trashed_at)
})
.collect())
}
async fn get_trash_item(
&self,
id: &Uuid,
user_id: &Uuid,
) -> Result<Option<TrashedItem>> {
let row = sqlx::query_as::<_, (Uuid, String, String, String, Option<DateTime<Utc>>)>(
r#"
SELECT id, name, item_type, user_id, trashed_at
FROM storage.trash_items
WHERE id = $1 AND user_id = $2
"#,
)
.bind(id)
.bind(user_id.to_string())
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("TrashDb", format!("get: {e}"))
})?;
Ok(row.map(|(id, name, item_type, uid, trashed_at)| {
self.row_to_trashed_item(id, name, item_type, uid, trashed_at)
}))
}
async fn restore_from_trash(&self, _id: &Uuid, _user_id: &Uuid) -> Result<()> {
// No-op: the actual restore is done by FileWritePort::restore_from_trash
// or FolderRepository::restore_from_trash. The TrashService also removes
// the index entry — which in the soft-delete model means the flag is
// already cleared.
Ok(())
}
async fn delete_permanently(&self, _id: &Uuid, _user_id: &Uuid) -> Result<()> {
// No-op: the actual delete is done by FileWritePort::delete_file_permanently
// or FolderRepository::delete_folder_permanently.
Ok(())
}
async fn clear_trash(&self, user_id: &Uuid) -> Result<()> {
// Delete all trashed files for this user
sqlx::query(
"DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE",
)
.bind(user_id.to_string())
.execute(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("TrashDb", format!("clear files: {e}"))
})?;
// Delete all trashed folders for this user
sqlx::query(
"DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE",
)
.bind(user_id.to_string())
.execute(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("TrashDb", format!("clear folders: {e}"))
})?;
Ok(())
}
async fn get_expired_items(&self) -> Result<Vec<TrashedItem>> {
let cutoff = Utc::now() - chrono::Duration::days(self.retention_days);
let rows = sqlx::query_as::<_, (Uuid, String, String, String, Option<DateTime<Utc>>)>(
r#"
SELECT id, name, item_type, user_id, trashed_at
FROM storage.trash_items
WHERE trashed_at < $1
ORDER BY trashed_at ASC
"#,
)
.bind(cutoff)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("TrashDb", format!("expired: {e}"))
})?;
Ok(rows
.into_iter()
.map(|(id, name, item_type, uid, trashed_at)| {
self.row_to_trashed_item(id, name, item_type, uid, trashed_at)
})
.collect())
}
}
src/infrastructure/repositories/repository_errors.rs
-110
View File
@@ -1,110 +0,0 @@
//! Infrastructure-layer error types for file and folder repository operations.
//!
//! These error types are used internally by the filesystem repository implementations
//! (FileFsReadRepository, FileFsWriteRepository, FolderFsRepository, etc.) to represent
//! errors that can occur during storage operations. They are converted to `DomainError`
//! at the port boundary before crossing into the application layer.
use crate::common::errors::DomainError;
/// Error types for file repository operations.
#[derive(Debug, thiserror::Error)]
pub enum FileRepositoryError {
#[error("File not found: {0}")]
NotFound(String),
#[error("File already exists: {0}")]
AlreadyExists(String),
#[error("Invalid file path: {0}")]
InvalidPath(String),
#[error("Operation not supported: {0}")]
OperationNotSupported(String),
#[error("Storage error: {0}")]
StorageError(String),
#[error("Domain error: {0}")]
DomainError(#[from] DomainError),
#[error("Other error: {0}")]
Other(String),
}
pub type FileRepositoryResult<T> = Result<T, FileRepositoryError>;
/// Error types for folder repository operations.
#[derive(Debug, thiserror::Error)]
pub enum FolderRepositoryError {
#[error("Folder not found: {0}")]
NotFound(String),
#[error("Folder already exists: {0}")]
AlreadyExists(String),
#[error("Invalid folder path: {0}")]
InvalidPath(String),
#[error("Operation not supported: {0}")]
OperationNotSupported(String),
#[error("Storage error: {0}")]
StorageError(String),
#[error("Validation error: {0}")]
ValidationError(String),
#[error("Domain error: {0}")]
DomainError(#[from] DomainError),
#[error("Other error: {0}")]
Other(String),
}
pub type FolderRepositoryResult<T> = Result<T, FolderRepositoryError>;
// ── Conversions to DomainError ──
impl From<FileRepositoryError> for DomainError {
fn from(err: FileRepositoryError) -> Self {
match err {
FileRepositoryError::NotFound(id) => DomainError::not_found("File", id),
FileRepositoryError::AlreadyExists(path) => DomainError::already_exists("File", path),
FileRepositoryError::InvalidPath(path) => {
DomainError::validation_error(format!("Invalid path: {}", path))
}
FileRepositoryError::StorageError(msg) => {
DomainError::internal_error("File", format!("Storage error: {}", msg))
}
FileRepositoryError::Other(msg) => DomainError::internal_error("File", msg),
FileRepositoryError::OperationNotSupported(msg) => {
DomainError::operation_not_supported("File", msg)
}
FileRepositoryError::DomainError(e) => e,
}
}
}
impl From<FolderRepositoryError> for DomainError {
fn from(err: FolderRepositoryError) -> Self {
match err {
FolderRepositoryError::NotFound(id) => DomainError::not_found("Folder", id),
FolderRepositoryError::AlreadyExists(path) => {
DomainError::already_exists("Folder", path)
}
FolderRepositoryError::InvalidPath(path) => {
DomainError::validation_error(format!("Invalid path: {}", path))
}
FolderRepositoryError::StorageError(msg) => {
DomainError::internal_error("Folder", format!("Storage error: {}", msg))
}
FolderRepositoryError::ValidationError(msg) => DomainError::validation_error(msg),
FolderRepositoryError::Other(msg) => DomainError::internal_error("Folder", msg),
FolderRepositoryError::OperationNotSupported(msg) => {
DomainError::operation_not_supported("Folder", msg)
}
FolderRepositoryError::DomainError(e) => e,
}
}
}
src/infrastructure/repositories/trash_fs_repository.rs
-441
View File
@@ -1,441 +0,0 @@
use async_trait::async_trait;
use chrono::Utc;
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::fs;
use tracing::{debug, error, instrument};
use uuid::Uuid;
use crate::application::ports::outbound::IdMappingPort;
use crate::common::errors::{DomainError, ErrorKind, Result};
use crate::domain::entities::trashed_item::{TrashedItem, TrashedItemType};
use crate::domain::repositories::trash_repository::TrashRepository;
/// Structure for storing trash items in JSON format
#[derive(Debug, Serialize, Deserialize)]
struct TrashedItemEntry {
id: String,
original_id: String,
user_id: String,
item_type: String,
name: String,
original_path: String,
trashed_at: String,
deletion_date: String,
}
/// Trash repository implementation using the file system
pub struct TrashFsRepository {
trash_dir: PathBuf,
trash_index_path: PathBuf,
}
impl TrashFsRepository {
pub fn new(
storage_root: impl AsRef<Path>,
_id_mapping_service: Arc<dyn IdMappingPort>,
) -> Self {
let trash_dir = storage_root.as_ref().join(".trash");
let trash_index_path = trash_dir.join("trash_index.json");
Self {
trash_dir,
trash_index_path,
}
}
/// Ensures the trash directory exists
async fn ensure_trash_dir(&self) -> Result<()> {
debug!(
"Checking if trash directory exists: {}",
self.trash_dir.display()
);
if !self.trash_dir.exists() {
debug!(
"Trash directory does not exist, creating it: {}",
self.trash_dir.display()
);
fs::create_dir_all(&self.trash_dir).await.map_err(|e| {
error!(
"Failed to create trash directory {}: {}",
self.trash_dir.display(),
e
);
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!(
"Failed to create trash directory {}: {}",
self.trash_dir.display(),
e
),
)
})?;
debug!("Trash directory created successfully");
} else {
debug!("Trash directory already exists");
}
// Ensure the files directory exists
let files_dir = self.trash_dir.join("files");
debug!(
"Checking if trash files directory exists: {}",
files_dir.display()
);
if !files_dir.exists() {
debug!(
"Trash files directory does not exist, creating it: {}",
files_dir.display()
);
fs::create_dir_all(&files_dir).await.map_err(|e| {
error!(
"Failed to create trash files directory {}: {}",
files_dir.display(),
e
);
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!(
"Failed to create trash files directory {}: {}",
files_dir.display(),
e
),
)
})?;
debug!("Trash files directory created successfully");
} else {
debug!("Trash files directory already exists");
}
// Also ensure the folders directory exists
let folders_dir = self.trash_dir.join("folders");
debug!(
"Checking if trash folders directory exists: {}",
folders_dir.display()
);
if !folders_dir.exists() {
debug!(
"Trash folders directory does not exist, creating it: {}",
folders_dir.display()
);
fs::create_dir_all(&folders_dir).await.map_err(|e| {
error!(
"Failed to create trash folders directory {}: {}",
folders_dir.display(),
e
);
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!(
"Failed to create trash folders directory {}: {}",
folders_dir.display(),
e
),
)
})?;
debug!("Trash folders directory created successfully");
} else {
debug!("Trash folders directory already exists");
}
Ok(())
}
/// Gets all entries from the trash index
async fn get_trash_entries(&self) -> Result<Vec<TrashedItemEntry>> {
self.ensure_trash_dir().await?;
if !self.trash_index_path.exists() {
return Ok(Vec::new());
}
let content = fs::read_to_string(&self.trash_index_path)
.await
.map_err(|e| {
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!("Failed to read trash index: {}", e),
)
})?;
if content.trim().is_empty() {
return Ok(Vec::new());
}
let entries: Vec<TrashedItemEntry> = serde_json::from_str(&content).map_err(|e| {
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!("Failed to parse trash index: {}", e),
)
})?;
Ok(entries)
}
/// Saves all entries to the trash index
async fn save_trash_entries(&self, entries: Vec<TrashedItemEntry>) -> Result<()> {
self.ensure_trash_dir().await?;
let json = serde_json::to_string_pretty(&entries).map_err(|e| {
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!("Failed to serialize trash index: {}", e),
)
})?;
fs::write(&self.trash_index_path, json).await.map_err(|e| {
DomainError::new(
ErrorKind::InternalError,
"Trash",
format!("Failed to write trash index: {}", e),
)
})?;
Ok(())
}
/// Converts a JSON entry to a TrashedItem entity
fn entry_to_trashed_item(&self, entry: TrashedItemEntry) -> Result<TrashedItem> {
let item_type = match entry.item_type.as_str() {
"file" => TrashedItemType::File,
"folder" => TrashedItemType::Folder,
_ => {
return Err(DomainError::new(
ErrorKind::InvalidInput,
"Trash",
format!("Invalid trashed item type: {}", entry.item_type),
));
}
};
let original_id = Uuid::parse_str(&entry.original_id).map_err(|e| {
DomainError::validation_error(format!("Invalid original ID format: {}", e))
})?;
let id = Uuid::parse_str(&entry.id)
.map_err(|e| DomainError::validation_error(format!("Invalid ID format: {}", e)))?;
let user_id = Uuid::parse_str(&entry.user_id)
.map_err(|e| DomainError::validation_error(format!("Invalid user ID format: {}", e)))?;
let trashed_at = chrono::DateTime::parse_from_rfc3339(&entry.trashed_at)
.map_err(|e| DomainError::validation_error(format!("Invalid trashed_at date: {}", e)))?
.with_timezone(&Utc);
let deletion_date = chrono::DateTime::parse_from_rfc3339(&entry.deletion_date)
.map_err(|e| DomainError::validation_error(format!("Invalid deletion_date: {}", e)))?
.with_timezone(&Utc);
Ok(TrashedItem::from_raw(
id,
original_id,
user_id,
item_type,
entry.name,
entry.original_path,
trashed_at,
deletion_date,
))
}
/// Converts a TrashedItem entity to a JSON entry
fn trashed_item_to_entry(&self, item: &TrashedItem) -> TrashedItemEntry {
TrashedItemEntry {
id: item.id().to_string(),
original_id: item.original_id().to_string(),
user_id: item.user_id().to_string(),
item_type: match item.item_type() {
TrashedItemType::File => "file".to_string(),
TrashedItemType::Folder => "folder".to_string(),
},
name: item.name().to_string(),
original_path: item.original_path().to_string(),
trashed_at: item.trashed_at().to_rfc3339(),
deletion_date: item.deletion_date().to_rfc3339(),
}
}
}
#[async_trait]
impl TrashRepository for TrashFsRepository {
#[instrument(skip(self))]
async fn add_to_trash(&self, item: &TrashedItem) -> Result<()> {
debug!(
"Adding item to trash: id={}, user={}",
item.id(),
item.user_id()
);
// Ensure the trash directory exists for this user
let user_trash_dir = self
.trash_dir
.join("files")
.join(item.user_id().to_string());
debug!("User trash directory path: {}", user_trash_dir.display());
// Create the user-specific trash directory
debug!(
"Creating user trash directory: {}",
user_trash_dir.display()
);
match fs::create_dir_all(&user_trash_dir).await {
Ok(_) => debug!("User trash directory created successfully"),
Err(e) => {
error!(
"Failed to create user trash directory {}: {}",
user_trash_dir.display(),
e
);
return Err(DomainError::new(
ErrorKind::InternalError,
"Trash",
format!("Failed to create user trash directory: {}", e),
));
}
}
// Log the current trash entries before adding the new one
let mut entries = self.get_trash_entries().await?;
debug!("Current trash entries count: {}", entries.len());
// Create the entry for the trash index
let entry = self.trashed_item_to_entry(item);
debug!(
"Created trash entry: id={}, original_id={}, name={}",
entry.id, entry.original_id, entry.name
);
// Add the entry to the index and save
entries.push(entry);
debug!("Saving updated trash index with {} entries", entries.len());
self.save_trash_entries(entries).await?;
debug!("Trash index updated successfully");
Ok(())
}
#[instrument(skip(self))]
async fn get_trash_items(&self, user_id: &Uuid) -> Result<Vec<TrashedItem>> {
debug!("Getting trash items for user: {}", user_id);
let entries = self.get_trash_entries().await?;
let user_id_str = user_id.to_string();
let user_entries = entries
.into_iter()
.filter(|entry| entry.user_id == user_id_str)
.collect::<Vec<_>>();
let mut items = Vec::new();
for entry in user_entries {
match self.entry_to_trashed_item(entry) {
Ok(item) => items.push(item),
Err(e) => error!("Error converting trash entry to item: {}", e),
}
}
Ok(items)
}
#[instrument(skip(self))]
async fn get_trash_item(&self, id: &Uuid, user_id: &Uuid) -> Result<Option<TrashedItem>> {
debug!("Looking for item in trash: id={}, user={}", id, user_id);
let entries = self.get_trash_entries().await?;
let id_str = id.to_string();
let user_id_str = user_id.to_string();
let item_entry = entries
.into_iter()
.find(|entry| entry.id == id_str && entry.user_id == user_id_str);
match item_entry {
Some(entry) => {
let item = self.entry_to_trashed_item(entry)?;
Ok(Some(item))
}
None => Ok(None),
}
}
#[instrument(skip(self))]
async fn restore_from_trash(&self, id: &Uuid, user_id: &Uuid) -> Result<()> {
debug!("Restoring item from trash: id={}, user={}", id, user_id);
let mut entries = self.get_trash_entries().await?;
let id_str = id.to_string();
let user_id_str = user_id.to_string();
let index = entries
.iter()
.position(|entry| entry.id == id_str && entry.user_id == user_id_str);
if let Some(index) = index {
entries.remove(index);
self.save_trash_entries(entries).await?;
Ok(())
} else {
Err(DomainError::not_found("TrashedItem", id.to_string()))
}
}
#[instrument(skip(self))]
async fn delete_permanently(&self, id: &Uuid, user_id: &Uuid) -> Result<()> {
debug!(
"Permanently deleting item from trash: id={}, user={}",
id, user_id
);
// Simply remove the entry from the index
// Physical files will be deleted through the corresponding repository
self.restore_from_trash(id, user_id).await
}
#[instrument(skip(self))]
async fn clear_trash(&self, user_id: &Uuid) -> Result<()> {
debug!("Clearing trash for user: {}", user_id);
let mut entries = self.get_trash_entries().await?;
let user_id_str = user_id.to_string();
entries.retain(|entry| entry.user_id != user_id_str);
self.save_trash_entries(entries).await?;
Ok(())
}
#[instrument(skip(self))]
async fn get_expired_items(&self) -> Result<Vec<TrashedItem>> {
debug!("Looking for expired trash items");
let entries = self.get_trash_entries().await?;
let now = Utc::now();
let mut expired_items = Vec::new();
for entry in entries {
match chrono::DateTime::parse_from_rfc3339(&entry.deletion_date) {
Ok(date) => {
let utc_date = date.with_timezone(&Utc);
if utc_date <= now {
match self.entry_to_trashed_item(entry) {
Ok(item) => expired_items.push(item),
Err(e) => error!("Error converting expired trash entry: {}", e),
}
}
}
Err(e) => error!("Invalid date format in trash entry: {}", e),
}
}
Ok(expired_items)
}
}
src/infrastructure/services/buffer_pool.rs
-504
View File
@@ -1,504 +0,0 @@
use std::cmp::min;
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Mutex, Semaphore};
use tracing::debug;
/// Default buffer size in the pool
pub const DEFAULT_BUFFER_SIZE: usize = 64 * 1024; // 64KB
/// Default maximum number of buffers in the pool
pub const DEFAULT_MAX_BUFFERS: usize = 100;
/// Default time-to-live for an inactive buffer (in seconds)
pub const DEFAULT_BUFFER_TTL: u64 = 60;
/// Buffer pooling to optimize read/write operations
pub struct BufferPool {
/// Pool of available buffers
pool: Mutex<VecDeque<PooledBuffer>>,
/// Semaphore to limit the maximum number of buffers
limit: Semaphore,
/// Size of buffers in the pool
buffer_size: usize,
/// Pool statistics
stats: Mutex<BufferPoolStats>,
/// Time-to-live for an inactive buffer
buffer_ttl: Duration,
}
/// Structure for tracking pool statistics
#[derive(Debug, Clone, Default)]
pub struct BufferPoolStats {
/// Total number of get operations
pub gets: usize,
/// Number of pool hits (successful reuse)
pub hits: usize,
/// Number of misses (new buffer creation)
pub misses: usize,
/// Number of returns to the pool
pub returns: usize,
/// Number of TTL evictions
pub evictions: usize,
/// Maximum number of buffers reached
pub max_buffers_reached: usize,
/// Semaphore waits
pub waits: usize,
}
/// Pool buffer with management metadata
struct PooledBuffer {
/// Actual byte buffer
buffer: Vec<u8>,
/// Timestamp of when it was added/returned to the pool
last_used: Instant,
}
/// Borrowed buffer from the pool with automatic cleanup
#[derive(Clone)]
pub struct BorrowedBuffer {
/// Current buffer
buffer: Vec<u8>,
/// Actual used size of the buffer
used_size: usize,
/// Reference to the pool for returning
pool: Arc<BufferPool>,
/// Whether the buffer should be returned to the pool or not
return_to_pool: bool,
}
impl BufferPool {
/// Creates a new buffer pool
pub fn new(buffer_size: usize, max_buffers: usize, buffer_ttl_secs: u64) -> Arc<Self> {
Arc::new(Self {
pool: Mutex::new(VecDeque::with_capacity(max_buffers)),
limit: Semaphore::new(max_buffers),
buffer_size,
stats: Mutex::new(BufferPoolStats::default()),
buffer_ttl: Duration::from_secs(buffer_ttl_secs),
})
}
/// Creates a pool with default configuration
pub fn default() -> Arc<Self> {
Self::new(DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFERS, DEFAULT_BUFFER_TTL)
}
/// Gets a buffer from the pool or creates a new one if needed.
/// This version takes an Arc<Self> to ensure the BorrowedBuffer keeps a proper
/// reference to the shared pool (not a clone).
#[allow(unused_variables)]
pub async fn get_buffer(self: &Arc<Self>) -> BorrowedBuffer {
// Increment get counter
{
let mut stats = self.stats.lock().await;
stats.gets += 1;
}
// Concurrency control
// Acquire a semaphore permit. If none available, wait.
// We forget() the permit so it doesn't auto-release on drop.
// Instead, the permit is manually released in return_buffer/Drop via add_permits(1).
match self.limit.try_acquire() {
Ok(permit) => permit.forget(),
Err(_) => {
// No permits available, waiting
{
let mut stats = self.stats.lock().await;
stats.waits += 1;
stats.max_buffers_reached += 1;
}
debug!("Buffer pool: waiting for available buffer");
let permit = self
.limit
.acquire()
.await
.expect("Semaphore should not be closed");
debug!("Buffer pool: acquired buffer after waiting");
permit.forget();
}
};
// Try to get an existing buffer from the pool
let mut pool_locked = self.pool.lock().await;
let pool_arc = Arc::clone(self);
if let Some(mut pooled_buffer) = pool_locked.pop_front() {
// Check if the buffer has expired
if pooled_buffer.last_used.elapsed() > self.buffer_ttl {
// Expired buffer, discard and create a new one
let mut stats = self.stats.lock().await;
stats.evictions += 1;
stats.misses += 1;
drop(stats);
debug!("Buffer pool: evicted expired buffer");
// Create new buffer (reusing the permit)
drop(pool_locked); // Release the lock before returning
BorrowedBuffer {
buffer: vec![0; self.buffer_size],
used_size: 0,
pool: pool_arc,
return_to_pool: true,
}
} else {
// Valid buffer, reuse it
let mut stats = self.stats.lock().await;
stats.hits += 1;
drop(stats);
// Release the lock before returning
drop(pool_locked);
// Clear buffer for security
pooled_buffer.buffer.fill(0);
BorrowedBuffer {
buffer: pooled_buffer.buffer,
used_size: 0,
pool: pool_arc,
return_to_pool: true,
}
}
} else {
// No buffers available, create a new one
let mut stats = self.stats.lock().await;
stats.misses += 1;
drop(stats);
// Release the lock before returning
drop(pool_locked);
debug!("Buffer pool: creating new buffer");
BorrowedBuffer {
buffer: vec![0; self.buffer_size],
used_size: 0,
pool: pool_arc,
return_to_pool: true,
}
}
}
/// Returns a buffer to the pool
async fn return_buffer(&self, mut buffer: Vec<u8>) {
// If the buffer is the wrong size, discard it
if buffer.capacity() != self.buffer_size {
debug!(
"Buffer pool: discarding buffer of wrong size: {} (expected {})",
buffer.capacity(),
self.buffer_size
);
// Release the semaphore permit even if we discard the buffer
self.limit.add_permits(1);
return;
}
// Resize to ensure correct capacity
buffer.resize(self.buffer_size, 0);
// Add to the pool
let mut pool_locked = self.pool.lock().await;
pool_locked.push_back(PooledBuffer {
buffer,
last_used: Instant::now(),
});
// Update statistics
let mut stats = self.stats.lock().await;
stats.returns += 1;
// Release the semaphore permit so another caller can acquire a buffer
drop(pool_locked);
drop(stats);
self.limit.add_permits(1);
}
/// Cleans expired buffers from the pool
pub async fn clean_expired_buffers(&self) {
let _now = Instant::now();
let mut pool_locked = self.pool.lock().await;
// Count expired
let count_before = pool_locked.len();
// Filter keeping only non-expired
pool_locked.retain(|buffer| buffer.last_used.elapsed() <= self.buffer_ttl);
// Count how many were removed
let removed = count_before - pool_locked.len();
if removed > 0 {
// Update statistics
let mut stats = self.stats.lock().await;
stats.evictions += removed;
debug!("Buffer pool: cleaned {} expired buffers", removed);
}
}
/// Gets current pool statistics
pub async fn get_stats(&self) -> BufferPoolStats {
self.stats.lock().await.clone()
}
/// Starts the periodic cleanup task
pub fn start_cleaner(pool: Arc<Self>) {
tokio::spawn(async move {
let interval = Duration::from_secs(30); // Clean every 30 seconds
loop {
tokio::time::sleep(interval).await;
pool.clean_expired_buffers().await;
// Log statistics periodically
let stats = pool.get_stats().await;
debug!(
"Buffer pool stats: gets={}, hits={}, misses={}, hit_ratio={:.2}%, returns={}, \
evictions={}, max_reached={}, waits={}",
stats.gets,
stats.hits,
stats.misses,
if stats.gets > 0 {
(stats.hits as f64 * 100.0) / stats.gets as f64
} else {
0.0
},
stats.returns,
stats.evictions,
stats.max_buffers_reached,
stats.waits
);
}
});
}
}
impl Clone for BufferPool {
fn clone(&self) -> Self {
Self {
pool: Mutex::new(VecDeque::new()),
limit: Semaphore::new(self.limit.available_permits()),
buffer_size: self.buffer_size,
stats: Mutex::new(BufferPoolStats::default()),
buffer_ttl: self.buffer_ttl,
}
}
}
impl BorrowedBuffer {
/// Accesses the internal buffer
pub fn as_mut_slice(&mut self) -> &mut [u8] {
&mut self.buffer
}
/// Gets a reference to the used data
pub fn as_slice(&self) -> &[u8] {
&self.buffer[..self.used_size]
}
/// Sets how many bytes were actually used
pub fn set_used(&mut self, size: usize) {
self.used_size = min(size, self.buffer.len());
}
/// Converts into a Vec<u8> that includes only the used data
pub fn into_vec(mut self) -> Vec<u8> {
// Mark to not return to pool
self.return_to_pool = false;
// Create a new vector with only the used data
self.buffer[..self.used_size].to_vec()
}
/// Copies data to this buffer and updates the used size
pub fn copy_from_slice(&mut self, data: &[u8]) -> usize {
let copy_size = min(data.len(), self.buffer.len());
self.buffer[..copy_size].copy_from_slice(&data[..copy_size]);
self.used_size = copy_size;
copy_size
}
/// Prevents the buffer from being returned to the pool on destruction
pub fn do_not_return(mut self) -> Self {
self.return_to_pool = false;
self
}
/// Gets the total buffer size
pub fn capacity(&self) -> usize {
self.buffer.len()
}
/// Gets the used buffer size
pub fn used_size(&self) -> usize {
self.used_size
}
}
// When a BorrowedBuffer is dropped, it is returned to the pool
impl Drop for BorrowedBuffer {
fn drop(&mut self) {
if self.return_to_pool {
// Take ownership of the buffer and create a clone of the pool
let buffer = std::mem::take(&mut self.buffer);
let pool = self.pool.clone();
// Spawn the return so that drop doesn't block
// return_buffer will release the semaphore permit
tokio::spawn(async move {
pool.return_buffer(buffer).await;
});
} else {
// Buffer not returned to pool, but we still need to release the semaphore permit
self.pool.limit.add_permits(1);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_buffer_pooling() {
// Create small pool for testing
let pool = BufferPool::new(1024, 5, 60);
// Get a buffer
let mut buffer1 = pool.get_buffer().await;
buffer1.copy_from_slice(b"test data");
assert_eq!(buffer1.as_slice(), b"test data");
// Get another buffer
let buffer2 = pool.get_buffer().await;
// Verify stats
let stats = pool.get_stats().await;
assert_eq!(stats.gets, 2);
assert_eq!(stats.hits, 0); // no hits yet
assert_eq!(stats.misses, 2); // all are misses
// Return buffer1 to pool (implicitly via drop)
drop(buffer1);
// Allow the async return to occur
tokio::time::sleep(Duration::from_millis(10)).await;
// Get another buffer (should reuse the returned one)
let buffer3 = pool.get_buffer().await;
// Verify updated stats
let stats = pool.get_stats().await;
assert_eq!(stats.gets, 3);
assert_eq!(stats.hits, 1); // now there should be a hit
assert_eq!(stats.returns, 1); // one buffer returned
// Cleanup
drop(buffer2);
drop(buffer3);
}
#[tokio::test]
async fn test_buffer_operations() {
let pool = BufferPool::new(1024, 10, 60);
// Get buffer
let mut buffer = pool.get_buffer().await;
// Write data
buffer.copy_from_slice(b"Hello, world!");
assert_eq!(buffer.used_size(), 13);
assert_eq!(buffer.as_slice(), b"Hello, world!");
// Convert to vec and verify
let vec = buffer.into_vec(); // This prevents returning to pool
assert_eq!(vec, b"Hello, world!");
// Verify that returns are not incremented (buffer not returned)
tokio::time::sleep(Duration::from_millis(10)).await;
let stats = pool.get_stats().await;
assert_eq!(stats.returns, 0);
}
#[tokio::test]
async fn test_pool_limit() {
// Pool with only 3 buffers
let pool = BufferPool::new(1024, 3, 60);
// Get 3 buffers (reaches the limit)
let buffer1 = pool.get_buffer().await;
let buffer2 = pool.get_buffer().await;
let buffer3 = pool.get_buffer().await;
// Verify stats
let stats = pool.get_stats().await;
assert_eq!(stats.gets, 3);
assert_eq!(stats.waits, 0); // no waits yet
// Try to get a 4th buffer in a separate task (should wait)
let pool_clone = pool.clone();
let handle = tokio::spawn(async move {
let _buffer4 = pool_clone.get_buffer().await;
true
});
// Give time for the task to try to take the buffer
tokio::time::sleep(Duration::from_millis(50)).await;
// Verify there is a wait
let stats = pool.get_stats().await;
assert_eq!(stats.waits, 1);
// Release a buffer
drop(buffer1);
// Give time for the async return and for the waiting task to get its buffer
tokio::time::sleep(Duration::from_millis(50)).await;
// Verify the task was able to continue
assert!(handle.await.unwrap());
// Cleanup
drop(buffer2);
drop(buffer3);
}
#[tokio::test]
async fn test_ttl_expiration() {
// Pool with very short TTL for testing
let pool = BufferPool::new(1024, 5, 1); // 1 second TTL
// Get and return a buffer
let buffer = pool.get_buffer().await;
drop(buffer);
// Allow the async return to occur
tokio::time::sleep(Duration::from_millis(50)).await;
// Verify there is a buffer in the pool
let stats = pool.get_stats().await;
assert_eq!(stats.returns, 1);
// Wait for the TTL to expire
tokio::time::sleep(Duration::from_secs(2)).await;
// Clean expired
pool.clean_expired_buffers().await;
// Get another buffer (should be a miss since the previous one expired)
let _buffer2 = pool.get_buffer().await;
// Verify stats
let stats = pool.get_stats().await;
assert_eq!(stats.evictions, 1); // one expired buffer
assert_eq!(stats.hits, 0); // no hits (the buffer expired)
assert_eq!(stats.misses, 2); // two misses (1st and 3rd get)
}
}
src/infrastructure/services/compression_service.rs
+2 -125
View File
@@ -6,14 +6,12 @@ use flate2::read::GzEncoder as GzEncoderRead;
use futures::{Stream, StreamExt};
use std::io;
use std::io::Read;
use std::sync::Arc;
use tracing::error;
use crate::application::ports::compression_ports::{
CompressionLevel as PortCompressionLevel, CompressionPort,
};
use crate::domain::errors::DomainError;
use crate::infrastructure::services::buffer_pool::BufferPool;
/// Compression level for files
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -73,22 +71,12 @@ pub trait CompressionService: Send + Sync {
}
/// Gzip compression service implementation
pub struct GzipCompressionService {
/// Buffer pool for memory optimization
buffer_pool: Option<Arc<BufferPool>>,
}
pub struct GzipCompressionService;
impl GzipCompressionService {
/// Creates a new service instance
pub fn new() -> Self {
Self { buffer_pool: None }
}
/// Creates a new service instance with buffer pool
pub fn new_with_buffer_pool(buffer_pool: Arc<BufferPool>) -> Self {
Self {
buffer_pool: Some(buffer_pool),
}
Self
}
}
@@ -96,63 +84,6 @@ impl GzipCompressionService {
impl CompressionService for GzipCompressionService {
/// Compresses data in memory using Gzip
async fn compress_data(&self, data: &[u8], level: CompressionLevel) -> io::Result<Vec<u8>> {
// If we have a buffer pool, use a borrowed buffer for compression
if let Some(pool) = &self.buffer_pool {
// Estimate the compression size (approximately 80% of original for typical cases)
let estimated_size = (data.len() as f64 * 0.8) as usize;
// Get a buffer from the pool
let buffer = pool.get_buffer().await;
// Check if the buffer is large enough
if buffer.capacity() >= estimated_size {
// Run compression in a worker thread using the buffer
let buffer_ptr = Arc::new(tokio::sync::Mutex::new(buffer));
let buffer_clone = buffer_ptr.clone();
// Compress data
// Clone the data to avoid lifetime issues
let data_owned = data.to_vec();
let result = tokio::task::spawn_blocking(move || {
let mut encoder = GzEncoderRead::new(&data_owned[..], level.into());
// Try to lock the mutex (should not fail since we are in a separate thread)
let mut buffer_guard = match futures::executor::block_on(buffer_clone.lock()) {
buffer => buffer,
};
// Read directly into the buffer
let read_bytes = encoder.read(buffer_guard.as_mut_slice())?;
buffer_guard.set_used(read_bytes);
Ok(()) as io::Result<()>
})
.await;
// Verify result
match result {
Ok(Ok(())) => {
// Get the buffer and convert it to Vec<u8>
let buffer = buffer_ptr.lock().await;
let cloned_buffer = buffer.clone();
drop(buffer); // Release the mutex first
return Ok(cloned_buffer.into_vec());
}
Ok(Err(e)) => {
error!("Compression error with buffer pool: {}", e);
// Fall back to standard implementation
}
Err(e) => {
error!("Compression task error with buffer pool: {}", e);
// Fall back to standard implementation
}
}
}
}
// Standard implementation if there is no buffer pool or the buffer is insufficient
// Clone the data to avoid lifetime issues
let data_owned = data.to_vec();
tokio::task::spawn_blocking(move || {
@@ -170,61 +101,7 @@ impl CompressionService for GzipCompressionService {
/// Decompresses data in memory
async fn decompress_data(&self, compressed_data: &[u8]) -> io::Result<Vec<u8>> {
// If we have a buffer pool, use a borrowed buffer for decompression
if let Some(pool) = &self.buffer_pool {
// Estimate the decompression size (approximately 5x of compressed for typical cases)
let estimated_size = compressed_data.len() * 5;
// Get a buffer from the pool
let buffer = pool.get_buffer().await;
// Check if the buffer is large enough
if buffer.capacity() >= estimated_size {
// Clone compressed data to move to the worker
let data = compressed_data.to_vec();
let buffer_ptr = Arc::new(tokio::sync::Mutex::new(buffer));
let buffer_clone = buffer_ptr.clone();
// Decompress data
let result = tokio::task::spawn_blocking(move || {
let mut decoder = GzDecoder::new(&data[..]);
// Try to lock the mutex
let mut buffer_guard = match futures::executor::block_on(buffer_clone.lock()) {
buffer => buffer,
};
// Read directly into the buffer
let read_bytes = decoder.read(buffer_guard.as_mut_slice())?;
buffer_guard.set_used(read_bytes);
Ok(()) as io::Result<()>
})
.await;
// Verify result
match result {
Ok(Ok(())) => {
// Get the buffer and convert it to Vec<u8>
let buffer = buffer_ptr.lock().await;
let cloned_buffer = buffer.clone();
drop(buffer); // Release the mutex first
return Ok(cloned_buffer.into_vec());
}
Ok(Err(e)) => {
error!("Decompression error with buffer pool: {}", e);
// Fall back to standard implementation
}
Err(e) => {
error!("Decompression task error with buffer pool: {}", e);
// Fall back to standard implementation
}
}
}
}
// Standard implementation if there is no buffer pool or the buffer is insufficient
let data = compressed_data.to_vec(); // Clone to move to the worker
tokio::task::spawn_blocking(move || {
let mut decoder = GzDecoder::new(&data[..]);
let mut decompressed = Vec::new();
src/infrastructure/services/file_metadata_cache.rs
-749
View File
@@ -1,749 +0,0 @@
use futures::future::BoxFuture;
use mime_guess::from_path;
use std::collections::{HashMap, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant, UNIX_EPOCH};
use tokio::fs;
use tokio::sync::RwLock;
use tokio::time;
use tracing::debug;
use crate::domain::entities::file::File;
use crate::common::config::AppConfig;
/// Cache entry types
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CacheEntryType {
/// File
File,
/// Directory
Directory,
/// Unknown type
Unknown,
}
/// Cache statistics for monitoring
#[derive(Debug, Clone, Default)]
pub struct CacheStats {
/// Number of cache hits
pub hits: usize,
/// Number of cache misses
pub misses: usize,
/// Number of manual invalidations
pub invalidations: usize,
/// Number of automatic expirations
pub expirations: usize,
/// Number of cache inserts
pub inserts: usize,
/// Total time saved (milliseconds)
pub time_saved_ms: u64,
}
/// Complete cached file metadata
#[derive(Debug, Clone)]
pub struct FileMetadata {
/// Absolute file path
pub path: PathBuf,
/// Whether the file physically exists
pub exists: bool,
/// Entry type (file, directory)
pub entry_type: CacheEntryType,
/// Size in bytes (for files)
pub size: Option<u64>,
/// MIME type (for files)
pub mime_type: Option<String>,
/// Creation timestamp (UNIX epoch seconds)
pub created_at: Option<u64>,
/// Modification timestamp (UNIX epoch seconds)
pub modified_at: Option<u64>,
/// Previous access (used for LRU)
pub last_access: Instant,
/// Cache expiration time
pub expires_at: Instant,
/// Number of accesses to this entry
pub access_count: usize,
}
impl FileMetadata {
/// Creates a new metadata entry
pub fn new(
path: PathBuf,
exists: bool,
entry_type: CacheEntryType,
size: Option<u64>,
mime_type: Option<String>,
created_at: Option<u64>,
modified_at: Option<u64>,
ttl: Duration,
) -> Self {
let now = Instant::now();
Self {
path,
exists,
entry_type,
size,
mime_type,
created_at,
modified_at,
last_access: now,
expires_at: now + ttl,
access_count: 1,
}
}
/// Updates the last access time
pub fn touch(&mut self) {
self.last_access = Instant::now();
self.access_count += 1;
}
/// Checks if the entry has expired
pub fn is_expired(&self) -> bool {
Instant::now() > self.expires_at
}
/// Updates the expiration time with a new TTL
pub fn update_expiry(&mut self, ttl: Duration) {
self.expires_at = Instant::now() + ttl;
}
}
/// Advanced file metadata cache
pub struct FileMetadataCache {
/// Main metadata cache
metadata_cache: RwLock<HashMap<PathBuf, FileMetadata>>,
/// LRU queue for cache management
lru_queue: RwLock<VecDeque<PathBuf>>,
/// Cache usage statistics
stats: RwLock<CacheStats>,
/// Global application configuration
config: AppConfig,
/// Adaptive TTL for popular entries
ttl_multiplier: f64,
/// Popularity threshold for extended TTL
popularity_threshold: usize,
/// Maximum cache size
max_entries: usize,
}
impl FileMetadataCache {
/// Creates a new metadata cache instance
pub fn new(config: AppConfig, max_entries: usize) -> Self {
Self {
metadata_cache: RwLock::new(HashMap::with_capacity(max_entries)),
lru_queue: RwLock::new(VecDeque::with_capacity(max_entries)),
stats: RwLock::new(CacheStats::default()),
config,
ttl_multiplier: 5.0, // Popular entries have 5x TTL
popularity_threshold: 10, // After 10 accesses it's considered popular
max_entries,
}
}
/// Creates a FileMetadata object from a File object
pub fn create_metadata_from_file(file: &File, abs_path: PathBuf) -> FileMetadata {
let entry_type = CacheEntryType::File;
let size = Some(file.size());
let mime_type = Some(file.mime_type().to_string());
let created_at = Some(file.created_at());
let modified_at = Some(file.modified_at());
// Use a standard TTL
let ttl = Duration::from_secs(60); // 1 minute
FileMetadata::new(
abs_path,
true,
entry_type,
size,
mime_type,
created_at,
modified_at,
ttl,
)
}
/// Creates a default instance
pub fn default() -> Self {
Self::new(AppConfig::default(), 10_000)
}
/// Creates a cache instance with default configuration
pub fn default_with_config(config: AppConfig) -> Self {
Self::new(config, 50_000) // Larger cache for production system
}
/// Gets file metadata if cached
pub async fn get_metadata(&self, path: &Path) -> Option<FileMetadata> {
let start_time = Instant::now();
let mut cache = self.metadata_cache.write().await;
if let Some(metadata) = cache.get_mut(path) {
// Check if expired
if metadata.is_expired() {
// Remove from cache if expired
cache.remove(path);
// Update statistics
let mut stats = self.stats.write().await;
stats.misses += 1;
stats.expirations += 1;
debug!("Cache entry expired for: {}", path.display());
return None;
}
// Update access time
metadata.touch();
// For popular entries, extend TTL
if metadata.access_count >= self.popularity_threshold {
let new_ttl = match metadata.entry_type {
CacheEntryType::File => Duration::from_millis(
(self.config.timeouts.file_operation_ms as f64 * self.ttl_multiplier)
as u64,
),
CacheEntryType::Directory => Duration::from_millis(
(self.config.timeouts.dir_operation_ms as f64 * self.ttl_multiplier) as u64,
),
_ => Duration::from_secs(60), // 1 minute by default
};
metadata.update_expiry(new_ttl);
debug!("Extended TTL for popular entry: {}", path.display());
}
// Calculate approximate time saved
let elapsed = start_time.elapsed().as_millis() as u64;
let estimated_io_time: u64 = 10; // We assume 10ms minimum for IO operation
let time_saved = estimated_io_time.saturating_sub(elapsed);
// Update statistics
let mut stats = self.stats.write().await;
stats.hits += 1;
stats.time_saved_ms += time_saved;
debug!("Cache hit for: {}", path.display());
// Also keep the LRU queue updated
self.update_lru(path.to_path_buf()).await;
// Clone to return
return Some(metadata.clone());
}
// Not found in cache
let mut stats = self.stats.write().await;
stats.misses += 1;
debug!("Cache miss for: {}", path.display());
None
}
/// Updates the LRU queue
async fn update_lru(&self, path: PathBuf) {
let mut lru = self.lru_queue.write().await;
// Remove if already exists
if let Some(pos) = lru.iter().position(|p| p == &path) {
lru.remove(pos);
}
// Add to the end (most recent)
lru.push_back(path);
}
/// Checks if a file exists
pub async fn exists(&self, path: &Path) -> Option<bool> {
if let Some(metadata) = self.get_metadata(path).await {
return Some(metadata.exists);
}
None
}
/// Checks if a path is a directory
pub async fn is_dir(&self, path: &Path) -> Option<bool> {
if let Some(metadata) = self.get_metadata(path).await {
return Some(metadata.entry_type == CacheEntryType::Directory);
}
None
}
/// Checks if a path is a file
pub async fn is_file(&self, path: &Path) -> Option<bool> {
if let Some(metadata) = self.get_metadata(path).await {
return Some(metadata.entry_type == CacheEntryType::File);
}
None
}
/// Gets the size of a file
pub async fn get_size(&self, path: &Path) -> Option<u64> {
if let Some(metadata) = self.get_metadata(path).await {
return metadata.size;
}
None
}
/// Gets the MIME type of a file
pub async fn get_mime_type(&self, path: &Path) -> Option<String> {
if let Some(metadata) = self.get_metadata(path).await {
return metadata.mime_type;
}
None
}
/// Refreshes metadata for a path
pub async fn refresh_metadata(&self, path: &Path) -> Result<FileMetadata, std::io::Error> {
// Perform actual filesystem read
let metadata = fs::metadata(path).await?;
// Determine entry type
let entry_type = if metadata.is_dir() {
CacheEntryType::Directory
} else if metadata.is_file() {
CacheEntryType::File
} else {
CacheEntryType::Unknown
};
// Get size for files
let size = if metadata.is_file() {
Some(metadata.len())
} else {
None
};
// Get MIME type for files
let mime_type = if metadata.is_file() {
Some(from_path(path).first_or_octet_stream().to_string())
} else {
None
};
// Get timestamps
let created_at = metadata
.created()
.map(|time| {
time.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.ok();
let modified_at = metadata
.modified()
.map(|time| {
time.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
})
.ok();
// Determine appropriate TTL
let ttl = if metadata.is_dir() {
Duration::from_millis(self.config.timeouts.dir_operation_ms)
} else {
Duration::from_millis(self.config.timeouts.file_operation_ms)
};
// Create metadata entry
let file_metadata = FileMetadata::new(
path.to_path_buf(),
true,
entry_type,
size,
mime_type,
created_at,
modified_at,
ttl,
);
// Update cache
self.update_cache(file_metadata.clone()).await;
Ok(file_metadata)
}
/// Updates the cache with new metadata
pub async fn update_cache(&self, metadata: FileMetadata) {
// Avoid full cache before inserting
self.ensure_capacity().await;
let path = metadata.path.clone();
// Insert into cache
{
let mut cache = self.metadata_cache.write().await;
cache.insert(path.clone(), metadata);
// Update statistics
let mut stats = self.stats.write().await;
stats.inserts += 1;
}
// Update the LRU queue
self.update_lru(path).await;
}
/// Ensures there is space in the cache
async fn ensure_capacity(&self) {
let cache_size = {
let cache = self.metadata_cache.read().await;
cache.len()
};
if cache_size >= self.max_entries {
self.evict_lru_entries(cache_size / 10).await; // Free up 10%
}
}
/// Removes least recently used entries
async fn evict_lru_entries(&self, count: usize) {
let mut paths_to_remove = Vec::with_capacity(count);
// Get entries to remove from the LRU queue
{
let mut lru = self.lru_queue.write().await;
for _ in 0..count {
if let Some(path) = lru.pop_front() {
paths_to_remove.push(path);
} else {
break;
}
}
}
// Remove from the main cache
{
let mut cache = self.metadata_cache.write().await;
for path in paths_to_remove {
cache.remove(&path);
}
}
debug!("Evicted {} LRU entries from cache", count);
}
/// Invalidate a specific cache entry
pub async fn invalidate(&self, path: &Path) {
// Remove from the main cache
{
let mut cache = self.metadata_cache.write().await;
cache.remove(path);
// Update statistics
let mut stats = self.stats.write().await;
stats.invalidations += 1;
}
// Remove from the LRU queue
let path_buf = path.to_path_buf();
{
let mut lru = self.lru_queue.write().await;
if let Some(pos) = lru.iter().position(|p| p == &path_buf) {
lru.remove(pos);
}
}
debug!("Invalidated cache entry for: {}", path.display());
}
/// Recursively invalidate entries under a directory
pub async fn invalidate_directory(&self, dir_path: &Path) {
let dir_str = dir_path.to_string_lossy().to_string();
let mut paths_to_remove = Vec::new();
// Find all paths that start with the directory
{
let cache = self.metadata_cache.read().await;
for path in cache.keys() {
let path_str = path.to_string_lossy().to_string();
if path_str.starts_with(&dir_str) {
paths_to_remove.push(path.clone());
}
}
}
// Update statistics
{
let mut stats = self.stats.write().await;
stats.invalidations += paths_to_remove.len();
}
// Remove each found path
for path in paths_to_remove {
self.invalidate(&path).await;
}
debug!("Invalidated directory and contents: {}", dir_path.display());
}
/// Get current cache statistics
pub async fn get_stats(&self) -> CacheStats {
let stats = self.stats.read().await;
stats.clone()
}
/// Clears all expired entries from the cache
pub async fn clear_expired(&self) {
let now = Instant::now();
let mut paths_to_remove = Vec::new();
// Find expired entries
{
let cache = self.metadata_cache.read().await;
for (path, metadata) in cache.iter() {
if now > metadata.expires_at {
paths_to_remove.push(path.clone());
}
}
}
// Update statistics
{
let mut stats = self.stats.write().await;
stats.expirations += paths_to_remove.len();
}
// Save the number of entries for logging
let num_paths = paths_to_remove.len();
// Remove expired entries
for path in paths_to_remove {
self.invalidate(&path).await;
}
debug!("Cleared {} expired entries from cache", num_paths);
}
/// Starts the periodic cleanup process
pub fn start_cleanup_task(cache: Arc<Self>) -> BoxFuture<'static, ()> {
Box::pin(async move {
let cleanup_interval = Duration::from_secs(60); // Every minute
loop {
// Wait for the interval
time::sleep(cleanup_interval).await;
// Clean expired entries
cache.clear_expired().await;
// Log statistics
let stats = cache.get_stats().await;
let cache_size = {
let cache_map = cache.metadata_cache.read().await;
cache_map.len()
};
debug!(
"Cache stats: size={}, hits={}, misses={}, hit_ratio={:.2}%, time_saved={}ms",
cache_size,
stats.hits,
stats.misses,
if stats.hits + stats.misses > 0 {
(stats.hits as f64 * 100.0) / (stats.hits + stats.misses) as f64
} else {
0.0
},
stats.time_saved_ms
);
}
})
}
/// Preloads metadata for entire directories (useful for initialization)
pub async fn preload_directory(
&self,
dir_path: &Path,
recursive: bool,
max_depth: usize,
) -> Result<usize, std::io::Error> {
self._preload_directory_internal(dir_path, recursive, max_depth, 0)
.await
}
/// Internal preload implementation with depth tracking
async fn _preload_directory_internal(
&self,
dir_path: &Path,
recursive: bool,
max_depth: usize,
current_depth: usize,
) -> Result<usize, std::io::Error> {
Box::pin(async move {
if current_depth > max_depth {
return Ok(0);
}
// Get directory entries
let mut entries = fs::read_dir(dir_path).await?;
let mut count = 0;
// Process each entry
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let metadata = fs::metadata(&path).await?;
// Refresh metadata for this entry
self.refresh_metadata(&path).await?;
count += 1;
// Recursively process subdirectories if needed
if recursive && metadata.is_dir() {
// Box to break recursion
count += self
._preload_directory_internal(&path, recursive, max_depth, current_depth + 1)
.await?;
}
}
Ok(count)
})
.await
}
}
// ─── MetadataCachePort implementation ────────────────────────
use crate::application::ports::cache_ports::{CachedMetadataDto, MetadataCachePort};
use crate::common::errors::DomainError;
use async_trait::async_trait;
#[async_trait]
impl MetadataCachePort for FileMetadataCache {
async fn get_metadata(&self, path: &Path) -> Option<CachedMetadataDto> {
// Delegate to the existing rich get_metadata, then project into the DTO.
let fm = FileMetadataCache::get_metadata(self, path).await?;
Some(CachedMetadataDto {
path: fm.path,
exists: fm.exists,
is_file: fm.entry_type == CacheEntryType::File,
size: fm.size,
mime_type: fm.mime_type,
created_at: fm.created_at,
modified_at: fm.modified_at,
})
}
async fn is_file(&self, path: &Path) -> Option<bool> {
FileMetadataCache::is_file(self, path).await
}
async fn refresh_metadata(&self, path: &Path) -> Result<CachedMetadataDto, DomainError> {
let fm = FileMetadataCache::refresh_metadata(self, path)
.await
.map_err(|e| DomainError::internal_error("MetadataCache", e.to_string()))?;
Ok(CachedMetadataDto {
path: fm.path,
exists: fm.exists,
is_file: fm.entry_type == CacheEntryType::File,
size: fm.size,
mime_type: fm.mime_type,
created_at: fm.created_at,
modified_at: fm.modified_at,
})
}
async fn invalidate(&self, path: &Path) {
FileMetadataCache::invalidate(self, path).await
}
async fn invalidate_directory(&self, dir_path: &Path) {
FileMetadataCache::invalidate_directory(self, dir_path).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
#[tokio::test]
async fn test_cache_operations() {
// Create temporary directory for tests
let temp_dir = tempdir().unwrap();
let file_path = temp_dir.path().join("test_file.txt");
// Create a test file
let mut file = File::create(&file_path).await.unwrap();
file.write_all(b"test content").await.unwrap();
file.flush().await.unwrap();
drop(file);
// Create cache
let config = AppConfig::default();
let cache = FileMetadataCache::new(config, 1000);
// Verify initial miss
assert!(cache.exists(&file_path).await.is_none());
// Refresh and verify hit
let metadata = cache.refresh_metadata(&file_path).await.unwrap();
assert_eq!(metadata.entry_type, CacheEntryType::File);
assert_eq!(metadata.size, Some(12)); // "test content" = 12 bytes
// Verify it now exists in cache
assert_eq!(cache.exists(&file_path).await, Some(true));
assert_eq!(cache.is_file(&file_path).await, Some(true));
// Invalidate and verify it no longer exists in cache
cache.invalidate(&file_path).await;
assert!(cache.exists(&file_path).await.is_none());
// Verify statistics
let stats = cache.get_stats().await;
assert_eq!(stats.inserts, 1);
assert_eq!(stats.invalidations, 1);
assert!(stats.hits > 0);
}
#[tokio::test]
async fn test_directory_operations() {
// Create directory structure for tests
let temp_dir = tempdir().unwrap();
// Canonicalize to handle macOS /var -> /private/var symlinks
let base_path = temp_dir.path().canonicalize().unwrap();
let sub_dir = base_path.join("subdir");
fs::create_dir(&sub_dir).await.unwrap();
let file1 = base_path.join("file1.txt");
let file2 = sub_dir.join("file2.txt");
File::create(&file1).await.unwrap();
File::create(&file2).await.unwrap();
// Create cache
let config = AppConfig::default();
let cache = FileMetadataCache::new(config, 1000);
// Preload directory recursively
// preload_directory caches the *contents* of the directory, not the root itself
let count = cache.preload_directory(&base_path, true, 2).await.unwrap();
assert_eq!(count, 3); // subdir, file1, file2
// Verify existence in cache (only contents, not the root)
assert_eq!(cache.is_dir(&sub_dir).await, Some(true));
assert_eq!(cache.is_file(&file1).await, Some(true));
assert_eq!(cache.is_file(&file2).await, Some(true));
// Invalidate directory and contents
cache.invalidate_directory(&base_path).await;
// Verify nothing exists in cache
assert!(cache.exists(&sub_dir).await.is_none());
assert!(cache.exists(&file1).await.is_none());
assert!(cache.exists(&file2).await.is_none());
}
}
src/infrastructure/services/file_system_utils.rs
-326
View File
@@ -1,326 +0,0 @@
use std::io::Error as IoError;
use std::path::Path;
use tempfile::NamedTempFile;
use tokio::fs::{self, File, OpenOptions};
use tokio::io::AsyncWriteExt;
use tracing::{error, warn};
/// Utility functions for file system operations with proper synchronization
pub struct FileSystemUtils;
impl FileSystemUtils {
/// Writes data to a file with fsync to ensure durability
/// Uses a safe atomic write pattern: write to temp file, fsync, rename
pub async fn atomic_write<P: AsRef<Path>>(path: P, contents: &[u8]) -> Result<(), IoError> {
let path = path.as_ref();
// Ensure parent directory exists
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
// Create a temporary file in the same directory
let dir = path.parent().unwrap_or_else(|| Path::new("."));
let temp_file = match NamedTempFile::new_in(dir) {
Ok(file) => file,
Err(e) => {
error!(
"Failed to create temporary file in {}: {}",
dir.display(),
e
);
return Err(IoError::other(format!(
"Failed to create temporary file: {}",
e
)));
}
};
let temp_path = temp_file.path().to_path_buf();
// Convert to tokio file and write contents
let std_file = temp_file.as_file().try_clone()?;
let mut file = File::from_std(std_file);
file.write_all(contents).await?;
// Ensure data is synced to disk
file.flush().await?;
file.sync_all().await?;
// Rename the temporary file to the target path (atomic operation on most filesystems)
fs::rename(&temp_path, path).await?;
// Sync the directory to ensure the rename is persisted
if let Some(parent) = path.parent() {
match Self::sync_directory(parent).await {
Ok(_) => {}
Err(e) => {
warn!(
"Failed to sync directory {}: {}. File was written but directory entry might not be durable.",
parent.display(),
e
);
}
}
}
Ok(())
}
/// Creates or appends to a file with fsync
pub async fn write_with_sync<P: AsRef<Path>>(
path: P,
contents: &[u8],
append: bool,
) -> Result<(), IoError> {
let path = path.as_ref();
// Ensure parent directory exists
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
// Open file with appropriate options
let mut file = OpenOptions::new()
.write(true)
.create(true)
.truncate(!append)
.append(append)
.open(path)
.await?;
// Write contents
file.write_all(contents).await?;
// Ensure data is synced to disk
file.flush().await?;
file.sync_all().await?;
Ok(())
}
/// Creates directories with fsync
pub async fn create_dir_with_sync<P: AsRef<Path>>(path: P) -> Result<(), IoError> {
let path = path.as_ref();
// Create directory
fs::create_dir_all(path).await?;
// Sync the directory
Self::sync_directory(path).await?;
// Sync parent directory to ensure directory creation is persisted
if let Some(parent) = path.parent() {
match Self::sync_directory(parent).await {
Ok(_) => {}
Err(e) => {
warn!(
"Failed to sync parent directory {}: {}. Directory was created but entry might not be durable.",
parent.display(),
e
);
}
}
}
Ok(())
}
/// Renames a file or directory with proper syncing
pub async fn rename_with_sync<P: AsRef<Path>, Q: AsRef<Path>>(
from: P,
to: Q,
) -> Result<(), IoError> {
let from = from.as_ref();
let to = to.as_ref();
// Ensure parent directory of destination exists
if let Some(parent) = to.parent() {
fs::create_dir_all(parent).await?;
}
// Perform rename
fs::rename(from, to).await?;
// Sync parent directories to ensure rename is persisted
if let Some(from_parent) = from.parent() {
match Self::sync_directory(from_parent).await {
Ok(_) => {}
Err(e) => {
warn!(
"Failed to sync source directory {}: {}. Rename completed but might not be durable.",
from_parent.display(),
e
);
}
}
}
if let Some(to_parent) = to.parent() {
match Self::sync_directory(to_parent).await {
Ok(_) => {}
Err(e) => {
warn!(
"Failed to sync destination directory {}: {}. Rename completed but might not be durable.",
to_parent.display(),
e
);
}
}
}
Ok(())
}
/// Removes a file with directory syncing
pub async fn remove_file_with_sync<P: AsRef<Path>>(path: P) -> Result<(), IoError> {
let path = path.as_ref();
// Remove file
fs::remove_file(path).await?;
// Sync parent directory to ensure removal is persisted
if let Some(parent) = path.parent() {
match Self::sync_directory(parent).await {
Ok(_) => {}
Err(e) => {
warn!(
"Failed to sync directory after file removal {}: {}. File was removed but entry might not be durable.",
parent.display(),
e
);
}
}
}
Ok(())
}
/// Removes a directory with parent directory syncing
pub async fn remove_dir_with_sync<P: AsRef<Path>>(
path: P,
recursive: bool,
) -> Result<(), IoError> {
let path = path.as_ref();
// Remove directory
if recursive {
fs::remove_dir_all(path).await?;
} else {
fs::remove_dir(path).await?;
}
// Sync parent directory to ensure removal is persisted
if let Some(parent) = path.parent() {
match Self::sync_directory(parent).await {
Ok(_) => {}
Err(e) => {
warn!(
"Failed to sync directory after directory removal {}: {}. Directory was removed but entry might not be durable.",
parent.display(),
e
);
}
}
}
Ok(())
}
/// Syncs a directory to ensure its contents are durable
async fn sync_directory<P: AsRef<Path>>(path: P) -> Result<(), IoError> {
let path = path.as_ref();
// Open directory with read permissions
let dir_file = match OpenOptions::new().read(true).open(path).await {
Ok(file) => file,
Err(e) => {
warn!(
"Failed to open directory for syncing {}: {}",
path.display(),
e
);
return Err(e);
}
};
// Sync the directory
dir_file.sync_all().await
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
use tokio::fs;
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn test_atomic_write() {
let temp_dir = tempdir().unwrap();
let file_path = temp_dir.path().join("test.txt");
// Write data atomically
FileSystemUtils::atomic_write(&file_path, b"Hello, world!")
.await
.unwrap();
// Read back the data
let mut file = fs::File::open(&file_path).await.unwrap();
let mut contents = String::new();
file.read_to_string(&mut contents).await.unwrap();
assert_eq!(contents, "Hello, world!");
}
#[tokio::test]
async fn test_write_with_sync() {
let temp_dir = tempdir().unwrap();
let file_path = temp_dir.path().join("test.txt");
// Write data with sync
FileSystemUtils::write_with_sync(&file_path, b"First line\n", false)
.await
.unwrap();
// Append data
FileSystemUtils::write_with_sync(&file_path, b"Second line", true)
.await
.unwrap();
// Read back the data
let mut file = fs::File::open(&file_path).await.unwrap();
let mut contents = String::new();
file.read_to_string(&mut contents).await.unwrap();
assert_eq!(contents, "First line\nSecond line");
}
#[tokio::test]
async fn test_rename_with_sync() {
let temp_dir = tempdir().unwrap();
let source_path = temp_dir.path().join("source.txt");
let dest_path = temp_dir.path().join("dest.txt");
// Create source file
FileSystemUtils::write_with_sync(&source_path, b"Test content", false)
.await
.unwrap();
// Rename file
FileSystemUtils::rename_with_sync(&source_path, &dest_path)
.await
.unwrap();
// Verify source doesn't exist
assert!(!source_path.exists());
// Verify destination exists
let mut file = fs::File::open(&dest_path).await.unwrap();
let mut contents = String::new();
file.read_to_string(&mut contents).await.unwrap();
assert_eq!(contents, "Test content");
}
}
src/infrastructure/services/id_mapping_optimizer.rs
-663
View File
@@ -1,663 +0,0 @@
use async_trait::async_trait;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Mutex, RwLock, Semaphore};
use tracing::{debug, error, info, warn};
use crate::application::ports::outbound::IdMappingPort;
use crate::common::errors::DomainError;
use crate::domain::services::path_service::StoragePath;
use crate::infrastructure::services::id_mapping_service::{IdMappingError, IdMappingService};
/// Maximum number of entries in the cache
const MAX_CACHE_SIZE: usize = 10_000;
/// Cache time-to-live (in seconds)
const CACHE_TTL_SECONDS: u64 = 60 * 5; // 5 minutes
/// Optimizer for batch ID mapping operations
pub struct IdMappingOptimizer {
/// Base ID mapping service
base_service: Arc<IdMappingService>,
/// Path to ID cache (path -> id)
path_to_id_cache: RwLock<HashMap<String, (String, Instant)>>,
/// ID to path cache (id -> path)
id_to_path_cache: RwLock<HashMap<String, (String, Instant)>>,
/// Hit counter
stats: RwLock<OptimizerStats>,
/// Semaphore to limit batch operations
batch_limiter: Semaphore,
/// Pending batch queue
pending_batch: Mutex<BatchQueue>,
}
/// Optimizer statistics
#[derive(Debug, Default, Clone)]
pub struct OptimizerStats {
/// Total number of get_path_by_id queries
pub path_by_id_queries: usize,
/// Number of cache hits for get_path_by_id
pub path_by_id_hits: usize,
/// Total number of get_or_create_id queries
pub get_id_queries: usize,
/// Number of cache hits for get_or_create_id
pub get_id_hits: usize,
/// Number of batch operations performed
pub batch_operations: usize,
/// Total number of IDs processed in batch
pub batch_items_processed: usize,
/// Last cache cleanup timestamp
pub last_cleanup: Option<Instant>,
}
/// Queue for batch operations
#[derive(Default)]
struct BatchQueue {
/// Pending paths to get/create ID
path_to_id_requests: HashSet<String>,
/// Pending IDs to get path
id_to_path_requests: HashSet<String>,
}
/// Result of a batch operation
struct BatchResult {
/// Path to ID mapping
path_to_id: HashMap<String, String>,
/// ID to path mapping
id_to_path: HashMap<String, String>,
}
impl IdMappingOptimizer {
/// Creates a new optimizer for the ID mapping service
pub fn new(base_service: Arc<IdMappingService>) -> Self {
Self {
base_service,
path_to_id_cache: RwLock::new(HashMap::with_capacity(1000)),
id_to_path_cache: RwLock::new(HashMap::with_capacity(1000)),
stats: RwLock::new(OptimizerStats::default()),
batch_limiter: Semaphore::new(2), // Limit to 2 concurrent batch operations
pending_batch: Mutex::new(BatchQueue::default()),
}
}
/// Gets optimizer statistics
pub async fn get_stats(&self) -> OptimizerStats {
self.stats.read().await.clone()
}
/// Cleans expired cache entries
pub async fn cleanup_cache(&self) {
let now = Instant::now();
let ttl = Duration::from_secs(CACHE_TTL_SECONDS);
// Clean path_to_id cache
{
let mut cache = self.path_to_id_cache.write().await;
let initial_size = cache.len();
// Retain only non-expired entries
cache.retain(|_, (_, timestamp)| now.duration_since(*timestamp) < ttl);
let removed = initial_size - cache.len();
if removed > 0 {
debug!("Cleaned {} expired entries from path_to_id cache", removed);
}
}
// Clean id_to_path cache
{
let mut cache = self.id_to_path_cache.write().await;
let initial_size = cache.len();
// Retain only non-expired entries
cache.retain(|_, (_, timestamp)| now.duration_since(*timestamp) < ttl);
let removed = initial_size - cache.len();
if removed > 0 {
debug!("Cleaned {} expired entries from id_to_path cache", removed);
}
}
// Update statistics
{
let mut stats = self.stats.write().await;
stats.last_cleanup = Some(now);
}
}
/// Starts periodic cleanup task
pub fn start_cleanup_task(optimizer: Arc<Self>) {
tokio::spawn(async move {
let cleanup_interval = Duration::from_secs(CACHE_TTL_SECONDS / 2);
loop {
tokio::time::sleep(cleanup_interval).await;
optimizer.cleanup_cache().await;
// Log statistics periodically
let stats = optimizer.get_stats().await;
info!(
"ID Mapping Optimizer stats - Path queries: {}, hits: {} ({}%), ID queries: {}, hits: {} ({}%), Batch ops: {}, items: {}",
stats.path_by_id_queries,
stats.path_by_id_hits,
if stats.path_by_id_queries > 0 {
stats.path_by_id_hits as f64 * 100.0 / stats.path_by_id_queries as f64
} else {
0.0
},
stats.get_id_queries,
stats.get_id_hits,
if stats.get_id_queries > 0 {
stats.get_id_hits as f64 * 100.0 / stats.get_id_queries as f64
} else {
0.0
},
stats.batch_operations,
stats.batch_items_processed
);
}
});
}
/// Adds a request to the pending queue for batch processing
async fn queue_path_to_id_request(
&self,
path: &StoragePath,
) -> Result<Option<String>, IdMappingError> {
let path_str = path.to_string();
// Check first in the cache
{
let cache = self.path_to_id_cache.read().await;
if let Some((id, _)) = cache.get(&path_str) {
// Update statistics
{
let mut stats = self.stats.write().await;
stats.get_id_hits += 1;
}
return Ok(Some(id.clone()));
}
}
// If not in cache, add to batch queue
{
let mut batch_queue = self.pending_batch.lock().await;
batch_queue.path_to_id_requests.insert(path_str);
}
// Not found in cache, must be processed in batch
Ok(None)
}
/// Processes pending requests in batch
async fn process_batch(&self) -> Result<BatchResult, IdMappingError> {
// Acquire permit for batch operation
let _permit = self.batch_limiter.acquire().await.unwrap();
// Get pending requests
let (path_requests, id_requests) = {
let mut batch_queue = self.pending_batch.lock().await;
let paths = std::mem::take(&mut batch_queue.path_to_id_requests);
let ids = std::mem::take(&mut batch_queue.id_to_path_requests);
(paths, ids)
};
// Create results
let mut result = BatchResult {
path_to_id: HashMap::with_capacity(path_requests.len()),
id_to_path: HashMap::with_capacity(id_requests.len()),
};
// Process path->id requests in batch
for path_str in path_requests {
let path = StoragePath::from_string(&path_str);
match self.base_service.get_or_create_id(&path).await {
Ok(id) => {
result.path_to_id.insert(path_str.clone(), id.clone());
result.id_to_path.insert(id, path_str);
}
Err(e) => {
error!("Error batch-processing path {}: {}", path_str, e);
// Continue with remaining requests
}
}
}
// Process id->path requests in batch
for id in id_requests {
match self.base_service.get_path_by_id(&id).await {
Ok(path) => {
let path_str = path.to_string();
result.id_to_path.insert(id.clone(), path_str.clone());
result.path_to_id.insert(path_str, id);
}
Err(e) => {
error!("Error batch-processing ID {}: {}", id, e);
// Continue with remaining requests
}
}
}
// Update cache with batch results
{
let mut path_cache = self.path_to_id_cache.write().await;
let mut id_cache = self.id_to_path_cache.write().await;
let now = Instant::now();
for (path, id) in &result.path_to_id {
path_cache.insert(path.clone(), (id.clone(), now));
}
for (id, path) in &result.id_to_path {
id_cache.insert(id.clone(), (path.clone(), now));
}
}
// Update statistics
{
let mut stats = self.stats.write().await;
stats.batch_operations += 1;
stats.batch_items_processed += result.path_to_id.len() + result.id_to_path.len();
}
// Save changes to disk in the background
let service_clone = self.base_service.clone();
tokio::spawn(async move {
if let Err(e) = service_clone.save_pending_changes().await {
error!("Error saving ID mapping changes: {}", e);
}
});
Ok(result)
}
/// Forces processing of pending requests if there are enough
async fn trigger_batch_if_needed(&self, min_batch_size: usize) -> Result<(), IdMappingError> {
// Check if there are enough pending requests
let should_process = {
let batch_queue = self.pending_batch.lock().await;
batch_queue.path_to_id_requests.len() + batch_queue.id_to_path_requests.len()
>= min_batch_size
};
// Process if necessary
if should_process {
self.process_batch().await?;
}
Ok(())
}
/// Preload a set of paths to get their IDs in batch
pub async fn preload_paths(&self, paths: Vec<StoragePath>) -> Result<(), IdMappingError> {
// Only proceed if there are paths to load
if paths.is_empty() {
return Ok(());
}
// Paths we need to load (those not in cache)
let mut paths_to_load = Vec::new();
// Check cache first
{
let cache = self.path_to_id_cache.read().await;
for path in paths {
let path_str = path.to_string();
if !cache.contains_key(&path_str) {
paths_to_load.push(path_str);
}
}
}
// If all were in cache, finish
if paths_to_load.is_empty() {
return Ok(());
}
// Add paths to queue for batch processing
{
let mut batch_queue = self.pending_batch.lock().await;
for path in paths_to_load {
batch_queue.path_to_id_requests.insert(path);
}
}
// Execute batch processing immediately
self.process_batch().await?;
Ok(())
}
/// Preload a set of IDs to get their paths in batch
pub async fn preload_ids(&self, ids: Vec<String>) -> Result<(), IdMappingError> {
// Only proceed if there are IDs to load
if ids.is_empty() {
return Ok(());
}
// IDs we need to load (those not in cache)
let mut ids_to_load = Vec::new();
// Check cache first
{
let cache = self.id_to_path_cache.read().await;
for id in ids {
if !cache.contains_key(&id) {
ids_to_load.push(id);
}
}
}
// If all were in cache, finish
if ids_to_load.is_empty() {
return Ok(());
}
// Add IDs to queue for batch processing
{
let mut batch_queue = self.pending_batch.lock().await;
for id in ids_to_load {
batch_queue.id_to_path_requests.insert(id);
}
}
// Execute batch processing immediately
self.process_batch().await?;
Ok(())
}
}
#[async_trait]
impl IdMappingPort for IdMappingOptimizer {
async fn get_or_create_id(&self, path: &StoragePath) -> Result<String, DomainError> {
// Update statistics
{
let mut stats = self.stats.write().await;
stats.get_id_queries += 1;
}
let path_str = path.to_string();
// Check cache first
{
let cache = self.path_to_id_cache.read().await;
if let Some((id, _)) = cache.get(&path_str) {
// Update statistics
{
let mut stats = self.stats.write().await;
stats.get_id_hits += 1;
}
return Ok(id.clone());
}
}
// If not in cache, try adding to batch queue first
let queued_result = self.queue_path_to_id_request(path).await?;
if let Some(id) = queued_result {
return Ok(id);
}
// Trigger batch processing if enough items accumulated
self.trigger_batch_if_needed(20).await?;
// Try to get from the base service
let id = self.base_service.get_or_create_id(path).await?;
// Update cache with the new ID
{
let mut path_cache = self.path_to_id_cache.write().await;
let mut id_cache = self.id_to_path_cache.write().await;
let now = Instant::now();
// Control cache size
if path_cache.len() >= MAX_CACHE_SIZE {
warn!(
"Path-to-ID cache size reached limit ({}), clearing oldest entries",
MAX_CACHE_SIZE
);
path_cache.clear();
}
if id_cache.len() >= MAX_CACHE_SIZE {
warn!(
"ID-to-path cache size reached limit ({}), clearing oldest entries",
MAX_CACHE_SIZE
);
id_cache.clear();
}
path_cache.insert(path_str.clone(), (id.clone(), now));
id_cache.insert(id.clone(), (path_str, now));
}
Ok(id)
}
async fn get_path_by_id(&self, id: &str) -> Result<StoragePath, DomainError> {
// Update statistics
{
let mut stats = self.stats.write().await;
stats.path_by_id_queries += 1;
}
// Check first in the cache
{
let cache = self.id_to_path_cache.read().await;
if let Some((path_str, _)) = cache.get(id) {
// Update statistics
{
let mut stats = self.stats.write().await;
stats.path_by_id_hits += 1;
}
return Ok(StoragePath::from_string(path_str));
}
}
// Get from the base service
let path = self.base_service.get_path_by_id(id).await?;
// Update cache
{
let mut id_cache = self.id_to_path_cache.write().await;
let mut path_cache = self.path_to_id_cache.write().await;
let now = Instant::now();
let path_str = path.to_string();
// Control cache size
if id_cache.len() >= MAX_CACHE_SIZE {
warn!(
"ID-to-path cache size reached limit ({}), clearing oldest entries",
MAX_CACHE_SIZE
);
id_cache.clear();
}
if path_cache.len() >= MAX_CACHE_SIZE {
warn!(
"Path-to-ID cache size reached limit ({}), clearing oldest entries",
MAX_CACHE_SIZE
);
path_cache.clear();
}
id_cache.insert(id.to_string(), (path_str.clone(), now));
path_cache.insert(path_str, (id.to_string(), now));
}
Ok(path)
}
async fn update_path(&self, id: &str, new_path: &StoragePath) -> Result<(), DomainError> {
// Invalidate cache for this ID
{
let mut id_cache = self.id_to_path_cache.write().await;
let mut path_cache = self.path_to_id_cache.write().await;
// Remove the ID entry
if let Some((old_path, _)) = id_cache.remove(id) {
path_cache.remove(&old_path);
}
}
// Update in the base service
let result = self.base_service.update_path(id, new_path).await?;
// Update cache with new mapping
{
let mut id_cache = self.id_to_path_cache.write().await;
let mut path_cache = self.path_to_id_cache.write().await;
let now = Instant::now();
let path_str = new_path.to_string();
id_cache.insert(id.to_string(), (path_str.clone(), now));
path_cache.insert(path_str, (id.to_string(), now));
}
Ok(result)
}
async fn remove_id(&self, id: &str) -> Result<(), DomainError> {
// Invalidate cache for this ID
{
let mut id_cache = self.id_to_path_cache.write().await;
let mut path_cache = self.path_to_id_cache.write().await;
// Remove the ID entry
if let Some((path, _)) = id_cache.remove(id) {
path_cache.remove(&path);
}
}
// Remove from the base service
self.base_service.remove_id(id).await?;
Ok(())
}
async fn save_changes(&self) -> Result<(), DomainError> {
// Delegate to the base service
self.base_service.save_changes().await?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
async fn create_test_service() -> (Arc<IdMappingService>, Arc<IdMappingOptimizer>) {
let temp_dir = tempdir().unwrap();
let map_path = temp_dir.path().join("id_map.json");
let base_service = Arc::new(IdMappingService::new(map_path).await.unwrap());
let optimizer = Arc::new(IdMappingOptimizer::new(base_service.clone()));
(base_service, optimizer)
}
#[tokio::test]
async fn test_basic_caching() {
let (_, optimizer) = create_test_service().await;
let path = StoragePath::from_string("/test/file.txt");
// First call should use the base service
let id = optimizer.get_or_create_id(&path).await.unwrap();
assert!(!id.is_empty(), "ID should not be empty");
// Second call should use cache
let id2 = optimizer.get_or_create_id(&path).await.unwrap();
assert_eq!(id, id2, "Same path should return same ID");
// Verify cache statistics
let stats = optimizer.get_stats().await;
assert_eq!(stats.get_id_queries, 2, "Should have 2 queries");
assert_eq!(stats.get_id_hits, 1, "Should have 1 hit");
}
#[tokio::test]
async fn test_batch_processing() {
let (_, optimizer) = create_test_service().await;
// Create a batch of paths
let mut paths = Vec::new();
for i in 0..50 {
paths.push(StoragePath::from_string(&format!(
"/test/batch/file{}.txt",
i
)));
}
// Preload the paths
optimizer.preload_paths(paths.clone()).await.unwrap();
// Verify all are in cache
for path in &paths {
let id = optimizer.get_or_create_id(path).await.unwrap();
assert!(!id.is_empty(), "ID should be available for path");
}
// Verify statistics
let stats = optimizer.get_stats().await;
assert_eq!(stats.batch_operations, 1, "Should have 1 batch operation");
assert!(
stats.batch_items_processed >= 50,
"Should have processed at least 50 items"
);
// Verify all subsequent queries are cache hits
assert_eq!(
stats.get_id_hits, 50,
"All subsequente queries should be cache hits"
);
}
#[tokio::test]
async fn test_cache_cleanup() {
let (_, optimizer) = create_test_service().await;
// Create some entries
let path = StoragePath::from_string("/test/cleanup.txt");
let id = optimizer.get_or_create_id(&path).await.unwrap();
// Verify initial statistics
{
let stats = optimizer.get_stats().await;
assert_eq!(stats.get_id_queries, 1, "Should have 1 query");
assert_eq!(stats.get_id_hits, 0, "Should have 0 hits");
}
// Run cleanup (should not remove anything yet)
optimizer.cleanup_cache().await;
// Verify cache is still working
let id2 = optimizer.get_or_create_id(&path).await.unwrap();
assert_eq!(id, id2, "Cache should still work after cleanup");
{
let stats = optimizer.get_stats().await;
assert_eq!(stats.get_id_hits, 1, "Should have 1 hit after cleanup");
}
}
}
src/infrastructure/services/id_mapping_service.rs
-712
View File
@@ -1,712 +0,0 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use tokio::fs;
use tokio::sync::{Mutex, RwLock};
use tokio::time;
use uuid::Uuid;
use crate::application::ports::outbound::IdMappingPort;
use crate::common::config::TimeoutConfig;
use crate::common::errors::{DomainError, ErrorKind};
use crate::domain::services::path_service::StoragePath;
/// Specific error for the ID mapping service
#[derive(Debug, thiserror::Error)]
pub enum IdMappingError {
#[error("ID not found: {0}")]
NotFound(String),
#[error("IO error: {0}")]
IoError(#[from] std::io::Error),
#[error("Timeout error: {0}")]
Timeout(String),
#[error("Serialization error: {0}")]
SerializationError(#[from] serde_json::Error),
#[error("Other error: {0}")]
Other(String),
}
// Implement conversion from IdMappingError to DomainError
impl From<IdMappingError> for DomainError {
fn from(err: IdMappingError) -> Self {
match err {
IdMappingError::NotFound(id) => DomainError::not_found("IdMapping", id),
IdMappingError::IoError(e) => DomainError::new(
ErrorKind::InternalError,
"IdMapping",
format!("IO error: {}", e),
)
.with_source(e),
IdMappingError::Timeout(msg) => {
DomainError::timeout("IdMapping", format!("Timeout: {}", msg))
}
IdMappingError::SerializationError(e) => DomainError::new(
ErrorKind::InternalError,
"IdMapping",
format!("Serialization error: {}", e),
)
.with_source(e),
IdMappingError::Other(msg) => DomainError::new(
ErrorKind::InternalError,
"IdMapping",
format!("Other error: {}", msg),
),
}
}
}
/// Structure to store IDs mapped to their paths
#[derive(Serialize, Deserialize, Debug, Default)]
struct IdMap {
path_to_id: HashMap<String, String>,
id_to_path: HashMap<String, String>, // Field for efficient bidirectional lookup
version: u32, // Version to detect changes
}
/// Service to manage mappings between paths and unique IDs
pub struct IdMappingService {
map_path: PathBuf,
id_map: RwLock<IdMap>,
save_mutex: Mutex<()>, // To prevent multiple concurrent saves
timeouts: TimeoutConfig,
pending_save: RwLock<bool>, // Indicates if there are pending changes
}
impl IdMappingService {
/// Creates a new ID mapping service
pub async fn new(map_path: PathBuf) -> Result<Self, DomainError> {
let timeouts = TimeoutConfig::default();
let id_map = Self::load_id_map(&map_path, &timeouts).await?;
Ok(Self {
map_path,
id_map: RwLock::new(id_map),
save_mutex: Mutex::new(()),
timeouts,
pending_save: RwLock::new(false),
})
}
/// Creates an in-memory ID mapping service (for testing)
///
/// Similar functionality as new_in_memory but with a simpler signature for dummy use
pub fn dummy() -> Self {
Self {
map_path: PathBuf::from("/tmp/dummy_id_map.json"),
id_map: RwLock::new(IdMap::default()),
save_mutex: Mutex::new(()),
timeouts: TimeoutConfig::default(),
pending_save: RwLock::new(false),
}
}
/// Creates an in-memory ID mapping service (for testing - original version)
pub fn new_in_memory() -> Self {
Self {
map_path: PathBuf::from("memory"),
id_map: RwLock::new(IdMap::default()),
save_mutex: Mutex::new(()),
timeouts: TimeoutConfig::default(),
pending_save: RwLock::new(false),
}
}
/// Loads the ID map from disk with robust error handling
async fn load_id_map(
map_path: &PathBuf,
timeouts: &TimeoutConfig,
) -> Result<IdMap, DomainError> {
if map_path.exists() {
// Try to read with timeout to avoid indefinite blocking
let read_result = time::timeout(timeouts.lock_timeout(), fs::read_to_string(map_path))
.await
.map_err(|_| {
DomainError::timeout(
"IdMapping",
format!("Timeout reading ID map from {}", map_path.display()),
)
})?;
let content = read_result.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!("Failed to read ID map from {}: {}", map_path.display(), e),
)
})?;
// Parse the JSON
match serde_json::from_str::<IdMap>(&content) {
Ok(mut map) => {
// Rebuild the inverse map if necessary
if map.id_to_path.is_empty() && !map.path_to_id.is_empty() {
let mut rebuild_count = 0;
for (path, id) in &map.path_to_id {
map.id_to_path.insert(id.clone(), path.clone());
rebuild_count += 1;
}
tracing::info!("Rebuilt inverse mapping with {} entries", rebuild_count);
}
tracing::info!(
"Loaded ID map with {} entries (version: {})",
map.path_to_id.len(),
map.version
);
return Ok(map);
}
Err(e) => {
tracing::error!("Error parsing ID map: {}", e);
// Try to backup the corrupted file
let backup_path = map_path.with_extension("json.bak");
if let Err(copy_err) = tokio::fs::copy(map_path, &backup_path).await {
tracing::error!("Failed to backup corrupted map file: {}", copy_err);
} else {
tracing::info!("Backed up corrupted ID map to {}", backup_path.display());
}
tracing::info!("Creating new empty map after error");
return Ok(IdMap {
path_to_id: HashMap::new(),
id_to_path: HashMap::new(),
version: 1, // Start with version 1
});
}
}
}
// Return an empty map if the file doesn't exist and create the file
tracing::info!("No existing ID map found, creating new empty map");
let empty_map = IdMap {
path_to_id: HashMap::new(),
id_to_path: HashMap::new(),
version: 1, // Start with version 1
};
// Ensure directory exists
if let Some(parent) = map_path.parent()
&& !parent.exists()
&& let Err(e) = fs::create_dir_all(parent).await
{
tracing::error!("Failed to create directory for ID map: {}", e);
}
// Write empty map to file (best-effort: the in-memory map is valid even if disk write fails)
match serde_json::to_string_pretty(&empty_map) {
Ok(json) => {
if let Err(e) = fs::write(map_path, json).await {
tracing::warn!(
"Could not write initial empty ID map (will retry on next save): {}",
e
);
} else {
tracing::info!("Created initial empty ID map at {}", map_path.display());
}
}
Err(e) => {
tracing::error!("Failed to serialize empty ID map: {}", e);
}
}
Ok(empty_map)
}
/// Saves the ID map to disk safely
async fn save_id_map(&self) -> Result<(), DomainError> {
// Acquire exclusive lock for saving
let _lock = time::timeout(self.timeouts.lock_timeout(), self.save_mutex.lock())
.await
.map_err(|_| {
DomainError::timeout("IdMapping", "Timeout acquiring save lock for ID mapping")
})?;
// Create JSON with read lock to minimize lock hold time
let json = {
let mut map = time::timeout(self.timeouts.lock_timeout(), self.id_map.write())
.await
.map_err(|_| {
DomainError::timeout("IdMapping", "Timeout acquiring write lock for ID mapping")
})?;
// Increment version only if there are pending changes to save
let pending = *self.pending_save.read().await;
if pending {
map.version += 1;
tracing::debug!("Incrementing ID map version to {}", map.version);
}
// Use serde with reasonably safe defaults
serde_json::to_string_pretty(&*map).map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!("Failed to serialize ID map to JSON: {}", e),
)
})?
};
// Write to a temporary file first to avoid corruption
let temp_path = self.map_path.with_extension("json.tmp");
fs::write(&temp_path, &json).await.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!(
"Failed to write temporary ID map to {}: {}",
temp_path.display(),
e
),
)
})?;
// Perform the atomic rename
fs::rename(&temp_path, &self.map_path).await.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!(
"Failed to rename temporary ID map to {}: {}",
self.map_path.display(),
e
),
)
})?;
// Reset pending flag
{
let mut pending = self.pending_save.write().await;
*pending = false;
}
tracing::info!("Saved ID map successfully to {}", self.map_path.display());
Ok(())
}
/// Generates a unique ID
fn generate_id(&self) -> String {
Uuid::new_v4().to_string()
}
/// Marks changes as pending
async fn mark_pending(&self) {
let mut pending = self.pending_save.write().await;
*pending = true;
}
/// Gets the ID for a path or generates a new one if it doesn't exist
pub async fn get_or_create_id(&self, path: &StoragePath) -> Result<String, IdMappingError> {
let path_str = path.to_string();
// First attempt with read lock (more efficient)
{
let read_result =
match time::timeout(self.timeouts.lock_timeout(), self.id_map.read()).await {
Ok(guard) => guard,
Err(_) => {
return Err(IdMappingError::Timeout(
"Timeout acquiring read lock for ID mapping".to_string(),
));
}
};
if let Some(id) = read_result.path_to_id.get(&path_str) {
return Ok(id.clone());
}
}
// If not found, acquire write lock
let write_result =
match time::timeout(self.timeouts.lock_timeout(), self.id_map.write()).await {
Ok(guard) => guard,
Err(_) => {
return Err(IdMappingError::Timeout(
"Timeout acquiring write lock for ID mapping".to_string(),
));
}
};
let mut map = write_result;
// Check again (it could have been added while we were waiting for the lock)
if let Some(id) = map.path_to_id.get(&path_str) {
return Ok(id.clone());
}
// Generate a new ID and store it
let id = self.generate_id();
map.path_to_id.insert(path_str.clone(), id.clone());
map.id_to_path.insert(id.clone(), path_str);
// Mark as pending for saving
drop(map); // Release the write lock before acquiring another
self.mark_pending().await;
tracing::debug!("Created new ID mapping: {} -> {}", path.to_string(), id);
Ok(id)
}
/// Gets a path by its ID with timeout handling
pub async fn get_path_by_id(&self, id: &str) -> Result<StoragePath, IdMappingError> {
let read_result =
match time::timeout(self.timeouts.lock_timeout(), self.id_map.read()).await {
Ok(guard) => guard,
Err(_) => {
return Err(IdMappingError::Timeout(
"Timeout acquiring read lock for ID lookup".to_string(),
));
}
};
if let Some(path_str) = read_result.id_to_path.get(id) {
return Ok(StoragePath::from_string(path_str));
}
Err(IdMappingError::NotFound(id.to_string()))
}
/// Updates the mapping of an existing ID to a new path
pub async fn update_path(
&self,
id: &str,
new_path: &StoragePath,
) -> Result<(), IdMappingError> {
let write_result =
match time::timeout(self.timeouts.lock_timeout(), self.id_map.write()).await {
Ok(guard) => guard,
Err(_) => {
return Err(IdMappingError::Timeout(
"Timeout acquiring write lock for ID update".to_string(),
));
}
};
let mut map = write_result;
// Find the previous path to remove it
if let Some(old_path) = map.id_to_path.get(id).cloned() {
map.path_to_id.remove(&old_path);
// Register the new path
let new_path_str = new_path.to_string();
map.path_to_id.insert(new_path_str.clone(), id.to_string());
map.id_to_path.insert(id.to_string(), new_path_str);
// Mark as pending
drop(map); // Release the write lock before acquiring another
self.mark_pending().await;
tracing::debug!(
"Updated path mapping for ID {}: {} -> {}",
id,
old_path,
new_path.to_string()
);
Ok(())
} else {
Err(IdMappingError::NotFound(id.to_string()))
}
}
/// Removes an ID from the map
pub async fn remove_id(&self, id: &str) -> Result<(), IdMappingError> {
let write_result =
match time::timeout(self.timeouts.lock_timeout(), self.id_map.write()).await {
Ok(guard) => guard,
Err(_) => {
return Err(IdMappingError::Timeout(
"Timeout acquiring write lock for ID removal".to_string(),
));
}
};
let mut map = write_result;
// Find the path to remove it
if let Some(path) = map.id_to_path.remove(id) {
map.path_to_id.remove(&path);
// Mark as pending
drop(map); // Release the write lock before acquiring another
self.mark_pending().await;
tracing::debug!("Removed ID mapping: {} -> {}", id, path);
Ok(())
} else {
Err(IdMappingError::NotFound(id.to_string()))
}
}
/// Saves pending changes to disk immediately, without debounce
pub async fn save_pending_changes(&self) -> Result<(), IdMappingError> {
// Check if there are pending changes
{
let pending = self.pending_save.read().await;
if !*pending {
return Ok(());
}
}
// Save immediately (without debounce or spawn)
match self.save_id_map().await {
Ok(_) => {
tracing::info!(
"ID mappings saved successfully to disk at {}",
self.map_path.display()
);
// Explicitly verify that the file exists and has size
match std::fs::metadata(&self.map_path) {
Ok(metadata) => {
if metadata.len() > 0 {
tracing::info!(
"Verified saved map file exists with size: {} bytes",
metadata.len()
);
} else {
tracing::warn!(
"Map file exists but has zero size - this might cause issues"
);
}
}
Err(e) => {
tracing::error!("Failed to verify saved map file: {}", e);
// Try a second save if verification fails
if let Err(retry_err) = self.save_id_map().await {
tracing::error!("Second save attempt also failed: {}", retry_err);
return Err(IdMappingError::IoError(std::io::Error::other(format!(
"Failed to verify and retry save: {}",
retry_err
))));
}
tracing::info!("Second save attempt succeeded");
}
}
Ok(())
}
Err(e) => {
tracing::error!(
"Failed to save ID map to {}: {}",
self.map_path.display(),
e
);
// Try a second save with delay in case of error
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
match self.save_id_map().await {
Ok(_) => {
tracing::info!("Second save attempt succeeded after initial failure");
Ok(())
}
Err(retry_e) => {
tracing::error!("Second save attempt also failed: {}", retry_e);
Err(IdMappingError::IoError(std::io::Error::other(format!(
"Failed to save ID mappings after retry: {}",
retry_e
))))
}
}
}
}
}
}
#[async_trait]
impl IdMappingPort for IdMappingService {
/// Gets the ID for a path or generates a new one if it doesn't exist
async fn get_or_create_id(&self, path: &StoragePath) -> Result<String, DomainError> {
self.get_or_create_id(path).await.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!(
"Failed to get or create ID for path: {}: {}",
path.to_string(),
e
),
)
})
}
/// Gets a path by its ID with timeout handling
async fn get_path_by_id(&self, id: &str) -> Result<StoragePath, DomainError> {
self.get_path_by_id(id).await.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!("Failed to get path for ID: {}: {}", id, e),
)
})
}
/// Updates the mapping of an existing ID to a new path
async fn update_path(&self, id: &str, new_path: &StoragePath) -> Result<(), DomainError> {
self.update_path(id, new_path).await.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!(
"Failed to update path for ID: {} to {}: {}",
id,
new_path.to_string(),
e
),
)
})
}
/// Removes an ID from the map
async fn remove_id(&self, id: &str) -> Result<(), DomainError> {
self.remove_id(id).await.map_err(|e| {
DomainError::internal_error("IdMapping", format!("Failed to remove ID: {}: {}", id, e))
})
}
/// Saves pending changes to disk
async fn save_changes(&self) -> Result<(), DomainError> {
self.save_pending_changes().await.map_err(|e| {
DomainError::internal_error(
"IdMapping",
format!("Failed to save pending ID mapping changes: {}", e),
)
})
}
}
// The extension methods were moved to the IdMappingPort trait as default implementations
// Implement Clone to allow use in tokio::spawn
/// Synchronous helper for contexts where we can't use async
impl IdMappingService {
/// Create a new service synchronously (only for stubs and initialization)
pub fn new_sync(map_path: PathBuf) -> Self {
// Create a minimal implementation for initialization purposes
Self {
map_path,
id_map: RwLock::new(IdMap::default()),
save_mutex: Mutex::new(()),
timeouts: TimeoutConfig::default(),
pending_save: RwLock::new(false),
}
}
}
impl Clone for IdMappingService {
fn clone(&self) -> Self {
// We cannot directly clone the RwLock/Mutex,
// but we can create new instances that point to the same internal Arc
// However, in this case we simply need the map_path
Self {
map_path: self.map_path.clone(),
id_map: RwLock::new(IdMap::default()), // This is not used in the async task
save_mutex: Mutex::new(()), // Neither is this
timeouts: self.timeouts.clone(),
pending_save: RwLock::new(false),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
use tempfile::tempdir;
#[tokio::test]
async fn test_get_or_create_id() {
let temp_dir = tempdir().unwrap();
let map_path = temp_dir.path().join("id_map.json");
let service = IdMappingService::new(map_path).await.unwrap();
let path = StoragePath::from_string("/test/file.txt");
let id = service.get_or_create_id(&path).await.unwrap();
assert!(!id.is_empty(), "ID should not be empty");
// Verify that the same ID is returned for the same path
let id2 = service.get_or_create_id(&path).await.unwrap();
assert_eq!(id, id2, "Same path should return same ID");
}
#[tokio::test]
async fn test_update_path() {
let temp_dir = tempdir().unwrap();
let map_path = temp_dir.path().join("id_map.json");
let service = IdMappingService::new(map_path).await.unwrap();
let old_path = StoragePath::from_string("/test/old.txt");
let id = service.get_or_create_id(&old_path).await.unwrap();
let new_path = StoragePath::from_string("/test/new.txt");
service.update_path(&id, &new_path).await.unwrap();
let retrieved_path = service.get_path_by_id(&id).await.unwrap();
assert_eq!(retrieved_path, new_path, "Path should be updated");
}
#[tokio::test]
async fn test_save_and_load() {
let temp_dir = tempdir().unwrap();
let map_path = temp_dir.path().join("id_map.json");
// Create and populate the service
let service = IdMappingService::new(map_path.clone()).await.unwrap();
let path1 = StoragePath::from_string("/test/file1.txt");
let path2 = StoragePath::from_string("/test/file2.txt");
let id1 = service.get_or_create_id(&path1).await.unwrap();
let id2 = service.get_or_create_id(&path2).await.unwrap();
// Save changes
service.save_pending_changes().await.unwrap();
// Wait to ensure the async save completes
tokio::time::sleep(Duration::from_millis(500)).await;
// Create a new service that should load the same map
let service2 = IdMappingService::new(map_path).await.unwrap();
// Verify that the IDs match
let loaded_id1 = service2.get_or_create_id(&path1).await.unwrap();
let loaded_id2 = service2.get_or_create_id(&path2).await.unwrap();
assert_eq!(id1, loaded_id1, "ID1 should be preserved");
assert_eq!(id2, loaded_id2, "ID2 should be preserved");
}
#[tokio::test]
async fn test_concurrent_operations() {
use futures::future::join_all;
let temp_dir = tempdir().unwrap();
let map_path = temp_dir.path().join("id_map.json");
let service = std::sync::Arc::new(IdMappingService::new(map_path).await.unwrap());
// Create multiple tasks that attempt simultaneous access
let mut tasks = Vec::new();
for i in 0..100 {
let path = StoragePath::from_string(&format!("/test/concurrent/file{}.txt", i));
let service_clone = service.clone();
tasks.push(tokio::spawn(async move {
service_clone.get_or_create_id(&path).await
}));
}
// Wait for all to finish
let results = join_all(tasks).await;
// Verify that all succeeded
for result in results {
assert!(
result.unwrap().is_ok(),
"Concurrent operations should succeed"
);
}
// Save changes
service.save_pending_changes().await.unwrap();
}
}
src/infrastructure/services/mod.rs
-5
View File
@@ -1,13 +1,8 @@
pub mod buffer_pool;
pub mod chunked_upload_service;
pub mod compression_service;
pub mod dedup_service;
pub mod file_content_cache;
pub mod file_metadata_cache;
pub mod file_system_i18n_service;
pub mod file_system_utils;
pub mod id_mapping_optimizer;
pub mod id_mapping_service;
pub mod image_transcode_service;
pub mod jwt_service;
pub mod oidc_service;
src/lib.rs
-5
View File
@@ -8,11 +8,6 @@ pub mod interfaces;
// Common public re-exports
pub use application::services::folder_service::FolderService;
pub use application::services::i18n_application_service::I18nApplicationService;
pub use application::services::storage_mediator::{FileSystemStorageMediator, StorageMediator};
pub use domain::services::path_service::StoragePath;
pub use infrastructure::repositories::CompositeFileRepository;
pub use infrastructure::repositories::folder_fs_repository::FolderFsRepository;
pub use infrastructure::repositories::parallel_file_processor::ParallelFileProcessor;
pub use infrastructure::services::buffer_pool::BufferPool;
pub use infrastructure::services::compression_service::GzipCompressionService;
pub use infrastructure::services::path_service::PathService;