From 3c7c16f07e0f593b5df47ca18906360aba1c33be Mon Sep 17 00:00:00 2001 From: Dionisio Date: Sat, 14 Feb 2026 17:54:25 +0100 Subject: [PATCH] =?UTF-8?q?feat(#113):=20100%=20blob=20storage=20model=20?= =?UTF-8?q?=E2=80=94=20PostgreSQL=20metadata=20+=20DedupService=20blobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: Storage model completely rewritten. All file/folder metadata now lives in PostgreSQL (storage schema). File content stored as content-addressable blobs via DedupService. Filesystem directories are no longer used for user storage. New components: - storage.folders / storage.files / storage.trash_items (PG schema) - FolderDbRepository: virtual folders backed by PG - FileBlobReadRepository: file reads via PG metadata + dedup blobs - FileBlobWriteRepository: file writes via PG metadata + dedup blobs - TrashDbRepository: soft-delete trash using is_trashed flags Removed legacy FS components (~5500 lines deleted): - FolderFsRepository, FileFsReadRepository, FileFsWriteRepository - CompositeFileRepository, ParallelFileProcessor - IdMappingService, IdMappingOptimizer, FileMetadataCache - BufferPool, FileSystemUtils, RepositoryErrors - TrashFsRepository, FolderFsRepositoryTrash DI rewired: build_app_state() now requires PgPool (no FS fallback). FileUploadService.new_with_read() and FileRetrievalService.new_with_cache() constructors added for blob model (no write-behind needed). Closes #113 --- db/schema.sql | 75 ++ .../services/file_retrieval_service.rs | 15 + .../services/file_upload_service.rs | 15 + src/common/di.rs | 706 +++++------- src/common/stubs.rs | 123 -- .../repositories/composite_file_repository.rs | 157 --- .../repositories/file_fs_read_repository.rs | 473 -------- .../repositories/file_fs_write_repository.rs | 800 ------------- .../repositories/folder_fs_repository.rs | 1010 ----------------- .../folder_fs_repository_trash.rs | 192 ---- src/infrastructure/repositories/mod.rs | 21 +- .../repositories/parallel_file_processor.rs | 571 ---------- .../pg/file_blob_read_repository.rs | 271 +++++ .../pg/file_blob_write_repository.rs | 435 +++++++ .../repositories/pg/folder_db_repository.rs | 603 ++++++++++ src/infrastructure/repositories/pg/mod.rs | 10 + .../repositories/pg/trash_db_repository.rs | 191 ++++ .../repositories/repository_errors.rs | 110 -- .../repositories/trash_fs_repository.rs | 441 ------- src/infrastructure/services/buffer_pool.rs | 504 -------- .../services/compression_service.rs | 129 +-- .../services/file_metadata_cache.rs | 749 ------------ .../services/file_system_utils.rs | 326 ------ .../services/id_mapping_optimizer.rs | 663 ----------- .../services/id_mapping_service.rs | 712 ------------ src/infrastructure/services/mod.rs | 5 - src/lib.rs | 5 - 27 files changed, 1883 insertions(+), 7429 deletions(-) delete mode 100644 src/infrastructure/repositories/composite_file_repository.rs delete mode 100644 src/infrastructure/repositories/file_fs_read_repository.rs delete mode 100644 src/infrastructure/repositories/file_fs_write_repository.rs delete mode 100644 src/infrastructure/repositories/folder_fs_repository.rs delete mode 100644 src/infrastructure/repositories/folder_fs_repository_trash.rs delete mode 100644 src/infrastructure/repositories/parallel_file_processor.rs create mode 100644 src/infrastructure/repositories/pg/file_blob_read_repository.rs create mode 100644 src/infrastructure/repositories/pg/file_blob_write_repository.rs create mode 100644 src/infrastructure/repositories/pg/folder_db_repository.rs create mode 100644 src/infrastructure/repositories/pg/trash_db_repository.rs delete mode 100644 src/infrastructure/repositories/repository_errors.rs delete mode 100644 src/infrastructure/repositories/trash_fs_repository.rs delete mode 100644 src/infrastructure/services/buffer_pool.rs delete mode 100644 src/infrastructure/services/file_metadata_cache.rs delete mode 100644 src/infrastructure/services/file_system_utils.rs delete mode 100644 src/infrastructure/services/id_mapping_optimizer.rs delete mode 100644 src/infrastructure/services/id_mapping_service.rs diff --git a/db/schema.sql b/db/schema.sql index a6a5ed7e..87ec7da7 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -303,3 +303,78 @@ COMMENT ON TABLE carddav.contacts IS 'Contacts stored with vCard data for round- COMMENT ON TABLE carddav.address_book_shares IS 'Address book sharing permissions between users'; COMMENT ON TABLE carddav.contact_groups IS 'Contact groups within address books'; COMMENT ON TABLE carddav.group_memberships IS 'Many-to-many relationship between contacts and groups'; + +-- ============================================================ +-- 4. STORAGE SCHEMA — 100% Blob Storage Model +-- ============================================================ +-- All file/folder metadata lives here. Actual file content is stored +-- as content-addressable blobs on the filesystem via DedupService +-- (.blobs/{prefix}/{hash}.blob). No physical directories are created +-- for user folders — they are virtual records in this schema. +-- ============================================================ +CREATE SCHEMA IF NOT EXISTS storage; + +-- Virtual folders (replaces physical directories on disk) +CREATE TABLE IF NOT EXISTS storage.folders ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + parent_id UUID REFERENCES storage.folders(id) ON DELETE CASCADE, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + is_trashed BOOLEAN NOT NULL DEFAULT FALSE, + trashed_at TIMESTAMP WITH TIME ZONE, + original_parent_id UUID, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- A user cannot have two non-trashed folders with the same name in the same parent +CREATE UNIQUE INDEX IF NOT EXISTS idx_folders_unique_name + ON storage.folders(parent_id, name, user_id) WHERE NOT is_trashed AND parent_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_folders_unique_name_root + ON storage.folders(name, user_id) WHERE NOT is_trashed AND parent_id IS NULL; + +CREATE INDEX IF NOT EXISTS idx_folders_user_id ON storage.folders(user_id); +CREATE INDEX IF NOT EXISTS idx_folders_parent_id ON storage.folders(parent_id); +CREATE INDEX IF NOT EXISTS idx_folders_trashed ON storage.folders(user_id, is_trashed); + +-- Files as references to content-addressable blobs +CREATE TABLE IF NOT EXISTS storage.files ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + folder_id UUID REFERENCES storage.folders(id) ON DELETE SET NULL, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + blob_hash VARCHAR(64) NOT NULL, + size BIGINT NOT NULL DEFAULT 0, + mime_type TEXT NOT NULL DEFAULT 'application/octet-stream', + is_trashed BOOLEAN NOT NULL DEFAULT FALSE, + trashed_at TIMESTAMP WITH TIME ZONE, + original_folder_id UUID, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- A user cannot have two non-trashed files with the same name in the same folder +CREATE UNIQUE INDEX IF NOT EXISTS idx_files_unique_name_in_folder + ON storage.files(folder_id, name, user_id) WHERE NOT is_trashed AND folder_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_files_unique_name_at_root + ON storage.files(name, user_id) WHERE NOT is_trashed AND folder_id IS NULL; + +CREATE INDEX IF NOT EXISTS idx_files_user_id ON storage.files(user_id); +CREATE INDEX IF NOT EXISTS idx_files_folder_id ON storage.files(folder_id); +CREATE INDEX IF NOT EXISTS idx_files_blob_hash ON storage.files(blob_hash); +CREATE INDEX IF NOT EXISTS idx_files_trashed ON storage.files(user_id, is_trashed); +CREATE INDEX IF NOT EXISTS idx_files_name_search ON storage.files(user_id, name text_pattern_ops); + +-- Trash view combining trashed files and folders for the TrashRepository +CREATE OR REPLACE VIEW storage.trash_items AS + SELECT id, name, 'file' AS item_type, user_id, trashed_at, + original_folder_id AS original_parent_id, created_at + FROM storage.files WHERE is_trashed = TRUE + UNION ALL + SELECT id, name, 'folder' AS item_type, user_id, trashed_at, + original_parent_id, created_at + FROM storage.folders WHERE is_trashed = TRUE; + +COMMENT ON TABLE storage.folders IS 'Virtual folder hierarchy — no physical directories on disk'; +COMMENT ON TABLE storage.files IS 'File metadata pointing to content-addressable blobs'; +COMMENT ON VIEW storage.trash_items IS 'Unified view of all trashed files and folders'; diff --git a/src/application/services/file_retrieval_service.rs b/src/application/services/file_retrieval_service.rs index d832cf3d..12d3679a 100644 --- a/src/application/services/file_retrieval_service.rs +++ b/src/application/services/file_retrieval_service.rs @@ -56,6 +56,21 @@ impl FileRetrievalService { } } + /// Constructor for blob-storage model: read + content cache + transcode. + /// No write-behind needed — dedup handled at the repository layer. + pub fn new_with_cache( + file_read: Arc, + content_cache: Arc, + transcode: Arc, + ) -> Self { + Self { + file_read, + write_behind: None, + content_cache: Some(content_cache), + transcode: Some(transcode), + } + } + // ── private helpers ────────────────────────────────────────── /// Try to transcode image content to WebP and return transcoded variant. diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 14d0f808..3387aa58 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -87,6 +87,21 @@ impl FileUploadService { } } + /// Constructor for blob-storage model: write + read ports only. + /// Dedup is handled at the repository layer — no write-behind needed. + pub fn new_with_read( + file_write: Arc, + file_read: Arc, + ) -> Self { + Self { + file_write, + file_read: Some(file_read), + write_behind: None, + dedup: None, + storage_usage_service: None, + } + } + /// Configures the storage usage service pub fn with_storage_usage_service( mut self, diff --git a/src/common/di.rs b/src/common/di.rs index e6233c0f..a439be5d 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -1,65 +1,54 @@ -use sqlx::PgPool; use std::path::PathBuf; use std::sync::Arc; +use sqlx::PgPool; -use crate::application::services::admin_settings_service::AdminSettingsService; use crate::application::services::auth_application_service::AuthApplicationService; +use crate::application::services::admin_settings_service::AdminSettingsService; -use crate::application::ports::cache_ports::{ContentCachePort, WriteBehindCachePort}; -use crate::application::ports::chunked_upload_ports::ChunkedUploadPort; -use crate::application::ports::compression_ports::CompressionPort; -use crate::application::ports::dedup_ports::DedupPort; -use crate::application::ports::favorites_ports::FavoritesUseCase; -use crate::application::ports::file_ports::{ - FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, FileUseCaseFactory, +use crate::infrastructure::services::path_service::PathService; +use crate::infrastructure::repositories::share_fs_repository::ShareFsRepository; +use crate::infrastructure::repositories::pg::{ + FolderDbRepository, FileBlobReadRepository, FileBlobWriteRepository, TrashDbRepository, }; -use crate::application::ports::inbound::{FolderUseCase, SearchUseCase}; -use crate::application::ports::outbound::FolderStoragePort; -use crate::application::ports::recent_ports::RecentItemsUseCase; -use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; -use crate::application::ports::thumbnail_ports::ThumbnailPort; -use crate::application::ports::transcode_ports::ImageTranscodePort; -use crate::application::ports::trash_ports::TrashUseCase; -use crate::application::ports::zip_ports::ZipPort; -use crate::application::services::favorites_service::FavoritesService; +use crate::infrastructure::services::file_system_i18n_service::FileSystemI18nService; +use crate::infrastructure::services::file_content_cache::{FileContentCache, FileContentCacheConfig}; +use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService; use crate::application::services::folder_service::FolderService; use crate::application::services::i18n_application_service::I18nApplicationService; -use crate::application::services::recent_service::RecentService; +use crate::application::services::trash_service::TrashService; use crate::application::services::search_service::SearchService; use crate::application::services::share_service::ShareService; -use crate::application::services::storage_mediator::{FileSystemStorageMediator, StorageMediator}; -use crate::application::services::trash_service::TrashService; -use crate::application::services::{ - AppFileUseCaseFactory, FileManagementService, FileRetrievalService, FileUploadService, -}; -use crate::common::config::AppConfig; +use crate::application::services::favorites_service::FavoritesService; +use crate::application::services::recent_service::RecentService; +use crate::application::ports::trash_ports::TrashUseCase; +use crate::application::ports::inbound::{FolderUseCase, SearchUseCase}; +use crate::application::ports::outbound::FolderStoragePort; +use crate::application::ports::favorites_ports::FavoritesUseCase; +use crate::application::ports::recent_ports::RecentItemsUseCase; +use crate::application::ports::file_ports::{FileUploadUseCase, FileRetrievalUseCase, FileManagementUseCase, FileUseCaseFactory}; +use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; +use crate::application::services::{FileUploadService, FileRetrievalService, FileManagementService, AppFileUseCaseFactory}; use crate::common::errors::DomainError; use crate::domain::services::i18n_service::I18nService; -use crate::infrastructure::repositories::folder_fs_repository::FolderFsRepository; -use crate::infrastructure::repositories::parallel_file_processor::ParallelFileProcessor; -use crate::infrastructure::repositories::share_fs_repository::ShareFsRepository; -use crate::infrastructure::repositories::trash_fs_repository::TrashFsRepository; -use crate::infrastructure::repositories::{FileFsReadRepository, FileFsWriteRepository}; -use crate::infrastructure::services::buffer_pool::BufferPool; -use crate::infrastructure::services::file_content_cache::{ - FileContentCache, FileContentCacheConfig, -}; -use crate::infrastructure::services::file_metadata_cache::FileMetadataCache; -use crate::infrastructure::services::file_system_i18n_service::FileSystemI18nService; -use crate::infrastructure::services::id_mapping_optimizer::IdMappingOptimizer; -use crate::infrastructure::services::id_mapping_service::IdMappingService; -use crate::infrastructure::services::path_service::PathService; -use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService; +use crate::common::config::AppConfig; +use crate::application::ports::cache_ports::ContentCachePort; +use crate::application::ports::thumbnail_ports::ThumbnailPort; +use crate::application::ports::transcode_ports::ImageTranscodePort; +use crate::application::ports::dedup_ports::DedupPort; +use crate::application::ports::chunked_upload_ports::ChunkedUploadPort; +use crate::application::ports::compression_ports::CompressionPort; +use crate::application::ports::zip_ports::ZipPort; use crate::common::stubs::{ - StubCompressionPort, StubFileManagementUseCase, StubFileReadPort, StubFileRetrievalUseCase, - StubFileUploadUseCase, StubFileUseCaseFactory, StubFileWritePort, StubFolderStoragePort, - StubFolderUseCase, StubI18nService, StubIdMappingService, StubSearchUseCase, - StubStorageMediator, StubZipPort, + StubZipPort, StubCompressionPort, + StubFileReadPort, StubFileWritePort, StubFolderStoragePort, + StubI18nService, StubFolderUseCase, StubFileUploadUseCase, + StubFileRetrievalUseCase, StubFileManagementUseCase, StubFileUseCaseFactory, + StubSearchUseCase, }; /// Factory for the different application components -/// +/// /// This factory centralizes the creation of all application services, /// ensuring the correct initialization order and resolving circular dependencies. pub struct AppServiceFactory { @@ -77,7 +66,7 @@ impl AppServiceFactory { config: AppConfig::default(), } } - + /// Creates a new service factory with custom configuration pub fn with_config(storage_path: PathBuf, locales_path: PathBuf, config: AppConfig) -> Self { Self { @@ -86,205 +75,131 @@ impl AppServiceFactory { config, } } - + /// Gets the configuration pub fn config(&self) -> &AppConfig { &self.config } - + /// Gets the storage path pub fn storage_path(&self) -> &PathBuf { &self.storage_path } - + /// Initializes the core system services pub async fn create_core_services(&self) -> Result { - // Path service + // Path service (still needed for blob storage root + thumbnails) let path_service = Arc::new(PathService::new(self.storage_path.clone())); - + // File content cache for ultra-fast file serving (hot files in RAM) let file_content_cache = Arc::new(FileContentCache::new(FileContentCacheConfig { - max_file_size: 10 * 1024 * 1024, // 10MB max per file - max_total_size: 512 * 1024 * 1024, // 512MB total cache - max_entries: 10000, // Up to 10k files + max_file_size: 10 * 1024 * 1024, // 10MB max per file + max_total_size: 512 * 1024 * 1024, // 512MB total cache + max_entries: 10000, // Up to 10k files })); tracing::info!("FileContentCache initialized: max 10MB/file, 512MB total, 10k entries"); - - // ID mapping service for folders - let folder_id_mapping_path = self.storage_path.join("folder_ids.json"); - let folder_id_mapping_service = - Arc::new(IdMappingService::new(folder_id_mapping_path).await?); - - // ID mapping service for files - let file_id_mapping_path = self.storage_path.join("file_ids.json"); - let file_id_mapping_service = Arc::new(IdMappingService::new(file_id_mapping_path).await?); - - // Optimizer with batch processing and caching - let id_mapping_optimizer = - Arc::new(IdMappingOptimizer::new(folder_id_mapping_service.clone())); - - // Start optimizer cleanup task - IdMappingOptimizer::start_cleanup_task(id_mapping_optimizer.clone()); - + // Thumbnail service for thumbnail generation let thumbnail_service = Arc::new( crate::infrastructure::services::thumbnail_service::ThumbnailService::new( &self.storage_path, - 5000, // max 5000 thumbnails in cache - 100 * 1024 * 1024, // max 100MB cache - ), + 5000, // max 5000 thumbnails in cache + 100 * 1024 * 1024, // max 100MB cache + ) ); // Initialize thumbnail directories thumbnail_service.initialize().await?; - - // Write-behind cache for instant uploads of small files - let write_behind_cache = - crate::infrastructure::services::write_behind_cache::WriteBehindCache::new(); - + // Chunked upload service for large files (>10MB) let chunked_temp_dir = std::path::PathBuf::from(&self.storage_path).join(".uploads"); let chunked_upload_service = Arc::new( - crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new( - chunked_temp_dir, - ), + crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new(chunked_temp_dir) ); - + // Image transcoding service for automatic WebP conversion let image_transcode_service = Arc::new( crate::infrastructure::services::image_transcode_service::ImageTranscodeService::new( &self.storage_path, - 2000, // max 2000 transcoded images in cache - 50 * 1024 * 1024, // max 50MB in-memory cache - ), + 2000, // max 2000 transcoded images in cache + 50 * 1024 * 1024, // max 50MB in-memory cache + ) ); image_transcode_service.initialize().await?; - - // Deduplication service for removing duplicate files + + // Deduplication service — PRIMARY blob storage engine let dedup_service = Arc::new( - crate::infrastructure::services::dedup_service::DedupService::new(&self.storage_path), + crate::infrastructure::services::dedup_service::DedupService::new(&self.storage_path) ); dedup_service.initialize().await?; - + // Compression service (gzip) let compression_service: Arc = Arc::new( - crate::infrastructure::services::compression_service::GzipCompressionService::new(), + crate::infrastructure::services::compression_service::GzipCompressionService::new() ); - - tracing::info!( - "Core services initialized: path service, cache manager, file content cache, ID mapping, thumbnails, write-behind cache, chunked upload, image transcode, dedup, compression" - ); - - // NOTE: zip_service requires ApplicationServices (FileRetrievalUseCase, FolderUseCase) which are - // created later. It will be set via AppState::with_zip_service() after application services are ready. - // For now we use a placeholder that will be replaced. - + + tracing::info!("Core services initialized: path service, file content cache, thumbnails, chunked upload, image transcode, dedup (PRIMARY blob storage), compression"); + Ok(CoreServices { path_service, file_content_cache, - id_mapping_service: folder_id_mapping_service, - file_id_mapping_service, - id_mapping_optimizer, thumbnail_service, - write_behind_cache, chunked_upload_service, image_transcode_service, dedup_service, compression_service, - zip_service: Arc::new(StubZipPort), // Placeholder - replaced after app services init + zip_service: Arc::new(StubZipPort), // Placeholder - replaced after app services init config: self.config.clone(), }) } - - /// Initializes the repository services - pub fn create_repository_services(&self, core: &CoreServices) -> RepositoryServices { - // Storage mediator - uses stub initially, will be replaced after folder repo is ready - let storage_mediator_stub: Arc = - Arc::new(FileSystemStorageMediator::new_stub()); - - // Folder repository — implements FolderStoragePort directly - let folder_repository = Arc::new(FolderFsRepository::new( - self.storage_path.clone(), - storage_mediator_stub.clone(), - core.id_mapping_service.clone(), - core.path_service.clone(), + + /// Initializes the repository services (blob-storage model). + /// + /// Requires a PgPool since all metadata lives in PostgreSQL. + pub fn create_repository_services(&self, core: &CoreServices, db_pool: &Arc) -> RepositoryServices { + // Folder repository — PostgreSQL-backed virtual folders + let folder_repo_concrete = Arc::new(FolderDbRepository::new(db_pool.clone())); + let folder_repository: Arc = folder_repo_concrete.clone(); + + // File repositories — PostgreSQL metadata + blob content via DedupService + let file_read_repository: Arc = Arc::new(FileBlobReadRepository::new( + db_pool.clone(), + core.dedup_service.clone(), + folder_repo_concrete.clone(), )); - - // Now create the real storage mediator with the folder repo (as FolderStoragePort) - let storage_mediator: Arc = Arc::new(FileSystemStorageMediator::new( - folder_repository.clone() as Arc, - core.path_service.clone(), - core.id_mapping_optimizer.clone(), + + let file_write_repository: Arc = Arc::new(FileBlobWriteRepository::new( + db_pool.clone(), + core.dedup_service.clone(), + folder_repo_concrete.clone(), )); - - // Metadata cache - let metadata_cache = Arc::new(FileMetadataCache::default_with_config(core.config.clone())); - - // Start metadata cache cleanup task - let cache_clone = metadata_cache.clone(); - tokio::spawn(async move { - FileMetadataCache::start_cleanup_task(cache_clone).await; - }); - - // Buffer pool for memory optimization - let buffer_pool = BufferPool::new(256 * 1024, 50, 120); // 256KB buffers, 50 max, 2 min TTL - BufferPool::start_cleaner(buffer_pool.clone()); - - // Parallel file processor - let parallel_processor = Arc::new(ParallelFileProcessor::new_with_buffer_pool( - core.config.clone(), - buffer_pool.clone(), - )); - - // Separate file repositories for reading and writing - let file_read_repository = Arc::new(FileFsReadRepository::new( - self.storage_path.clone(), - storage_mediator.clone(), - core.file_id_mapping_service.clone(), - core.path_service.clone(), - metadata_cache.clone(), - core.config.clone(), - Some(parallel_processor.clone()), - )); - - let file_write_repository = Arc::new(FileFsWriteRepository::new( - self.storage_path.clone(), - storage_mediator.clone(), - core.file_id_mapping_service.clone(), - core.path_service.clone(), - metadata_cache.clone(), - core.config.clone(), - Some(parallel_processor.clone()), - )); - + // I18n repository - let i18n_repository = Arc::new(FileSystemI18nService::new(self.locales_path.clone())); - - // Trash repository + let i18n_repository = Arc::new(FileSystemI18nService::new( + self.locales_path.clone() + )); + + // Trash repository — reads soft-delete flags from storage.files/folders let trash_repository = if core.config.features.enable_trash { - Some(Arc::new(TrashFsRepository::new( - self.storage_path.as_path(), - core.id_mapping_service.clone(), - )) - as Arc< - dyn crate::domain::repositories::trash_repository::TrashRepository, - >) + Some(Arc::new(TrashDbRepository::new( + db_pool.clone(), + core.config.storage.trash_retention_days, + )) as Arc) } else { None }; - - tracing::info!("Repository services initialized with parallel processing and buffer pool"); - + + tracing::info!("Repository services initialized with 100% blob storage model (PG metadata + DedupService blobs)"); + RepositoryServices { folder_repository, + folder_repo_concrete, file_read_repository, file_write_repository, i18n_repository, - storage_mediator, - metadata_cache, trash_repository, } } - + /// Initializes the application services pub fn create_application_services( &self, @@ -293,23 +208,23 @@ impl AppServiceFactory { trash_service: Option>, ) -> ApplicationServices { // Main services - let folder_service = Arc::new(FolderService::new(repos.folder_repository.clone())); - + let folder_service = Arc::new(FolderService::new( + repos.folder_repository.clone() + )); + // Refactored services with all infrastructure ports - let file_upload_service = Arc::new(FileUploadService::new_full( + // In blob model, dedup is handled by the repository — no separate write-behind needed + let file_upload_service = Arc::new(FileUploadService::new_with_read( repos.file_write_repository.clone(), repos.file_read_repository.clone(), - core.write_behind_cache.clone(), - core.dedup_service.clone(), )); - - let file_retrieval_service = Arc::new(FileRetrievalService::new_full( + + let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( repos.file_read_repository.clone(), - core.write_behind_cache.clone(), core.file_content_cache.clone(), core.image_transcode_service.clone(), )); - + // FileManagementService with dedup and trash let file_management_service = Arc::new(FileManagementService::new_full( repos.file_write_repository.clone(), @@ -317,24 +232,26 @@ impl AppServiceFactory { trash_service.clone(), core.dedup_service.clone(), )); - + let file_use_case_factory = Arc::new(AppFileUseCaseFactory::new( repos.file_read_repository.clone(), - repos.file_write_repository.clone(), + repos.file_write_repository.clone() )); - - let i18n_service = Arc::new(I18nApplicationService::new(repos.i18n_repository.clone())); - + + let i18n_service = Arc::new(I18nApplicationService::new( + repos.i18n_repository.clone() + )); + // Search service with cache let search_service: Option> = Some(Arc::new(SearchService::new( repos.file_read_repository.clone(), repos.folder_repository.clone(), - 300, // Cache TTL in seconds (5 minutes) + 300, // Cache TTL in seconds (5 minutes) 1000, // Maximum cache entries ))); - + tracing::info!("Application services initialized"); - + ApplicationServices { // Concrete types for handlers that need them folder_service_concrete: folder_service.clone(), @@ -347,12 +264,12 @@ impl AppServiceFactory { i18n_service, trash_service, // Already set via parameter search_service, - share_service: None, // Configured later with create_share_service + share_service: None, // Configured later with create_share_service favorites_service: None, // Configured later with create_favorites_service - recent_service: None, // Configured later with create_recent_service + recent_service: None, // Configured later with create_recent_service } } - + /// Creates the trash service pub async fn create_trash_service( &self, @@ -362,9 +279,9 @@ impl AppServiceFactory { tracing::info!("Trash service is disabled in configuration"); return None; } - + let trash_repo = repos.trash_repository.as_ref()?; - + // Wire ports directly to TrashService — no adapter layer needed let service = Arc::new(TrashService::new( trash_repo.clone(), @@ -373,20 +290,20 @@ impl AppServiceFactory { repos.folder_repository.clone(), self.config.storage.trash_retention_days, )); - + // Initialize cleanup service let cleanup_service = TrashCleanupService::new( service.clone(), trash_repo.clone(), 24, // Run cleanup every 24 hours ); - + cleanup_service.start_cleanup_job().await; tracing::info!("Trash service initialized with daily cleanup schedule"); - + Some(service as Arc) } - + /// Creates the sharing service pub fn create_share_service( &self, @@ -396,9 +313,11 @@ impl AppServiceFactory { tracing::info!("File sharing service is disabled in configuration"); return None; } - - let share_repository = Arc::new(ShareFsRepository::new(Arc::new(self.config.clone()))); - + + let share_repository = Arc::new(ShareFsRepository::new( + Arc::new(self.config.clone()) + )); + // Build a password hasher for share password verification let password_hasher: Arc = Arc::new(crate::infrastructure::services::password_hasher::Argon2PasswordHasher::new()); @@ -410,37 +329,44 @@ impl AppServiceFactory { repos.folder_repository.clone(), password_hasher, )); - + tracing::info!("File sharing service initialized"); Some(service) } - + /// Creates the favorites service (requires database) - pub fn create_favorites_service(&self, db_pool: &Arc) -> Arc { + pub fn create_favorites_service( + &self, + db_pool: &Arc, + ) -> Arc { let repo = Arc::new( - crate::infrastructure::repositories::pg::FavoritesPgRepository::new(db_pool.clone()), + crate::infrastructure::repositories::pg::FavoritesPgRepository::new(db_pool.clone()) ); let service = Arc::new(FavoritesService::new(repo)); tracing::info!("Favorites service initialized"); service } - + /// Creates the recent items service (requires database) - pub fn create_recent_service(&self, db_pool: &Arc) -> Arc { + pub fn create_recent_service( + &self, + db_pool: &Arc, + ) -> Arc { let repo = Arc::new( - crate::infrastructure::repositories::pg::RecentItemsPgRepository::new(db_pool.clone()), + crate::infrastructure::repositories::pg::RecentItemsPgRepository::new(db_pool.clone()) ); let service = Arc::new(RecentService::new( - repo, 50, // Maximum recent items per user + repo, + 50 // Maximum recent items per user )); tracing::info!("Recent items service initialized"); service } - + /// Preloads translations pub async fn preload_translations(&self, i18n_service: &I18nApplicationService) { use crate::domain::services::i18n_service::Locale; - + if let Err(e) = i18n_service.load_translations(Locale::English).await { tracing::warn!("Failed to load English translations: {}", e); } @@ -459,17 +385,6 @@ impl AppServiceFactory { tracing::info!("Translations preloaded"); } - /// Preloads directories into cache - pub async fn preload_cache(&self, metadata_cache: &FileMetadataCache) { - tracing::info!("Preloading common directories to warm up cache..."); - if let Ok(count) = metadata_cache - .preload_directory(&self.storage_path, true, 1) - .await - { - tracing::info!("Preloaded {} directory entries into cache", count); - } - } - /// Creates the storage usage service (requires database) pub fn create_storage_usage_service( &self, @@ -477,13 +392,13 @@ impl AppServiceFactory { db_pool: &Arc, ) -> Arc { let user_repository = Arc::new( - crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone()), + crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone()) ); let service = Arc::new( crate::application::services::storage_usage_service::StorageUsageService::new( repos.file_read_repository.clone(), user_repository, - ), + ) ); tracing::info!("Storage usage service initialized"); service @@ -496,11 +411,16 @@ impl AppServiceFactory { &self, db_pool: Option>, ) -> Result { + // Database is REQUIRED in 100% blob storage model + let pool = db_pool.clone().ok_or_else(|| { + DomainError::internal_error("Database", "PostgreSQL database is required for blob storage model") + })?; + // 1. Core services let core = self.create_core_services().await?; - // 2. Repository services - let repos = self.create_repository_services(&core); + // 2. Repository services (requires PgPool for all metadata) + let repos = self.create_repository_services(&core, &pool); // 3. Trash service (needed before application services) let trash_service = self.create_trash_service(&repos).await; @@ -512,24 +432,22 @@ impl AppServiceFactory { let share_service = self.create_share_service(&repos); apps.share_service = share_service.clone(); - // 6. Database-dependent services - let mut favorites_service: Option> = None; - let mut recent_service: Option> = None; - let mut storage_usage_service: Option< - Arc, - > = None; + // 6. Database-dependent services (PgPool always available in blob model) + let favorites_service: Option>; + let recent_service: Option>; + let storage_usage_service: Option>; let mut auth_services: Option = None; - if let Some(ref pool) = db_pool { - let favs = self.create_favorites_service(pool); + { + let favs = self.create_favorites_service(&pool); favorites_service = Some(favs.clone()); apps.favorites_service = Some(favs); - let recent = self.create_recent_service(pool); + let recent = self.create_recent_service(&pool); recent_service = Some(recent.clone()); apps.recent_service = Some(recent); - storage_usage_service = Some(self.create_storage_usage_service(&repos, pool)); + storage_usage_service = Some(self.create_storage_usage_service(&repos, &pool)); // Auth services if self.config.features.enable_auth { @@ -537,9 +455,7 @@ impl AppServiceFactory { &self.config, pool.clone(), Some(apps.folder_service_concrete.clone()), - ) - .await - { + ).await { Ok(services) => { tracing::info!("Authentication services initialized successfully"); auth_services = Some(services); @@ -554,20 +470,17 @@ impl AppServiceFactory { // 7. Preload translations self.preload_translations(&apps.i18n_service).await; - // 8. Preload cache - self.preload_cache(&repos.metadata_cache).await; - - // 9. Build the ZipService with real application services + // 8. Build the ZipService with real application services let zip_service: Arc = Arc::new( crate::infrastructure::services::zip_service::ZipService::new( apps.file_retrieval_service.clone(), apps.folder_service.clone(), - ), + ) ); let mut core = core; core.zip_service = zip_service; - // 10. Assemble final AppState + // 9. Assemble final AppState let mut app_state = AppState { core, repositories: repos, @@ -586,11 +499,11 @@ impl AppServiceFactory { addressbook_use_case: None, contact_use_case: None, }; - - // 10b. Wire admin settings service when auth + DB are available - if let (Some(auth_svc), Some(pool)) = (&app_state.auth_service, &db_pool) { + + // 9b. Wire admin settings service when auth is available + if let Some(auth_svc) = &app_state.auth_service { let settings_repo = Arc::new( - crate::infrastructure::repositories::pg::SettingsPgRepository::new(pool.clone()), + crate::infrastructure::repositories::pg::SettingsPgRepository::new(pool.clone()) ); let server_base_url = self.config.base_url(); @@ -606,50 +519,34 @@ impl AppServiceFactory { // Hot-reload OIDC from DB settings if configured match admin_svc.load_effective_oidc_config().await { - Ok(eff) - if eff.enabled - && !eff.issuer_url.is_empty() - && !eff.client_id.is_empty() - && !eff.client_secret.is_empty() => + Ok(eff) if eff.enabled && !eff.issuer_url.is_empty() + && !eff.client_id.is_empty() && !eff.client_secret.is_empty() => { let oidc_svc = Arc::new( - crate::infrastructure::services::oidc_service::OidcService::new( - eff.clone(), - ), + crate::infrastructure::services::oidc_service::OidcService::new(eff.clone()) ); auth_svc.auth_application_service.reload_oidc(oidc_svc, eff); tracing::info!("OIDC config loaded from admin settings (database)"); } Ok(_) => { - tracing::info!( - "No active OIDC config in admin settings — using env vars or defaults" - ); + tracing::info!("No active OIDC config in admin settings — using env vars or defaults"); } Err(e) => { - tracing::warn!( - "Failed to load OIDC settings from database (table may not exist yet): {}", - e - ); + tracing::warn!("Failed to load OIDC settings from database (table may not exist yet): {}", e); } } app_state.admin_settings_service = Some(admin_svc); } - - // 11. Wire CalDAV/CardDAV services when database is available - if let Some(ref pool) = db_pool { + + // 10. Wire CalDAV/CardDAV services + { // CalDAV - let calendar_repo: Arc< - dyn crate::domain::repositories::calendar_repository::CalendarRepository, - > = Arc::new( - crate::infrastructure::repositories::pg::CalendarPgRepository::new(pool.clone()), + let calendar_repo: Arc = Arc::new( + crate::infrastructure::repositories::pg::CalendarPgRepository::new(pool.clone()) ); - let event_repo: Arc< - dyn crate::domain::repositories::calendar_event_repository::CalendarEventRepository, - > = Arc::new( - crate::infrastructure::repositories::pg::CalendarEventPgRepository::new( - pool.clone(), - ), + let event_repo: Arc = Arc::new( + crate::infrastructure::repositories::pg::CalendarEventPgRepository::new(pool.clone()) ); let calendar_storage = Arc::new( crate::infrastructure::adapters::calendar_storage_adapter::CalendarStorageAdapter::new( @@ -658,32 +555,19 @@ impl AppServiceFactory { ) ); let calendar_service = Arc::new( - crate::application::services::calendar_service::CalendarService::new( - calendar_storage, - ), + crate::application::services::calendar_service::CalendarService::new(calendar_storage) ); - app_state.calendar_use_case = Some( - calendar_service - as Arc, - ); - + app_state.calendar_use_case = Some(calendar_service as Arc); + // CardDAV - let address_book_repo: Arc< - dyn crate::domain::repositories::address_book_repository::AddressBookRepository, - > = Arc::new( - crate::infrastructure::repositories::pg::AddressBookPgRepository::new(pool.clone()), + let address_book_repo: Arc = Arc::new( + crate::infrastructure::repositories::pg::AddressBookPgRepository::new(pool.clone()) ); - let contact_repo: Arc< - dyn crate::domain::repositories::contact_repository::ContactRepository, - > = Arc::new( - crate::infrastructure::repositories::pg::ContactPgRepository::new(pool.clone()), + let contact_repo: Arc = Arc::new( + crate::infrastructure::repositories::pg::ContactPgRepository::new(pool.clone()) ); - let group_repo: Arc< - dyn crate::domain::repositories::contact_repository::ContactGroupRepository, - > = Arc::new( - crate::infrastructure::repositories::pg::ContactGroupPgRepository::new( - pool.clone(), - ), + let group_repo: Arc = Arc::new( + crate::infrastructure::repositories::pg::ContactGroupPgRepository::new(pool.clone()) ); let contact_storage = Arc::new( crate::infrastructure::adapters::contact_storage_adapter::ContactStorageAdapter::new( @@ -692,13 +576,9 @@ impl AppServiceFactory { group_repo, ) ); - app_state.addressbook_use_case = Some(contact_storage.clone() - as Arc); - app_state.contact_use_case = Some( - contact_storage - as Arc, - ); - + app_state.addressbook_use_case = Some(contact_storage.clone() as Arc); + app_state.contact_use_case = Some(contact_storage as Arc); + tracing::info!("CalDAV and CardDAV services initialized with PostgreSQL repositories"); } @@ -711,11 +591,7 @@ impl AppServiceFactory { pub struct CoreServices { pub path_service: Arc, pub file_content_cache: Arc, - pub id_mapping_service: Arc, - pub file_id_mapping_service: Arc, - pub id_mapping_optimizer: Arc, pub thumbnail_service: Arc, - pub write_behind_cache: Arc, pub chunked_upload_service: Arc, pub image_transcode_service: Arc, pub dedup_service: Arc, @@ -728,13 +604,11 @@ pub struct CoreServices { #[derive(Clone)] pub struct RepositoryServices { pub folder_repository: Arc, + pub folder_repo_concrete: Arc, pub file_read_repository: Arc, pub file_write_repository: Arc, pub i18n_repository: Arc, - pub storage_mediator: Arc, - pub metadata_cache: Arc, - pub trash_repository: - Option>, + pub trash_repository: Option>, } /// Container for application services @@ -776,50 +650,32 @@ pub struct AppState { pub share_service: Option>, pub favorites_service: Option>, pub recent_service: Option>, - pub storage_usage_service: - Option>, + pub storage_usage_service: Option>, pub calendar_service: Option>, pub contact_service: Option>, - pub calendar_use_case: - Option>, - pub addressbook_use_case: - Option>, + pub calendar_use_case: Option>, + pub addressbook_use_case: Option>, pub contact_use_case: Option>, } impl Default for AppState { fn default() -> Self { // Minimal stub version for auth middleware and route construction. - // Real services are wired in main.rs via AppServiceFactory or manual init. + // Real services are wired in main.rs via AppServiceFactory. let config = crate::common::config::AppConfig::default(); let path_service = Arc::new( crate::infrastructure::services::path_service::PathService::new( - std::path::PathBuf::from("./storage"), - ), + std::path::PathBuf::from("./storage") + ) ); - // Create service instances from the stubs module - let id_mapping_service = Arc::new(StubIdMappingService) - as Arc; - let storage_mediator = Arc::new(StubStorageMediator) - as Arc; - let i18n_repository = Arc::new(StubI18nService) - as Arc; - let folder_service = Arc::new(StubFolderUseCase) - as Arc; - let file_upload_service = Arc::new(StubFileUploadUseCase) - as Arc; - let file_retrieval_service = Arc::new(StubFileRetrievalUseCase) - as Arc; - let file_management_service = Arc::new(StubFileManagementUseCase) - as Arc; - let file_use_case_factory = Arc::new(StubFileUseCaseFactory) - as Arc; - - // Create dummy ID mapping service for files - let dummy_file_id_mapping = Arc::new(IdMappingService::dummy()); - let dummy_id_optimizer = Arc::new(IdMappingOptimizer::new(dummy_file_id_mapping.clone())); + let i18n_repository = Arc::new(StubI18nService) as Arc; + let folder_service = Arc::new(StubFolderUseCase) as Arc; + let file_upload_service = Arc::new(StubFileUploadUseCase) as Arc; + let file_retrieval_service = Arc::new(StubFileRetrievalUseCase) as Arc; + let file_management_service = Arc::new(StubFileManagementUseCase) as Arc; + let file_use_case_factory = Arc::new(StubFileUseCaseFactory) as Arc; // Create file content cache for stub let file_content_cache = Arc::new(FileContentCache::new(FileContentCacheConfig::default())); @@ -830,18 +686,14 @@ impl Default for AppState { &std::path::PathBuf::from("./storage"), 100, 10 * 1024 * 1024, - ), + ) ); - // Create dummy write-behind cache - let dummy_write_behind_cache: Arc = - crate::infrastructure::services::write_behind_cache::WriteBehindCache::new(); - // Create dummy chunked upload service let dummy_chunked_upload_service: Arc = Arc::new( crate::infrastructure::services::chunked_upload_service::ChunkedUploadService::new( - std::path::PathBuf::from("./storage/.uploads"), - ), + std::path::PathBuf::from("./storage/.uploads") + ) ); // Create dummy image transcode service @@ -850,25 +702,21 @@ impl Default for AppState { &std::path::PathBuf::from("./storage"), 100, 10 * 1024 * 1024, - ), + ) ); // Create dummy dedup service let dummy_dedup_service: Arc = Arc::new( crate::infrastructure::services::dedup_service::DedupService::new( - &std::path::PathBuf::from("./storage"), - ), + &std::path::PathBuf::from("./storage") + ) ); // Core services using stubs let core_services = CoreServices { path_service: path_service.clone(), file_content_cache, - id_mapping_service: id_mapping_service.clone(), - file_id_mapping_service: dummy_file_id_mapping, - id_mapping_optimizer: dummy_id_optimizer, thumbnail_service: dummy_thumbnail_service, - write_behind_cache: dummy_write_behind_cache, chunked_upload_service: dummy_chunked_upload_service, image_transcode_service: dummy_image_transcode_service, dedup_service: dummy_dedup_service, @@ -877,34 +725,28 @@ impl Default for AppState { config: config.clone(), }; - // Dummy metadata cache - let dummy_metadata_cache = Arc::new(FileMetadataCache::default_with_config(config.clone())); + // Stub folder repo concrete — uses a dummy PgPool-less constructor + // In practice, Default is only used for routing stubs; real app always goes through build_app_state + let dummy_folder_repo_concrete = Arc::new(FolderDbRepository::new_stub()); // Repository services using stubs let repository_services = RepositoryServices { - folder_repository: Arc::new(StubFolderStoragePort) - as Arc, - file_read_repository: Arc::new(StubFileReadPort) - as Arc, - file_write_repository: Arc::new(StubFileWritePort) - as Arc, + folder_repository: Arc::new(StubFolderStoragePort) as Arc, + folder_repo_concrete: dummy_folder_repo_concrete, + file_read_repository: Arc::new(StubFileReadPort) as Arc, + file_write_repository: Arc::new(StubFileWritePort) as Arc, i18n_repository, - storage_mediator: storage_mediator.clone(), - metadata_cache: dummy_metadata_cache, trash_repository: None, }; // Dummy concrete services for compatibility - let dummy_folder_storage = Arc::new(StubFolderStoragePort) - as Arc; + let dummy_folder_storage = Arc::new(StubFolderStoragePort) as Arc; let folder_service_concrete = Arc::new(FolderService::new(dummy_folder_storage)); // Dummy I18nApplicationService - let dummy_i18n_app_service = - crate::application::services::i18n_application_service::I18nApplicationService::new( - Arc::new(StubI18nService) - as Arc, - ); + let dummy_i18n_app_service = crate::application::services::i18n_application_service::I18nApplicationService::new( + Arc::new(StubI18nService) as Arc + ); // Application services using stubs let application_services = ApplicationServices { @@ -916,8 +758,7 @@ impl Default for AppState { file_use_case_factory, i18n_service: Arc::new(dummy_i18n_app_service), trash_service: None, - search_service: Some(Arc::new(StubSearchUseCase) - as Arc), + search_service: Some(Arc::new(StubSearchUseCase) as Arc), share_service: None, favorites_service: None, recent_service: None, @@ -969,12 +810,12 @@ impl AppState { contact_use_case: None, } } - + pub fn with_database(mut self, db_pool: Arc) -> Self { self.db_pool = Some(db_pool); self } - + /// Creates a minimal AppState for route construction. /// /// Uses `Default` stubs for infrastructure services, then overlays the real @@ -982,15 +823,11 @@ impl AppState { /// This keeps `routes.rs` free of any `crate::infrastructure` references. pub fn for_routing( folder_service: Arc, - file_retrieval_service: Arc< - dyn crate::application::ports::file_ports::FileRetrievalUseCase, - >, + file_retrieval_service: Arc, file_upload_service: Arc, file_management_service: Arc, folder_use_case: Arc, - i18n_service: Option< - Arc, - >, + i18n_service: Option>, trash_service: Option>, search_service: Option>, share_service: Option>, @@ -998,121 +835,98 @@ impl AppState { recent_service: Option>, ) -> Self { let mut state = Self::default(); - + // Override application services with real ones state.applications.folder_service_concrete = folder_service.clone(); state.applications.folder_service = folder_use_case; state.applications.file_upload_service = file_upload_service; state.applications.file_retrieval_service = file_retrieval_service.clone(); state.applications.file_management_service = file_management_service; - + if let Some(i18n) = i18n_service { state.applications.i18n_service = i18n; } - + state.applications.trash_service = trash_service.clone(); state.applications.search_service = search_service.clone(); state.applications.share_service = share_service.clone(); state.applications.favorites_service = favorites_service.clone(); state.applications.recent_service = recent_service.clone(); - + // Also set top-level optional services state.trash_service = trash_service; state.share_service = share_service; state.favorites_service = favorites_service; state.recent_service = recent_service; - + // Create real ZipService with the actual file/folder services state.core.zip_service = Arc::new( crate::infrastructure::services::zip_service::ZipService::new( - file_retrieval_service - as Arc, - folder_service.clone() - as Arc, - ), + file_retrieval_service as Arc, + folder_service.clone() as Arc, + ) ); - + state } - + pub fn with_auth_services(mut self, auth_services: AuthServices) -> Self { self.auth_service = Some(auth_services); self } - + pub fn with_trash_service(mut self, trash_service: Arc) -> Self { self.trash_service = Some(trash_service); self } - - pub fn with_share_service( - mut self, - share_service: Arc, - ) -> Self { + + pub fn with_share_service(mut self, share_service: Arc) -> Self { self.share_service = Some(share_service); self } - + pub fn with_favorites_service(mut self, favorites_service: Arc) -> Self { self.favorites_service = Some(favorites_service); self } - + pub fn with_recent_service(mut self, recent_service: Arc) -> Self { self.recent_service = Some(recent_service); self } - - pub fn with_storage_usage_service( - mut self, - storage_usage_service: Arc, - ) -> Self { + + pub fn with_storage_usage_service(mut self, storage_usage_service: Arc) -> Self { self.storage_usage_service = Some(storage_usage_service); self } - - pub fn with_calendar_service( - mut self, - calendar_service: Arc, - ) -> Self { + + pub fn with_calendar_service(mut self, calendar_service: Arc) -> Self { self.calendar_service = Some(calendar_service); self } - - pub fn with_contact_service( - mut self, - contact_service: Arc, - ) -> Self { + + pub fn with_contact_service(mut self, contact_service: Arc) -> Self { self.contact_service = Some(contact_service); self } - - pub fn with_calendar_use_case( - mut self, - calendar_use_case: Arc, - ) -> Self { + + pub fn with_calendar_use_case(mut self, calendar_use_case: Arc) -> Self { self.calendar_use_case = Some(calendar_use_case); self } - - pub fn with_addressbook_use_case( - mut self, - addressbook_use_case: Arc, - ) -> Self { + + pub fn with_addressbook_use_case(mut self, addressbook_use_case: Arc) -> Self { self.addressbook_use_case = Some(addressbook_use_case); self } - - pub fn with_contact_use_case( - mut self, - contact_use_case: Arc, - ) -> Self { + + pub fn with_contact_use_case(mut self, contact_use_case: Arc) -> Self { self.contact_use_case = Some(contact_use_case); self } - + pub fn with_zip_service(mut self, zip_service: Arc) -> Self { self.core.zip_service = zip_service; self } -} +} \ No newline at end of file diff --git a/src/common/stubs.rs b/src/common/stubs.rs index 0b43707a..ad296f98 100644 --- a/src/common/stubs.rs +++ b/src/common/stubs.rs @@ -25,10 +25,8 @@ use crate::application::ports::file_ports::{ OptimizedFileContent, UploadStrategy, }; use crate::application::ports::inbound::{FolderUseCase, SearchUseCase}; -use crate::application::ports::outbound::IdMappingPort; use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; use crate::application::ports::zip_ports::ZipPort; -use crate::application::services::storage_mediator::{StorageMediator, StorageMediatorError}; use crate::common::errors::DomainError; use crate::domain::entities::file::File; use crate::domain::entities::folder::Folder; @@ -83,127 +81,6 @@ impl CompressionPort for StubCompressionPort { } } -// --------------------------------------------------------------------------- -// IdMappingPort -// --------------------------------------------------------------------------- - -pub struct StubIdMappingService; - -#[async_trait] -impl IdMappingPort for StubIdMappingService { - async fn get_or_create_id(&self, _path: &StoragePath) -> Result { - Ok("dummy-id".to_string()) - } - - async fn get_path_by_id(&self, _id: &str) -> Result { - Ok(StoragePath::from_string("/")) - } - - async fn update_path(&self, _id: &str, _new_path: &StoragePath) -> Result<(), DomainError> { - Ok(()) - } - - async fn remove_id(&self, _id: &str) -> Result<(), DomainError> { - Ok(()) - } - - async fn save_changes(&self) -> Result<(), DomainError> { - Ok(()) - } -} - -// --------------------------------------------------------------------------- -// StorageMediator -// --------------------------------------------------------------------------- - -pub struct StubStorageMediator; - -#[async_trait] -impl StorageMediator for StubStorageMediator { - async fn get_folder_path(&self, _folder_id: &str) -> Result { - Ok(PathBuf::from("/tmp")) - } - - async fn get_folder_storage_path( - &self, - _folder_id: &str, - ) -> Result { - Ok(StoragePath::root()) - } - - async fn get_folder(&self, _folder_id: &str) -> Result { - Err(StorageMediatorError::NotFound( - "Stub not implemented".to_string(), - )) - } - - async fn file_exists_at_path(&self, _path: &Path) -> Result { - Ok(false) - } - - async fn file_exists_at_storage_path( - &self, - _storage_path: &StoragePath, - ) -> Result { - Ok(false) - } - - async fn folder_exists_at_path(&self, _path: &Path) -> Result { - Ok(false) - } - - async fn folder_exists_at_storage_path( - &self, - _storage_path: &StoragePath, - ) -> Result { - Ok(false) - } - - fn resolve_path(&self, _relative_path: &Path) -> PathBuf { - PathBuf::from("/tmp") - } - - fn resolve_storage_path(&self, _storage_path: &StoragePath) -> PathBuf { - PathBuf::from("/tmp") - } - - async fn ensure_directory(&self, _path: &Path) -> Result<(), StorageMediatorError> { - Ok(()) - } - - async fn ensure_storage_directory( - &self, - _storage_path: &StoragePath, - ) -> Result<(), StorageMediatorError> { - Ok(()) - } -} - -// --------------------------------------------------------------------------- -// IdMappingPort -// --------------------------------------------------------------------------- - -pub struct StubIdMappingPort; - -#[async_trait] -impl IdMappingPort for StubIdMappingPort { - async fn get_or_create_id(&self, _path: &StoragePath) -> Result { - Ok("stub-id".to_string()) - } - async fn get_path_by_id(&self, _id: &str) -> Result { - Ok(StoragePath::from_string("/")) - } - async fn update_path(&self, _id: &str, _new_path: &StoragePath) -> Result<(), DomainError> { - Ok(()) - } - async fn remove_id(&self, _id: &str) -> Result<(), DomainError> { - Ok(()) - } - async fn save_changes(&self) -> Result<(), DomainError> { - Ok(()) - } -} - // --------------------------------------------------------------------------- // FileReadPort // --------------------------------------------------------------------------- diff --git a/src/infrastructure/repositories/composite_file_repository.rs b/src/infrastructure/repositories/composite_file_repository.rs deleted file mode 100644 index bd1ccd82..00000000 --- a/src/infrastructure/repositories/composite_file_repository.rs +++ /dev/null @@ -1,157 +0,0 @@ -use std::path::PathBuf; -use std::sync::Arc; - -use async_trait::async_trait; -use bytes::Bytes; -use futures::Stream; - -use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; -use crate::common::errors::DomainError; -use crate::domain::entities::file::File; -use crate::domain::services::path_service::StoragePath; - -/// Composite that wraps `Arc` + `Arc` -/// and delegates each method to the corresponding port. -/// -/// Thanks to the blanket impl `impl FileStoragePort for T {}` -/// this type automatically gets `FileStoragePort`. -pub struct CompositeFileRepository { - read: Arc, - write: Arc, -} - -impl CompositeFileRepository { - pub fn new(read: Arc, write: Arc) -> Self { - Self { read, write } - } -} - -// ───────────────────────────────────────────────────── -// FileReadPort — delegate to self.read -// ───────────────────────────────────────────────────── - -#[async_trait] -impl FileReadPort for CompositeFileRepository { - async fn get_file(&self, id: &str) -> Result { - self.read.get_file(id).await - } - - async fn list_files(&self, folder_id: Option<&str>) -> Result, DomainError> { - self.read.list_files(folder_id).await - } - - async fn get_file_content(&self, id: &str) -> Result, DomainError> { - self.read.get_file_content(id).await - } - - async fn get_file_stream( - &self, - id: &str, - ) -> Result> + Send>, DomainError> { - self.read.get_file_stream(id).await - } - - async fn get_file_range_stream( - &self, - id: &str, - start: u64, - end: Option, - ) -> Result> + Send>, DomainError> { - self.read.get_file_range_stream(id, start, end).await - } - - async fn get_file_mmap(&self, id: &str) -> Result { - self.read.get_file_mmap(id).await - } - - async fn get_file_path(&self, id: &str) -> Result { - self.read.get_file_path(id).await - } - - async fn get_parent_folder_id(&self, path: &str) -> Result { - self.read.get_parent_folder_id(path).await - } -} - -// ───────────────────────────────────────────────────── -// FileWritePort — delegate to self.write -// ───────────────────────────────────────────────────── - -#[async_trait] -impl FileWritePort for CompositeFileRepository { - async fn save_file( - &self, - name: String, - folder_id: Option, - content_type: String, - content: Vec, - ) -> Result { - self.write - .save_file(name, folder_id, content_type, content) - .await - } - - async fn save_file_from_stream( - &self, - name: String, - folder_id: Option, - content_type: String, - stream: std::pin::Pin> + Send>>, - ) -> Result { - self.write - .save_file_from_stream(name, folder_id, content_type, stream) - .await - } - - async fn move_file( - &self, - file_id: &str, - target_folder_id: Option, - ) -> Result { - self.write.move_file(file_id, target_folder_id).await - } - - async fn rename_file(&self, file_id: &str, new_name: &str) -> Result { - self.write.rename_file(file_id, new_name).await - } - - async fn delete_file(&self, id: &str) -> Result<(), DomainError> { - self.write.delete_file(id).await - } - - async fn update_file_content( - &self, - file_id: &str, - content: Vec, - ) -> Result<(), DomainError> { - self.write.update_file_content(file_id, content).await - } - - async fn register_file_deferred( - &self, - name: String, - folder_id: Option, - content_type: String, - size: u64, - ) -> Result<(File, PathBuf), DomainError> { - self.write - .register_file_deferred(name, folder_id, content_type, size) - .await - } - - async fn move_to_trash(&self, file_id: &str) -> Result<(), DomainError> { - self.write.move_to_trash(file_id).await - } - - async fn restore_from_trash( - &self, - file_id: &str, - original_path: &str, - ) -> Result<(), DomainError> { - self.write.restore_from_trash(file_id, original_path).await - } - - async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> { - self.write.delete_file_permanently(file_id).await - } -} diff --git a/src/infrastructure/repositories/file_fs_read_repository.rs b/src/infrastructure/repositories/file_fs_read_repository.rs deleted file mode 100644 index ee018f73..00000000 --- a/src/infrastructure/repositories/file_fs_read_repository.rs +++ /dev/null @@ -1,473 +0,0 @@ -use std::path::PathBuf; -use std::sync::Arc; - -use async_trait::async_trait; -use bytes::Bytes; -use futures::{Stream, StreamExt}; -use mime_guess::from_path; -use tokio::fs::File as TokioFile; -use tokio::task; -use tokio::{fs, time}; -use tokio_util::codec::{BytesCodec, FramedRead}; - -use crate::application::ports::cache_ports::MetadataCachePort; -use crate::application::ports::storage_ports::FileReadPort; -use crate::application::services::storage_mediator::StorageMediator; -use crate::common::config::AppConfig; -use crate::common::errors::DomainError; -use crate::domain::entities::file::File; -use crate::domain::services::path_service::StoragePath; -use crate::infrastructure::repositories::parallel_file_processor::ParallelFileProcessor; -use crate::infrastructure::repositories::repository_errors::{ - FileRepositoryError, FileRepositoryResult, -}; -use crate::infrastructure::services::path_service::PathService; - -/// Repository implementation for file **read** operations. -/// -/// Implements `FileReadPort`: -/// get_file, list_files, get_file_content, get_file_stream, -/// get_file_range_stream, get_file_mmap, get_file_path, get_parent_folder_id. -pub struct FileFsReadRepository { - root_path: PathBuf, - storage_mediator: Arc, - id_mapping_service: Arc, - path_service: Arc, - metadata_cache: Arc, - config: AppConfig, - parallel_processor: Option>, -} - -impl FileFsReadRepository { - /// Full constructor with all infrastructure dependencies. - pub fn new( - root_path: PathBuf, - storage_mediator: Arc, - id_mapping_service: Arc, - path_service: Arc, - metadata_cache: Arc, - config: AppConfig, - parallel_processor: Option>, - ) -> Self { - Self { - root_path, - storage_mediator, - id_mapping_service, - path_service, - metadata_cache, - config, - parallel_processor, - } - } - - /// Stub for testing (does not perform real I/O). - pub fn default_stub() -> Self { - Self { - root_path: PathBuf::from("./storage"), - storage_mediator: Arc::new( - crate::application::services::storage_mediator::FileSystemStorageMediator::new_stub( - ), - ), - id_mapping_service: Arc::new(crate::common::stubs::StubIdMappingPort), - path_service: Arc::new(PathService::new(PathBuf::from("./storage"))), - metadata_cache: Arc::new( - crate::infrastructure::services::file_metadata_cache::FileMetadataCache::default(), - ) as Arc, - config: AppConfig::default(), - parallel_processor: None, - } - } - - // ─── internal helpers ───────────────────────────────────── - - fn resolve_storage_path(&self, storage_path: &StoragePath) -> PathBuf { - self.path_service.resolve_path(storage_path) - } - - async fn get_file_metadata_raw( - &self, - abs_path: &PathBuf, - ) -> FileRepositoryResult<(u64, u64, u64)> { - // Cache first - if let Some(cached) = self.metadata_cache.get_metadata(abs_path).await - && let (Some(s), Some(c), Some(m)) = - (cached.size, cached.created_at, cached.modified_at) - { - return Ok((s, c, m)); - } - let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(abs_path)) - .await - .map_err(|_| { - FileRepositoryError::StorageError(format!( - "Timeout metadata: {}", - abs_path.display() - )) - })? - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - let size = metadata.len(); - let created_at = metadata - .created() - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or(0); - let modified_at = metadata - .modified() - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or(0); - let _ = self.metadata_cache.refresh_metadata(abs_path).await; - Ok((size, created_at, modified_at)) - } - - async fn get_file_by_id(&self, id: &str) -> FileRepositoryResult { - let storage_path = self - .id_mapping_service - .get_path_by_id(id) - .await - .map_err(|e| FileRepositoryError::Other(e.to_string()))?; - let abs_path = self.resolve_storage_path(&storage_path); - - if !abs_path.exists() || !abs_path.is_file() { - return Err(FileRepositoryError::NotFound(format!( - "File {} not found at {}", - id, - storage_path.to_string() - ))); - } - - let (size, created_at, modified_at) = self.get_file_metadata_raw(&abs_path).await?; - let name = storage_path - .file_name() - .ok_or_else(|| FileRepositoryError::InvalidPath(storage_path.to_string()))?; - let mime_type = from_path(&abs_path).first_or_octet_stream().to_string(); - - File::with_timestamps( - id.to_string(), - name, - storage_path, - size, - mime_type, - None, - created_at, - modified_at, - ) - .map_err(|e| FileRepositoryError::Other(e.to_string())) - } -} - -impl Clone for FileFsReadRepository { - fn clone(&self) -> Self { - Self { - root_path: self.root_path.clone(), - storage_mediator: self.storage_mediator.clone(), - id_mapping_service: self.id_mapping_service.clone(), - path_service: self.path_service.clone(), - metadata_cache: self.metadata_cache.clone(), - config: self.config.clone(), - parallel_processor: self.parallel_processor.clone(), - } - } -} - -#[async_trait] -impl FileReadPort for FileFsReadRepository { - async fn get_file(&self, id: &str) -> Result { - self.get_file_by_id(id).await.map_err(|e| match e { - FileRepositoryError::NotFound(msg) => DomainError::not_found("File", msg), - FileRepositoryError::StorageError(msg) => DomainError::internal_error("File", msg), - other => DomainError::internal_error("File", other.to_string()), - }) - } - - async fn list_files(&self, folder_id: Option<&str>) -> Result, DomainError> { - let folder_storage_path = match folder_id { - Some(id) => match self.storage_mediator.get_folder_path(id).await { - Ok(path) => { - let lossy = path.to_string_lossy().to_string(); - let folder_name = path.file_name().and_then(|f| f.to_str()).unwrap_or(&lossy); - StoragePath::from_string(folder_name) - } - Err(_) => return Ok(Vec::new()), - }, - None => StoragePath::root(), - }; - - let abs_folder_path = self.path_service.resolve_path(&folder_storage_path); - if !abs_folder_path.exists() || !abs_folder_path.is_dir() { - return Ok(Vec::new()); - } - - let mut files_result = Vec::new(); - let mut entries = fs::read_dir(&abs_folder_path) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - while let Some(entry) = entries - .next_entry() - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))? - { - let path = entry.path(); - if !path.is_file() { - continue; - } - let file_name = entry.file_name().to_string_lossy().to_string(); - if file_name.starts_with('.') - || file_name == "folder_ids.json" - || file_name == "file_ids.json" - { - continue; - } - let metadata = match fs::metadata(&path).await { - Ok(m) => m, - Err(_) => continue, - }; - let file_storage_path = folder_storage_path.join(&file_name); - let id = match self - .id_mapping_service - .get_or_create_id(&file_storage_path) - .await - { - Ok(id) => id, - Err(_) => continue, - }; - let size = metadata.len(); - let created_at = metadata - .created() - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or(0); - let modified_at = metadata - .modified() - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or(0); - let mime_type = from_path(&path).first_or_octet_stream().to_string(); - - match File::with_timestamps( - id, - file_name, - file_storage_path, - size, - mime_type, - folder_id.map(String::from), - created_at, - modified_at, - ) { - Ok(file) => files_result.push(file), - Err(_) => continue, - } - } - - // Persist any new ID mappings - let _ = self.id_mapping_service.save_changes().await; - Ok(files_result) - } - - async fn get_file_content(&self, id: &str) -> Result, DomainError> { - let file = self - .get_file_by_id(id) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let abs_path = self.resolve_storage_path(file.storage_path()); - - let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs_path)) - .await - .map_err(|_| { - DomainError::internal_error( - "File", - format!("Timeout metadata: {}", abs_path.display()), - ) - })? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let file_size = metadata.len(); - - if !self.config.resources.can_load_in_memory(file_size) { - return Err(DomainError::internal_error( - "File", - format!( - "File too large for memory: {} MB", - file_size / (1024 * 1024) - ), - )); - } - - // Parallel read for very large files - if self - .config - .resources - .needs_parallel_processing(file_size, &self.config.concurrency) - { - let content = if let Some(processor) = &self.parallel_processor { - processor.read_file_parallel(&abs_path).await - } else { - let processor = ParallelFileProcessor::new(self.config.clone()); - processor.read_file_parallel(&abs_path).await - }; - return content.map_err(|e| DomainError::internal_error("File", e.to_string())); - } - - // spawn_blocking for large-ish files - if self.config.resources.is_large_file(file_size) { - let abs_clone = abs_path.clone(); - let chunk_size = self.config.resources.chunk_size_bytes; - let content = task::spawn_blocking(move || -> std::io::Result> { - use std::io::{BufReader, Read}; - let file = std::fs::File::open(&abs_clone)?; - let mut reader = BufReader::with_capacity(chunk_size, file); - let mut buf = Vec::with_capacity(file_size as usize); - reader.read_to_end(&mut buf)?; - Ok(buf) - }) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - return Ok(content); - } - - // Small files — async read - time::timeout(self.config.timeouts.file_timeout(), fs::read(&abs_path)) - .await - .map_err(|_| { - DomainError::internal_error( - "File", - format!("Timeout reading: {}", abs_path.display()), - ) - })? - .map_err(|e| DomainError::internal_error("File", e.to_string())) - } - - async fn get_file_stream( - &self, - id: &str, - ) -> Result> + Send>, DomainError> { - let file = self - .get_file_by_id(id) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let abs_path = self.resolve_storage_path(file.storage_path()); - - let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs_path)) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout getting metadata"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let file_size = metadata.len(); - let is_large = self.config.resources.is_large_file(file_size); - - let fh = time::timeout( - self.config.timeouts.file_timeout(), - TokioFile::open(&abs_path), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout opening file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - let chunk_size = if is_large { - self.config.resources.chunk_size_bytes - } else { - 4096 - }; - let codec = BytesCodec::new(); - let stream = - FramedRead::with_capacity(fh, codec, chunk_size).map(|r| r.map(|bm| bm.freeze())); - Ok(Box::new(stream)) - } - - async fn get_file_range_stream( - &self, - id: &str, - start: u64, - end: Option, - ) -> Result> + Send>, DomainError> { - use tokio::io::AsyncSeekExt; - - let file = self - .get_file_by_id(id) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let abs_path = self.resolve_storage_path(file.storage_path()); - - let metadata = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs_path)) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let file_size = metadata.len(); - if start >= file_size { - return Err(DomainError::internal_error( - "File", - format!("Range start {} beyond file size {}", start, file_size), - )); - } - let actual_end = end.map(|e| e.min(file_size - 1)).unwrap_or(file_size - 1); - let range_length = actual_end - start + 1; - - let mut fh = time::timeout( - self.config.timeouts.file_timeout(), - TokioFile::open(&abs_path), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout opening file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - fh.seek(std::io::SeekFrom::Start(start)) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - let chunk_size = if range_length > 1024 * 1024 { - self.config.resources.chunk_size_bytes - } else { - 8192 - }; - use tokio::io::AsyncReadExt; - let limited = fh.take(range_length); - let codec = BytesCodec::new(); - let stream = - FramedRead::with_capacity(limited, codec, chunk_size).map(|r| r.map(|bm| bm.freeze())); - Ok(Box::new(stream)) - } - - async fn get_file_mmap(&self, id: &str) -> Result { - use memmap2::Mmap; - let file = self - .get_file_by_id(id) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let abs_path = self.resolve_storage_path(file.storage_path()); - let path_clone = abs_path.clone(); - - task::spawn_blocking(move || -> Result { - let fh = std::fs::File::open(&path_clone) - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let mmap = unsafe { Mmap::map(&fh) } - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - Ok(Bytes::copy_from_slice(&mmap[..])) - }) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))? - } - - async fn get_file_path(&self, id: &str) -> Result { - self.id_mapping_service.get_path_by_id(id).await - } - - async fn get_parent_folder_id(&self, path: &str) -> Result { - let storage_path = StoragePath::from_string(path); - match storage_path.parent() { - Some(parent) if !parent.is_empty() => { - self.id_mapping_service.get_or_create_id(&parent).await - } - _ => Ok("root".to_string()), - } - } -} diff --git a/src/infrastructure/repositories/file_fs_write_repository.rs b/src/infrastructure/repositories/file_fs_write_repository.rs deleted file mode 100644 index fbf66d19..00000000 --- a/src/infrastructure/repositories/file_fs_write_repository.rs +++ /dev/null @@ -1,800 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use futures::{Stream, StreamExt}; -use mime_guess::from_path; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::fs::File as TokioFile; -use tokio::io::AsyncWriteExt; -use tokio::task; -use tokio::{fs, time}; - -use crate::application::ports::cache_ports::MetadataCachePort; -use crate::application::ports::storage_ports::FileWritePort; -use crate::application::services::storage_mediator::StorageMediator; -use crate::common::config::AppConfig; -use crate::common::errors::DomainError; -use crate::domain::entities::file::File; -use crate::domain::services::path_service::StoragePath; -use crate::infrastructure::repositories::parallel_file_processor::ParallelFileProcessor; -use crate::infrastructure::repositories::repository_errors::{ - FileRepositoryError, FileRepositoryResult, -}; -use crate::infrastructure::services::file_system_utils::FileSystemUtils; -use crate::infrastructure::services::path_service::PathService; - -/// Repository implementation for file **write** operations. -/// -/// Implements `FileWritePort`: -/// save_file, save_file_from_stream, move_file, delete_file, -/// update_file_content, register_file_deferred. -pub struct FileFsWriteRepository { - root_path: PathBuf, - storage_mediator: Arc, - id_mapping_service: Arc, - path_service: Arc, - metadata_cache: Arc, - config: AppConfig, - parallel_processor: Option>, -} - -impl FileFsWriteRepository { - /// Full constructor with all dependencies. - pub fn new( - root_path: PathBuf, - storage_mediator: Arc, - id_mapping_service: Arc, - path_service: Arc, - metadata_cache: Arc, - config: AppConfig, - parallel_processor: Option>, - ) -> Self { - Self { - root_path, - storage_mediator, - id_mapping_service, - path_service, - metadata_cache, - config, - parallel_processor, - } - } - - /// Stub for testing (does not perform real I/O). - pub fn default_stub() -> Self { - Self { - root_path: PathBuf::from("./storage"), - storage_mediator: Arc::new( - crate::application::services::storage_mediator::FileSystemStorageMediator::new_stub( - ), - ), - id_mapping_service: Arc::new(crate::common::stubs::StubIdMappingPort), - path_service: Arc::new(PathService::new(PathBuf::from("./storage"))), - metadata_cache: Arc::new( - crate::infrastructure::services::file_metadata_cache::FileMetadataCache::default(), - ) as Arc, - config: AppConfig::default(), - parallel_processor: None, - } - } - - // ─── helpers ───────────────────────────────────────────── - - fn resolve_storage_path(&self, storage_path: &StoragePath) -> PathBuf { - self.path_service.resolve_path(storage_path) - } - - async fn ensure_parent_directory(&self, abs_path: &PathBuf) -> FileRepositoryResult<()> { - if let Some(parent) = abs_path.parent() { - time::timeout( - self.config.timeouts.dir_timeout(), - FileSystemUtils::create_dir_with_sync(parent), - ) - .await - .map_err(|_| { - FileRepositoryError::StorageError(format!( - "Timeout creating dir: {}", - parent.display() - )) - })? - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - } - Ok(()) - } - - async fn file_exists_at_storage_path( - &self, - storage_path: &StoragePath, - ) -> FileRepositoryResult { - let abs = self.resolve_storage_path(storage_path); - if let Some(is_file) = self.metadata_cache.is_file(&abs).await { - return Ok(is_file); - } - match time::timeout(self.config.timeouts.file_timeout(), fs::metadata(&abs)).await { - Ok(Ok(m)) => { - let _ = self.metadata_cache.refresh_metadata(&abs).await; - Ok(m.is_file()) - } - Ok(Err(_)) => Ok(false), - Err(_) => Err(FileRepositoryError::StorageError(format!( - "Timeout: {}", - abs.display() - ))), - } - } - - async fn get_file_metadata_raw( - &self, - abs_path: &PathBuf, - ) -> FileRepositoryResult<(u64, u64, u64)> { - if let Some(cached) = self.metadata_cache.get_metadata(abs_path).await - && let (Some(s), Some(c), Some(m)) = - (cached.size, cached.created_at, cached.modified_at) - { - return Ok((s, c, m)); - } - let meta = time::timeout(self.config.timeouts.file_timeout(), fs::metadata(abs_path)) - .await - .map_err(|_| { - FileRepositoryError::StorageError(format!("Timeout: {}", abs_path.display())) - })? - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - let s = meta.len(); - let c = meta - .created() - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or(0); - let m = meta - .modified() - .map(|t| { - t.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or(0); - let _ = self.metadata_cache.refresh_metadata(abs_path).await; - Ok((s, c, m)) - } - - /// Resolve folder to StoragePath - async fn resolve_folder_path(&self, folder_id: &Option) -> StoragePath { - match folder_id { - Some(id) => match self.storage_mediator.get_folder_path(id).await { - Ok(path) => { - let lossy = path.to_string_lossy().to_string(); - let folder_name = path.file_name().and_then(|f| f.to_str()).unwrap_or(&lossy); - StoragePath::from_string(folder_name) - } - Err(_) => StoragePath::root(), - }, - None => StoragePath::root(), - } - } - - /// Generate unique file path avoiding name collisions. - async fn unique_file_path( - &self, - folder_path: &StoragePath, - name: &str, - ) -> FileRepositoryResult<(StoragePath, String)> { - let mut file_path = folder_path.join(name); - let mut actual_name = name.to_string(); - let mut counter = 1; - while self.file_exists_at_storage_path(&file_path).await? { - let (stem, ext) = if let Some(dot) = name.rfind('.') { - (name[..dot].to_string(), name[dot..].to_string()) - } else { - (name.to_string(), String::new()) - }; - actual_name = format!("{}_{}{}", stem, counter, ext); - file_path = folder_path.join(&actual_name); - counter += 1; - } - Ok((file_path, actual_name)) - } - - async fn delete_file_non_blocking(&self, abs_path: PathBuf) -> FileRepositoryResult<()> { - let file_size = match fs::metadata(&abs_path).await { - Ok(m) => m.len(), - Err(_) => 0, - }; - if self.config.resources.is_large_file(file_size) { - task::spawn_blocking(move || { - let _ = std::fs::remove_file(&abs_path); - }) - .await - .map_err(|e| FileRepositoryError::Other(e.to_string()))?; - } else { - time::timeout( - self.config.timeouts.file_timeout(), - fs::remove_file(&abs_path), - ) - .await - .map_err(|_| FileRepositoryError::StorageError("Timeout deleting file".into()))? - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - } - Ok(()) - } - - /// Persist ID mapping with retry + verification. - async fn persist_id_mapping(&self, id: &str, expected_path: &str) -> FileRepositoryResult<()> { - for attempt in 1..=3 { - match self.id_mapping_service.save_changes().await { - Ok(_) => { - if let Ok(verified) = self.id_mapping_service.get_path_by_id(id).await - && verified.to_string() == expected_path - { - return Ok(()); - } - if attempt == 3 { - return Err(FileRepositoryError::Other( - "Failed to verify ID mapping after 3 attempts".into(), - )); - } - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - Err(e) if attempt < 3 => { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - tracing::warn!("ID mapping save retry {}: {}", attempt, e); - } - Err(e) => { - return Err(FileRepositoryError::Other(format!( - "Save ID mapping failed: {}", - e - ))); - } - } - } - Ok(()) - } -} - -impl Clone for FileFsWriteRepository { - fn clone(&self) -> Self { - Self { - root_path: self.root_path.clone(), - storage_mediator: self.storage_mediator.clone(), - id_mapping_service: self.id_mapping_service.clone(), - path_service: self.path_service.clone(), - metadata_cache: self.metadata_cache.clone(), - config: self.config.clone(), - parallel_processor: self.parallel_processor.clone(), - } - } -} - -fn map_repo_err(e: FileRepositoryError) -> DomainError { - match e { - FileRepositoryError::NotFound(m) => DomainError::not_found("File", m), - FileRepositoryError::AlreadyExists(m) => DomainError::already_exists("File", m), - FileRepositoryError::StorageError(m) => DomainError::internal_error("File", m), - other => DomainError::internal_error("File", other.to_string()), - } -} - -#[async_trait] -impl FileWritePort for FileFsWriteRepository { - async fn save_file( - &self, - name: String, - folder_id: Option, - content_type: String, - content: Vec, - ) -> Result { - let folder_path = self.resolve_folder_path(&folder_id).await; - let (file_storage_path, actual_name) = self - .unique_file_path(&folder_path, &name) - .await - .map_err(map_repo_err)?; - let abs_path = self.resolve_storage_path(&file_storage_path); - self.ensure_parent_directory(&abs_path) - .await - .map_err(map_repo_err)?; - - let content_size = content.len() as u64; - - // Write strategy based on file size - if self - .config - .resources - .needs_parallel_processing(content_size, &self.config.concurrency) - { - if let Some(proc) = &self.parallel_processor { - proc.write_file_parallel(&abs_path, &content) - .await - .map_err(map_repo_err)?; - } else { - let proc = ParallelFileProcessor::new(self.config.clone()); - proc.write_file_parallel(&abs_path, &content) - .await - .map_err(map_repo_err)?; - } - } else if content_size > self.config.resources.large_file_threshold_mb * 1024 * 1024 { - let mut fh = time::timeout( - self.config.timeouts.file_timeout(), - TokioFile::create(&abs_path), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout creating file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let chunk_size = self.config.resources.chunk_size_bytes; - for chunk in content.chunks(chunk_size) { - fh.write_all(chunk) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - } - fh.flush() - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - } else { - let mut fh = time::timeout( - self.config.timeouts.file_timeout(), - TokioFile::create(&abs_path), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout creating file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - fh.write_all(&content) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - fh.flush() - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - } - - let (size, created_at, modified_at) = self - .get_file_metadata_raw(&abs_path) - .await - .map_err(map_repo_err)?; - let mime = if content_type.is_empty() { - from_path(&abs_path).first_or_octet_stream().to_string() - } else { - content_type - }; - let id = self - .id_mapping_service - .get_or_create_id(&file_storage_path) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let path_string = file_storage_path.to_string(); - - let file = File::with_timestamps( - id.clone(), - actual_name, - file_storage_path, - size, - mime, - folder_id, - created_at, - modified_at, - ) - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - self.persist_id_mapping(&id, &path_string) - .await - .map_err(map_repo_err)?; - if let Some(parent) = abs_path.parent() { - self.metadata_cache.invalidate_directory(parent).await; - } - Ok(file) - } - - async fn save_file_from_stream( - &self, - name: String, - folder_id: Option, - content_type: String, - mut stream: std::pin::Pin> + Send>>, - ) -> Result { - let folder_path = self.resolve_folder_path(&folder_id).await; - let (file_storage_path, actual_name) = self - .unique_file_path(&folder_path, &name) - .await - .map_err(map_repo_err)?; - let abs_path = self.resolve_storage_path(&file_storage_path); - self.ensure_parent_directory(&abs_path) - .await - .map_err(map_repo_err)?; - - let temp_path = abs_path.with_extension("tmp.upload"); - let mut fh = time::timeout( - self.config.timeouts.file_timeout(), - TokioFile::create(&temp_path), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout creating temp file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - let mut total_bytes: u64 = 0; - while let Some(chunk_result) = stream.next().await { - let chunk = - chunk_result.map_err(|e| DomainError::internal_error("File", e.to_string()))?; - fh.write_all(&chunk) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - total_bytes += chunk.len() as u64; - } - fh.flush() - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - fh.sync_all() - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - drop(fh); - - // Atomic rename - fs::rename(&temp_path, &abs_path) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - let (size, created_at, modified_at) = self - .get_file_metadata_raw(&abs_path) - .await - .map_err(map_repo_err)?; - let mime = if content_type.is_empty() { - from_path(&abs_path).first_or_octet_stream().to_string() - } else { - content_type - }; - let id = self - .id_mapping_service - .get_or_create_id(&file_storage_path) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let path_string = file_storage_path.to_string(); - let log_name = actual_name.clone(); - - let file = File::with_timestamps( - id.clone(), - actual_name, - file_storage_path, - size, - mime, - folder_id, - created_at, - modified_at, - ) - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - self.persist_id_mapping(&id, &path_string) - .await - .map_err(map_repo_err)?; - if let Some(parent) = abs_path.parent() { - self.metadata_cache.invalidate_directory(parent).await; - } - tracing::info!( - "✅ STREAMING UPLOAD COMPLETE: {} ({} bytes)", - log_name, - total_bytes - ); - Ok(file) - } - - async fn move_file( - &self, - file_id: &str, - target_folder_id: Option, - ) -> Result { - // Get original file - let original_path = self.id_mapping_service.get_path_by_id(file_id).await?; - let old_abs = self.resolve_storage_path(&original_path); - if !old_abs.exists() || !old_abs.is_file() { - return Err(DomainError::not_found("File", file_id.to_string())); - } - let (size, created_at, modified_at) = self - .get_file_metadata_raw(&old_abs) - .await - .map_err(map_repo_err)?; - let name = original_path - .file_name() - .ok_or_else(|| DomainError::internal_error("File", "Invalid path"))?; - let mime = from_path(&old_abs).first_or_octet_stream().to_string(); - - // Build target path - let target_folder_path = self.resolve_folder_path(&target_folder_id).await; - let new_storage_path = target_folder_path.join(&name); - if self - .file_exists_at_storage_path(&new_storage_path) - .await - .map_err(map_repo_err)? - { - return Err(DomainError::already_exists( - "File", - format!("File already exists at {}", new_storage_path.to_string()), - )); - } - let new_abs = self.resolve_storage_path(&new_storage_path); - self.ensure_parent_directory(&new_abs) - .await - .map_err(map_repo_err)?; - - // Rename - time::timeout( - self.config.timeouts.file_timeout(), - FileSystemUtils::rename_with_sync(&old_abs, &new_abs), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout moving file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - // Update mapping - self.id_mapping_service - .update_path(file_id, &new_storage_path) - .await?; - let _ = self.id_mapping_service.save_changes().await; - - File::with_timestamps( - file_id.to_string(), - name, - new_storage_path, - size, - mime, - target_folder_id, - created_at, - modified_at, - ) - .map_err(|e| DomainError::internal_error("File", e.to_string())) - } - - async fn rename_file(&self, file_id: &str, new_name: &str) -> Result { - // 1. Get current file info - let original_path = self.id_mapping_service.get_path_by_id(file_id).await?; - let old_abs = self.resolve_storage_path(&original_path); - if !old_abs.exists() || !old_abs.is_file() { - return Err(DomainError::not_found("File", file_id.to_string())); - } - let (size, created_at, modified_at) = self - .get_file_metadata_raw(&old_abs) - .await - .map_err(map_repo_err)?; - - // 2. Build new path (same parent directory, different filename) - let parent = original_path - .parent() - .unwrap_or_else(|| StoragePath::new(vec![])); - let new_storage_path = parent.join(new_name); - if self - .file_exists_at_storage_path(&new_storage_path) - .await - .map_err(map_repo_err)? - { - return Err(DomainError::already_exists( - "File", - format!("File already exists: {}", new_name), - )); - } - let new_abs = self.resolve_storage_path(&new_storage_path); - let mime = from_path(&new_abs).first_or_octet_stream().to_string(); - - // 3. Rename on disk - time::timeout( - self.config.timeouts.file_timeout(), - FileSystemUtils::rename_with_sync(&old_abs, &new_abs), - ) - .await - .map_err(|_| DomainError::internal_error("File", "Timeout renaming file"))? - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - // 4. Update id→path mapping - self.id_mapping_service - .update_path(file_id, &new_storage_path) - .await?; - let _ = self.id_mapping_service.save_changes().await; - - File::with_timestamps( - file_id.to_string(), - new_name.to_string(), - new_storage_path, - size, - mime, - None, - created_at, - modified_at, - ) - .map_err(|e| DomainError::internal_error("File", e.to_string())) - } - - async fn delete_file(&self, id: &str) -> Result<(), DomainError> { - let storage_path = self.id_mapping_service.get_path_by_id(id).await?; - let abs_path = self.resolve_storage_path(&storage_path); - - self.metadata_cache.invalidate(&abs_path).await; - if let Some(parent) = abs_path.parent() { - self.metadata_cache.invalidate_directory(parent).await; - } - - self.delete_file_non_blocking(abs_path) - .await - .map_err(map_repo_err)?; - - // Clean up the ID mapping so we don't leave orphaned entries - if let Err(e) = self.id_mapping_service.remove_id(id).await { - tracing::warn!("Failed to remove ID mapping for deleted file {}: {}", id, e); - } - let _ = self.id_mapping_service.save_changes().await; - - Ok(()) - } - - async fn update_file_content( - &self, - file_id: &str, - content: Vec, - ) -> Result<(), DomainError> { - let storage_path = self.id_mapping_service.get_path_by_id(file_id).await?; - let physical_path = self.resolve_storage_path(&storage_path); - - FileSystemUtils::atomic_write(&physical_path, &content) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - // Refresh cache - let _ = self.metadata_cache.refresh_metadata(&physical_path).await; - Ok(()) - } - - async fn register_file_deferred( - &self, - name: String, - folder_id: Option, - content_type: String, - size: u64, - ) -> Result<(File, PathBuf), DomainError> { - let folder_path = self.resolve_folder_path(&folder_id).await; - let (file_storage_path, actual_name) = self - .unique_file_path(&folder_path, &name) - .await - .map_err(map_repo_err)?; - let abs_path = self.resolve_storage_path(&file_storage_path); - self.ensure_parent_directory(&abs_path) - .await - .map_err(map_repo_err)?; - - let mime = if content_type.is_empty() { - from_path(&abs_path).first_or_octet_stream().to_string() - } else { - content_type - }; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - - let id = self - .id_mapping_service - .get_or_create_id(&file_storage_path) - .await - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - let _ = self.id_mapping_service.save_changes().await; - - let file = File::with_timestamps( - id.clone(), - actual_name, - file_storage_path, - size, - mime, - folder_id, - now, - now, - ) - .map_err(|e| DomainError::internal_error("File", e.to_string()))?; - - tracing::debug!("⚡ Registered deferred file: {} -> {:?}", id, abs_path); - Ok((file, abs_path)) - } - - async fn move_to_trash(&self, file_id: &str) -> Result<(), DomainError> { - // Get the file's current path - let storage_path = self.id_mapping_service.get_path_by_id(file_id).await?; - let abs_path = self.resolve_storage_path(&storage_path); - - if !abs_path.exists() || !abs_path.is_file() { - return Err(DomainError::not_found("File", file_id.to_string())); - } - - // Create trash directory - let trash_dir = self.root_path.join(".trash").join("files"); - fs::create_dir_all(&trash_dir).await.map_err(|e| { - DomainError::internal_error("File", format!("Failed to create trash dir: {}", e)) - })?; - - // Move file to trash - let trash_path = trash_dir.join(file_id); - fs::rename(&abs_path, &trash_path).await.map_err(|e| { - DomainError::internal_error("File", format!("Failed to move file to trash: {}", e)) - })?; - - // Update mapping to trash location - let trash_storage_path = StoragePath::from_string(&format!(".trash/files/{}", file_id)); - self.id_mapping_service - .update_path(file_id, &trash_storage_path) - .await?; - let _ = self.id_mapping_service.save_changes().await; - - // Invalidate cache - self.metadata_cache.invalidate(&abs_path).await; - if let Some(parent) = abs_path.parent() { - self.metadata_cache.invalidate_directory(parent).await; - } - - tracing::debug!( - "File moved to trash: {} -> {}", - file_id, - trash_path.display() - ); - Ok(()) - } - - async fn restore_from_trash( - &self, - file_id: &str, - original_path: &str, - ) -> Result<(), DomainError> { - // Get current path (should be in trash) - let current_storage_path = self.id_mapping_service.get_path_by_id(file_id).await?; - let current_abs_path = self.resolve_storage_path(¤t_storage_path); - - if !current_abs_path.exists() { - return Err(DomainError::not_found( - "File", - format!("File {} not found in trash", file_id), - )); - } - - // Ensure parent directory exists for original location - let original_storage_path = StoragePath::from_string(original_path); - let original_abs_path = self.resolve_storage_path(&original_storage_path); - if let Some(parent) = original_abs_path.parent() { - fs::create_dir_all(parent).await.map_err(|e| { - DomainError::internal_error("File", format!("Failed to create parent dir: {}", e)) - })?; - } - - // Move file back to original location - fs::rename(¤t_abs_path, &original_abs_path) - .await - .map_err(|e| { - DomainError::internal_error("File", format!("Failed to restore file: {}", e)) - })?; - - // Update mapping back to original path - self.id_mapping_service - .update_path(file_id, &original_storage_path) - .await?; - let _ = self.id_mapping_service.save_changes().await; - - tracing::debug!( - "File restored from trash: {} -> {}", - file_id, - original_abs_path.display() - ); - Ok(()) - } - - async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> { - // Get current path (could be in trash or original location) - let storage_path = self.id_mapping_service.get_path_by_id(file_id).await?; - let abs_path = self.resolve_storage_path(&storage_path); - - // Delete the physical file if it exists - if abs_path.exists() { - self.delete_file_non_blocking(abs_path.clone()) - .await - .map_err(map_repo_err)?; - } - - // Remove ID mapping - self.id_mapping_service.remove_id(file_id).await?; - let _ = self.id_mapping_service.save_changes().await; - - // Invalidate cache - self.metadata_cache.invalidate(&abs_path).await; - - tracing::debug!("File permanently deleted: {}", file_id); - Ok(()) - } -} diff --git a/src/infrastructure/repositories/folder_fs_repository.rs b/src/infrastructure/repositories/folder_fs_repository.rs deleted file mode 100644 index 62f24887..00000000 --- a/src/infrastructure/repositories/folder_fs_repository.rs +++ /dev/null @@ -1,1010 +0,0 @@ -use async_trait::async_trait; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::Duration; -use tokio::fs; -use tokio::time::timeout; - -use crate::domain::entities::folder::{Folder, FolderError}; -use crate::domain::services::path_service::StoragePath; -use crate::infrastructure::repositories::repository_errors::{ - FolderRepositoryError, FolderRepositoryResult, -}; -use crate::infrastructure::services::path_service::PathService; -// use crate::application::ports::outbound::IdMappingPort; -use crate::application::services::storage_mediator::StorageMediator; -use crate::common::errors::DomainError; -use crate::domain::repositories::folder_repository::FolderRepository; -use crate::infrastructure::services::id_mapping_service::{IdMappingError, IdMappingService}; - -// To be able to use streams in the list_folders function -use tokio_stream; - -/// Filesystem implementation of the FolderRepository interface -pub struct FolderFsRepository { - root_path: PathBuf, - storage_mediator: Arc, - id_mapping_service: Arc, - path_service: Arc, -} - -impl FolderFsRepository { - /// Creates a new filesystem-based folder repository - pub fn new( - root_path: PathBuf, - storage_mediator: Arc, - id_mapping_service: Arc, - path_service: Arc, - ) -> Self { - Self { - root_path, - storage_mediator, - id_mapping_service, - path_service, - } - } - - /// Returns the root path of the storage - pub fn get_root_path(&self) -> &PathBuf { - &self.root_path - } - - /// Creates a stub repository for initialization purposes - /// This is used temporarily during dependency injection setup - pub fn new_stub() -> Self { - let root_path = PathBuf::from("/tmp"); - let path_service = Arc::new(PathService::new(root_path.clone())); - - // Create minimal implementations just to satisfy initialization - // Since we can't easily block on an async function in a sync context, create with a stub - let id_mapping_service = Arc::new(IdMappingService::new_sync(root_path.clone())); - - // Create a self-referential stub (only used for initialization) - let storage_mediator_stub = - Arc::new(crate::application::services::storage_mediator::StubStorageMediator::new()); - - Self { - root_path, - storage_mediator: storage_mediator_stub, - id_mapping_service, - path_service, - } - } - - /// Gets the count of items in a directory efficiently - async fn count_directory_items(&self, directory_path: &Path) -> FolderRepositoryResult { - use tokio::fs::read_dir; - - // Timeout to avoid blocking - let read_dir_timeout = Duration::from_secs(30); - let read_dir_result = timeout(read_dir_timeout, read_dir(directory_path)).await; - - match read_dir_result { - Ok(result) => { - let mut entries = - result.map_err(|e| FolderRepositoryError::StorageError(e.to_string()))?; - let mut count = 0; - - // Count entries manually, skipping hidden/system directories - while let Ok(Some(entry)) = entries.next_entry().await { - let name = entry.file_name().to_string_lossy().to_string(); - if name.starts_with('.') { - continue; - } - count += 1; - } - - Ok(count) - } - Err(_) => Err(FolderRepositoryError::Other(format!( - "Timeout counting items in directory: {}", - directory_path.display() - ))), - } - } - - /// Resolves a domain storage path to an absolute filesystem path - fn resolve_storage_path(&self, storage_path: &StoragePath) -> PathBuf { - self.path_service.resolve_path(storage_path) - } - - /// Returns a reference to the ID mapping service - pub fn id_mapping_service( - &self, - ) -> &Arc { - &self.id_mapping_service - } - - /// Gets the storage path for a folder by its ID (internal helper) - async fn _get_folder_storage_path(&self, id: &str) -> FolderRepositoryResult { - let storage_path = self - .id_mapping_service - .get_path_by_id(id) - .await - .map_err(FolderRepositoryError::from)?; - Ok(storage_path) - } - - /// Gets a folder path from the ID mapping service - pub async fn get_mapped_folder_path(&self, folder_id: &str) -> FolderRepositoryResult { - let storage_path = self - .id_mapping_service - .get_path_by_id(folder_id) - .await - .map_err(|e| { - FolderRepositoryError::StorageError(format!("Failed to get folder path: {}", e)) - })?; - Ok(storage_path.to_string()) - } - - /// Updates a folder path in the ID mapping service - pub async fn update_mapped_folder_path( - &self, - folder_id: &str, - new_path: &PathBuf, - ) -> FolderRepositoryResult<()> { - let storage_path = StoragePath::from_string(new_path.to_string_lossy().as_ref()); - self.id_mapping_service - .update_path(folder_id, &storage_path) - .await - .map_err(|e| { - FolderRepositoryError::StorageError(format!("Failed to update folder path: {}", e)) - }) - } - - /// Removes a folder ID from the ID mapping service - pub async fn remove_mapped_folder_id(&self, folder_id: &str) -> FolderRepositoryResult<()> { - self.id_mapping_service - .remove_id(folder_id) - .await - .map_err(|e| { - FolderRepositoryError::StorageError(format!("Failed to remove folder ID: {}", e)) - }) - } - - /// Checks if a folder exists at a given storage path - async fn check_folder_exists_at_storage_path( - &self, - storage_path: &StoragePath, - ) -> FolderRepositoryResult { - let abs_path = self.resolve_storage_path(storage_path); - - // Check if folder exists and is a directory - let exists = abs_path.exists() && abs_path.is_dir(); - - tracing::debug!( - "Checking if folder exists: {} - path: {}", - exists, - abs_path.display() - ); - - Ok(exists) - } - - /// Creates the physical directory on the filesystem - async fn create_directory(&self, path: &Path) -> Result<(), std::io::Error> { - fs::create_dir_all(path).await - } - - /// Helper method to create a Folder entity from a storage path and metadata - async fn create_folder_entity( - &self, - id: String, - name: String, - storage_path: StoragePath, - parent_id: Option, - created_at: Option, - modified_at: Option, - ) -> FolderRepositoryResult { - // If timestamps are provided, use them; otherwise, let Folder::new create default timestamps - let folder = if let (Some(created), Some(modified)) = (created_at, modified_at) { - Folder::with_timestamps(id, name, storage_path, parent_id, created, modified) - } else { - Folder::new(id, name, storage_path, parent_id) - }; - - // Convert domain error to repository error - folder.map_err(|e| match e { - FolderError::InvalidFolderName(name) => { - FolderRepositoryError::ValidationError(format!("Invalid folder name: {}", name)) - } - FolderError::ValidationError(msg) => FolderRepositoryError::ValidationError(msg), - }) - } - - /// Extracts folder metadata from a physical path - async fn get_folder_metadata(&self, abs_path: &PathBuf) -> FolderRepositoryResult<(u64, u64)> { - let metadata = fs::metadata(&abs_path) - .await - .map_err(|e| FolderRepositoryError::StorageError(e.to_string()))?; - - // Get creation timestamp - let created_at = metadata - .created() - .map(|time| { - time.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or_else(|_| 0); - - // Get modification timestamp - let modified_at = metadata - .modified() - .map(|time| { - time.duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .unwrap_or_else(|_| 0); - - Ok((created_at, modified_at)) - } -} - -// Convert IdMappingError to FolderRepositoryError -impl From for FolderRepositoryError { - fn from(err: IdMappingError) -> Self { - match err { - IdMappingError::NotFound(id) => FolderRepositoryError::NotFound(id), - IdMappingError::IoError(e) => FolderRepositoryError::StorageError(e.to_string()), - IdMappingError::Timeout(msg) => { - FolderRepositoryError::StorageError(format!("Timeout: {}", msg)) - } - _ => FolderRepositoryError::StorageError(err.to_string()), - } - } -} - -// Implement Clone to use in concurrent processing -impl Clone for FolderFsRepository { - fn clone(&self) -> Self { - // Clone the Arcs, which only increments the reference counter - Self { - root_path: self.root_path.clone(), - storage_mediator: self.storage_mediator.clone(), - id_mapping_service: self.id_mapping_service.clone(), - path_service: self.path_service.clone(), - } - } -} - -#[async_trait] -impl FolderRepository for FolderFsRepository { - async fn create_folder( - &self, - name: String, - parent_id: Option, - ) -> Result { - // Get the parent folder path (if any) - let parent_storage_path = match &parent_id { - Some(id) => match self._get_folder_storage_path(id).await { - Ok(path) => { - tracing::info!( - "Using folder path: {:?} for parent_id: {:?}", - path.to_string(), - id - ); - Some(path) - } - Err(e) => { - tracing::error!("Error getting parent folder: {}", e); - return Err(DomainError::from(e)); - } - }, - None => None, - }; - - // Create the storage path for the new folder - let folder_storage_path = match parent_storage_path { - Some(parent) => parent.join(&name), - None => StoragePath::from_string(&name), - }; - tracing::info!( - "Creating folder at path: {:?}", - folder_storage_path.to_string() - ); - - // Check if folder already exists - if self - .check_folder_exists_at_storage_path(&folder_storage_path) - .await - .map_err(DomainError::from)? - { - return Err(DomainError::already_exists( - "Folder", - folder_storage_path.to_string(), - )); - } - - // Create the physical directory - let abs_path = self.resolve_storage_path(&folder_storage_path); - self.create_directory(&abs_path) - .await - .map_err(|e| DomainError::internal_error("Folder", e.to_string()))?; - - // Create and return the folder entity with a persisted ID - let id = self - .id_mapping_service - .get_or_create_id(&folder_storage_path) - .await - .map_err(|e| DomainError::internal_error("Folder", e.to_string()))?; - let folder = self - .create_folder_entity( - id.clone(), - name.clone(), - folder_storage_path.clone(), - parent_id.clone(), - None, - None, - ) - .await - .map_err(DomainError::from)?; - - // Ensure ID mapping is persisted - let save_result = self.id_mapping_service.save_changes().await; - if let Err(e) = &save_result { - tracing::error!("Failed to save ID mapping for folder {}: {}", id, e); - } else { - tracing::info!( - "Successfully saved ID mapping for folder ID: {} -> path: {} (name: {})", - id, - folder_storage_path.to_string(), - name - ); - } - save_result?; - - tracing::debug!("Created folder with ID: {}", folder.id()); - Ok(folder) - } - - async fn get_folder(&self, id: &str) -> Result { - tracing::debug!("Looking for folder with ID: {}", id); - - // Find path by ID using the mapping service - let storage_path = self.id_mapping_service.get_path_by_id(id).await?; - - // Check if folder exists physically - let abs_path = self.resolve_storage_path(&storage_path); - if !abs_path.exists() || !abs_path.is_dir() { - tracing::error!("Folder not found at path: {}", abs_path.display()); - return Err(DomainError::not_found( - "Folder", - format!("Folder {} not found at {}", id, storage_path.to_string()), - )); - } - - // Get folder metadata - let (created_at, modified_at) = self - .get_folder_metadata(&abs_path) - .await - .map_err(DomainError::from)?; - - // Get folder name from the storage path - let name = match storage_path.file_name() { - Some(name) => name, - None => { - tracing::error!("Invalid folder path: {}", storage_path.to_string()); - return Err(DomainError::validation_error(format!( - "Invalid path: {}", - storage_path.to_string() - ))); - } - }; - - // Determine parent ID if any - let parent = storage_path.parent(); - let parent_id: Option = if parent.is_none() || parent.as_ref().unwrap().is_empty() { - None - } else { - self.id_mapping_service - .get_or_create_id(parent.as_ref().unwrap()) - .await - .ok() - }; - - // Create folder entity - let folder = self - .create_folder_entity( - id.to_string(), - name, - storage_path, - parent_id, - Some(created_at), - Some(modified_at), - ) - .await - .map_err(DomainError::from)?; - - Ok(folder) - } - - async fn get_folder_by_path(&self, storage_path: &StoragePath) -> Result { - // Check if the physical directory exists - let abs_path = self.resolve_storage_path(storage_path); - if !abs_path.exists() || !abs_path.is_dir() { - return Err(DomainError::not_found("Folder", storage_path.to_string())); - } - - // Extract folder name from storage path - let name = match storage_path.file_name() { - Some(name) => name, - None => { - return Err(DomainError::validation_error(format!( - "Invalid path: {}", - storage_path.to_string() - ))); - } - }; - - // Determine parent ID if any - let parent = storage_path.parent(); - let parent_id: Option = if parent.is_none() || parent.as_ref().unwrap().is_empty() { - None - } else { - self.id_mapping_service - .get_or_create_id(parent.as_ref().unwrap()) - .await - .ok() - }; - - // Get folder metadata - let (created_at, modified_at) = self - .get_folder_metadata(&abs_path) - .await - .map_err(DomainError::from)?; - - // Get or create an ID for this path - let id = self - .id_mapping_service - .get_or_create_id(storage_path) - .await?; - tracing::debug!( - "Found folder with path: {:?}, assigned ID: {}", - storage_path.to_string(), - id - ); - - // Create folder entity - let folder = self - .create_folder_entity( - id, - name, - storage_path.clone(), - parent_id, - Some(created_at), - Some(modified_at), - ) - .await - .map_err(DomainError::from)?; - - // Ensure ID mapping is persisted - self.id_mapping_service.save_changes().await?; - - Ok(folder) - } - - async fn list_folders(&self, parent_id: Option<&str>) -> Result, DomainError> { - use futures::stream::StreamExt; - use tokio::time::{Duration, timeout}; - - tracing::info!("Listing folders in parent_id: {:?}", parent_id); - - // Get the parent storage path - let parent_storage_path = match parent_id { - Some(id) => match self._get_folder_storage_path(id).await { - Ok(path) => { - tracing::info!("Found parent folder with path: {:?}", path.to_string()); - path - } - Err(e) => { - tracing::error!("Error getting parent folder by ID: {}: {}", id, e); - return Ok(Vec::new()); - } - }, - None => StoragePath::root(), - }; - - // Get the absolute folder path - let abs_parent_path = self.resolve_storage_path(&parent_storage_path); - tracing::info!("Absolute parent path: {:?}", &abs_parent_path); - - // Ensure the directory exists - if !abs_parent_path.exists() || !abs_parent_path.is_dir() { - tracing::error!( - "Directory does not exist or is not a directory: {:?}", - &abs_parent_path - ); - return Ok(Vec::new()); - } - - // Read the directory with a timeout - let read_dir_timeout = Duration::from_secs(30); - let read_dir_result = match timeout(read_dir_timeout, fs::read_dir(&abs_parent_path)).await - { - Ok(result) => { - result.map_err(|e| DomainError::internal_error("Folder", e.to_string()))? - } - Err(_) => { - return Err(DomainError::internal_error( - "Folder", - format!("Timeout reading directory: {}", abs_parent_path.display()), - )); - } - }; - - let mut folders = Vec::new(); - let mut entries = tokio_stream::wrappers::ReadDirStream::new(read_dir_result); - - while let Some(entry_result) = entries.next().await { - let entry = match entry_result { - Ok(e) => e, - Err(err) => { - tracing::error!("Error reading directory entry: {}", err); - continue; - } - }; - - let metadata = match entry.metadata().await { - Ok(m) => m, - Err(err) => { - tracing::error!( - "Error getting metadata for {}: {}", - entry.path().display(), - err - ); - continue; - } - }; - - if !metadata.is_dir() { - continue; - } - - let folder_name = entry.file_name().to_string_lossy().to_string(); - - // Skip hidden/system directories (e.g. .blobs, .trash, .dedup_temp) - if folder_name.starts_with('.') { - continue; - } - - let folder_storage_path = parent_storage_path.join(&folder_name); - - let get_folder_timeout = Duration::from_secs(5); - let folder_result = timeout( - get_folder_timeout, - self.get_folder_by_path(&folder_storage_path), - ) - .await; - - match folder_result { - Ok(result) => match result { - Ok(folder) => { - tracing::debug!("Found folder: {}", folder.name()); - folders.push(folder); - } - Err(e) => { - tracing::warn!("Could not get folder entity for {}: {}", folder_name, e); - } - }, - Err(_) => { - tracing::warn!("Timeout getting folder entity for {}", folder_name); - } - } - } - - if let Err(e) = self.id_mapping_service.save_changes().await { - tracing::error!("Failed to save ID mappings: {}", e); - } - - tracing::info!("Found {} folders in parent {:?}", folders.len(), parent_id); - Ok(folders) - } - - async fn list_folders_paginated( - &self, - parent_id: Option<&str>, - offset: usize, - limit: usize, - include_total: bool, - ) -> Result<(Vec, Option), DomainError> { - use futures::stream::StreamExt; - use tokio::time::{Duration, timeout}; - - tracing::info!( - "Listing folders in parent_id: {:?} with pagination (offset={}, limit={})", - parent_id, - offset, - limit - ); - - let parent_storage_path = match parent_id { - Some(id) => match self._get_folder_storage_path(id).await { - Ok(path) => path, - Err(e) => { - tracing::error!("Error getting parent folder by ID: {}: {}", id, e); - return Ok((Vec::new(), Some(0))); - } - }, - None => StoragePath::root(), - }; - - let abs_parent_path = self.resolve_storage_path(&parent_storage_path); - - if !abs_parent_path.exists() || !abs_parent_path.is_dir() { - return Ok((Vec::new(), Some(0))); - } - - let total_count = if include_total { - match self.count_directory_items(&abs_parent_path).await { - Ok(count) => Some(count), - Err(e) => { - tracing::warn!("Error counting directory items: {}", e); - None - } - } - } else { - None - }; - - let read_dir_timeout = Duration::from_secs(30); - let read_dir_result = match timeout(read_dir_timeout, fs::read_dir(&abs_parent_path)).await - { - Ok(result) => { - result.map_err(|e| DomainError::internal_error("Folder", e.to_string()))? - } - Err(_) => { - return Err(DomainError::internal_error( - "Folder", - format!("Timeout reading directory: {}", abs_parent_path.display()), - )); - } - }; - - let mut entries = tokio_stream::wrappers::ReadDirStream::new(read_dir_result); - let mut folders = Vec::new(); - let mut current_idx = 0; - - while let Some(entry_result) = entries.next().await { - if current_idx < offset { - current_idx += 1; - continue; - } - - if folders.len() >= limit { - break; - } - - let entry = match entry_result { - Ok(e) => e, - Err(err) => { - tracing::error!("Error reading directory entry: {}", err); - current_idx += 1; - continue; - } - }; - - let file_type = match entry.file_type().await { - Ok(ft) => ft, - Err(e) => { - tracing::error!("Error getting file type: {}", e); - current_idx += 1; - continue; - } - }; - - if !file_type.is_dir() { - current_idx += 1; - continue; - } - - // Skip hidden/system directories (e.g. .blobs, .trash, .dedup_temp) - let dir_name = entry.file_name().to_string_lossy().to_string(); - if dir_name.starts_with('.') { - current_idx += 1; - continue; - } - - let path = entry.path(); - let rel_path = match path.strip_prefix(&self.root_path) { - Ok(rel) => StoragePath::from(rel.to_path_buf()), - Err(_) => { - tracing::error!("Error stripping prefix from path: {}", path.display()); - current_idx += 1; - continue; - } - }; - - let folder_result = - timeout(Duration::from_secs(10), self.get_folder_by_path(&rel_path)).await; - - match folder_result { - Ok(result) => match result { - Ok(folder) => { - folders.push(folder); - } - Err(e) => { - tracing::error!( - "Error getting folder by path: {}: {}", - rel_path.to_string(), - e - ); - } - }, - Err(_) => { - tracing::error!("Timeout getting folder by path: {}", rel_path.to_string()); - } - } - - current_idx += 1; - } - - if !folders.is_empty() - && let Err(e) = self.id_mapping_service.save_changes().await - { - tracing::error!("Error saving ID mappings: {}", e); - } - - Ok((folders, total_count)) - } - - async fn rename_folder(&self, id: &str, new_name: String) -> Result { - let original_folder = self.get_folder(id).await?; - tracing::debug!( - "Renaming folder with ID: {}, Name: {}", - id, - original_folder.name() - ); - - let renamed_folder = original_folder - .with_name(new_name) - .map_err(|e| DomainError::validation_error(e.to_string()))?; - - if self - .check_folder_exists_at_storage_path(renamed_folder.storage_path()) - .await - .map_err(DomainError::from)? - { - return Err(DomainError::already_exists( - "Folder", - renamed_folder.storage_path().to_string(), - )); - } - - let abs_old_path = self.resolve_storage_path(original_folder.storage_path()); - let abs_new_path = self.resolve_storage_path(renamed_folder.storage_path()); - - fs::rename(&abs_old_path, &abs_new_path) - .await - .map_err(|e| DomainError::internal_error("Folder", e.to_string()))?; - - self.id_mapping_service - .update_path(id, renamed_folder.storage_path()) - .await?; - self.id_mapping_service.save_changes().await?; - - tracing::debug!( - "Folder renamed successfully: ID={}, New name={}", - id, - renamed_folder.name() - ); - Ok(renamed_folder) - } - - async fn move_folder( - &self, - id: &str, - new_parent_id: Option<&str>, - ) -> Result { - let original_folder = self.get_folder(id).await?; - tracing::debug!( - "Moving folder with ID: {}, Name: {}", - id, - original_folder.name() - ); - - if original_folder.parent_id() == new_parent_id { - tracing::info!("Folder is already in the target parent, no need to move"); - return Ok(original_folder); - } - - let target_parent_storage_path = match new_parent_id { - Some(parent_id) => match self._get_folder_storage_path(parent_id).await { - Ok(path) => Some(path), - Err(e) => { - return Err(DomainError::internal_error( - "Folder", - format!("Could not get target folder: {}", e), - )); - } - }, - None => None, - }; - - let new_parent_id_option = new_parent_id.map(String::from); - let moved_folder = original_folder - .with_parent(new_parent_id_option, target_parent_storage_path) - .map_err(|e| DomainError::validation_error(e.to_string()))?; - - if self - .check_folder_exists_at_storage_path(moved_folder.storage_path()) - .await - .map_err(DomainError::from)? - { - return Err(DomainError::already_exists( - "Folder", - format!( - "Folder already exists at destination: {}", - moved_folder.storage_path().to_string() - ), - )); - } - - let old_abs_path = self.resolve_storage_path(original_folder.storage_path()); - let new_abs_path = self.resolve_storage_path(moved_folder.storage_path()); - - if let Some(parent) = new_abs_path.parent() { - fs::create_dir_all(parent) - .await - .map_err(|e| DomainError::internal_error("Folder", e.to_string()))?; - } - - fs::rename(&old_abs_path, &new_abs_path) - .await - .map_err(|e| DomainError::internal_error("Folder", e.to_string()))?; - - self.id_mapping_service - .update_path(id, moved_folder.storage_path()) - .await?; - self.id_mapping_service.save_changes().await?; - - tracing::debug!( - "Folder moved successfully: ID={}, New path={:?}", - id, - moved_folder.storage_path().to_string() - ); - Ok(moved_folder) - } - - async fn delete_folder(&self, id: &str) -> Result<(), DomainError> { - use tokio::time::{Duration, timeout}; - - let folder = self.get_folder(id).await?; - let folder_name = folder.name().to_string(); - let storage_path = folder.storage_path().clone(); - - tracing::info!("Deleting folder with ID: {}, Name: {}", id, folder_name); - - let abs_path = self.resolve_storage_path(&storage_path); - let path_for_display = abs_path.display().to_string(); - let path_for_deletion = abs_path.clone(); - - let delete_task = tokio::spawn(async move { - tracing::debug!("Starting removal of folder: {}", path_for_display); - - let path_for_counting = path_for_deletion.clone(); - let entry_count = tokio::task::spawn_blocking(move || { - let mut count = 0; - if let Ok(entries) = std::fs::read_dir(&path_for_counting) { - for _ in entries { - count += 1; - if count > 1000 { - break; - } - } - } - count - }) - .await - .unwrap_or(0); - - if entry_count > 1000 { - tracing::info!("Large folder detected with >1000 entries, using blocking removal"); - let path_for_large_removal = path_for_deletion.clone(); - tokio::task::spawn_blocking(move || { - if let Err(e) = std::fs::remove_dir_all(&path_for_large_removal) { - tracing::error!("Error removing large directory: {}", e); - return Err(std::io::Error::other(format!( - "Failed to remove large directory: {}", - e - ))); - } - Ok(()) - }) - .await - .unwrap_or_else(|e| { - Err(std::io::Error::other(format!( - "Task panicked during directory removal: {}", - e - ))) - }) - } else { - fs::remove_dir_all(&path_for_deletion).await - } - }); - - const DELETE_TIMEOUT_SECS: u64 = 60; - - let delete_result = timeout(Duration::from_secs(DELETE_TIMEOUT_SECS), delete_task).await; - - match delete_result { - Ok(task_result) => match task_result { - Ok(fs_result) => { - if let Err(e) = fs_result { - return Err(DomainError::internal_error("Folder", e.to_string())); - } - } - Err(join_err) => { - return Err(DomainError::internal_error( - "Folder", - format!("Task panicked during folder deletion: {}", join_err), - )); - } - }, - Err(_) => { - tracing::warn!("Timeout waiting for folder deletion, continuing with ID removal"); - } - } - - const MAPPING_TIMEOUT_SECS: u64 = 5; - let remove_id_result = timeout( - Duration::from_secs(MAPPING_TIMEOUT_SECS), - self.id_mapping_service.remove_id(id), - ) - .await; - - match remove_id_result { - Ok(result) => result?, - Err(_) => { - return Err(DomainError::internal_error( - "Folder", - "Timeout removing folder ID from mapping".to_string(), - )); - } - } - - let _ = self.id_mapping_service.save_changes().await; - - tracing::info!( - "Folder deleted successfully: ID={}, Name={}", - id, - folder_name - ); - Ok(()) - } - - async fn folder_exists(&self, storage_path: &StoragePath) -> Result { - self.check_folder_exists_at_storage_path(storage_path) - .await - .map_err(DomainError::from) - } - - async fn get_folder_path(&self, id: &str) -> Result { - self._get_folder_storage_path(id) - .await - .map_err(DomainError::from) - } - - async fn move_to_trash(&self, folder_id: &str) -> Result<(), DomainError> { - self._trash_move_to_trash(folder_id) - .await - .map_err(DomainError::from) - } - - async fn restore_from_trash( - &self, - folder_id: &str, - original_path: &str, - ) -> Result<(), DomainError> { - self._trash_restore_from_trash(folder_id, original_path) - .await - .map_err(DomainError::from) - } - - async fn delete_folder_permanently(&self, folder_id: &str) -> Result<(), DomainError> { - self._trash_delete_folder_permanently(folder_id) - .await - .map_err(DomainError::from) - } -} diff --git a/src/infrastructure/repositories/folder_fs_repository_trash.rs b/src/infrastructure/repositories/folder_fs_repository_trash.rs deleted file mode 100644 index 78913a1e..00000000 --- a/src/infrastructure/repositories/folder_fs_repository_trash.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::path::PathBuf; -use tokio::fs; -use tracing::{debug, error}; - -use crate::infrastructure::repositories::folder_fs_repository::FolderFsRepository; -use crate::infrastructure::repositories::repository_errors::FolderRepositoryResult; - -// This file contains the implementation of trash-related methods -// for the FolderFsRepository folder repository - -// Implementation of trash methods for the folder repository -impl FolderFsRepository { - // Gets the full path to the trash directory - fn get_trash_dir(&self) -> PathBuf { - self.get_root_path().join(".trash").join("folders") - } - - // Creates a unique path in the trash for the folder - async fn create_trash_folder_path(&self, folder_id: &str) -> FolderRepositoryResult { - let trash_dir = self.get_trash_dir(); - - // Ensure the trash directory exists - if !trash_dir.exists() { - fs::create_dir_all(&trash_dir) - .await - .map_err(|e| FolderRepositoryError::StorageError(e.to_string()))?; - } - - // Create a unique path for the folder in the trash - Ok(trash_dir.join(folder_id)) - } -} - -// Implementation of public FolderRepository trait methods related to trash -// Implementation of internal methods for trash functionality -// These will be enabled when the trash feature is re-enabled -impl FolderFsRepository { - /// Helper method that will be used for trash functionality - pub(crate) async fn _trash_move_to_trash(&self, folder_id: &str) -> FolderRepositoryResult<()> { - debug!("Moving folder to trash: {}", folder_id); - - // Get the physical path of the folder - let folder_path = match self.get_mapped_folder_path(folder_id).await { - Ok(path) => path, - Err(e) => { - error!("Error getting folder path {}: {:?}", folder_id, e); - return Err(e); - } - }; - - let folder_path_buf = PathBuf::from(folder_path.to_string()); - - // Verify the folder exists - if !folder_path_buf.exists() { - return Err(FolderRepositoryError::NotFound(format!( - "Folder not found: {}", - folder_id - ))); - } - - // Create directory in the trash - let trash_folder_path = self.create_trash_folder_path(folder_id).await?; - - // Physically move the folder to the trash - match fs::rename(&folder_path_buf, &trash_folder_path).await { - Ok(_) => { - debug!( - "Folder moved to trash: {} -> {}", - folder_path_buf.display(), - trash_folder_path.display() - ); - - // Update the mapping to the new path in the trash - if let Err(e) = self - .update_mapped_folder_path(folder_id, &trash_folder_path) - .await - { - error!("Error updating folder mapping in trash: {}", e); - return Err(e); - } - - Ok(()) - } - Err(e) => { - error!("Error moving folder to trash: {}", e); - Err(FolderRepositoryError::StorageError(e.to_string())) - } - } - } - - /// Restores a folder from the trash to its original location - pub(crate) async fn _trash_restore_from_trash( - &self, - folder_id: &str, - original_path: &str, - ) -> FolderRepositoryResult<()> { - debug!("Restoring folder {} to {}", folder_id, original_path); - - // Get the current path in the trash - let current_path = match self.get_mapped_folder_path(folder_id).await { - Ok(path) => PathBuf::from(path), - Err(e) => { - error!("Error getting current folder path {}: {:?}", folder_id, e); - return Err(e); - } - }; - - // Convert the original path to PathBuf - let original_path_buf = PathBuf::from(original_path); - - // Ensure the destination parent directory exists - if let Some(parent) = original_path_buf.parent() - && !parent.exists() - { - fs::create_dir_all(parent).await.map_err(|e| { - error!("Error creating parent directory for restoration: {}", e); - FolderRepositoryError::StorageError(e.to_string()) - })?; - } - - // Move the folder from the trash to its original location - match fs::rename(¤t_path, &original_path_buf).await { - Ok(_) => { - debug!( - "Folder restored: {} -> {}", - current_path.display(), - original_path_buf.display() - ); - - // Update the mapping to the original path - if let Err(e) = self - .update_mapped_folder_path(folder_id, &original_path_buf) - .await - { - error!("Error updating restored folder mapping: {}", e); - return Err(e); - } - - Ok(()) - } - Err(e) => { - error!("Error restoring folder: {}", e); - Err(FolderRepositoryError::StorageError(e.to_string())) - } - } - } - - /// Permanently deletes a folder (used by the trash) - pub(crate) async fn _trash_delete_folder_permanently( - &self, - folder_id: &str, - ) -> FolderRepositoryResult<()> { - debug!("Permanently deleting folder: {}", folder_id); - - // Similar to delete_folder but without additional validations - let folder_path = match self.get_mapped_folder_path(folder_id).await { - Ok(path) => PathBuf::from(path), - Err(e) => { - error!("Error getting folder path {}: {:?}", folder_id, e); - return Err(e); - } - }; - - // Delete the folder recursively - if folder_path.exists() { - match fs::remove_dir_all(&folder_path).await { - Ok(_) => { - debug!("Folder permanently deleted: {}", folder_path.display()); - } - Err(e) => { - error!("Error permanently deleting folder: {}", e); - // Don't report error if the folder no longer exists - if e.kind() != std::io::ErrorKind::NotFound { - return Err(FolderRepositoryError::StorageError(e.to_string())); - } - } - } - } - - // Remove the mapping - if let Err(e) = self.remove_mapped_folder_id(folder_id).await { - error!("Error removing folder mapping: {}", e); - return Err(e); - } - - debug!("Folder permanently deleted successfully: {}", folder_id); - Ok(()) - } -} - -// Re-exports needed by the compiler -use crate::infrastructure::repositories::repository_errors::FolderRepositoryError; diff --git a/src/infrastructure/repositories/mod.rs b/src/infrastructure/repositories/mod.rs index 4823837f..9f75fa0a 100644 --- a/src/infrastructure/repositories/mod.rs +++ b/src/infrastructure/repositories/mod.rs @@ -1,21 +1,10 @@ -pub mod folder_fs_repository; -pub mod parallel_file_processor; -pub mod repository_errors; - -// Repositorios CQRS (Read/Write) + composite -pub mod composite_file_repository; -pub mod file_fs_read_repository; -pub mod file_fs_write_repository; - -pub mod folder_fs_repository_trash; pub mod share_fs_repository; -pub mod trash_fs_repository; -// Repositorios PostgreSQL +// Repositorios PostgreSQL (blob-storage model) pub mod pg; // Re-exportar para facilitar acceso -pub use composite_file_repository::CompositeFileRepository; -pub use file_fs_read_repository::FileFsReadRepository; -pub use file_fs_write_repository::FileFsWriteRepository; -pub use pg::{SessionPgRepository, UserPgRepository}; +pub use pg::{ + FileBlobReadRepository, FileBlobWriteRepository, FolderDbRepository, + SessionPgRepository, TrashDbRepository, UserPgRepository, +}; diff --git a/src/infrastructure/repositories/parallel_file_processor.rs b/src/infrastructure/repositories/parallel_file_processor.rs deleted file mode 100644 index c7c45562..00000000 --- a/src/infrastructure/repositories/parallel_file_processor.rs +++ /dev/null @@ -1,571 +0,0 @@ -use bytes::{Bytes, BytesMut}; -use futures::future::join_all; -use std::io::{self, SeekFrom}; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::fs::File; -use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; -use tokio::sync::{Mutex, Semaphore}; -use tokio::task; -use tracing::{debug, error, info}; - -use crate::common::config::AppConfig; -use crate::infrastructure::repositories::repository_errors::FileRepositoryError; -use crate::infrastructure::services::buffer_pool::BufferPool; - -/// Structure for the byte range to process -#[derive(Debug, Clone, Copy)] -pub struct ChunkRange { - /// Chunk index - pub index: usize, - /// Start position in bytes - pub start: u64, - /// Chunk size in bytes - pub size: usize, -} - -/// Specific buffer pooling for BytesMut -pub struct BytesBufferPool { - buffers: Mutex>, - buffer_size: usize, - max_buffers: usize, -} - -impl BytesBufferPool { - pub fn new(buffer_size: usize, max_buffers: usize) -> Self { - Self { - buffers: Mutex::new(Vec::with_capacity(max_buffers)), - buffer_size, - max_buffers, - } - } - - /// Get a buffer from the pool or create a new one - pub async fn get_buffer(&self) -> BytesMut { - let mut buffers = self.buffers.lock().await; - - if let Some(mut buffer) = buffers.pop() { - // Reuse existing buffer - buffer.clear(); // Keep capacity, clear content - buffer - } else { - // Create new buffer if the pool is empty - BytesMut::with_capacity(self.buffer_size) - } - } - - /// Return a buffer to the pool for reuse - pub async fn return_buffer(&self, mut buffer: BytesMut) { - // Reset the buffer for reuse - buffer.clear(); - - let mut buffers = self.buffers.lock().await; - - // Only keep up to max_buffers - if buffers.len() < self.max_buffers { - buffers.push(buffer); - } - // If we already have enough buffers, this one will be discarded - } -} - -/// Parallel file processor for IO-intensive operations -pub struct ParallelFileProcessor { - /// Application configuration - config: AppConfig, - /// Semaphore to limit global concurrency - concurrency_limiter: Arc, - /// Buffer pool to optimize memory - buffer_pool: Option>, - /// BytesMut buffer pool for zero-copy operations - bytes_pool: Arc, -} - -impl ParallelFileProcessor { - /// Creates a new processor instance - pub fn new(config: AppConfig) -> Self { - let concurrency_limiter = Arc::new(Semaphore::new(config.concurrency.max_concurrent_io)); - - // Create BytesMut pool for efficient operations - let chunk_size = config.resources.chunk_size_bytes; - let max_chunks = config.concurrency.max_parallel_chunks; - let bytes_pool = Arc::new(BytesBufferPool::new(chunk_size, max_chunks * 2)); - - Self { - config, - concurrency_limiter, - buffer_pool: None, - bytes_pool, - } - } - - /// Creates a new processor instance with a buffer pool - pub fn new_with_buffer_pool(config: AppConfig, buffer_pool: Arc) -> Self { - let concurrency_limiter = Arc::new(Semaphore::new(config.concurrency.max_concurrent_io)); - - // Create BytesMut pool for efficient operations - let chunk_size = config.resources.chunk_size_bytes; - let max_chunks = config.concurrency.max_parallel_chunks; - let bytes_pool = Arc::new(BytesBufferPool::new(chunk_size, max_chunks * 2)); - - Self { - config, - concurrency_limiter, - buffer_pool: Some(buffer_pool), - bytes_pool, - } - } - - /// Divides a file into chunks for parallel processing - pub fn calculate_chunks(&self, file_size: u64) -> Vec { - // Determine if the file needs parallel processing - let needs_parallel = self - .config - .resources - .needs_parallel_processing(file_size, &self.config.concurrency); - - if !needs_parallel { - // For small files, use a single chunk - return vec![ChunkRange { - index: 0, - start: 0, - size: file_size as usize, - }]; - } - - // Calculate optimal number of chunks - let chunk_count = self - .config - .resources - .calculate_optimal_chunks(file_size, &self.config.concurrency); - - // Calculate size of each chunk - let chunk_size = self - .config - .resources - .calculate_chunk_size(file_size, chunk_count); - - // Create chunk ranges - let mut chunks = Vec::with_capacity(chunk_count); - - let mut start = 0; - for i in 0..chunk_count { - let current_chunk_size = if i == chunk_count - 1 { - // Last chunk might be smaller - (file_size - start) as usize - } else { - chunk_size - }; - - chunks.push(ChunkRange { - index: i, - start, - size: current_chunk_size, - }); - - start += current_chunk_size as u64; - } - - debug!( - "File size: {} bytes, divided into {} chunks of ~{} bytes each", - file_size, - chunks.len(), - chunk_size - ); - - chunks - } - - /// Reads a file in parallel and returns the complete content - /// Optimized implementation using BytesMut to reduce memory copies - pub async fn read_file_parallel( - &self, - file_path: &PathBuf, - ) -> Result, FileRepositoryError> { - // Get file size - let metadata = tokio::fs::metadata(file_path) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - let file_size = metadata.len(); - - // Check if the file is too large for memory - if !self.config.resources.can_load_in_memory(file_size) { - return Err(FileRepositoryError::Other(format!( - "File too large to load in memory: {} MB (max: {} MB)", - file_size / (1024 * 1024), - self.config.resources.max_in_memory_file_size_mb - ))); - } - - // Calculate chunks - let chunks = self.calculate_chunks(file_size); - - if chunks.len() == 1 { - // For a single chunk, use simple reading with buffer pool if available - info!( - "Reading file with size {}MB as a single chunk", - file_size / (1024 * 1024) - ); - - if let Some(pool) = &self.buffer_pool { - // Use buffer from the pool for efficient reading - debug!("Using buffer pool for single chunk read"); - let mut buffer = pool.get_buffer().await; - - // If the buffer is too small, revert to standard implementation - if buffer.capacity() < file_size as usize { - debug!( - "Buffer from pool too small ({}), using standard read", - buffer.capacity() - ); - let content = tokio::fs::read(file_path) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - return Ok(content); - } - - // Use memory buffer from the pool - let mut file = File::open(file_path) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - let read_size = file - .read(buffer.as_mut_slice()) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - buffer.set_used(read_size); - - // Convert to Vec - let content = buffer.into_vec(); - return Ok(content); - } else { - // Standard implementation without pool - let content = tokio::fs::read(file_path) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - return Ok(content); - } - } - - // For multiple chunks, use parallel reading - info!( - "Reading file with size {}MB in {} parallel chunks using BytesMut", - file_size / (1024 * 1024), - chunks.len() - ); - - // Create final result buffer (pre-allocated) - let mut result = BytesMut::with_capacity(file_size as usize); - result.resize(file_size as usize, 0); - let result_mutex = Arc::new(Mutex::new(result)); - - // Create tasks for each chunk - let mut tasks = Vec::with_capacity(chunks.len()); - - // Open file once and share it - let file = Arc::new( - File::open(file_path) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?, - ); - - // Reference to BytesMut pool - let bytes_pool = self.bytes_pool.clone(); - - // Process chunks in parallel - for chunk in chunks { - let file_clone = file.clone(); - let result_clone = result_mutex.clone(); - let semaphore_clone = self.concurrency_limiter.clone(); - let bytes_pool_clone = bytes_pool.clone(); - - // Spawn task for this chunk - no need to copy the original data - let task = task::spawn(async move { - // Acquire semaphore permit - let _permit = semaphore_clone.acquire().await.unwrap(); - - // Get a reusable buffer from the BytesMut pool - let mut chunk_buffer = bytes_pool_clone.get_buffer().await; - - // Ensure it has sufficient capacity - if chunk_buffer.capacity() < chunk.size { - chunk_buffer = BytesMut::with_capacity(chunk.size); - } - // Resize to the exact size needed - chunk_buffer.resize(chunk.size, 0); - - // Create a duplicate file descriptor for independent use - let mut file_handle = file_clone.try_clone().await?; - - // Position and read directly into the BytesMut - file_handle.seek(SeekFrom::Start(chunk.start)).await?; - let bytes_read = file_handle - .read_exact(&mut chunk_buffer[..chunk.size]) - .await?; - - if bytes_read != chunk.size { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!( - "Expected to read {} bytes but got {}", - chunk.size, bytes_read - ), - )); - } - - // Write to final result - let mut result_lock = result_clone.lock().await; - let start_pos = chunk.start as usize; - let end_pos = start_pos + chunk.size; - - // Use copy_from_slice to copy from BytesMut to result buffer - result_lock[start_pos..end_pos].copy_from_slice(&chunk_buffer[..chunk.size]); - - // Return the buffer to the pool for reuse - bytes_pool_clone.return_buffer(chunk_buffer).await; - - // Log progress - debug!( - "Chunk {} processed: {} bytes from offset {}", - chunk.index, chunk.size, chunk.start - ); - - Ok::<_, io::Error>(()) - }); - - tasks.push(task); - } - - // Wait for all tasks to complete - let results = join_all(tasks).await; - - // Check for errors - for (i, task_result) in results.into_iter().enumerate() { - match task_result { - Ok(Ok(())) => {} - Ok(Err(e)) => { - error!("Error in chunk {}: {}", i, e); - return Err(FileRepositoryError::StorageError(e.to_string())); - } - Err(e) => { - error!("Task error in chunk {}: {}", i, e); - return Err(FileRepositoryError::Other(format!("Task error: {}", e))); - } - } - } - - // Get the final result and convert to Vec - let result_buffer = result_mutex.lock().await; - let result_vec = result_buffer.to_vec(); - - info!( - "Successfully read file of {}MB in parallel with optimized BytesMut", - file_size / (1024 * 1024) - ); - Ok(result_vec) - } - - /// Writes a file in parallel from a buffer - /// Optimized implementation using BytesMut/Bytes to reduce memory copies - pub async fn write_file_parallel( - &self, - file_path: &PathBuf, - content: &[u8], - ) -> Result<(), FileRepositoryError> { - let file_size = content.len() as u64; - - // Calculate chunks - let chunks = self.calculate_chunks(file_size); - - if chunks.len() == 1 { - // For a single chunk, use simple writing - info!( - "Writing file with size {}MB as a single chunk", - file_size / (1024 * 1024) - ); - - // Standard implementation (buffer pooling offers no advantages for simple writing) - tokio::fs::write(file_path, content) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - return Ok(()); - } - - // For multiple chunks, use parallel writing - info!( - "Writing file with size {}MB in {} parallel chunks using Bytes", - file_size / (1024 * 1024), - chunks.len() - ); - - // Create file (we don't use Mutex to reduce contention) - let file = File::create(file_path) - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - // Convert content to Bytes (single copy step) - let content_bytes = Bytes::copy_from_slice(content); - - // Create tasks for each chunk - let mut tasks = Vec::with_capacity(chunks.len()); - - // Process chunks in parallel - for chunk in chunks { - let file_clone = file - .try_clone() - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - let semaphore_clone = self.concurrency_limiter.clone(); - - // Create Bytes slice (doesn't copy data, only references) - let start_idx = chunk.start as usize; - let end_idx = start_idx + chunk.size; - let chunk_data = content_bytes.slice(start_idx..end_idx); - - // Create and launch task - let task = task::spawn(async move { - // Acquire semaphore permit - let _permit = semaphore_clone.acquire().await.unwrap(); - - // Position and write - let mut file_handle = file_clone; - file_handle.seek(SeekFrom::Start(chunk.start)).await?; - file_handle.write_all(&chunk_data).await?; - - // Log progress - debug!( - "Chunk {} written: {} bytes at offset {}", - chunk.index, chunk.size, chunk.start - ); - - Ok::<_, io::Error>(()) - }); - - tasks.push(task); - } - - // Wait for all tasks to complete - let results = join_all(tasks).await; - - // Check for errors - for (i, task_result) in results.into_iter().enumerate() { - match task_result { - Ok(Ok(())) => {} - Ok(Err(e)) => { - error!("Error in chunk {}: {}", i, e); - return Err(FileRepositoryError::StorageError(e.to_string())); - } - Err(e) => { - error!("Task error in chunk {}: {}", i, e); - return Err(FileRepositoryError::Other(format!("Task error: {}", e))); - } - } - } - - // Ensure everything has been written correctly - let mut file_handle = file; - file_handle - .flush() - .await - .map_err(|e| FileRepositoryError::StorageError(e.to_string()))?; - - info!( - "Successfully wrote file of {}MB in parallel with optimized Bytes", - file_size / (1024 * 1024) - ); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::BufMut; - use tempfile::tempdir; - - #[tokio::test] - async fn test_parallel_read_write() { - // Create configuration with low threshold for testing - let mut config = AppConfig::default(); - config.concurrency.min_size_for_parallel_chunks_mb = 1; // 1MB for testing - config.concurrency.max_parallel_chunks = 4; - - let processor = ParallelFileProcessor::new(config); - - // Create temporary directory - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("test_file.bin"); - - // Create test data (2MB) - let size = 2 * 1024 * 1024; - let mut test_data = Vec::with_capacity(size); - for i in 0..size { - test_data.push((i % 256) as u8); - } - - // Write file in parallel - processor - .write_file_parallel(&file_path, &test_data) - .await - .unwrap(); - - // Read file in parallel - let read_data = processor.read_file_parallel(&file_path).await.unwrap(); - - // Verify that the data is identical - assert_eq!(test_data.len(), read_data.len()); - assert_eq!(test_data, read_data); - } - - #[tokio::test] - async fn test_bytesmut_pool() { - // Create pool - let pool = BytesBufferPool::new(1024, 5); - - // Get buffer - let mut buffer1 = pool.get_buffer().await; - buffer1.put_slice(b"test data"); - assert_eq!(&buffer1[..9], b"test data"); - - // Return buffer to the pool - pool.return_buffer(buffer1).await; - - // Get another buffer (should be the same one) - let buffer2 = pool.get_buffer().await; - assert_eq!(buffer2.capacity(), 1024); - - // The buffer should be empty (cleared) - assert_eq!(buffer2.len(), 0); - } - - #[test] - fn test_chunk_calculation() { - // Create test configuration - let mut config = AppConfig::default(); - config.concurrency.min_size_for_parallel_chunks_mb = 100; // 100MB - config.concurrency.max_parallel_chunks = 4; - config.concurrency.parallel_chunk_size_bytes = 50 * 1024 * 1024; // 50MB - - let processor = ParallelFileProcessor::new(config); - - // Small file (10MB) - let small_file_size = 10 * 1024 * 1024; - let chunks = processor.calculate_chunks(small_file_size); - assert_eq!(chunks.len(), 1); - assert_eq!(chunks[0].size as u64, small_file_size); - - // Large file (300MB) - let large_file_size = 300 * 1024 * 1024; - let chunks = processor.calculate_chunks(large_file_size); - assert_eq!(chunks.len(), 4); // Limited to max_parallel_chunks - - // Verify that all chunks add up to the total size - let total_size: u64 = chunks.iter().map(|c| c.size as u64).sum(); - assert_eq!(total_size, large_file_size); - } -} diff --git a/src/infrastructure/repositories/pg/file_blob_read_repository.rs b/src/infrastructure/repositories/pg/file_blob_read_repository.rs new file mode 100644 index 00000000..51015fc3 --- /dev/null +++ b/src/infrastructure/repositories/pg/file_blob_read_repository.rs @@ -0,0 +1,271 @@ +//! PostgreSQL + Blob-backed file read repository. +//! +//! Implements `FileReadPort` using: +//! - `storage.files` table for metadata lookups +//! - `DedupPort` for reading content-addressable blobs from the filesystem + +use async_trait::async_trait; +use bytes::Bytes; +use futures::Stream; +use sqlx::PgPool; +use std::sync::Arc; + +use crate::application::ports::dedup_ports::DedupPort; +use crate::application::ports::storage_ports::FileReadPort; +use crate::common::errors::DomainError; +use crate::domain::entities::file::File; +use crate::domain::repositories::folder_repository::FolderRepository; +use crate::domain::services::path_service::StoragePath; + +use super::folder_db_repository::FolderDbRepository; + +/// File read repository backed by PostgreSQL metadata + blob storage. +pub struct FileBlobReadRepository { + pool: Arc, + dedup: Arc, + folder_repo: Arc, +} + +impl FileBlobReadRepository { + pub fn new( + pool: Arc, + dedup: Arc, + folder_repo: Arc, + ) -> Self { + Self { + pool, + dedup, + folder_repo, + } + } + + /// Build a virtual StoragePath for a file. + async fn build_file_path( + &self, + folder_id: Option<&str>, + file_name: &str, + ) -> Result { + if let Some(fid) = folder_id { + let folder_path = self.folder_repo.get_folder_path(fid).await?; + Ok(folder_path.join(file_name)) + } else { + Ok(StoragePath::from_string(file_name)) + } + } + + /// Convert a database row into a `File` domain entity. + async fn row_to_file( + &self, + id: String, + name: String, + folder_id: Option, + size: i64, + mime_type: String, + created_at: i64, + modified_at: i64, + ) -> Result { + let storage_path = self + .build_file_path(folder_id.as_deref(), &name) + .await?; + File::with_timestamps( + id, + name, + storage_path, + size as u64, + mime_type, + folder_id, + created_at as u64, + modified_at as u64, + ) + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("entity: {e}"))) + } + + /// Get the blob hash for a file. + async fn get_blob_hash(&self, file_id: &str) -> Result { + sqlx::query_scalar::<_, String>( + "SELECT blob_hash FROM storage.files WHERE id = $1::uuid AND NOT is_trashed", + ) + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("hash lookup: {e}")))? + .ok_or_else(|| DomainError::not_found("File", file_id)) + } +} + +#[async_trait] +impl FileReadPort for FileBlobReadRepository { + async fn get_file(&self, id: &str) -> Result { + let row = sqlx::query_as::<_, (String, String, Option, i64, String, i64, i64)>( + r#" + SELECT id::text, name, folder_id::text, size, mime_type, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.files + WHERE id = $1::uuid AND NOT is_trashed + "#, + ) + .bind(id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("get: {e}")))? + .ok_or_else(|| DomainError::not_found("File", id))?; + + self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6) + .await + } + + async fn list_files( + &self, + folder_id: Option<&str>, + ) -> Result, DomainError> { + let rows: Vec<(String, String, Option, i64, String, i64, i64)> = + if let Some(fid) = folder_id { + sqlx::query_as( + r#" + SELECT id::text, name, folder_id::text, size, mime_type, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.files + WHERE folder_id = $1::uuid AND NOT is_trashed + ORDER BY name + "#, + ) + .bind(fid) + .fetch_all(self.pool.as_ref()) + .await + } else { + sqlx::query_as( + r#" + SELECT id::text, name, folder_id::text, size, mime_type, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.files + WHERE folder_id IS NULL AND NOT is_trashed + ORDER BY name + "#, + ) + .fetch_all(self.pool.as_ref()) + .await + } + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("list: {e}")))?; + + let mut files = Vec::with_capacity(rows.len()); + for (id, name, fid, size, mime, ca, ma) in rows { + files.push(self.row_to_file(id, name, fid, size, mime, ca, ma).await?); + } + Ok(files) + } + + async fn get_file_content(&self, id: &str) -> Result, DomainError> { + let blob_hash = self.get_blob_hash(id).await?; + self.dedup.read_blob(&blob_hash).await + } + + async fn get_file_stream( + &self, + id: &str, + ) -> Result> + Send>, DomainError> { + // Read blob as bytes and wrap in a single-chunk stream. + // For very large files, a true streaming implementation from the + // blob file would be better, but DedupPort API currently returns bytes. + let blob_hash = self.get_blob_hash(id).await?; + let content = self.dedup.read_blob_bytes(&blob_hash).await?; + + let stream = futures::stream::once(async move { Ok(content) }); + Ok(Box::new(stream)) + } + + async fn get_file_range_stream( + &self, + id: &str, + start: u64, + end: Option, + ) -> Result> + Send>, DomainError> { + let blob_hash = self.get_blob_hash(id).await?; + let content = self.dedup.read_blob_bytes(&blob_hash).await?; + + let start = start as usize; + let end = end.map_or(content.len(), |e| e as usize).min(content.len()); + + if start >= content.len() { + return Ok(Box::new(futures::stream::empty())); + } + + let slice = content.slice(start..end); + let stream = futures::stream::once(async move { Ok(slice) }); + Ok(Box::new(stream)) + } + + async fn get_file_mmap(&self, id: &str) -> Result { + let blob_hash = self.get_blob_hash(id).await?; + self.dedup.read_blob_bytes(&blob_hash).await + } + + async fn get_file_path(&self, id: &str) -> Result { + let row = sqlx::query_as::<_, (String, Option)>( + r#" + SELECT name, folder_id::text + FROM storage.files + WHERE id = $1::uuid AND NOT is_trashed + "#, + ) + .bind(id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("path: {e}")))? + .ok_or_else(|| DomainError::not_found("File", id))?; + + self.build_file_path(row.1.as_deref(), &row.0).await + } + + async fn get_parent_folder_id(&self, path: &str) -> Result { + // Walk the path to find the parent folder, searching by folder names + let path = path.trim_start_matches('/').trim_end_matches('/'); + let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + + if segments.is_empty() { + return Err(DomainError::not_found("Folder", "empty path")); + } + + // For path "a/b/c/file.txt", the parent folder path is "a/b/c" + // But we don't know which part is folders vs filename. + // Walk segments trying to find matching folders. + let mut current_parent: Option = None; + + for segment in &segments { + let row = if let Some(ref pid) = current_parent { + sqlx::query_as::<_, (String,)>( + r#" + SELECT id::text FROM storage.folders + WHERE name = $1 AND parent_id = $2::uuid AND NOT is_trashed + "#, + ) + .bind(segment) + .bind(pid) + .fetch_optional(self.pool.as_ref()) + .await + } else { + sqlx::query_as::<_, (String,)>( + r#" + SELECT id::text FROM storage.folders + WHERE name = $1 AND parent_id IS NULL AND NOT is_trashed + "#, + ) + .bind(segment) + .fetch_optional(self.pool.as_ref()) + .await + } + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("path walk: {e}")))?; + + match row { + Some(r) => current_parent = Some(r.0), + None => break, // This segment is not a folder → it's the filename + } + } + + current_parent.ok_or_else(|| { + DomainError::not_found("Folder", format!("parent for path: {path}")) + }) + } +} diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs new file mode 100644 index 00000000..c545c97a --- /dev/null +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -0,0 +1,435 @@ +//! PostgreSQL + Blob-backed file write repository. +//! +//! Implements `FileWritePort` using: +//! - `storage.files` table for metadata +//! - `DedupPort` for content-addressable blob storage on the filesystem + +use async_trait::async_trait; +use bytes::Bytes; +use futures::Stream; +use sqlx::PgPool; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; + +use crate::application::ports::dedup_ports::DedupPort; +use crate::application::ports::storage_ports::FileWritePort; +use crate::common::errors::DomainError; +use crate::domain::entities::file::File; +use crate::domain::repositories::folder_repository::FolderRepository; +use crate::domain::services::path_service::StoragePath; + +use super::folder_db_repository::FolderDbRepository; + +/// File write repository backed by PostgreSQL metadata + blob storage. +pub struct FileBlobWriteRepository { + pool: Arc, + dedup: Arc, + folder_repo: Arc, +} + +impl FileBlobWriteRepository { + pub fn new( + pool: Arc, + dedup: Arc, + folder_repo: Arc, + ) -> Self { + Self { + pool, + dedup, + folder_repo, + } + } + + /// Build a virtual StoragePath for a file from its DB metadata. + async fn build_file_path( + &self, + folder_id: Option<&str>, + file_name: &str, + ) -> Result { + if let Some(fid) = folder_id { + let folder_path = self.folder_repo.get_folder_path(fid).await?; + Ok(folder_path.join(file_name)) + } else { + Ok(StoragePath::from_string(file_name)) + } + } + + /// Convert a database row into a `File` domain entity. + async fn row_to_file( + &self, + id: String, + name: String, + folder_id: Option, + size: i64, + mime_type: String, + created_at: i64, + modified_at: i64, + ) -> Result { + let storage_path = self + .build_file_path(folder_id.as_deref(), &name) + .await?; + File::with_timestamps( + id, + name, + storage_path, + size as u64, + mime_type, + folder_id, + created_at as u64, + modified_at as u64, + ) + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("entity: {e}"))) + } + + /// Derive user_id from the parent folder, or error if folder_id is None. + async fn resolve_user_id(&self, folder_id: Option<&str>) -> Result { + match folder_id { + Some(fid) => self.folder_repo.get_folder_user_id(fid).await, + None => Err(DomainError::internal_error( + "FileBlobWrite", + "folder_id is required to determine file owner", + )), + } + } +} + +#[async_trait] +impl FileWritePort for FileBlobWriteRepository { + async fn save_file( + &self, + name: String, + folder_id: Option, + content_type: String, + content: Vec, + ) -> Result { + let user_id = self.resolve_user_id(folder_id.as_deref()).await?; + let size = content.len() as i64; + + // Store content in blob store + let dedup_result = self + .dedup + .store_bytes(&content, Some(content_type.clone())) + .await?; + let blob_hash = dedup_result.hash().to_string(); + + // Insert file metadata + let row = sqlx::query_as::<_, (String, i64, i64)>( + r#" + INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type) + VALUES ($1, $2::uuid, $3, $4, $5, $6) + RETURNING id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + "#, + ) + .bind(&name) + .bind(&folder_id) + .bind(&user_id) + .bind(&blob_hash) + .bind(size) + .bind(&content_type) + .fetch_one(self.pool.as_ref()) + .await + .map_err(|e| { + if let sqlx::Error::Database(ref db_err) = e { + if db_err.code().as_deref() == Some("23505") { + return DomainError::already_exists( + "File", + format!("{name} already exists in folder"), + ); + } + } + DomainError::internal_error("FileBlobWrite", format!("insert: {e}")) + })?; + + tracing::info!( + "💾 BLOB WRITE: {} ({} bytes, hash: {})", + name, + size, + &blob_hash[..12] + ); + + self.row_to_file( + row.0, + name, + folder_id, + size, + content_type, + row.1, + row.2, + ) + .await + } + + async fn save_file_from_stream( + &self, + name: String, + folder_id: Option, + content_type: String, + stream: Pin> + Send>>, + ) -> Result { + use futures::StreamExt; + + // Collect stream into bytes (blobs are content-addressed, need full content for hash) + let mut content = Vec::new(); + let mut stream = stream; + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| { + DomainError::internal_error("FileBlobWrite", format!("stream read: {e}")) + })?; + content.extend_from_slice(&chunk); + } + + self.save_file(name, folder_id, content_type, content).await + } + + async fn move_file( + &self, + file_id: &str, + target_folder_id: Option, + ) -> Result { + // If moving to a different folder, get the new user_id (must be same user) + let row = sqlx::query_as::<_, (String, String, Option, i64, String, i64, i64)>( + r#" + UPDATE storage.files + SET folder_id = $1::uuid, updated_at = NOW() + WHERE id = $2::uuid AND NOT is_trashed + RETURNING id::text, name, folder_id::text, size, mime_type, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + "#, + ) + .bind(&target_folder_id) + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("move: {e}")))? + .ok_or_else(|| DomainError::not_found("File", file_id))?; + + self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6) + .await + } + + async fn rename_file( + &self, + file_id: &str, + new_name: &str, + ) -> Result { + let row = sqlx::query_as::<_, (String, String, Option, i64, String, i64, i64)>( + r#" + UPDATE storage.files + SET name = $1, updated_at = NOW() + WHERE id = $2::uuid AND NOT is_trashed + RETURNING id::text, name, folder_id::text, size, mime_type, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + "#, + ) + .bind(new_name) + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + if let sqlx::Error::Database(ref db_err) = e { + if db_err.code().as_deref() == Some("23505") { + return DomainError::already_exists( + "File", + format!("{new_name} already exists"), + ); + } + } + DomainError::internal_error("FileBlobWrite", format!("rename: {e}")) + })? + .ok_or_else(|| DomainError::not_found("File", file_id))?; + + self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6) + .await + } + + async fn delete_file(&self, id: &str) -> Result<(), DomainError> { + // Get blob_hash before deleting so we can decrement ref + let hash = sqlx::query_scalar::<_, String>( + "SELECT blob_hash FROM storage.files WHERE id = $1::uuid", + ) + .bind(id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("hash lookup: {e}")))?; + + let result = sqlx::query("DELETE FROM storage.files WHERE id = $1::uuid") + .bind(id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("delete: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("File", id)); + } + + // Decrement blob reference + if let Some(h) = hash { + if let Err(e) = self.dedup.remove_reference(&h).await { + tracing::warn!("Failed to decrement blob ref for {}: {}", &h[..12], e); + } + } + + Ok(()) + } + + async fn update_file_content( + &self, + file_id: &str, + content: Vec, + ) -> Result<(), DomainError> { + // Get old blob hash to decrement ref + let old_hash = sqlx::query_scalar::<_, String>( + "SELECT blob_hash FROM storage.files WHERE id = $1::uuid", + ) + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("old hash: {e}")))? + .ok_or_else(|| DomainError::not_found("File", file_id))?; + + // Store new content + let new_size = content.len() as i64; + let dedup_result = self.dedup.store_bytes(&content, None).await?; + let new_hash = dedup_result.hash().to_string(); + + // Update file metadata + sqlx::query( + r#" + UPDATE storage.files + SET blob_hash = $1, size = $2, updated_at = NOW() + WHERE id = $3::uuid + "#, + ) + .bind(&new_hash) + .bind(new_size) + .bind(file_id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("update: {e}")))?; + + // Decrement old blob ref (only if hash changed) + if old_hash != new_hash { + if let Err(e) = self.dedup.remove_reference(&old_hash).await { + tracing::warn!( + "Failed to decrement old blob ref {}: {}", + &old_hash[..12], + e + ); + } + } + + Ok(()) + } + + async fn register_file_deferred( + &self, + name: String, + folder_id: Option, + content_type: String, + size: u64, + ) -> Result<(File, PathBuf), DomainError> { + let user_id = self.resolve_user_id(folder_id.as_deref()).await?; + + // For deferred registration we use a placeholder hash. + // The write-behind cache will call update_file_content later. + let placeholder_hash = "0000000000000000000000000000000000000000000000000000000000000000"; + + let row = sqlx::query_as::<_, (String, i64, i64)>( + r#" + INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type) + VALUES ($1, $2::uuid, $3, $4, $5, $6) + RETURNING id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + "#, + ) + .bind(&name) + .bind(&folder_id) + .bind(&user_id) + .bind(placeholder_hash) + .bind(size as i64) + .bind(&content_type) + .fetch_one(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("deferred: {e}")))?; + + let file = self + .row_to_file( + row.0.clone(), + name, + folder_id, + size as i64, + content_type, + row.1, + row.2, + ) + .await?; + + // The target_path is not meaningful for blob storage (content goes to .blobs/) + // but the WriteBehindCache API requires it. We return a synthetic path. + let target_path = PathBuf::from(format!(".pending/{}", row.0)); + + Ok((file, target_path)) + } + + // ── Trash operations ── + + async fn move_to_trash(&self, file_id: &str) -> Result<(), DomainError> { + let result = sqlx::query( + r#" + UPDATE storage.files + SET is_trashed = TRUE, + trashed_at = NOW(), + original_folder_id = folder_id, + updated_at = NOW() + WHERE id = $1::uuid AND NOT is_trashed + "#, + ) + .bind(file_id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("trash: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("File", file_id)); + } + Ok(()) + } + + async fn restore_from_trash( + &self, + file_id: &str, + _original_path: &str, + ) -> Result<(), DomainError> { + let result = sqlx::query( + r#" + UPDATE storage.files + SET is_trashed = FALSE, + trashed_at = NULL, + folder_id = COALESCE(original_folder_id, folder_id), + original_folder_id = NULL, + updated_at = NOW() + WHERE id = $1::uuid AND is_trashed + "#, + ) + .bind(file_id) + .execute(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobWrite", format!("restore: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("File", file_id)); + } + Ok(()) + } + + async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> { + // Same as delete_file — removes from DB and decrements blob ref + self.delete_file(file_id).await + } +} diff --git a/src/infrastructure/repositories/pg/folder_db_repository.rs b/src/infrastructure/repositories/pg/folder_db_repository.rs new file mode 100644 index 00000000..cf285710 --- /dev/null +++ b/src/infrastructure/repositories/pg/folder_db_repository.rs @@ -0,0 +1,603 @@ +//! PostgreSQL-backed folder repository. +//! +//! Implements `FolderRepository` (and thus `FolderStoragePort`) using the +//! `storage.folders` table. Folders are purely virtual — no physical +//! directories are created on the filesystem. + +use async_trait::async_trait; +use sqlx::PgPool; +use std::sync::Arc; + +use crate::common::errors::DomainError; +use crate::domain::entities::folder::Folder; +use crate::domain::repositories::folder_repository::FolderRepository; +use crate::domain::services::path_service::StoragePath; + +/// PostgreSQL-backed folder repository. +/// +/// All folder metadata lives in the `storage.folders` table. The physical +/// filesystem is never touched for folder operations. +pub struct FolderDbRepository { + pool: Option>, +} + +impl FolderDbRepository { + pub fn new(pool: Arc) -> Self { + Self { pool: Some(pool) } + } + + /// Creates a stub instance for `AppState::default()`. + /// This is never called in production — only used for route scaffolding. + pub fn new_stub() -> Self { + Self { pool: None } + } + + /// Get the pool, panicking if stub. + fn pool(&self) -> &PgPool { + self.pool.as_deref().expect("FolderDbRepository: pool not available (stub instance)") + } + + // ── helpers ────────────────────────────────────────────────── + + /// Build the full virtual path for a folder by walking up the `parent_id` chain. + async fn build_folder_path(&self, folder_id: &str) -> Result { + // CTE-based recursive query to build path segments + let _rows = sqlx::query_as::<_, (String,)>( + r#" + WITH RECURSIVE ancestors AS ( + SELECT id, name, parent_id + FROM storage.folders + WHERE id = $1::uuid + UNION ALL + SELECT f.id, f.name, f.parent_id + FROM storage.folders f + JOIN ancestors a ON f.id = a.parent_id + ) + SELECT name FROM ancestors ORDER BY name + "#, + ) + .bind(folder_id) + .fetch_all(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("path query: {e}")))?; + + // Actually we need a proper ordering. Let me rewrite with depth tracking. + // Re-query with depth. + let rows = sqlx::query_as::<_, (String, i32)>( + r#" + WITH RECURSIVE ancestors AS ( + SELECT id, name, parent_id, 0 AS depth + FROM storage.folders + WHERE id = $1::uuid + UNION ALL + SELECT f.id, f.name, f.parent_id, a.depth + 1 + FROM storage.folders f + JOIN ancestors a ON f.id = a.parent_id + ) + SELECT name, depth FROM ancestors ORDER BY depth DESC + "#, + ) + .bind(folder_id) + .fetch_all(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("path query: {e}")))?; + + let path_parts: Vec<&str> = rows.iter().map(|(name, _)| name.as_str()).collect(); + let path_str = path_parts.join("/"); + Ok(StoragePath::from_string(&path_str)) + } + + /// Convert a database row into a `Folder` domain entity. + async fn row_to_folder( + &self, + id: String, + name: String, + parent_id: Option, + created_at: i64, + modified_at: i64, + ) -> Result { + let storage_path = self.build_folder_path(&id).await?; + Folder::with_timestamps( + id, + name, + storage_path, + parent_id, + created_at as u64, + modified_at as u64, + ) + .map_err(|e| DomainError::internal_error("FolderDb", format!("entity: {e}"))) + } +} + +#[async_trait] +impl FolderRepository for FolderDbRepository { + async fn create_folder( + &self, + name: String, + parent_id: Option, + ) -> Result { + // Derive user_id from parent folder. Root-level folders require the + // caller to have set up the home folder beforehand (done during user + // registration). + let user_id: String = if let Some(ref pid) = parent_id { + sqlx::query_scalar::<_, String>( + "SELECT user_id FROM storage.folders WHERE id = $1::uuid", + ) + .bind(pid) + .fetch_optional(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("parent lookup: {e}")))? + .ok_or_else(|| DomainError::not_found("Folder", pid))? + } else { + return Err(DomainError::internal_error( + "FolderDb", + "Cannot create root folder without user_id — use create_home_folder instead", + )); + }; + + let row = sqlx::query_as::<_, (String, i64, i64)>( + r#" + INSERT INTO storage.folders (name, parent_id, user_id) + VALUES ($1, $2::uuid, $3) + RETURNING id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + "#, + ) + .bind(&name) + .bind(&parent_id) + .bind(&user_id) + .fetch_one(self.pool()) + .await + .map_err(|e| { + if let sqlx::Error::Database(ref db_err) = e { + if db_err.code().as_deref() == Some("23505") { + return DomainError::already_exists( + "Folder", + format!("{name} already exists in parent"), + ); + } + } + DomainError::internal_error("FolderDb", format!("insert: {e}")) + })?; + + self.row_to_folder(row.0, name, parent_id, row.1, row.2) + .await + } + + async fn get_folder(&self, id: &str) -> Result { + let row = sqlx::query_as::<_, (String, String, Option, i64, i64)>( + r#" + SELECT id::text, name, parent_id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.folders + WHERE id = $1::uuid AND NOT is_trashed + "#, + ) + .bind(id) + .fetch_optional(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("get: {e}")))? + .ok_or_else(|| DomainError::not_found("Folder", id))?; + + self.row_to_folder(row.0, row.1, row.2, row.3, row.4) + .await + } + + async fn get_folder_by_path( + &self, + storage_path: &StoragePath, + ) -> Result { + // Walk the path segments to find the folder. + let path_str = storage_path.to_string(); + let segments: Vec<&str> = path_str.split('/').filter(|s| !s.is_empty()).collect(); + + if segments.is_empty() { + return Err(DomainError::not_found("Folder", "empty path")); + } + + let mut current_parent: Option = None; + let mut current_id = String::new(); + + for segment in &segments { + let row = if let Some(ref pid) = current_parent { + sqlx::query_as::<_, (String,)>( + r#" + SELECT id::text FROM storage.folders + WHERE name = $1 AND parent_id = $2::uuid AND NOT is_trashed + "#, + ) + .bind(segment) + .bind(pid) + .fetch_optional(self.pool()) + .await + } else { + sqlx::query_as::<_, (String,)>( + r#" + SELECT id::text FROM storage.folders + WHERE name = $1 AND parent_id IS NULL AND NOT is_trashed + "#, + ) + .bind(segment) + .fetch_optional(self.pool()) + .await + } + .map_err(|e| DomainError::internal_error("FolderDb", format!("path walk: {e}")))? + .ok_or_else(|| { + DomainError::not_found("Folder", format!("segment '{segment}' in path")) + })?; + + current_id = row.0; + current_parent = Some(current_id.clone()); + } + + self.get_folder(¤t_id).await + } + + async fn list_folders( + &self, + parent_id: Option<&str>, + ) -> Result, DomainError> { + let rows: Vec<(String, String, Option, i64, i64)> = if let Some(pid) = parent_id { + sqlx::query_as( + r#" + SELECT id::text, name, parent_id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.folders + WHERE parent_id = $1::uuid AND NOT is_trashed + ORDER BY name + "#, + ) + .bind(pid) + .fetch_all(self.pool()) + .await + } else { + sqlx::query_as( + r#" + SELECT id::text, name, parent_id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.folders + WHERE parent_id IS NULL AND NOT is_trashed + ORDER BY name + "#, + ) + .fetch_all(self.pool()) + .await + } + .map_err(|e| DomainError::internal_error("FolderDb", format!("list: {e}")))?; + + let mut folders = Vec::with_capacity(rows.len()); + for (id, name, pid, ca, ma) in rows { + folders.push(self.row_to_folder(id, name, pid, ca, ma).await?); + } + Ok(folders) + } + + async fn list_folders_paginated( + &self, + parent_id: Option<&str>, + offset: usize, + limit: usize, + include_total: bool, + ) -> Result<(Vec, Option), DomainError> { + let total = if include_total { + let count: i64 = if let Some(pid) = parent_id { + sqlx::query_scalar( + "SELECT COUNT(*) FROM storage.folders WHERE parent_id = $1::uuid AND NOT is_trashed", + ) + .bind(pid) + .fetch_one(self.pool()) + .await + } else { + sqlx::query_scalar( + "SELECT COUNT(*) FROM storage.folders WHERE parent_id IS NULL AND NOT is_trashed", + ) + .fetch_one(self.pool()) + .await + } + .map_err(|e| DomainError::internal_error("FolderDb", format!("count: {e}")))?; + Some(count as usize) + } else { + None + }; + + let rows: Vec<(String, String, Option, i64, i64)> = if let Some(pid) = parent_id { + sqlx::query_as( + r#" + SELECT id::text, name, parent_id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.folders + WHERE parent_id = $1::uuid AND NOT is_trashed + ORDER BY name + LIMIT $2 OFFSET $3 + "#, + ) + .bind(pid) + .bind(limit as i64) + .bind(offset as i64) + .fetch_all(self.pool()) + .await + } else { + sqlx::query_as( + r#" + SELECT id::text, name, parent_id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.folders + WHERE parent_id IS NULL AND NOT is_trashed + ORDER BY name + LIMIT $1 OFFSET $2 + "#, + ) + .bind(limit as i64) + .bind(offset as i64) + .fetch_all(self.pool()) + .await + } + .map_err(|e| DomainError::internal_error("FolderDb", format!("paginate: {e}")))?; + + let mut folders = Vec::with_capacity(rows.len()); + for (id, name, pid, ca, ma) in rows { + folders.push(self.row_to_folder(id, name, pid, ca, ma).await?); + } + Ok((folders, total)) + } + + async fn rename_folder( + &self, + id: &str, + new_name: String, + ) -> Result { + sqlx::query( + r#" + UPDATE storage.folders + SET name = $1, updated_at = NOW() + WHERE id = $2::uuid AND NOT is_trashed + "#, + ) + .bind(&new_name) + .bind(id) + .execute(self.pool()) + .await + .map_err(|e| { + if let sqlx::Error::Database(ref db_err) = e { + if db_err.code().as_deref() == Some("23505") { + return DomainError::already_exists( + "Folder", + format!("{new_name} already exists"), + ); + } + } + DomainError::internal_error("FolderDb", format!("rename: {e}")) + })?; + + self.get_folder(id).await + } + + async fn move_folder( + &self, + id: &str, + new_parent_id: Option<&str>, + ) -> Result { + sqlx::query( + r#" + UPDATE storage.folders + SET parent_id = $1::uuid, updated_at = NOW() + WHERE id = $2::uuid AND NOT is_trashed + "#, + ) + .bind(new_parent_id) + .bind(id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("move: {e}")))?; + + self.get_folder(id).await + } + + async fn delete_folder(&self, id: &str) -> Result<(), DomainError> { + // Hard delete folder and all descendants (CASCADE handles children) + let result = sqlx::query("DELETE FROM storage.folders WHERE id = $1::uuid") + .bind(id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("delete: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("Folder", id)); + } + Ok(()) + } + + async fn folder_exists(&self, storage_path: &StoragePath) -> Result { + // Try to find by walking the path + match self.get_folder_by_path(storage_path).await { + Ok(_) => Ok(true), + Err(e) if e.to_string().contains("not found") => Ok(false), + Err(e) => Err(e), + } + } + + async fn get_folder_path(&self, id: &str) -> Result { + self.build_folder_path(id).await + } + + // ── Trash operations ── + + async fn move_to_trash(&self, folder_id: &str) -> Result<(), DomainError> { + // Soft-delete: set is_trashed = true and remember original parent + let result = sqlx::query( + r#" + UPDATE storage.folders + SET is_trashed = TRUE, + trashed_at = NOW(), + original_parent_id = parent_id, + updated_at = NOW() + WHERE id = $1::uuid AND NOT is_trashed + "#, + ) + .bind(folder_id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("trash: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("Folder", folder_id)); + } + + // Also trash all files inside the folder (recursively) + sqlx::query( + r#" + WITH RECURSIVE descendants AS ( + SELECT id FROM storage.folders WHERE id = $1::uuid + UNION ALL + SELECT f.id FROM storage.folders f JOIN descendants d ON f.parent_id = d.id + ) + UPDATE storage.files + SET is_trashed = TRUE, trashed_at = NOW(), original_folder_id = folder_id + WHERE folder_id IN (SELECT id FROM descendants) AND NOT is_trashed + "#, + ) + .bind(folder_id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("trash files: {e}")))?; + + Ok(()) + } + + async fn restore_from_trash( + &self, + folder_id: &str, + _original_path: &str, + ) -> Result<(), DomainError> { + // Restore: set is_trashed = false, restore parent_id from original_parent_id + let result = sqlx::query( + r#" + UPDATE storage.folders + SET is_trashed = FALSE, + trashed_at = NULL, + parent_id = COALESCE(original_parent_id, parent_id), + original_parent_id = NULL, + updated_at = NOW() + WHERE id = $1::uuid AND is_trashed + "#, + ) + .bind(folder_id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("restore: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("Folder", folder_id)); + } + + // Also restore files that were trashed with this folder + sqlx::query( + r#" + WITH RECURSIVE descendants AS ( + SELECT id FROM storage.folders WHERE id = $1::uuid + UNION ALL + SELECT f.id FROM storage.folders f JOIN descendants d ON f.parent_id = d.id + ) + UPDATE storage.files + SET is_trashed = FALSE, + trashed_at = NULL, + folder_id = COALESCE(original_folder_id, folder_id), + original_folder_id = NULL + WHERE folder_id IN (SELECT id FROM descendants) AND is_trashed + "#, + ) + .bind(folder_id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("restore files: {e}")))?; + + Ok(()) + } + + async fn delete_folder_permanently(&self, folder_id: &str) -> Result<(), DomainError> { + // Permanently delete — CASCADE handles children + let result = sqlx::query("DELETE FROM storage.folders WHERE id = $1::uuid") + .bind(folder_id) + .execute(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("perm delete: {e}")))?; + + if result.rows_affected() == 0 { + return Err(DomainError::not_found("Folder", folder_id)); + } + Ok(()) + } +} + +// ── Extra helpers for blob-storage bootstrap ── + +impl FolderDbRepository { + /// Creates a root-level home folder for a user. + /// This is called during user registration. + pub async fn create_home_folder( + &self, + user_id: &str, + name: &str, + ) -> Result { + let row = sqlx::query_as::<_, (String, i64, i64)>( + r#" + INSERT INTO storage.folders (name, parent_id, user_id) + VALUES ($1, NULL, $2) + ON CONFLICT DO NOTHING + RETURNING id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + "#, + ) + .bind(name) + .bind(user_id) + .fetch_optional(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("home folder: {e}")))?; + + match row { + Some((id, ca, ma)) => { + self.row_to_folder(id, name.to_string(), None, ca, ma).await + } + None => { + // Already exists — fetch it + let existing = sqlx::query_as::<_, (String, i64, i64)>( + r#" + SELECT id::text, + EXTRACT(EPOCH FROM created_at)::bigint, + EXTRACT(EPOCH FROM updated_at)::bigint + FROM storage.folders + WHERE name = $1 AND user_id = $2 AND parent_id IS NULL + "#, + ) + .bind(name) + .bind(user_id) + .fetch_one(self.pool()) + .await + .map_err(|e| { + DomainError::internal_error("FolderDb", format!("home fetch: {e}")) + })?; + self.row_to_folder(existing.0, name.to_string(), None, existing.1, existing.2) + .await + } + } + } + + /// Returns user_id for a given folder. Used by file repositories. + pub async fn get_folder_user_id(&self, folder_id: &str) -> Result { + sqlx::query_scalar::<_, String>( + "SELECT user_id FROM storage.folders WHERE id = $1::uuid", + ) + .bind(folder_id) + .fetch_optional(self.pool()) + .await + .map_err(|e| DomainError::internal_error("FolderDb", format!("user_id lookup: {e}")))? + .ok_or_else(|| DomainError::not_found("Folder", folder_id)) + } +} diff --git a/src/infrastructure/repositories/pg/mod.rs b/src/infrastructure/repositories/pg/mod.rs index 72c62ec1..ce95afc9 100644 --- a/src/infrastructure/repositories/pg/mod.rs +++ b/src/infrastructure/repositories/pg/mod.rs @@ -11,6 +11,12 @@ mod settings_pg_repository; mod transaction_utils; mod user_pg_repository; +// ── Blob-storage repositories ── +pub mod file_blob_read_repository; +pub mod file_blob_write_repository; +pub mod folder_db_repository; +pub mod trash_db_repository; + pub use address_book_pg_repository::AddressBookPgRepository; pub use calendar_event_pg_repository::CalendarEventPgRepository; pub use calendar_pg_repository::CalendarPgRepository; @@ -18,7 +24,11 @@ pub use contact_group_pg_repository::ContactGroupPgRepository; pub use contact_persistence_dto::*; pub use contact_pg_repository::ContactPgRepository; pub use favorites_pg_repository::FavoritesPgRepository; +pub use file_blob_read_repository::FileBlobReadRepository; +pub use file_blob_write_repository::FileBlobWriteRepository; +pub use folder_db_repository::FolderDbRepository; pub use recent_items_pg_repository::RecentItemsPgRepository; pub use session_pg_repository::SessionPgRepository; pub use settings_pg_repository::SettingsPgRepository; +pub use trash_db_repository::TrashDbRepository; pub use user_pg_repository::UserPgRepository; diff --git a/src/infrastructure/repositories/pg/trash_db_repository.rs b/src/infrastructure/repositories/pg/trash_db_repository.rs new file mode 100644 index 00000000..6adf0a17 --- /dev/null +++ b/src/infrastructure/repositories/pg/trash_db_repository.rs @@ -0,0 +1,191 @@ +//! PostgreSQL-backed trash repository. +//! +//! Implements `TrashRepository` using soft-delete columns in `storage.files` +//! and `storage.folders`. There is no separate trash table — trashed items +//! are files/folders with `is_trashed = TRUE`. + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use sqlx::PgPool; +use std::sync::Arc; +use uuid::Uuid; + +use crate::common::errors::{DomainError, Result}; +use crate::domain::entities::trashed_item::{TrashedItem, TrashedItemType}; +use crate::domain::repositories::trash_repository::TrashRepository; + +/// Default retention period (days) used when computing deletion_date. +const _DEFAULT_RETENTION_DAYS: i64 = 30; + +/// PostgreSQL-backed trash repository using soft-delete flags. +pub struct TrashDbRepository { + pool: Arc, + retention_days: i64, +} + +impl TrashDbRepository { + pub fn new(pool: Arc, retention_days: u32) -> Self { + Self { + pool, + retention_days: retention_days as i64, + } + } + + /// Convert a trash_items view row into a TrashedItem entity. + fn row_to_trashed_item( + &self, + id: Uuid, + name: String, + item_type: String, + user_id: String, + trashed_at: Option>, + ) -> TrashedItem { + let trashed_at = trashed_at.unwrap_or_else(Utc::now); + let deletion_date = trashed_at + chrono::Duration::days(self.retention_days); + + let item_type_enum = match item_type.as_str() { + "folder" => TrashedItemType::Folder, + _ => TrashedItemType::File, + }; + + let user_uuid = Uuid::parse_str(&user_id).unwrap_or_else(|_| Uuid::nil()); + + // In the soft-delete model, the trash entry ID is the same as the + // original item ID since there is no separate trash table. + TrashedItem::from_raw( + id, // trash entry id (same as original) + id, // original item id + user_uuid, // owner + item_type_enum, + name.clone(), + String::new(), // original_path — not stored separately in soft-delete model + trashed_at, + deletion_date, + ) + } +} + +#[async_trait] +impl TrashRepository for TrashDbRepository { + async fn add_to_trash(&self, _item: &TrashedItem) -> Result<()> { + // No-op: the actual flagging is done by FileWritePort::move_to_trash + // or FolderRepository::move_to_trash. This method exists for interface + // compatibility with the TrashService. + Ok(()) + } + + async fn get_trash_items(&self, user_id: &Uuid) -> Result> { + let rows = sqlx::query_as::<_, (Uuid, String, String, String, Option>)>( + r#" + SELECT id, name, item_type, user_id, trashed_at + FROM storage.trash_items + WHERE user_id = $1 + ORDER BY trashed_at DESC + "#, + ) + .bind(user_id.to_string()) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("TrashDb", format!("list: {e}")) + })?; + + Ok(rows + .into_iter() + .map(|(id, name, item_type, uid, trashed_at)| { + self.row_to_trashed_item(id, name, item_type, uid, trashed_at) + }) + .collect()) + } + + async fn get_trash_item( + &self, + id: &Uuid, + user_id: &Uuid, + ) -> Result> { + let row = sqlx::query_as::<_, (Uuid, String, String, String, Option>)>( + r#" + SELECT id, name, item_type, user_id, trashed_at + FROM storage.trash_items + WHERE id = $1 AND user_id = $2 + "#, + ) + .bind(id) + .bind(user_id.to_string()) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("TrashDb", format!("get: {e}")) + })?; + + Ok(row.map(|(id, name, item_type, uid, trashed_at)| { + self.row_to_trashed_item(id, name, item_type, uid, trashed_at) + })) + } + + async fn restore_from_trash(&self, _id: &Uuid, _user_id: &Uuid) -> Result<()> { + // No-op: the actual restore is done by FileWritePort::restore_from_trash + // or FolderRepository::restore_from_trash. The TrashService also removes + // the index entry — which in the soft-delete model means the flag is + // already cleared. + Ok(()) + } + + async fn delete_permanently(&self, _id: &Uuid, _user_id: &Uuid) -> Result<()> { + // No-op: the actual delete is done by FileWritePort::delete_file_permanently + // or FolderRepository::delete_folder_permanently. + Ok(()) + } + + async fn clear_trash(&self, user_id: &Uuid) -> Result<()> { + // Delete all trashed files for this user + sqlx::query( + "DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE", + ) + .bind(user_id.to_string()) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("TrashDb", format!("clear files: {e}")) + })?; + + // Delete all trashed folders for this user + sqlx::query( + "DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE", + ) + .bind(user_id.to_string()) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("TrashDb", format!("clear folders: {e}")) + })?; + + Ok(()) + } + + async fn get_expired_items(&self) -> Result> { + let cutoff = Utc::now() - chrono::Duration::days(self.retention_days); + + let rows = sqlx::query_as::<_, (Uuid, String, String, String, Option>)>( + r#" + SELECT id, name, item_type, user_id, trashed_at + FROM storage.trash_items + WHERE trashed_at < $1 + ORDER BY trashed_at ASC + "#, + ) + .bind(cutoff) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("TrashDb", format!("expired: {e}")) + })?; + + Ok(rows + .into_iter() + .map(|(id, name, item_type, uid, trashed_at)| { + self.row_to_trashed_item(id, name, item_type, uid, trashed_at) + }) + .collect()) + } +} diff --git a/src/infrastructure/repositories/repository_errors.rs b/src/infrastructure/repositories/repository_errors.rs deleted file mode 100644 index 6fa7167a..00000000 --- a/src/infrastructure/repositories/repository_errors.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! Infrastructure-layer error types for file and folder repository operations. -//! -//! These error types are used internally by the filesystem repository implementations -//! (FileFsReadRepository, FileFsWriteRepository, FolderFsRepository, etc.) to represent -//! errors that can occur during storage operations. They are converted to `DomainError` -//! at the port boundary before crossing into the application layer. - -use crate::common::errors::DomainError; - -/// Error types for file repository operations. -#[derive(Debug, thiserror::Error)] -pub enum FileRepositoryError { - #[error("File not found: {0}")] - NotFound(String), - - #[error("File already exists: {0}")] - AlreadyExists(String), - - #[error("Invalid file path: {0}")] - InvalidPath(String), - - #[error("Operation not supported: {0}")] - OperationNotSupported(String), - - #[error("Storage error: {0}")] - StorageError(String), - - #[error("Domain error: {0}")] - DomainError(#[from] DomainError), - - #[error("Other error: {0}")] - Other(String), -} - -pub type FileRepositoryResult = Result; - -/// Error types for folder repository operations. -#[derive(Debug, thiserror::Error)] -pub enum FolderRepositoryError { - #[error("Folder not found: {0}")] - NotFound(String), - - #[error("Folder already exists: {0}")] - AlreadyExists(String), - - #[error("Invalid folder path: {0}")] - InvalidPath(String), - - #[error("Operation not supported: {0}")] - OperationNotSupported(String), - - #[error("Storage error: {0}")] - StorageError(String), - - #[error("Validation error: {0}")] - ValidationError(String), - - #[error("Domain error: {0}")] - DomainError(#[from] DomainError), - - #[error("Other error: {0}")] - Other(String), -} - -pub type FolderRepositoryResult = Result; - -// ── Conversions to DomainError ── - -impl From for DomainError { - fn from(err: FileRepositoryError) -> Self { - match err { - FileRepositoryError::NotFound(id) => DomainError::not_found("File", id), - FileRepositoryError::AlreadyExists(path) => DomainError::already_exists("File", path), - FileRepositoryError::InvalidPath(path) => { - DomainError::validation_error(format!("Invalid path: {}", path)) - } - FileRepositoryError::StorageError(msg) => { - DomainError::internal_error("File", format!("Storage error: {}", msg)) - } - FileRepositoryError::Other(msg) => DomainError::internal_error("File", msg), - FileRepositoryError::OperationNotSupported(msg) => { - DomainError::operation_not_supported("File", msg) - } - FileRepositoryError::DomainError(e) => e, - } - } -} - -impl From for DomainError { - fn from(err: FolderRepositoryError) -> Self { - match err { - FolderRepositoryError::NotFound(id) => DomainError::not_found("Folder", id), - FolderRepositoryError::AlreadyExists(path) => { - DomainError::already_exists("Folder", path) - } - FolderRepositoryError::InvalidPath(path) => { - DomainError::validation_error(format!("Invalid path: {}", path)) - } - FolderRepositoryError::StorageError(msg) => { - DomainError::internal_error("Folder", format!("Storage error: {}", msg)) - } - FolderRepositoryError::ValidationError(msg) => DomainError::validation_error(msg), - FolderRepositoryError::Other(msg) => DomainError::internal_error("Folder", msg), - FolderRepositoryError::OperationNotSupported(msg) => { - DomainError::operation_not_supported("Folder", msg) - } - FolderRepositoryError::DomainError(e) => e, - } - } -} diff --git a/src/infrastructure/repositories/trash_fs_repository.rs b/src/infrastructure/repositories/trash_fs_repository.rs deleted file mode 100644 index d36453ca..00000000 --- a/src/infrastructure/repositories/trash_fs_repository.rs +++ /dev/null @@ -1,441 +0,0 @@ -use async_trait::async_trait; -use chrono::Utc; -use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::fs; -use tracing::{debug, error, instrument}; -use uuid::Uuid; - -use crate::application::ports::outbound::IdMappingPort; -use crate::common::errors::{DomainError, ErrorKind, Result}; -use crate::domain::entities::trashed_item::{TrashedItem, TrashedItemType}; -use crate::domain::repositories::trash_repository::TrashRepository; - -/// Structure for storing trash items in JSON format -#[derive(Debug, Serialize, Deserialize)] -struct TrashedItemEntry { - id: String, - original_id: String, - user_id: String, - item_type: String, - name: String, - original_path: String, - trashed_at: String, - deletion_date: String, -} - -/// Trash repository implementation using the file system -pub struct TrashFsRepository { - trash_dir: PathBuf, - trash_index_path: PathBuf, -} - -impl TrashFsRepository { - pub fn new( - storage_root: impl AsRef, - _id_mapping_service: Arc, - ) -> Self { - let trash_dir = storage_root.as_ref().join(".trash"); - let trash_index_path = trash_dir.join("trash_index.json"); - - Self { - trash_dir, - trash_index_path, - } - } - - /// Ensures the trash directory exists - async fn ensure_trash_dir(&self) -> Result<()> { - debug!( - "Checking if trash directory exists: {}", - self.trash_dir.display() - ); - if !self.trash_dir.exists() { - debug!( - "Trash directory does not exist, creating it: {}", - self.trash_dir.display() - ); - fs::create_dir_all(&self.trash_dir).await.map_err(|e| { - error!( - "Failed to create trash directory {}: {}", - self.trash_dir.display(), - e - ); - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!( - "Failed to create trash directory {}: {}", - self.trash_dir.display(), - e - ), - ) - })?; - debug!("Trash directory created successfully"); - } else { - debug!("Trash directory already exists"); - } - - // Ensure the files directory exists - let files_dir = self.trash_dir.join("files"); - debug!( - "Checking if trash files directory exists: {}", - files_dir.display() - ); - if !files_dir.exists() { - debug!( - "Trash files directory does not exist, creating it: {}", - files_dir.display() - ); - fs::create_dir_all(&files_dir).await.map_err(|e| { - error!( - "Failed to create trash files directory {}: {}", - files_dir.display(), - e - ); - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!( - "Failed to create trash files directory {}: {}", - files_dir.display(), - e - ), - ) - })?; - debug!("Trash files directory created successfully"); - } else { - debug!("Trash files directory already exists"); - } - - // Also ensure the folders directory exists - let folders_dir = self.trash_dir.join("folders"); - debug!( - "Checking if trash folders directory exists: {}", - folders_dir.display() - ); - if !folders_dir.exists() { - debug!( - "Trash folders directory does not exist, creating it: {}", - folders_dir.display() - ); - fs::create_dir_all(&folders_dir).await.map_err(|e| { - error!( - "Failed to create trash folders directory {}: {}", - folders_dir.display(), - e - ); - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!( - "Failed to create trash folders directory {}: {}", - folders_dir.display(), - e - ), - ) - })?; - debug!("Trash folders directory created successfully"); - } else { - debug!("Trash folders directory already exists"); - } - - Ok(()) - } - - /// Gets all entries from the trash index - async fn get_trash_entries(&self) -> Result> { - self.ensure_trash_dir().await?; - - if !self.trash_index_path.exists() { - return Ok(Vec::new()); - } - - let content = fs::read_to_string(&self.trash_index_path) - .await - .map_err(|e| { - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!("Failed to read trash index: {}", e), - ) - })?; - - if content.trim().is_empty() { - return Ok(Vec::new()); - } - - let entries: Vec = serde_json::from_str(&content).map_err(|e| { - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!("Failed to parse trash index: {}", e), - ) - })?; - - Ok(entries) - } - - /// Saves all entries to the trash index - async fn save_trash_entries(&self, entries: Vec) -> Result<()> { - self.ensure_trash_dir().await?; - - let json = serde_json::to_string_pretty(&entries).map_err(|e| { - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!("Failed to serialize trash index: {}", e), - ) - })?; - - fs::write(&self.trash_index_path, json).await.map_err(|e| { - DomainError::new( - ErrorKind::InternalError, - "Trash", - format!("Failed to write trash index: {}", e), - ) - })?; - - Ok(()) - } - - /// Converts a JSON entry to a TrashedItem entity - fn entry_to_trashed_item(&self, entry: TrashedItemEntry) -> Result { - let item_type = match entry.item_type.as_str() { - "file" => TrashedItemType::File, - "folder" => TrashedItemType::Folder, - _ => { - return Err(DomainError::new( - ErrorKind::InvalidInput, - "Trash", - format!("Invalid trashed item type: {}", entry.item_type), - )); - } - }; - - let original_id = Uuid::parse_str(&entry.original_id).map_err(|e| { - DomainError::validation_error(format!("Invalid original ID format: {}", e)) - })?; - - let id = Uuid::parse_str(&entry.id) - .map_err(|e| DomainError::validation_error(format!("Invalid ID format: {}", e)))?; - - let user_id = Uuid::parse_str(&entry.user_id) - .map_err(|e| DomainError::validation_error(format!("Invalid user ID format: {}", e)))?; - - let trashed_at = chrono::DateTime::parse_from_rfc3339(&entry.trashed_at) - .map_err(|e| DomainError::validation_error(format!("Invalid trashed_at date: {}", e)))? - .with_timezone(&Utc); - - let deletion_date = chrono::DateTime::parse_from_rfc3339(&entry.deletion_date) - .map_err(|e| DomainError::validation_error(format!("Invalid deletion_date: {}", e)))? - .with_timezone(&Utc); - - Ok(TrashedItem::from_raw( - id, - original_id, - user_id, - item_type, - entry.name, - entry.original_path, - trashed_at, - deletion_date, - )) - } - - /// Converts a TrashedItem entity to a JSON entry - fn trashed_item_to_entry(&self, item: &TrashedItem) -> TrashedItemEntry { - TrashedItemEntry { - id: item.id().to_string(), - original_id: item.original_id().to_string(), - user_id: item.user_id().to_string(), - item_type: match item.item_type() { - TrashedItemType::File => "file".to_string(), - TrashedItemType::Folder => "folder".to_string(), - }, - name: item.name().to_string(), - original_path: item.original_path().to_string(), - trashed_at: item.trashed_at().to_rfc3339(), - deletion_date: item.deletion_date().to_rfc3339(), - } - } -} - -#[async_trait] -impl TrashRepository for TrashFsRepository { - #[instrument(skip(self))] - async fn add_to_trash(&self, item: &TrashedItem) -> Result<()> { - debug!( - "Adding item to trash: id={}, user={}", - item.id(), - item.user_id() - ); - - // Ensure the trash directory exists for this user - let user_trash_dir = self - .trash_dir - .join("files") - .join(item.user_id().to_string()); - debug!("User trash directory path: {}", user_trash_dir.display()); - - // Create the user-specific trash directory - debug!( - "Creating user trash directory: {}", - user_trash_dir.display() - ); - match fs::create_dir_all(&user_trash_dir).await { - Ok(_) => debug!("User trash directory created successfully"), - Err(e) => { - error!( - "Failed to create user trash directory {}: {}", - user_trash_dir.display(), - e - ); - return Err(DomainError::new( - ErrorKind::InternalError, - "Trash", - format!("Failed to create user trash directory: {}", e), - )); - } - } - - // Log the current trash entries before adding the new one - let mut entries = self.get_trash_entries().await?; - debug!("Current trash entries count: {}", entries.len()); - - // Create the entry for the trash index - let entry = self.trashed_item_to_entry(item); - debug!( - "Created trash entry: id={}, original_id={}, name={}", - entry.id, entry.original_id, entry.name - ); - - // Add the entry to the index and save - entries.push(entry); - debug!("Saving updated trash index with {} entries", entries.len()); - self.save_trash_entries(entries).await?; - debug!("Trash index updated successfully"); - - Ok(()) - } - - #[instrument(skip(self))] - async fn get_trash_items(&self, user_id: &Uuid) -> Result> { - debug!("Getting trash items for user: {}", user_id); - - let entries = self.get_trash_entries().await?; - - let user_id_str = user_id.to_string(); - let user_entries = entries - .into_iter() - .filter(|entry| entry.user_id == user_id_str) - .collect::>(); - - let mut items = Vec::new(); - for entry in user_entries { - match self.entry_to_trashed_item(entry) { - Ok(item) => items.push(item), - Err(e) => error!("Error converting trash entry to item: {}", e), - } - } - - Ok(items) - } - - #[instrument(skip(self))] - async fn get_trash_item(&self, id: &Uuid, user_id: &Uuid) -> Result> { - debug!("Looking for item in trash: id={}, user={}", id, user_id); - - let entries = self.get_trash_entries().await?; - - let id_str = id.to_string(); - let user_id_str = user_id.to_string(); - - let item_entry = entries - .into_iter() - .find(|entry| entry.id == id_str && entry.user_id == user_id_str); - - match item_entry { - Some(entry) => { - let item = self.entry_to_trashed_item(entry)?; - Ok(Some(item)) - } - None => Ok(None), - } - } - - #[instrument(skip(self))] - async fn restore_from_trash(&self, id: &Uuid, user_id: &Uuid) -> Result<()> { - debug!("Restoring item from trash: id={}, user={}", id, user_id); - - let mut entries = self.get_trash_entries().await?; - - let id_str = id.to_string(); - let user_id_str = user_id.to_string(); - - let index = entries - .iter() - .position(|entry| entry.id == id_str && entry.user_id == user_id_str); - - if let Some(index) = index { - entries.remove(index); - self.save_trash_entries(entries).await?; - Ok(()) - } else { - Err(DomainError::not_found("TrashedItem", id.to_string())) - } - } - - #[instrument(skip(self))] - async fn delete_permanently(&self, id: &Uuid, user_id: &Uuid) -> Result<()> { - debug!( - "Permanently deleting item from trash: id={}, user={}", - id, user_id - ); - - // Simply remove the entry from the index - // Physical files will be deleted through the corresponding repository - self.restore_from_trash(id, user_id).await - } - - #[instrument(skip(self))] - async fn clear_trash(&self, user_id: &Uuid) -> Result<()> { - debug!("Clearing trash for user: {}", user_id); - - let mut entries = self.get_trash_entries().await?; - let user_id_str = user_id.to_string(); - - entries.retain(|entry| entry.user_id != user_id_str); - self.save_trash_entries(entries).await?; - - Ok(()) - } - - #[instrument(skip(self))] - async fn get_expired_items(&self) -> Result> { - debug!("Looking for expired trash items"); - - let entries = self.get_trash_entries().await?; - let now = Utc::now(); - - let mut expired_items = Vec::new(); - - for entry in entries { - match chrono::DateTime::parse_from_rfc3339(&entry.deletion_date) { - Ok(date) => { - let utc_date = date.with_timezone(&Utc); - if utc_date <= now { - match self.entry_to_trashed_item(entry) { - Ok(item) => expired_items.push(item), - Err(e) => error!("Error converting expired trash entry: {}", e), - } - } - } - Err(e) => error!("Invalid date format in trash entry: {}", e), - } - } - - Ok(expired_items) - } -} diff --git a/src/infrastructure/services/buffer_pool.rs b/src/infrastructure/services/buffer_pool.rs deleted file mode 100644 index 41bb7c72..00000000 --- a/src/infrastructure/services/buffer_pool.rs +++ /dev/null @@ -1,504 +0,0 @@ -use std::cmp::min; -use std::collections::VecDeque; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Mutex, Semaphore}; -use tracing::debug; - -/// Default buffer size in the pool -pub const DEFAULT_BUFFER_SIZE: usize = 64 * 1024; // 64KB - -/// Default maximum number of buffers in the pool -pub const DEFAULT_MAX_BUFFERS: usize = 100; - -/// Default time-to-live for an inactive buffer (in seconds) -pub const DEFAULT_BUFFER_TTL: u64 = 60; - -/// Buffer pooling to optimize read/write operations -pub struct BufferPool { - /// Pool of available buffers - pool: Mutex>, - /// Semaphore to limit the maximum number of buffers - limit: Semaphore, - /// Size of buffers in the pool - buffer_size: usize, - /// Pool statistics - stats: Mutex, - /// Time-to-live for an inactive buffer - buffer_ttl: Duration, -} - -/// Structure for tracking pool statistics -#[derive(Debug, Clone, Default)] -pub struct BufferPoolStats { - /// Total number of get operations - pub gets: usize, - /// Number of pool hits (successful reuse) - pub hits: usize, - /// Number of misses (new buffer creation) - pub misses: usize, - /// Number of returns to the pool - pub returns: usize, - /// Number of TTL evictions - pub evictions: usize, - /// Maximum number of buffers reached - pub max_buffers_reached: usize, - /// Semaphore waits - pub waits: usize, -} - -/// Pool buffer with management metadata -struct PooledBuffer { - /// Actual byte buffer - buffer: Vec, - /// Timestamp of when it was added/returned to the pool - last_used: Instant, -} - -/// Borrowed buffer from the pool with automatic cleanup -#[derive(Clone)] -pub struct BorrowedBuffer { - /// Current buffer - buffer: Vec, - /// Actual used size of the buffer - used_size: usize, - /// Reference to the pool for returning - pool: Arc, - /// Whether the buffer should be returned to the pool or not - return_to_pool: bool, -} - -impl BufferPool { - /// Creates a new buffer pool - pub fn new(buffer_size: usize, max_buffers: usize, buffer_ttl_secs: u64) -> Arc { - Arc::new(Self { - pool: Mutex::new(VecDeque::with_capacity(max_buffers)), - limit: Semaphore::new(max_buffers), - buffer_size, - stats: Mutex::new(BufferPoolStats::default()), - buffer_ttl: Duration::from_secs(buffer_ttl_secs), - }) - } - - /// Creates a pool with default configuration - pub fn default() -> Arc { - Self::new(DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFERS, DEFAULT_BUFFER_TTL) - } - - /// Gets a buffer from the pool or creates a new one if needed. - /// This version takes an Arc to ensure the BorrowedBuffer keeps a proper - /// reference to the shared pool (not a clone). - #[allow(unused_variables)] - pub async fn get_buffer(self: &Arc) -> BorrowedBuffer { - // Increment get counter - { - let mut stats = self.stats.lock().await; - stats.gets += 1; - } - - // Concurrency control - // Acquire a semaphore permit. If none available, wait. - // We forget() the permit so it doesn't auto-release on drop. - // Instead, the permit is manually released in return_buffer/Drop via add_permits(1). - match self.limit.try_acquire() { - Ok(permit) => permit.forget(), - Err(_) => { - // No permits available, waiting - { - let mut stats = self.stats.lock().await; - stats.waits += 1; - stats.max_buffers_reached += 1; - } - - debug!("Buffer pool: waiting for available buffer"); - let permit = self - .limit - .acquire() - .await - .expect("Semaphore should not be closed"); - debug!("Buffer pool: acquired buffer after waiting"); - permit.forget(); - } - }; - - // Try to get an existing buffer from the pool - let mut pool_locked = self.pool.lock().await; - - let pool_arc = Arc::clone(self); - - if let Some(mut pooled_buffer) = pool_locked.pop_front() { - // Check if the buffer has expired - if pooled_buffer.last_used.elapsed() > self.buffer_ttl { - // Expired buffer, discard and create a new one - let mut stats = self.stats.lock().await; - stats.evictions += 1; - stats.misses += 1; - drop(stats); - - debug!("Buffer pool: evicted expired buffer"); - - // Create new buffer (reusing the permit) - drop(pool_locked); // Release the lock before returning - - BorrowedBuffer { - buffer: vec![0; self.buffer_size], - used_size: 0, - pool: pool_arc, - return_to_pool: true, - } - } else { - // Valid buffer, reuse it - let mut stats = self.stats.lock().await; - stats.hits += 1; - drop(stats); - - // Release the lock before returning - drop(pool_locked); - - // Clear buffer for security - pooled_buffer.buffer.fill(0); - - BorrowedBuffer { - buffer: pooled_buffer.buffer, - used_size: 0, - pool: pool_arc, - return_to_pool: true, - } - } - } else { - // No buffers available, create a new one - let mut stats = self.stats.lock().await; - stats.misses += 1; - drop(stats); - - // Release the lock before returning - drop(pool_locked); - - debug!("Buffer pool: creating new buffer"); - - BorrowedBuffer { - buffer: vec![0; self.buffer_size], - used_size: 0, - pool: pool_arc, - return_to_pool: true, - } - } - } - - /// Returns a buffer to the pool - async fn return_buffer(&self, mut buffer: Vec) { - // If the buffer is the wrong size, discard it - if buffer.capacity() != self.buffer_size { - debug!( - "Buffer pool: discarding buffer of wrong size: {} (expected {})", - buffer.capacity(), - self.buffer_size - ); - // Release the semaphore permit even if we discard the buffer - self.limit.add_permits(1); - return; - } - - // Resize to ensure correct capacity - buffer.resize(self.buffer_size, 0); - - // Add to the pool - let mut pool_locked = self.pool.lock().await; - - pool_locked.push_back(PooledBuffer { - buffer, - last_used: Instant::now(), - }); - - // Update statistics - let mut stats = self.stats.lock().await; - stats.returns += 1; - - // Release the semaphore permit so another caller can acquire a buffer - drop(pool_locked); - drop(stats); - self.limit.add_permits(1); - } - - /// Cleans expired buffers from the pool - pub async fn clean_expired_buffers(&self) { - let _now = Instant::now(); - let mut pool_locked = self.pool.lock().await; - - // Count expired - let count_before = pool_locked.len(); - - // Filter keeping only non-expired - pool_locked.retain(|buffer| buffer.last_used.elapsed() <= self.buffer_ttl); - - // Count how many were removed - let removed = count_before - pool_locked.len(); - - if removed > 0 { - // Update statistics - let mut stats = self.stats.lock().await; - stats.evictions += removed; - - debug!("Buffer pool: cleaned {} expired buffers", removed); - } - } - - /// Gets current pool statistics - pub async fn get_stats(&self) -> BufferPoolStats { - self.stats.lock().await.clone() - } - - /// Starts the periodic cleanup task - pub fn start_cleaner(pool: Arc) { - tokio::spawn(async move { - let interval = Duration::from_secs(30); // Clean every 30 seconds - - loop { - tokio::time::sleep(interval).await; - pool.clean_expired_buffers().await; - - // Log statistics periodically - let stats = pool.get_stats().await; - debug!( - "Buffer pool stats: gets={}, hits={}, misses={}, hit_ratio={:.2}%, returns={}, \ - evictions={}, max_reached={}, waits={}", - stats.gets, - stats.hits, - stats.misses, - if stats.gets > 0 { - (stats.hits as f64 * 100.0) / stats.gets as f64 - } else { - 0.0 - }, - stats.returns, - stats.evictions, - stats.max_buffers_reached, - stats.waits - ); - } - }); - } -} - -impl Clone for BufferPool { - fn clone(&self) -> Self { - Self { - pool: Mutex::new(VecDeque::new()), - limit: Semaphore::new(self.limit.available_permits()), - buffer_size: self.buffer_size, - stats: Mutex::new(BufferPoolStats::default()), - buffer_ttl: self.buffer_ttl, - } - } -} - -impl BorrowedBuffer { - /// Accesses the internal buffer - pub fn as_mut_slice(&mut self) -> &mut [u8] { - &mut self.buffer - } - - /// Gets a reference to the used data - pub fn as_slice(&self) -> &[u8] { - &self.buffer[..self.used_size] - } - - /// Sets how many bytes were actually used - pub fn set_used(&mut self, size: usize) { - self.used_size = min(size, self.buffer.len()); - } - - /// Converts into a Vec that includes only the used data - pub fn into_vec(mut self) -> Vec { - // Mark to not return to pool - self.return_to_pool = false; - - // Create a new vector with only the used data - self.buffer[..self.used_size].to_vec() - } - - /// Copies data to this buffer and updates the used size - pub fn copy_from_slice(&mut self, data: &[u8]) -> usize { - let copy_size = min(data.len(), self.buffer.len()); - self.buffer[..copy_size].copy_from_slice(&data[..copy_size]); - self.used_size = copy_size; - copy_size - } - - /// Prevents the buffer from being returned to the pool on destruction - pub fn do_not_return(mut self) -> Self { - self.return_to_pool = false; - self - } - - /// Gets the total buffer size - pub fn capacity(&self) -> usize { - self.buffer.len() - } - - /// Gets the used buffer size - pub fn used_size(&self) -> usize { - self.used_size - } -} - -// When a BorrowedBuffer is dropped, it is returned to the pool -impl Drop for BorrowedBuffer { - fn drop(&mut self) { - if self.return_to_pool { - // Take ownership of the buffer and create a clone of the pool - let buffer = std::mem::take(&mut self.buffer); - let pool = self.pool.clone(); - - // Spawn the return so that drop doesn't block - // return_buffer will release the semaphore permit - tokio::spawn(async move { - pool.return_buffer(buffer).await; - }); - } else { - // Buffer not returned to pool, but we still need to release the semaphore permit - self.pool.limit.add_permits(1); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_buffer_pooling() { - // Create small pool for testing - let pool = BufferPool::new(1024, 5, 60); - - // Get a buffer - let mut buffer1 = pool.get_buffer().await; - buffer1.copy_from_slice(b"test data"); - assert_eq!(buffer1.as_slice(), b"test data"); - - // Get another buffer - let buffer2 = pool.get_buffer().await; - - // Verify stats - let stats = pool.get_stats().await; - assert_eq!(stats.gets, 2); - assert_eq!(stats.hits, 0); // no hits yet - assert_eq!(stats.misses, 2); // all are misses - - // Return buffer1 to pool (implicitly via drop) - drop(buffer1); - - // Allow the async return to occur - tokio::time::sleep(Duration::from_millis(10)).await; - - // Get another buffer (should reuse the returned one) - let buffer3 = pool.get_buffer().await; - - // Verify updated stats - let stats = pool.get_stats().await; - assert_eq!(stats.gets, 3); - assert_eq!(stats.hits, 1); // now there should be a hit - assert_eq!(stats.returns, 1); // one buffer returned - - // Cleanup - drop(buffer2); - drop(buffer3); - } - - #[tokio::test] - async fn test_buffer_operations() { - let pool = BufferPool::new(1024, 10, 60); - - // Get buffer - let mut buffer = pool.get_buffer().await; - - // Write data - buffer.copy_from_slice(b"Hello, world!"); - assert_eq!(buffer.used_size(), 13); - assert_eq!(buffer.as_slice(), b"Hello, world!"); - - // Convert to vec and verify - let vec = buffer.into_vec(); // This prevents returning to pool - assert_eq!(vec, b"Hello, world!"); - - // Verify that returns are not incremented (buffer not returned) - tokio::time::sleep(Duration::from_millis(10)).await; - let stats = pool.get_stats().await; - assert_eq!(stats.returns, 0); - } - - #[tokio::test] - async fn test_pool_limit() { - // Pool with only 3 buffers - let pool = BufferPool::new(1024, 3, 60); - - // Get 3 buffers (reaches the limit) - let buffer1 = pool.get_buffer().await; - let buffer2 = pool.get_buffer().await; - let buffer3 = pool.get_buffer().await; - - // Verify stats - let stats = pool.get_stats().await; - assert_eq!(stats.gets, 3); - assert_eq!(stats.waits, 0); // no waits yet - - // Try to get a 4th buffer in a separate task (should wait) - let pool_clone = pool.clone(); - let handle = tokio::spawn(async move { - let _buffer4 = pool_clone.get_buffer().await; - true - }); - - // Give time for the task to try to take the buffer - tokio::time::sleep(Duration::from_millis(50)).await; - - // Verify there is a wait - let stats = pool.get_stats().await; - assert_eq!(stats.waits, 1); - - // Release a buffer - drop(buffer1); - - // Give time for the async return and for the waiting task to get its buffer - tokio::time::sleep(Duration::from_millis(50)).await; - - // Verify the task was able to continue - assert!(handle.await.unwrap()); - - // Cleanup - drop(buffer2); - drop(buffer3); - } - - #[tokio::test] - async fn test_ttl_expiration() { - // Pool with very short TTL for testing - let pool = BufferPool::new(1024, 5, 1); // 1 second TTL - - // Get and return a buffer - let buffer = pool.get_buffer().await; - drop(buffer); - - // Allow the async return to occur - tokio::time::sleep(Duration::from_millis(50)).await; - - // Verify there is a buffer in the pool - let stats = pool.get_stats().await; - assert_eq!(stats.returns, 1); - - // Wait for the TTL to expire - tokio::time::sleep(Duration::from_secs(2)).await; - - // Clean expired - pool.clean_expired_buffers().await; - - // Get another buffer (should be a miss since the previous one expired) - let _buffer2 = pool.get_buffer().await; - - // Verify stats - let stats = pool.get_stats().await; - assert_eq!(stats.evictions, 1); // one expired buffer - assert_eq!(stats.hits, 0); // no hits (the buffer expired) - assert_eq!(stats.misses, 2); // two misses (1st and 3rd get) - } -} diff --git a/src/infrastructure/services/compression_service.rs b/src/infrastructure/services/compression_service.rs index de816bb6..b337b273 100644 --- a/src/infrastructure/services/compression_service.rs +++ b/src/infrastructure/services/compression_service.rs @@ -6,14 +6,12 @@ use flate2::read::GzEncoder as GzEncoderRead; use futures::{Stream, StreamExt}; use std::io; use std::io::Read; -use std::sync::Arc; use tracing::error; use crate::application::ports::compression_ports::{ CompressionLevel as PortCompressionLevel, CompressionPort, }; use crate::domain::errors::DomainError; -use crate::infrastructure::services::buffer_pool::BufferPool; /// Compression level for files #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -73,22 +71,12 @@ pub trait CompressionService: Send + Sync { } /// Gzip compression service implementation -pub struct GzipCompressionService { - /// Buffer pool for memory optimization - buffer_pool: Option>, -} +pub struct GzipCompressionService; impl GzipCompressionService { /// Creates a new service instance pub fn new() -> Self { - Self { buffer_pool: None } - } - - /// Creates a new service instance with buffer pool - pub fn new_with_buffer_pool(buffer_pool: Arc) -> Self { - Self { - buffer_pool: Some(buffer_pool), - } + Self } } @@ -96,63 +84,6 @@ impl GzipCompressionService { impl CompressionService for GzipCompressionService { /// Compresses data in memory using Gzip async fn compress_data(&self, data: &[u8], level: CompressionLevel) -> io::Result> { - // If we have a buffer pool, use a borrowed buffer for compression - if let Some(pool) = &self.buffer_pool { - // Estimate the compression size (approximately 80% of original for typical cases) - let estimated_size = (data.len() as f64 * 0.8) as usize; - - // Get a buffer from the pool - let buffer = pool.get_buffer().await; - - // Check if the buffer is large enough - if buffer.capacity() >= estimated_size { - // Run compression in a worker thread using the buffer - let buffer_ptr = Arc::new(tokio::sync::Mutex::new(buffer)); - let buffer_clone = buffer_ptr.clone(); - - // Compress data - // Clone the data to avoid lifetime issues - let data_owned = data.to_vec(); - - let result = tokio::task::spawn_blocking(move || { - let mut encoder = GzEncoderRead::new(&data_owned[..], level.into()); - - // Try to lock the mutex (should not fail since we are in a separate thread) - let mut buffer_guard = match futures::executor::block_on(buffer_clone.lock()) { - buffer => buffer, - }; - - // Read directly into the buffer - let read_bytes = encoder.read(buffer_guard.as_mut_slice())?; - buffer_guard.set_used(read_bytes); - - Ok(()) as io::Result<()> - }) - .await; - - // Verify result - match result { - Ok(Ok(())) => { - // Get the buffer and convert it to Vec - let buffer = buffer_ptr.lock().await; - let cloned_buffer = buffer.clone(); - drop(buffer); // Release the mutex first - return Ok(cloned_buffer.into_vec()); - } - Ok(Err(e)) => { - error!("Compression error with buffer pool: {}", e); - // Fall back to standard implementation - } - Err(e) => { - error!("Compression task error with buffer pool: {}", e); - // Fall back to standard implementation - } - } - } - } - - // Standard implementation if there is no buffer pool or the buffer is insufficient - // Clone the data to avoid lifetime issues let data_owned = data.to_vec(); tokio::task::spawn_blocking(move || { @@ -170,61 +101,7 @@ impl CompressionService for GzipCompressionService { /// Decompresses data in memory async fn decompress_data(&self, compressed_data: &[u8]) -> io::Result> { - // If we have a buffer pool, use a borrowed buffer for decompression - if let Some(pool) = &self.buffer_pool { - // Estimate the decompression size (approximately 5x of compressed for typical cases) - let estimated_size = compressed_data.len() * 5; - - // Get a buffer from the pool - let buffer = pool.get_buffer().await; - - // Check if the buffer is large enough - if buffer.capacity() >= estimated_size { - // Clone compressed data to move to the worker - let data = compressed_data.to_vec(); - let buffer_ptr = Arc::new(tokio::sync::Mutex::new(buffer)); - let buffer_clone = buffer_ptr.clone(); - - // Decompress data - let result = tokio::task::spawn_blocking(move || { - let mut decoder = GzDecoder::new(&data[..]); - - // Try to lock the mutex - let mut buffer_guard = match futures::executor::block_on(buffer_clone.lock()) { - buffer => buffer, - }; - - // Read directly into the buffer - let read_bytes = decoder.read(buffer_guard.as_mut_slice())?; - buffer_guard.set_used(read_bytes); - - Ok(()) as io::Result<()> - }) - .await; - - // Verify result - match result { - Ok(Ok(())) => { - // Get the buffer and convert it to Vec - let buffer = buffer_ptr.lock().await; - let cloned_buffer = buffer.clone(); - drop(buffer); // Release the mutex first - return Ok(cloned_buffer.into_vec()); - } - Ok(Err(e)) => { - error!("Decompression error with buffer pool: {}", e); - // Fall back to standard implementation - } - Err(e) => { - error!("Decompression task error with buffer pool: {}", e); - // Fall back to standard implementation - } - } - } - } - - // Standard implementation if there is no buffer pool or the buffer is insufficient - let data = compressed_data.to_vec(); // Clone to move to the worker + let data = compressed_data.to_vec(); tokio::task::spawn_blocking(move || { let mut decoder = GzDecoder::new(&data[..]); let mut decompressed = Vec::new(); diff --git a/src/infrastructure/services/file_metadata_cache.rs b/src/infrastructure/services/file_metadata_cache.rs deleted file mode 100644 index becdbe81..00000000 --- a/src/infrastructure/services/file_metadata_cache.rs +++ /dev/null @@ -1,749 +0,0 @@ -use futures::future::BoxFuture; -use mime_guess::from_path; -use std::collections::{HashMap, VecDeque}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant, UNIX_EPOCH}; -use tokio::fs; -use tokio::sync::RwLock; -use tokio::time; -use tracing::debug; - -use crate::domain::entities::file::File; - -use crate::common::config::AppConfig; - -/// Cache entry types -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CacheEntryType { - /// File - File, - /// Directory - Directory, - /// Unknown type - Unknown, -} - -/// Cache statistics for monitoring -#[derive(Debug, Clone, Default)] -pub struct CacheStats { - /// Number of cache hits - pub hits: usize, - /// Number of cache misses - pub misses: usize, - /// Number of manual invalidations - pub invalidations: usize, - /// Number of automatic expirations - pub expirations: usize, - /// Number of cache inserts - pub inserts: usize, - /// Total time saved (milliseconds) - pub time_saved_ms: u64, -} - -/// Complete cached file metadata -#[derive(Debug, Clone)] -pub struct FileMetadata { - /// Absolute file path - pub path: PathBuf, - /// Whether the file physically exists - pub exists: bool, - /// Entry type (file, directory) - pub entry_type: CacheEntryType, - /// Size in bytes (for files) - pub size: Option, - /// MIME type (for files) - pub mime_type: Option, - /// Creation timestamp (UNIX epoch seconds) - pub created_at: Option, - /// Modification timestamp (UNIX epoch seconds) - pub modified_at: Option, - /// Previous access (used for LRU) - pub last_access: Instant, - /// Cache expiration time - pub expires_at: Instant, - /// Number of accesses to this entry - pub access_count: usize, -} - -impl FileMetadata { - /// Creates a new metadata entry - pub fn new( - path: PathBuf, - exists: bool, - entry_type: CacheEntryType, - size: Option, - mime_type: Option, - created_at: Option, - modified_at: Option, - ttl: Duration, - ) -> Self { - let now = Instant::now(); - - Self { - path, - exists, - entry_type, - size, - mime_type, - created_at, - modified_at, - last_access: now, - expires_at: now + ttl, - access_count: 1, - } - } - - /// Updates the last access time - pub fn touch(&mut self) { - self.last_access = Instant::now(); - self.access_count += 1; - } - - /// Checks if the entry has expired - pub fn is_expired(&self) -> bool { - Instant::now() > self.expires_at - } - - /// Updates the expiration time with a new TTL - pub fn update_expiry(&mut self, ttl: Duration) { - self.expires_at = Instant::now() + ttl; - } -} - -/// Advanced file metadata cache -pub struct FileMetadataCache { - /// Main metadata cache - metadata_cache: RwLock>, - /// LRU queue for cache management - lru_queue: RwLock>, - /// Cache usage statistics - stats: RwLock, - /// Global application configuration - config: AppConfig, - /// Adaptive TTL for popular entries - ttl_multiplier: f64, - /// Popularity threshold for extended TTL - popularity_threshold: usize, - /// Maximum cache size - max_entries: usize, -} - -impl FileMetadataCache { - /// Creates a new metadata cache instance - pub fn new(config: AppConfig, max_entries: usize) -> Self { - Self { - metadata_cache: RwLock::new(HashMap::with_capacity(max_entries)), - lru_queue: RwLock::new(VecDeque::with_capacity(max_entries)), - stats: RwLock::new(CacheStats::default()), - config, - ttl_multiplier: 5.0, // Popular entries have 5x TTL - popularity_threshold: 10, // After 10 accesses it's considered popular - max_entries, - } - } - - /// Creates a FileMetadata object from a File object - pub fn create_metadata_from_file(file: &File, abs_path: PathBuf) -> FileMetadata { - let entry_type = CacheEntryType::File; - let size = Some(file.size()); - let mime_type = Some(file.mime_type().to_string()); - let created_at = Some(file.created_at()); - let modified_at = Some(file.modified_at()); - - // Use a standard TTL - let ttl = Duration::from_secs(60); // 1 minute - - FileMetadata::new( - abs_path, - true, - entry_type, - size, - mime_type, - created_at, - modified_at, - ttl, - ) - } - - /// Creates a default instance - pub fn default() -> Self { - Self::new(AppConfig::default(), 10_000) - } - - /// Creates a cache instance with default configuration - pub fn default_with_config(config: AppConfig) -> Self { - Self::new(config, 50_000) // Larger cache for production system - } - - /// Gets file metadata if cached - pub async fn get_metadata(&self, path: &Path) -> Option { - let start_time = Instant::now(); - let mut cache = self.metadata_cache.write().await; - - if let Some(metadata) = cache.get_mut(path) { - // Check if expired - if metadata.is_expired() { - // Remove from cache if expired - cache.remove(path); - - // Update statistics - let mut stats = self.stats.write().await; - stats.misses += 1; - stats.expirations += 1; - - debug!("Cache entry expired for: {}", path.display()); - - return None; - } - - // Update access time - metadata.touch(); - - // For popular entries, extend TTL - if metadata.access_count >= self.popularity_threshold { - let new_ttl = match metadata.entry_type { - CacheEntryType::File => Duration::from_millis( - (self.config.timeouts.file_operation_ms as f64 * self.ttl_multiplier) - as u64, - ), - CacheEntryType::Directory => Duration::from_millis( - (self.config.timeouts.dir_operation_ms as f64 * self.ttl_multiplier) as u64, - ), - _ => Duration::from_secs(60), // 1 minute by default - }; - - metadata.update_expiry(new_ttl); - debug!("Extended TTL for popular entry: {}", path.display()); - } - - // Calculate approximate time saved - let elapsed = start_time.elapsed().as_millis() as u64; - let estimated_io_time: u64 = 10; // We assume 10ms minimum for IO operation - let time_saved = estimated_io_time.saturating_sub(elapsed); - - // Update statistics - let mut stats = self.stats.write().await; - stats.hits += 1; - stats.time_saved_ms += time_saved; - - debug!("Cache hit for: {}", path.display()); - - // Also keep the LRU queue updated - self.update_lru(path.to_path_buf()).await; - - // Clone to return - return Some(metadata.clone()); - } - - // Not found in cache - let mut stats = self.stats.write().await; - stats.misses += 1; - - debug!("Cache miss for: {}", path.display()); - None - } - - /// Updates the LRU queue - async fn update_lru(&self, path: PathBuf) { - let mut lru = self.lru_queue.write().await; - - // Remove if already exists - if let Some(pos) = lru.iter().position(|p| p == &path) { - lru.remove(pos); - } - - // Add to the end (most recent) - lru.push_back(path); - } - - /// Checks if a file exists - pub async fn exists(&self, path: &Path) -> Option { - if let Some(metadata) = self.get_metadata(path).await { - return Some(metadata.exists); - } - - None - } - - /// Checks if a path is a directory - pub async fn is_dir(&self, path: &Path) -> Option { - if let Some(metadata) = self.get_metadata(path).await { - return Some(metadata.entry_type == CacheEntryType::Directory); - } - - None - } - - /// Checks if a path is a file - pub async fn is_file(&self, path: &Path) -> Option { - if let Some(metadata) = self.get_metadata(path).await { - return Some(metadata.entry_type == CacheEntryType::File); - } - - None - } - - /// Gets the size of a file - pub async fn get_size(&self, path: &Path) -> Option { - if let Some(metadata) = self.get_metadata(path).await { - return metadata.size; - } - - None - } - - /// Gets the MIME type of a file - pub async fn get_mime_type(&self, path: &Path) -> Option { - if let Some(metadata) = self.get_metadata(path).await { - return metadata.mime_type; - } - - None - } - - /// Refreshes metadata for a path - pub async fn refresh_metadata(&self, path: &Path) -> Result { - // Perform actual filesystem read - let metadata = fs::metadata(path).await?; - - // Determine entry type - let entry_type = if metadata.is_dir() { - CacheEntryType::Directory - } else if metadata.is_file() { - CacheEntryType::File - } else { - CacheEntryType::Unknown - }; - - // Get size for files - let size = if metadata.is_file() { - Some(metadata.len()) - } else { - None - }; - - // Get MIME type for files - let mime_type = if metadata.is_file() { - Some(from_path(path).first_or_octet_stream().to_string()) - } else { - None - }; - - // Get timestamps - let created_at = metadata - .created() - .map(|time| { - time.duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .ok(); - - let modified_at = metadata - .modified() - .map(|time| { - time.duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - }) - .ok(); - - // Determine appropriate TTL - let ttl = if metadata.is_dir() { - Duration::from_millis(self.config.timeouts.dir_operation_ms) - } else { - Duration::from_millis(self.config.timeouts.file_operation_ms) - }; - - // Create metadata entry - let file_metadata = FileMetadata::new( - path.to_path_buf(), - true, - entry_type, - size, - mime_type, - created_at, - modified_at, - ttl, - ); - - // Update cache - self.update_cache(file_metadata.clone()).await; - - Ok(file_metadata) - } - - /// Updates the cache with new metadata - pub async fn update_cache(&self, metadata: FileMetadata) { - // Avoid full cache before inserting - self.ensure_capacity().await; - - let path = metadata.path.clone(); - - // Insert into cache - { - let mut cache = self.metadata_cache.write().await; - cache.insert(path.clone(), metadata); - - // Update statistics - let mut stats = self.stats.write().await; - stats.inserts += 1; - } - - // Update the LRU queue - self.update_lru(path).await; - } - - /// Ensures there is space in the cache - async fn ensure_capacity(&self) { - let cache_size = { - let cache = self.metadata_cache.read().await; - cache.len() - }; - - if cache_size >= self.max_entries { - self.evict_lru_entries(cache_size / 10).await; // Free up 10% - } - } - - /// Removes least recently used entries - async fn evict_lru_entries(&self, count: usize) { - let mut paths_to_remove = Vec::with_capacity(count); - - // Get entries to remove from the LRU queue - { - let mut lru = self.lru_queue.write().await; - for _ in 0..count { - if let Some(path) = lru.pop_front() { - paths_to_remove.push(path); - } else { - break; - } - } - } - - // Remove from the main cache - { - let mut cache = self.metadata_cache.write().await; - for path in paths_to_remove { - cache.remove(&path); - } - } - - debug!("Evicted {} LRU entries from cache", count); - } - - /// Invalidate a specific cache entry - pub async fn invalidate(&self, path: &Path) { - // Remove from the main cache - { - let mut cache = self.metadata_cache.write().await; - cache.remove(path); - - // Update statistics - let mut stats = self.stats.write().await; - stats.invalidations += 1; - } - - // Remove from the LRU queue - let path_buf = path.to_path_buf(); - { - let mut lru = self.lru_queue.write().await; - if let Some(pos) = lru.iter().position(|p| p == &path_buf) { - lru.remove(pos); - } - } - - debug!("Invalidated cache entry for: {}", path.display()); - } - - /// Recursively invalidate entries under a directory - pub async fn invalidate_directory(&self, dir_path: &Path) { - let dir_str = dir_path.to_string_lossy().to_string(); - let mut paths_to_remove = Vec::new(); - - // Find all paths that start with the directory - { - let cache = self.metadata_cache.read().await; - for path in cache.keys() { - let path_str = path.to_string_lossy().to_string(); - if path_str.starts_with(&dir_str) { - paths_to_remove.push(path.clone()); - } - } - } - - // Update statistics - { - let mut stats = self.stats.write().await; - stats.invalidations += paths_to_remove.len(); - } - - // Remove each found path - for path in paths_to_remove { - self.invalidate(&path).await; - } - - debug!("Invalidated directory and contents: {}", dir_path.display()); - } - - /// Get current cache statistics - pub async fn get_stats(&self) -> CacheStats { - let stats = self.stats.read().await; - stats.clone() - } - - /// Clears all expired entries from the cache - pub async fn clear_expired(&self) { - let now = Instant::now(); - let mut paths_to_remove = Vec::new(); - - // Find expired entries - { - let cache = self.metadata_cache.read().await; - for (path, metadata) in cache.iter() { - if now > metadata.expires_at { - paths_to_remove.push(path.clone()); - } - } - } - - // Update statistics - { - let mut stats = self.stats.write().await; - stats.expirations += paths_to_remove.len(); - } - - // Save the number of entries for logging - let num_paths = paths_to_remove.len(); - - // Remove expired entries - for path in paths_to_remove { - self.invalidate(&path).await; - } - - debug!("Cleared {} expired entries from cache", num_paths); - } - - /// Starts the periodic cleanup process - pub fn start_cleanup_task(cache: Arc) -> BoxFuture<'static, ()> { - Box::pin(async move { - let cleanup_interval = Duration::from_secs(60); // Every minute - - loop { - // Wait for the interval - time::sleep(cleanup_interval).await; - - // Clean expired entries - cache.clear_expired().await; - - // Log statistics - let stats = cache.get_stats().await; - let cache_size = { - let cache_map = cache.metadata_cache.read().await; - cache_map.len() - }; - - debug!( - "Cache stats: size={}, hits={}, misses={}, hit_ratio={:.2}%, time_saved={}ms", - cache_size, - stats.hits, - stats.misses, - if stats.hits + stats.misses > 0 { - (stats.hits as f64 * 100.0) / (stats.hits + stats.misses) as f64 - } else { - 0.0 - }, - stats.time_saved_ms - ); - } - }) - } - - /// Preloads metadata for entire directories (useful for initialization) - pub async fn preload_directory( - &self, - dir_path: &Path, - recursive: bool, - max_depth: usize, - ) -> Result { - self._preload_directory_internal(dir_path, recursive, max_depth, 0) - .await - } - - /// Internal preload implementation with depth tracking - async fn _preload_directory_internal( - &self, - dir_path: &Path, - recursive: bool, - max_depth: usize, - current_depth: usize, - ) -> Result { - Box::pin(async move { - if current_depth > max_depth { - return Ok(0); - } - - // Get directory entries - let mut entries = fs::read_dir(dir_path).await?; - let mut count = 0; - - // Process each entry - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - let metadata = fs::metadata(&path).await?; - - // Refresh metadata for this entry - self.refresh_metadata(&path).await?; - count += 1; - - // Recursively process subdirectories if needed - if recursive && metadata.is_dir() { - // Box to break recursion - count += self - ._preload_directory_internal(&path, recursive, max_depth, current_depth + 1) - .await?; - } - } - - Ok(count) - }) - .await - } -} - -// ─── MetadataCachePort implementation ──────────────────────── - -use crate::application::ports::cache_ports::{CachedMetadataDto, MetadataCachePort}; -use crate::common::errors::DomainError; -use async_trait::async_trait; - -#[async_trait] -impl MetadataCachePort for FileMetadataCache { - async fn get_metadata(&self, path: &Path) -> Option { - // Delegate to the existing rich get_metadata, then project into the DTO. - let fm = FileMetadataCache::get_metadata(self, path).await?; - Some(CachedMetadataDto { - path: fm.path, - exists: fm.exists, - is_file: fm.entry_type == CacheEntryType::File, - size: fm.size, - mime_type: fm.mime_type, - created_at: fm.created_at, - modified_at: fm.modified_at, - }) - } - - async fn is_file(&self, path: &Path) -> Option { - FileMetadataCache::is_file(self, path).await - } - - async fn refresh_metadata(&self, path: &Path) -> Result { - let fm = FileMetadataCache::refresh_metadata(self, path) - .await - .map_err(|e| DomainError::internal_error("MetadataCache", e.to_string()))?; - Ok(CachedMetadataDto { - path: fm.path, - exists: fm.exists, - is_file: fm.entry_type == CacheEntryType::File, - size: fm.size, - mime_type: fm.mime_type, - created_at: fm.created_at, - modified_at: fm.modified_at, - }) - } - - async fn invalidate(&self, path: &Path) { - FileMetadataCache::invalidate(self, path).await - } - - async fn invalidate_directory(&self, dir_path: &Path) { - FileMetadataCache::invalidate_directory(self, dir_path).await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - use tokio::fs::File; - use tokio::io::AsyncWriteExt; - - #[tokio::test] - async fn test_cache_operations() { - // Create temporary directory for tests - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("test_file.txt"); - - // Create a test file - let mut file = File::create(&file_path).await.unwrap(); - file.write_all(b"test content").await.unwrap(); - file.flush().await.unwrap(); - drop(file); - - // Create cache - let config = AppConfig::default(); - let cache = FileMetadataCache::new(config, 1000); - - // Verify initial miss - assert!(cache.exists(&file_path).await.is_none()); - - // Refresh and verify hit - let metadata = cache.refresh_metadata(&file_path).await.unwrap(); - assert_eq!(metadata.entry_type, CacheEntryType::File); - assert_eq!(metadata.size, Some(12)); // "test content" = 12 bytes - - // Verify it now exists in cache - assert_eq!(cache.exists(&file_path).await, Some(true)); - assert_eq!(cache.is_file(&file_path).await, Some(true)); - - // Invalidate and verify it no longer exists in cache - cache.invalidate(&file_path).await; - assert!(cache.exists(&file_path).await.is_none()); - - // Verify statistics - let stats = cache.get_stats().await; - assert_eq!(stats.inserts, 1); - assert_eq!(stats.invalidations, 1); - assert!(stats.hits > 0); - } - - #[tokio::test] - async fn test_directory_operations() { - // Create directory structure for tests - let temp_dir = tempdir().unwrap(); - // Canonicalize to handle macOS /var -> /private/var symlinks - let base_path = temp_dir.path().canonicalize().unwrap(); - let sub_dir = base_path.join("subdir"); - fs::create_dir(&sub_dir).await.unwrap(); - - let file1 = base_path.join("file1.txt"); - let file2 = sub_dir.join("file2.txt"); - - File::create(&file1).await.unwrap(); - File::create(&file2).await.unwrap(); - - // Create cache - let config = AppConfig::default(); - let cache = FileMetadataCache::new(config, 1000); - - // Preload directory recursively - // preload_directory caches the *contents* of the directory, not the root itself - let count = cache.preload_directory(&base_path, true, 2).await.unwrap(); - assert_eq!(count, 3); // subdir, file1, file2 - - // Verify existence in cache (only contents, not the root) - assert_eq!(cache.is_dir(&sub_dir).await, Some(true)); - assert_eq!(cache.is_file(&file1).await, Some(true)); - assert_eq!(cache.is_file(&file2).await, Some(true)); - - // Invalidate directory and contents - cache.invalidate_directory(&base_path).await; - - // Verify nothing exists in cache - assert!(cache.exists(&sub_dir).await.is_none()); - assert!(cache.exists(&file1).await.is_none()); - assert!(cache.exists(&file2).await.is_none()); - } -} diff --git a/src/infrastructure/services/file_system_utils.rs b/src/infrastructure/services/file_system_utils.rs deleted file mode 100644 index 6cea8302..00000000 --- a/src/infrastructure/services/file_system_utils.rs +++ /dev/null @@ -1,326 +0,0 @@ -use std::io::Error as IoError; -use std::path::Path; -use tempfile::NamedTempFile; -use tokio::fs::{self, File, OpenOptions}; -use tokio::io::AsyncWriteExt; -use tracing::{error, warn}; - -/// Utility functions for file system operations with proper synchronization -pub struct FileSystemUtils; - -impl FileSystemUtils { - /// Writes data to a file with fsync to ensure durability - /// Uses a safe atomic write pattern: write to temp file, fsync, rename - pub async fn atomic_write>(path: P, contents: &[u8]) -> Result<(), IoError> { - let path = path.as_ref(); - - // Ensure parent directory exists - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).await?; - } - - // Create a temporary file in the same directory - let dir = path.parent().unwrap_or_else(|| Path::new(".")); - let temp_file = match NamedTempFile::new_in(dir) { - Ok(file) => file, - Err(e) => { - error!( - "Failed to create temporary file in {}: {}", - dir.display(), - e - ); - return Err(IoError::other(format!( - "Failed to create temporary file: {}", - e - ))); - } - }; - - let temp_path = temp_file.path().to_path_buf(); - - // Convert to tokio file and write contents - let std_file = temp_file.as_file().try_clone()?; - let mut file = File::from_std(std_file); - file.write_all(contents).await?; - - // Ensure data is synced to disk - file.flush().await?; - file.sync_all().await?; - - // Rename the temporary file to the target path (atomic operation on most filesystems) - fs::rename(&temp_path, path).await?; - - // Sync the directory to ensure the rename is persisted - if let Some(parent) = path.parent() { - match Self::sync_directory(parent).await { - Ok(_) => {} - Err(e) => { - warn!( - "Failed to sync directory {}: {}. File was written but directory entry might not be durable.", - parent.display(), - e - ); - } - } - } - - Ok(()) - } - - /// Creates or appends to a file with fsync - pub async fn write_with_sync>( - path: P, - contents: &[u8], - append: bool, - ) -> Result<(), IoError> { - let path = path.as_ref(); - - // Ensure parent directory exists - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).await?; - } - - // Open file with appropriate options - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(!append) - .append(append) - .open(path) - .await?; - - // Write contents - file.write_all(contents).await?; - - // Ensure data is synced to disk - file.flush().await?; - file.sync_all().await?; - - Ok(()) - } - - /// Creates directories with fsync - pub async fn create_dir_with_sync>(path: P) -> Result<(), IoError> { - let path = path.as_ref(); - - // Create directory - fs::create_dir_all(path).await?; - - // Sync the directory - Self::sync_directory(path).await?; - - // Sync parent directory to ensure directory creation is persisted - if let Some(parent) = path.parent() { - match Self::sync_directory(parent).await { - Ok(_) => {} - Err(e) => { - warn!( - "Failed to sync parent directory {}: {}. Directory was created but entry might not be durable.", - parent.display(), - e - ); - } - } - } - - Ok(()) - } - - /// Renames a file or directory with proper syncing - pub async fn rename_with_sync, Q: AsRef>( - from: P, - to: Q, - ) -> Result<(), IoError> { - let from = from.as_ref(); - let to = to.as_ref(); - - // Ensure parent directory of destination exists - if let Some(parent) = to.parent() { - fs::create_dir_all(parent).await?; - } - - // Perform rename - fs::rename(from, to).await?; - - // Sync parent directories to ensure rename is persisted - if let Some(from_parent) = from.parent() { - match Self::sync_directory(from_parent).await { - Ok(_) => {} - Err(e) => { - warn!( - "Failed to sync source directory {}: {}. Rename completed but might not be durable.", - from_parent.display(), - e - ); - } - } - } - - if let Some(to_parent) = to.parent() { - match Self::sync_directory(to_parent).await { - Ok(_) => {} - Err(e) => { - warn!( - "Failed to sync destination directory {}: {}. Rename completed but might not be durable.", - to_parent.display(), - e - ); - } - } - } - - Ok(()) - } - - /// Removes a file with directory syncing - pub async fn remove_file_with_sync>(path: P) -> Result<(), IoError> { - let path = path.as_ref(); - - // Remove file - fs::remove_file(path).await?; - - // Sync parent directory to ensure removal is persisted - if let Some(parent) = path.parent() { - match Self::sync_directory(parent).await { - Ok(_) => {} - Err(e) => { - warn!( - "Failed to sync directory after file removal {}: {}. File was removed but entry might not be durable.", - parent.display(), - e - ); - } - } - } - - Ok(()) - } - - /// Removes a directory with parent directory syncing - pub async fn remove_dir_with_sync>( - path: P, - recursive: bool, - ) -> Result<(), IoError> { - let path = path.as_ref(); - - // Remove directory - if recursive { - fs::remove_dir_all(path).await?; - } else { - fs::remove_dir(path).await?; - } - - // Sync parent directory to ensure removal is persisted - if let Some(parent) = path.parent() { - match Self::sync_directory(parent).await { - Ok(_) => {} - Err(e) => { - warn!( - "Failed to sync directory after directory removal {}: {}. Directory was removed but entry might not be durable.", - parent.display(), - e - ); - } - } - } - - Ok(()) - } - - /// Syncs a directory to ensure its contents are durable - async fn sync_directory>(path: P) -> Result<(), IoError> { - let path = path.as_ref(); - - // Open directory with read permissions - let dir_file = match OpenOptions::new().read(true).open(path).await { - Ok(file) => file, - Err(e) => { - warn!( - "Failed to open directory for syncing {}: {}", - path.display(), - e - ); - return Err(e); - } - }; - - // Sync the directory - dir_file.sync_all().await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - use tokio::fs; - use tokio::io::AsyncReadExt; - - #[tokio::test] - async fn test_atomic_write() { - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("test.txt"); - - // Write data atomically - FileSystemUtils::atomic_write(&file_path, b"Hello, world!") - .await - .unwrap(); - - // Read back the data - let mut file = fs::File::open(&file_path).await.unwrap(); - let mut contents = String::new(); - file.read_to_string(&mut contents).await.unwrap(); - - assert_eq!(contents, "Hello, world!"); - } - - #[tokio::test] - async fn test_write_with_sync() { - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("test.txt"); - - // Write data with sync - FileSystemUtils::write_with_sync(&file_path, b"First line\n", false) - .await - .unwrap(); - - // Append data - FileSystemUtils::write_with_sync(&file_path, b"Second line", true) - .await - .unwrap(); - - // Read back the data - let mut file = fs::File::open(&file_path).await.unwrap(); - let mut contents = String::new(); - file.read_to_string(&mut contents).await.unwrap(); - - assert_eq!(contents, "First line\nSecond line"); - } - - #[tokio::test] - async fn test_rename_with_sync() { - let temp_dir = tempdir().unwrap(); - let source_path = temp_dir.path().join("source.txt"); - let dest_path = temp_dir.path().join("dest.txt"); - - // Create source file - FileSystemUtils::write_with_sync(&source_path, b"Test content", false) - .await - .unwrap(); - - // Rename file - FileSystemUtils::rename_with_sync(&source_path, &dest_path) - .await - .unwrap(); - - // Verify source doesn't exist - assert!(!source_path.exists()); - - // Verify destination exists - let mut file = fs::File::open(&dest_path).await.unwrap(); - let mut contents = String::new(); - file.read_to_string(&mut contents).await.unwrap(); - - assert_eq!(contents, "Test content"); - } -} diff --git a/src/infrastructure/services/id_mapping_optimizer.rs b/src/infrastructure/services/id_mapping_optimizer.rs deleted file mode 100644 index cdb74907..00000000 --- a/src/infrastructure/services/id_mapping_optimizer.rs +++ /dev/null @@ -1,663 +0,0 @@ -use async_trait::async_trait; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Mutex, RwLock, Semaphore}; -use tracing::{debug, error, info, warn}; - -use crate::application::ports::outbound::IdMappingPort; -use crate::common::errors::DomainError; -use crate::domain::services::path_service::StoragePath; -use crate::infrastructure::services::id_mapping_service::{IdMappingError, IdMappingService}; - -/// Maximum number of entries in the cache -const MAX_CACHE_SIZE: usize = 10_000; - -/// Cache time-to-live (in seconds) -const CACHE_TTL_SECONDS: u64 = 60 * 5; // 5 minutes - -/// Optimizer for batch ID mapping operations -pub struct IdMappingOptimizer { - /// Base ID mapping service - base_service: Arc, - - /// Path to ID cache (path -> id) - path_to_id_cache: RwLock>, - - /// ID to path cache (id -> path) - id_to_path_cache: RwLock>, - - /// Hit counter - stats: RwLock, - - /// Semaphore to limit batch operations - batch_limiter: Semaphore, - - /// Pending batch queue - pending_batch: Mutex, -} - -/// Optimizer statistics -#[derive(Debug, Default, Clone)] -pub struct OptimizerStats { - /// Total number of get_path_by_id queries - pub path_by_id_queries: usize, - /// Number of cache hits for get_path_by_id - pub path_by_id_hits: usize, - - /// Total number of get_or_create_id queries - pub get_id_queries: usize, - /// Number of cache hits for get_or_create_id - pub get_id_hits: usize, - - /// Number of batch operations performed - pub batch_operations: usize, - /// Total number of IDs processed in batch - pub batch_items_processed: usize, - - /// Last cache cleanup timestamp - pub last_cleanup: Option, -} - -/// Queue for batch operations -#[derive(Default)] -struct BatchQueue { - /// Pending paths to get/create ID - path_to_id_requests: HashSet, - /// Pending IDs to get path - id_to_path_requests: HashSet, -} - -/// Result of a batch operation -struct BatchResult { - /// Path to ID mapping - path_to_id: HashMap, - /// ID to path mapping - id_to_path: HashMap, -} - -impl IdMappingOptimizer { - /// Creates a new optimizer for the ID mapping service - pub fn new(base_service: Arc) -> Self { - Self { - base_service, - path_to_id_cache: RwLock::new(HashMap::with_capacity(1000)), - id_to_path_cache: RwLock::new(HashMap::with_capacity(1000)), - stats: RwLock::new(OptimizerStats::default()), - batch_limiter: Semaphore::new(2), // Limit to 2 concurrent batch operations - pending_batch: Mutex::new(BatchQueue::default()), - } - } - - /// Gets optimizer statistics - pub async fn get_stats(&self) -> OptimizerStats { - self.stats.read().await.clone() - } - - /// Cleans expired cache entries - pub async fn cleanup_cache(&self) { - let now = Instant::now(); - let ttl = Duration::from_secs(CACHE_TTL_SECONDS); - - // Clean path_to_id cache - { - let mut cache = self.path_to_id_cache.write().await; - let initial_size = cache.len(); - - // Retain only non-expired entries - cache.retain(|_, (_, timestamp)| now.duration_since(*timestamp) < ttl); - - let removed = initial_size - cache.len(); - if removed > 0 { - debug!("Cleaned {} expired entries from path_to_id cache", removed); - } - } - - // Clean id_to_path cache - { - let mut cache = self.id_to_path_cache.write().await; - let initial_size = cache.len(); - - // Retain only non-expired entries - cache.retain(|_, (_, timestamp)| now.duration_since(*timestamp) < ttl); - - let removed = initial_size - cache.len(); - if removed > 0 { - debug!("Cleaned {} expired entries from id_to_path cache", removed); - } - } - - // Update statistics - { - let mut stats = self.stats.write().await; - stats.last_cleanup = Some(now); - } - } - - /// Starts periodic cleanup task - pub fn start_cleanup_task(optimizer: Arc) { - tokio::spawn(async move { - let cleanup_interval = Duration::from_secs(CACHE_TTL_SECONDS / 2); - - loop { - tokio::time::sleep(cleanup_interval).await; - optimizer.cleanup_cache().await; - - // Log statistics periodically - let stats = optimizer.get_stats().await; - info!( - "ID Mapping Optimizer stats - Path queries: {}, hits: {} ({}%), ID queries: {}, hits: {} ({}%), Batch ops: {}, items: {}", - stats.path_by_id_queries, - stats.path_by_id_hits, - if stats.path_by_id_queries > 0 { - stats.path_by_id_hits as f64 * 100.0 / stats.path_by_id_queries as f64 - } else { - 0.0 - }, - stats.get_id_queries, - stats.get_id_hits, - if stats.get_id_queries > 0 { - stats.get_id_hits as f64 * 100.0 / stats.get_id_queries as f64 - } else { - 0.0 - }, - stats.batch_operations, - stats.batch_items_processed - ); - } - }); - } - - /// Adds a request to the pending queue for batch processing - async fn queue_path_to_id_request( - &self, - path: &StoragePath, - ) -> Result, IdMappingError> { - let path_str = path.to_string(); - - // Check first in the cache - { - let cache = self.path_to_id_cache.read().await; - if let Some((id, _)) = cache.get(&path_str) { - // Update statistics - { - let mut stats = self.stats.write().await; - stats.get_id_hits += 1; - } - - return Ok(Some(id.clone())); - } - } - - // If not in cache, add to batch queue - { - let mut batch_queue = self.pending_batch.lock().await; - batch_queue.path_to_id_requests.insert(path_str); - } - - // Not found in cache, must be processed in batch - Ok(None) - } - - /// Processes pending requests in batch - async fn process_batch(&self) -> Result { - // Acquire permit for batch operation - let _permit = self.batch_limiter.acquire().await.unwrap(); - - // Get pending requests - let (path_requests, id_requests) = { - let mut batch_queue = self.pending_batch.lock().await; - - let paths = std::mem::take(&mut batch_queue.path_to_id_requests); - let ids = std::mem::take(&mut batch_queue.id_to_path_requests); - - (paths, ids) - }; - - // Create results - let mut result = BatchResult { - path_to_id: HashMap::with_capacity(path_requests.len()), - id_to_path: HashMap::with_capacity(id_requests.len()), - }; - - // Process path->id requests in batch - for path_str in path_requests { - let path = StoragePath::from_string(&path_str); - match self.base_service.get_or_create_id(&path).await { - Ok(id) => { - result.path_to_id.insert(path_str.clone(), id.clone()); - result.id_to_path.insert(id, path_str); - } - Err(e) => { - error!("Error batch-processing path {}: {}", path_str, e); - // Continue with remaining requests - } - } - } - - // Process id->path requests in batch - for id in id_requests { - match self.base_service.get_path_by_id(&id).await { - Ok(path) => { - let path_str = path.to_string(); - result.id_to_path.insert(id.clone(), path_str.clone()); - result.path_to_id.insert(path_str, id); - } - Err(e) => { - error!("Error batch-processing ID {}: {}", id, e); - // Continue with remaining requests - } - } - } - - // Update cache with batch results - { - let mut path_cache = self.path_to_id_cache.write().await; - let mut id_cache = self.id_to_path_cache.write().await; - - let now = Instant::now(); - - for (path, id) in &result.path_to_id { - path_cache.insert(path.clone(), (id.clone(), now)); - } - - for (id, path) in &result.id_to_path { - id_cache.insert(id.clone(), (path.clone(), now)); - } - } - - // Update statistics - { - let mut stats = self.stats.write().await; - stats.batch_operations += 1; - stats.batch_items_processed += result.path_to_id.len() + result.id_to_path.len(); - } - - // Save changes to disk in the background - let service_clone = self.base_service.clone(); - tokio::spawn(async move { - if let Err(e) = service_clone.save_pending_changes().await { - error!("Error saving ID mapping changes: {}", e); - } - }); - - Ok(result) - } - - /// Forces processing of pending requests if there are enough - async fn trigger_batch_if_needed(&self, min_batch_size: usize) -> Result<(), IdMappingError> { - // Check if there are enough pending requests - let should_process = { - let batch_queue = self.pending_batch.lock().await; - batch_queue.path_to_id_requests.len() + batch_queue.id_to_path_requests.len() - >= min_batch_size - }; - - // Process if necessary - if should_process { - self.process_batch().await?; - } - - Ok(()) - } - - /// Preload a set of paths to get their IDs in batch - pub async fn preload_paths(&self, paths: Vec) -> Result<(), IdMappingError> { - // Only proceed if there are paths to load - if paths.is_empty() { - return Ok(()); - } - - // Paths we need to load (those not in cache) - let mut paths_to_load = Vec::new(); - - // Check cache first - { - let cache = self.path_to_id_cache.read().await; - for path in paths { - let path_str = path.to_string(); - if !cache.contains_key(&path_str) { - paths_to_load.push(path_str); - } - } - } - - // If all were in cache, finish - if paths_to_load.is_empty() { - return Ok(()); - } - - // Add paths to queue for batch processing - { - let mut batch_queue = self.pending_batch.lock().await; - for path in paths_to_load { - batch_queue.path_to_id_requests.insert(path); - } - } - - // Execute batch processing immediately - self.process_batch().await?; - - Ok(()) - } - - /// Preload a set of IDs to get their paths in batch - pub async fn preload_ids(&self, ids: Vec) -> Result<(), IdMappingError> { - // Only proceed if there are IDs to load - if ids.is_empty() { - return Ok(()); - } - - // IDs we need to load (those not in cache) - let mut ids_to_load = Vec::new(); - - // Check cache first - { - let cache = self.id_to_path_cache.read().await; - for id in ids { - if !cache.contains_key(&id) { - ids_to_load.push(id); - } - } - } - - // If all were in cache, finish - if ids_to_load.is_empty() { - return Ok(()); - } - - // Add IDs to queue for batch processing - { - let mut batch_queue = self.pending_batch.lock().await; - for id in ids_to_load { - batch_queue.id_to_path_requests.insert(id); - } - } - - // Execute batch processing immediately - self.process_batch().await?; - - Ok(()) - } -} - -#[async_trait] -impl IdMappingPort for IdMappingOptimizer { - async fn get_or_create_id(&self, path: &StoragePath) -> Result { - // Update statistics - { - let mut stats = self.stats.write().await; - stats.get_id_queries += 1; - } - - let path_str = path.to_string(); - - // Check cache first - { - let cache = self.path_to_id_cache.read().await; - if let Some((id, _)) = cache.get(&path_str) { - // Update statistics - { - let mut stats = self.stats.write().await; - stats.get_id_hits += 1; - } - - return Ok(id.clone()); - } - } - - // If not in cache, try adding to batch queue first - let queued_result = self.queue_path_to_id_request(path).await?; - if let Some(id) = queued_result { - return Ok(id); - } - - // Trigger batch processing if enough items accumulated - self.trigger_batch_if_needed(20).await?; - - // Try to get from the base service - let id = self.base_service.get_or_create_id(path).await?; - - // Update cache with the new ID - { - let mut path_cache = self.path_to_id_cache.write().await; - let mut id_cache = self.id_to_path_cache.write().await; - - let now = Instant::now(); - - // Control cache size - if path_cache.len() >= MAX_CACHE_SIZE { - warn!( - "Path-to-ID cache size reached limit ({}), clearing oldest entries", - MAX_CACHE_SIZE - ); - path_cache.clear(); - } - - if id_cache.len() >= MAX_CACHE_SIZE { - warn!( - "ID-to-path cache size reached limit ({}), clearing oldest entries", - MAX_CACHE_SIZE - ); - id_cache.clear(); - } - - path_cache.insert(path_str.clone(), (id.clone(), now)); - id_cache.insert(id.clone(), (path_str, now)); - } - - Ok(id) - } - - async fn get_path_by_id(&self, id: &str) -> Result { - // Update statistics - { - let mut stats = self.stats.write().await; - stats.path_by_id_queries += 1; - } - - // Check first in the cache - { - let cache = self.id_to_path_cache.read().await; - if let Some((path_str, _)) = cache.get(id) { - // Update statistics - { - let mut stats = self.stats.write().await; - stats.path_by_id_hits += 1; - } - - return Ok(StoragePath::from_string(path_str)); - } - } - - // Get from the base service - let path = self.base_service.get_path_by_id(id).await?; - - // Update cache - { - let mut id_cache = self.id_to_path_cache.write().await; - let mut path_cache = self.path_to_id_cache.write().await; - - let now = Instant::now(); - let path_str = path.to_string(); - - // Control cache size - if id_cache.len() >= MAX_CACHE_SIZE { - warn!( - "ID-to-path cache size reached limit ({}), clearing oldest entries", - MAX_CACHE_SIZE - ); - id_cache.clear(); - } - - if path_cache.len() >= MAX_CACHE_SIZE { - warn!( - "Path-to-ID cache size reached limit ({}), clearing oldest entries", - MAX_CACHE_SIZE - ); - path_cache.clear(); - } - - id_cache.insert(id.to_string(), (path_str.clone(), now)); - path_cache.insert(path_str, (id.to_string(), now)); - } - - Ok(path) - } - - async fn update_path(&self, id: &str, new_path: &StoragePath) -> Result<(), DomainError> { - // Invalidate cache for this ID - { - let mut id_cache = self.id_to_path_cache.write().await; - let mut path_cache = self.path_to_id_cache.write().await; - - // Remove the ID entry - if let Some((old_path, _)) = id_cache.remove(id) { - path_cache.remove(&old_path); - } - } - - // Update in the base service - let result = self.base_service.update_path(id, new_path).await?; - - // Update cache with new mapping - { - let mut id_cache = self.id_to_path_cache.write().await; - let mut path_cache = self.path_to_id_cache.write().await; - - let now = Instant::now(); - let path_str = new_path.to_string(); - - id_cache.insert(id.to_string(), (path_str.clone(), now)); - path_cache.insert(path_str, (id.to_string(), now)); - } - - Ok(result) - } - - async fn remove_id(&self, id: &str) -> Result<(), DomainError> { - // Invalidate cache for this ID - { - let mut id_cache = self.id_to_path_cache.write().await; - let mut path_cache = self.path_to_id_cache.write().await; - - // Remove the ID entry - if let Some((path, _)) = id_cache.remove(id) { - path_cache.remove(&path); - } - } - - // Remove from the base service - self.base_service.remove_id(id).await?; - - Ok(()) - } - - async fn save_changes(&self) -> Result<(), DomainError> { - // Delegate to the base service - self.base_service.save_changes().await?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - async fn create_test_service() -> (Arc, Arc) { - let temp_dir = tempdir().unwrap(); - let map_path = temp_dir.path().join("id_map.json"); - - let base_service = Arc::new(IdMappingService::new(map_path).await.unwrap()); - let optimizer = Arc::new(IdMappingOptimizer::new(base_service.clone())); - - (base_service, optimizer) - } - - #[tokio::test] - async fn test_basic_caching() { - let (_, optimizer) = create_test_service().await; - - let path = StoragePath::from_string("/test/file.txt"); - - // First call should use the base service - let id = optimizer.get_or_create_id(&path).await.unwrap(); - assert!(!id.is_empty(), "ID should not be empty"); - - // Second call should use cache - let id2 = optimizer.get_or_create_id(&path).await.unwrap(); - assert_eq!(id, id2, "Same path should return same ID"); - - // Verify cache statistics - let stats = optimizer.get_stats().await; - assert_eq!(stats.get_id_queries, 2, "Should have 2 queries"); - assert_eq!(stats.get_id_hits, 1, "Should have 1 hit"); - } - - #[tokio::test] - async fn test_batch_processing() { - let (_, optimizer) = create_test_service().await; - - // Create a batch of paths - let mut paths = Vec::new(); - for i in 0..50 { - paths.push(StoragePath::from_string(&format!( - "/test/batch/file{}.txt", - i - ))); - } - - // Preload the paths - optimizer.preload_paths(paths.clone()).await.unwrap(); - - // Verify all are in cache - for path in &paths { - let id = optimizer.get_or_create_id(path).await.unwrap(); - assert!(!id.is_empty(), "ID should be available for path"); - } - - // Verify statistics - let stats = optimizer.get_stats().await; - assert_eq!(stats.batch_operations, 1, "Should have 1 batch operation"); - assert!( - stats.batch_items_processed >= 50, - "Should have processed at least 50 items" - ); - - // Verify all subsequent queries are cache hits - assert_eq!( - stats.get_id_hits, 50, - "All subsequente queries should be cache hits" - ); - } - - #[tokio::test] - async fn test_cache_cleanup() { - let (_, optimizer) = create_test_service().await; - - // Create some entries - let path = StoragePath::from_string("/test/cleanup.txt"); - let id = optimizer.get_or_create_id(&path).await.unwrap(); - - // Verify initial statistics - { - let stats = optimizer.get_stats().await; - assert_eq!(stats.get_id_queries, 1, "Should have 1 query"); - assert_eq!(stats.get_id_hits, 0, "Should have 0 hits"); - } - - // Run cleanup (should not remove anything yet) - optimizer.cleanup_cache().await; - - // Verify cache is still working - let id2 = optimizer.get_or_create_id(&path).await.unwrap(); - assert_eq!(id, id2, "Cache should still work after cleanup"); - - { - let stats = optimizer.get_stats().await; - assert_eq!(stats.get_id_hits, 1, "Should have 1 hit after cleanup"); - } - } -} diff --git a/src/infrastructure/services/id_mapping_service.rs b/src/infrastructure/services/id_mapping_service.rs deleted file mode 100644 index 19eb81cf..00000000 --- a/src/infrastructure/services/id_mapping_service.rs +++ /dev/null @@ -1,712 +0,0 @@ -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::PathBuf; -use tokio::fs; -use tokio::sync::{Mutex, RwLock}; -use tokio::time; -use uuid::Uuid; - -use crate::application::ports::outbound::IdMappingPort; -use crate::common::config::TimeoutConfig; -use crate::common::errors::{DomainError, ErrorKind}; -use crate::domain::services::path_service::StoragePath; - -/// Specific error for the ID mapping service -#[derive(Debug, thiserror::Error)] -pub enum IdMappingError { - #[error("ID not found: {0}")] - NotFound(String), - - #[error("IO error: {0}")] - IoError(#[from] std::io::Error), - - #[error("Timeout error: {0}")] - Timeout(String), - - #[error("Serialization error: {0}")] - SerializationError(#[from] serde_json::Error), - - #[error("Other error: {0}")] - Other(String), -} - -// Implement conversion from IdMappingError to DomainError -impl From for DomainError { - fn from(err: IdMappingError) -> Self { - match err { - IdMappingError::NotFound(id) => DomainError::not_found("IdMapping", id), - IdMappingError::IoError(e) => DomainError::new( - ErrorKind::InternalError, - "IdMapping", - format!("IO error: {}", e), - ) - .with_source(e), - IdMappingError::Timeout(msg) => { - DomainError::timeout("IdMapping", format!("Timeout: {}", msg)) - } - IdMappingError::SerializationError(e) => DomainError::new( - ErrorKind::InternalError, - "IdMapping", - format!("Serialization error: {}", e), - ) - .with_source(e), - IdMappingError::Other(msg) => DomainError::new( - ErrorKind::InternalError, - "IdMapping", - format!("Other error: {}", msg), - ), - } - } -} - -/// Structure to store IDs mapped to their paths -#[derive(Serialize, Deserialize, Debug, Default)] -struct IdMap { - path_to_id: HashMap, - id_to_path: HashMap, // Field for efficient bidirectional lookup - version: u32, // Version to detect changes -} - -/// Service to manage mappings between paths and unique IDs -pub struct IdMappingService { - map_path: PathBuf, - id_map: RwLock, - save_mutex: Mutex<()>, // To prevent multiple concurrent saves - timeouts: TimeoutConfig, - pending_save: RwLock, // Indicates if there are pending changes -} - -impl IdMappingService { - /// Creates a new ID mapping service - pub async fn new(map_path: PathBuf) -> Result { - let timeouts = TimeoutConfig::default(); - let id_map = Self::load_id_map(&map_path, &timeouts).await?; - - Ok(Self { - map_path, - id_map: RwLock::new(id_map), - save_mutex: Mutex::new(()), - timeouts, - pending_save: RwLock::new(false), - }) - } - - /// Creates an in-memory ID mapping service (for testing) - /// - /// Similar functionality as new_in_memory but with a simpler signature for dummy use - pub fn dummy() -> Self { - Self { - map_path: PathBuf::from("/tmp/dummy_id_map.json"), - id_map: RwLock::new(IdMap::default()), - save_mutex: Mutex::new(()), - timeouts: TimeoutConfig::default(), - pending_save: RwLock::new(false), - } - } - - /// Creates an in-memory ID mapping service (for testing - original version) - pub fn new_in_memory() -> Self { - Self { - map_path: PathBuf::from("memory"), - id_map: RwLock::new(IdMap::default()), - save_mutex: Mutex::new(()), - timeouts: TimeoutConfig::default(), - pending_save: RwLock::new(false), - } - } - - /// Loads the ID map from disk with robust error handling - async fn load_id_map( - map_path: &PathBuf, - timeouts: &TimeoutConfig, - ) -> Result { - if map_path.exists() { - // Try to read with timeout to avoid indefinite blocking - let read_result = time::timeout(timeouts.lock_timeout(), fs::read_to_string(map_path)) - .await - .map_err(|_| { - DomainError::timeout( - "IdMapping", - format!("Timeout reading ID map from {}", map_path.display()), - ) - })?; - - let content = read_result.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!("Failed to read ID map from {}: {}", map_path.display(), e), - ) - })?; - - // Parse the JSON - match serde_json::from_str::(&content) { - Ok(mut map) => { - // Rebuild the inverse map if necessary - if map.id_to_path.is_empty() && !map.path_to_id.is_empty() { - let mut rebuild_count = 0; - for (path, id) in &map.path_to_id { - map.id_to_path.insert(id.clone(), path.clone()); - rebuild_count += 1; - } - tracing::info!("Rebuilt inverse mapping with {} entries", rebuild_count); - } - - tracing::info!( - "Loaded ID map with {} entries (version: {})", - map.path_to_id.len(), - map.version - ); - return Ok(map); - } - Err(e) => { - tracing::error!("Error parsing ID map: {}", e); - // Try to backup the corrupted file - let backup_path = map_path.with_extension("json.bak"); - if let Err(copy_err) = tokio::fs::copy(map_path, &backup_path).await { - tracing::error!("Failed to backup corrupted map file: {}", copy_err); - } else { - tracing::info!("Backed up corrupted ID map to {}", backup_path.display()); - } - - tracing::info!("Creating new empty map after error"); - return Ok(IdMap { - path_to_id: HashMap::new(), - id_to_path: HashMap::new(), - version: 1, // Start with version 1 - }); - } - } - } - - // Return an empty map if the file doesn't exist and create the file - tracing::info!("No existing ID map found, creating new empty map"); - let empty_map = IdMap { - path_to_id: HashMap::new(), - id_to_path: HashMap::new(), - version: 1, // Start with version 1 - }; - - // Ensure directory exists - if let Some(parent) = map_path.parent() - && !parent.exists() - && let Err(e) = fs::create_dir_all(parent).await - { - tracing::error!("Failed to create directory for ID map: {}", e); - } - - // Write empty map to file (best-effort: the in-memory map is valid even if disk write fails) - match serde_json::to_string_pretty(&empty_map) { - Ok(json) => { - if let Err(e) = fs::write(map_path, json).await { - tracing::warn!( - "Could not write initial empty ID map (will retry on next save): {}", - e - ); - } else { - tracing::info!("Created initial empty ID map at {}", map_path.display()); - } - } - Err(e) => { - tracing::error!("Failed to serialize empty ID map: {}", e); - } - } - - Ok(empty_map) - } - - /// Saves the ID map to disk safely - async fn save_id_map(&self) -> Result<(), DomainError> { - // Acquire exclusive lock for saving - let _lock = time::timeout(self.timeouts.lock_timeout(), self.save_mutex.lock()) - .await - .map_err(|_| { - DomainError::timeout("IdMapping", "Timeout acquiring save lock for ID mapping") - })?; - - // Create JSON with read lock to minimize lock hold time - let json = { - let mut map = time::timeout(self.timeouts.lock_timeout(), self.id_map.write()) - .await - .map_err(|_| { - DomainError::timeout("IdMapping", "Timeout acquiring write lock for ID mapping") - })?; - - // Increment version only if there are pending changes to save - let pending = *self.pending_save.read().await; - if pending { - map.version += 1; - tracing::debug!("Incrementing ID map version to {}", map.version); - } - - // Use serde with reasonably safe defaults - serde_json::to_string_pretty(&*map).map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!("Failed to serialize ID map to JSON: {}", e), - ) - })? - }; - - // Write to a temporary file first to avoid corruption - let temp_path = self.map_path.with_extension("json.tmp"); - fs::write(&temp_path, &json).await.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!( - "Failed to write temporary ID map to {}: {}", - temp_path.display(), - e - ), - ) - })?; - - // Perform the atomic rename - fs::rename(&temp_path, &self.map_path).await.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!( - "Failed to rename temporary ID map to {}: {}", - self.map_path.display(), - e - ), - ) - })?; - - // Reset pending flag - { - let mut pending = self.pending_save.write().await; - *pending = false; - } - - tracing::info!("Saved ID map successfully to {}", self.map_path.display()); - Ok(()) - } - - /// Generates a unique ID - fn generate_id(&self) -> String { - Uuid::new_v4().to_string() - } - - /// Marks changes as pending - async fn mark_pending(&self) { - let mut pending = self.pending_save.write().await; - *pending = true; - } - - /// Gets the ID for a path or generates a new one if it doesn't exist - pub async fn get_or_create_id(&self, path: &StoragePath) -> Result { - let path_str = path.to_string(); - - // First attempt with read lock (more efficient) - { - let read_result = - match time::timeout(self.timeouts.lock_timeout(), self.id_map.read()).await { - Ok(guard) => guard, - Err(_) => { - return Err(IdMappingError::Timeout( - "Timeout acquiring read lock for ID mapping".to_string(), - )); - } - }; - - if let Some(id) = read_result.path_to_id.get(&path_str) { - return Ok(id.clone()); - } - } - - // If not found, acquire write lock - let write_result = - match time::timeout(self.timeouts.lock_timeout(), self.id_map.write()).await { - Ok(guard) => guard, - Err(_) => { - return Err(IdMappingError::Timeout( - "Timeout acquiring write lock for ID mapping".to_string(), - )); - } - }; - - let mut map = write_result; - - // Check again (it could have been added while we were waiting for the lock) - if let Some(id) = map.path_to_id.get(&path_str) { - return Ok(id.clone()); - } - - // Generate a new ID and store it - let id = self.generate_id(); - map.path_to_id.insert(path_str.clone(), id.clone()); - map.id_to_path.insert(id.clone(), path_str); - - // Mark as pending for saving - drop(map); // Release the write lock before acquiring another - self.mark_pending().await; - - tracing::debug!("Created new ID mapping: {} -> {}", path.to_string(), id); - - Ok(id) - } - - /// Gets a path by its ID with timeout handling - pub async fn get_path_by_id(&self, id: &str) -> Result { - let read_result = - match time::timeout(self.timeouts.lock_timeout(), self.id_map.read()).await { - Ok(guard) => guard, - Err(_) => { - return Err(IdMappingError::Timeout( - "Timeout acquiring read lock for ID lookup".to_string(), - )); - } - }; - - if let Some(path_str) = read_result.id_to_path.get(id) { - return Ok(StoragePath::from_string(path_str)); - } - - Err(IdMappingError::NotFound(id.to_string())) - } - - /// Updates the mapping of an existing ID to a new path - pub async fn update_path( - &self, - id: &str, - new_path: &StoragePath, - ) -> Result<(), IdMappingError> { - let write_result = - match time::timeout(self.timeouts.lock_timeout(), self.id_map.write()).await { - Ok(guard) => guard, - Err(_) => { - return Err(IdMappingError::Timeout( - "Timeout acquiring write lock for ID update".to_string(), - )); - } - }; - - let mut map = write_result; - - // Find the previous path to remove it - if let Some(old_path) = map.id_to_path.get(id).cloned() { - map.path_to_id.remove(&old_path); - - // Register the new path - let new_path_str = new_path.to_string(); - map.path_to_id.insert(new_path_str.clone(), id.to_string()); - map.id_to_path.insert(id.to_string(), new_path_str); - - // Mark as pending - drop(map); // Release the write lock before acquiring another - self.mark_pending().await; - - tracing::debug!( - "Updated path mapping for ID {}: {} -> {}", - id, - old_path, - new_path.to_string() - ); - - Ok(()) - } else { - Err(IdMappingError::NotFound(id.to_string())) - } - } - - /// Removes an ID from the map - pub async fn remove_id(&self, id: &str) -> Result<(), IdMappingError> { - let write_result = - match time::timeout(self.timeouts.lock_timeout(), self.id_map.write()).await { - Ok(guard) => guard, - Err(_) => { - return Err(IdMappingError::Timeout( - "Timeout acquiring write lock for ID removal".to_string(), - )); - } - }; - - let mut map = write_result; - - // Find the path to remove it - if let Some(path) = map.id_to_path.remove(id) { - map.path_to_id.remove(&path); - - // Mark as pending - drop(map); // Release the write lock before acquiring another - self.mark_pending().await; - - tracing::debug!("Removed ID mapping: {} -> {}", id, path); - Ok(()) - } else { - Err(IdMappingError::NotFound(id.to_string())) - } - } - - /// Saves pending changes to disk immediately, without debounce - pub async fn save_pending_changes(&self) -> Result<(), IdMappingError> { - // Check if there are pending changes - { - let pending = self.pending_save.read().await; - if !*pending { - return Ok(()); - } - } - - // Save immediately (without debounce or spawn) - match self.save_id_map().await { - Ok(_) => { - tracing::info!( - "ID mappings saved successfully to disk at {}", - self.map_path.display() - ); - - // Explicitly verify that the file exists and has size - match std::fs::metadata(&self.map_path) { - Ok(metadata) => { - if metadata.len() > 0 { - tracing::info!( - "Verified saved map file exists with size: {} bytes", - metadata.len() - ); - } else { - tracing::warn!( - "Map file exists but has zero size - this might cause issues" - ); - } - } - Err(e) => { - tracing::error!("Failed to verify saved map file: {}", e); - // Try a second save if verification fails - if let Err(retry_err) = self.save_id_map().await { - tracing::error!("Second save attempt also failed: {}", retry_err); - return Err(IdMappingError::IoError(std::io::Error::other(format!( - "Failed to verify and retry save: {}", - retry_err - )))); - } - tracing::info!("Second save attempt succeeded"); - } - } - - Ok(()) - } - Err(e) => { - tracing::error!( - "Failed to save ID map to {}: {}", - self.map_path.display(), - e - ); - // Try a second save with delay in case of error - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - match self.save_id_map().await { - Ok(_) => { - tracing::info!("Second save attempt succeeded after initial failure"); - Ok(()) - } - Err(retry_e) => { - tracing::error!("Second save attempt also failed: {}", retry_e); - Err(IdMappingError::IoError(std::io::Error::other(format!( - "Failed to save ID mappings after retry: {}", - retry_e - )))) - } - } - } - } - } -} - -#[async_trait] -impl IdMappingPort for IdMappingService { - /// Gets the ID for a path or generates a new one if it doesn't exist - async fn get_or_create_id(&self, path: &StoragePath) -> Result { - self.get_or_create_id(path).await.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!( - "Failed to get or create ID for path: {}: {}", - path.to_string(), - e - ), - ) - }) - } - - /// Gets a path by its ID with timeout handling - async fn get_path_by_id(&self, id: &str) -> Result { - self.get_path_by_id(id).await.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!("Failed to get path for ID: {}: {}", id, e), - ) - }) - } - - /// Updates the mapping of an existing ID to a new path - async fn update_path(&self, id: &str, new_path: &StoragePath) -> Result<(), DomainError> { - self.update_path(id, new_path).await.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!( - "Failed to update path for ID: {} to {}: {}", - id, - new_path.to_string(), - e - ), - ) - }) - } - - /// Removes an ID from the map - async fn remove_id(&self, id: &str) -> Result<(), DomainError> { - self.remove_id(id).await.map_err(|e| { - DomainError::internal_error("IdMapping", format!("Failed to remove ID: {}: {}", id, e)) - }) - } - - /// Saves pending changes to disk - async fn save_changes(&self) -> Result<(), DomainError> { - self.save_pending_changes().await.map_err(|e| { - DomainError::internal_error( - "IdMapping", - format!("Failed to save pending ID mapping changes: {}", e), - ) - }) - } -} - -// The extension methods were moved to the IdMappingPort trait as default implementations - -// Implement Clone to allow use in tokio::spawn -/// Synchronous helper for contexts where we can't use async -impl IdMappingService { - /// Create a new service synchronously (only for stubs and initialization) - pub fn new_sync(map_path: PathBuf) -> Self { - // Create a minimal implementation for initialization purposes - Self { - map_path, - id_map: RwLock::new(IdMap::default()), - save_mutex: Mutex::new(()), - timeouts: TimeoutConfig::default(), - pending_save: RwLock::new(false), - } - } -} - -impl Clone for IdMappingService { - fn clone(&self) -> Self { - // We cannot directly clone the RwLock/Mutex, - // but we can create new instances that point to the same internal Arc - // However, in this case we simply need the map_path - Self { - map_path: self.map_path.clone(), - id_map: RwLock::new(IdMap::default()), // This is not used in the async task - save_mutex: Mutex::new(()), // Neither is this - timeouts: self.timeouts.clone(), - pending_save: RwLock::new(false), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - use tempfile::tempdir; - - #[tokio::test] - async fn test_get_or_create_id() { - let temp_dir = tempdir().unwrap(); - let map_path = temp_dir.path().join("id_map.json"); - - let service = IdMappingService::new(map_path).await.unwrap(); - - let path = StoragePath::from_string("/test/file.txt"); - let id = service.get_or_create_id(&path).await.unwrap(); - - assert!(!id.is_empty(), "ID should not be empty"); - - // Verify that the same ID is returned for the same path - let id2 = service.get_or_create_id(&path).await.unwrap(); - assert_eq!(id, id2, "Same path should return same ID"); - } - - #[tokio::test] - async fn test_update_path() { - let temp_dir = tempdir().unwrap(); - let map_path = temp_dir.path().join("id_map.json"); - - let service = IdMappingService::new(map_path).await.unwrap(); - - let old_path = StoragePath::from_string("/test/old.txt"); - let id = service.get_or_create_id(&old_path).await.unwrap(); - - let new_path = StoragePath::from_string("/test/new.txt"); - service.update_path(&id, &new_path).await.unwrap(); - - let retrieved_path = service.get_path_by_id(&id).await.unwrap(); - assert_eq!(retrieved_path, new_path, "Path should be updated"); - } - - #[tokio::test] - async fn test_save_and_load() { - let temp_dir = tempdir().unwrap(); - let map_path = temp_dir.path().join("id_map.json"); - - // Create and populate the service - let service = IdMappingService::new(map_path.clone()).await.unwrap(); - - let path1 = StoragePath::from_string("/test/file1.txt"); - let path2 = StoragePath::from_string("/test/file2.txt"); - let id1 = service.get_or_create_id(&path1).await.unwrap(); - let id2 = service.get_or_create_id(&path2).await.unwrap(); - - // Save changes - service.save_pending_changes().await.unwrap(); - - // Wait to ensure the async save completes - tokio::time::sleep(Duration::from_millis(500)).await; - - // Create a new service that should load the same map - let service2 = IdMappingService::new(map_path).await.unwrap(); - - // Verify that the IDs match - let loaded_id1 = service2.get_or_create_id(&path1).await.unwrap(); - let loaded_id2 = service2.get_or_create_id(&path2).await.unwrap(); - - assert_eq!(id1, loaded_id1, "ID1 should be preserved"); - assert_eq!(id2, loaded_id2, "ID2 should be preserved"); - } - - #[tokio::test] - async fn test_concurrent_operations() { - use futures::future::join_all; - - let temp_dir = tempdir().unwrap(); - let map_path = temp_dir.path().join("id_map.json"); - - let service = std::sync::Arc::new(IdMappingService::new(map_path).await.unwrap()); - - // Create multiple tasks that attempt simultaneous access - let mut tasks = Vec::new(); - for i in 0..100 { - let path = StoragePath::from_string(&format!("/test/concurrent/file{}.txt", i)); - let service_clone = service.clone(); - - tasks.push(tokio::spawn(async move { - service_clone.get_or_create_id(&path).await - })); - } - - // Wait for all to finish - let results = join_all(tasks).await; - - // Verify that all succeeded - for result in results { - assert!( - result.unwrap().is_ok(), - "Concurrent operations should succeed" - ); - } - - // Save changes - service.save_pending_changes().await.unwrap(); - } -} diff --git a/src/infrastructure/services/mod.rs b/src/infrastructure/services/mod.rs index 62bc703c..db72e32c 100644 --- a/src/infrastructure/services/mod.rs +++ b/src/infrastructure/services/mod.rs @@ -1,13 +1,8 @@ -pub mod buffer_pool; pub mod chunked_upload_service; pub mod compression_service; pub mod dedup_service; pub mod file_content_cache; -pub mod file_metadata_cache; pub mod file_system_i18n_service; -pub mod file_system_utils; -pub mod id_mapping_optimizer; -pub mod id_mapping_service; pub mod image_transcode_service; pub mod jwt_service; pub mod oidc_service; diff --git a/src/lib.rs b/src/lib.rs index e51779c6..9379f9ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,11 +8,6 @@ pub mod interfaces; // Common public re-exports pub use application::services::folder_service::FolderService; pub use application::services::i18n_application_service::I18nApplicationService; -pub use application::services::storage_mediator::{FileSystemStorageMediator, StorageMediator}; pub use domain::services::path_service::StoragePath; -pub use infrastructure::repositories::CompositeFileRepository; -pub use infrastructure::repositories::folder_fs_repository::FolderFsRepository; -pub use infrastructure::repositories::parallel_file_processor::ParallelFileProcessor; -pub use infrastructure::services::buffer_pool::BufferPool; pub use infrastructure::services::compression_service::GzipCompressionService; pub use infrastructure::services::path_service::PathService;