diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 654c980b..0f0a3c3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,7 +157,7 @@ jobs: retention-days: 1 api-test: - name: API tests (via Hurl) + name: API & Webdav tests needs: build if: github.event_name == 'pull_request' timeout-minutes: 30 @@ -173,7 +173,7 @@ jobs: - name: Set execute bit on pre-built binary run: chmod +x target/release/oxicloud - - name: Install Hurl + - name: Install Hurl and b3sum env: HURL_MAJOR: "8" run: | @@ -181,13 +181,18 @@ jobs: https://api.github.com/repos/Orange-OpenSource/hurl/releases \ | jq -r "map(select(.tag_name | startswith(\"${HURL_MAJOR}.\"))) | first | .tag_name") curl -fLO "https://github.com/Orange-OpenSource/hurl/releases/download/${HURL_VERSION}/hurl_${HURL_VERSION}_amd64.deb" - sudo apt-get install -y "./hurl_${HURL_VERSION}_amd64.deb" + sudo apt-get install -y "./hurl_${HURL_VERSION}_amd64.deb" b3sum - name: Run Hurl API tests run: bash tests/api/run.sh env: BUILD_TARGET: release + - name: Run Webdav tests + run: bash tests/webdav/run.sh + env: + BUILD_TARGET: release + - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: diff --git a/CLAUDE.md b/CLAUDE.md index d51dedc3..c267348a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,7 +78,7 @@ Hexagonal / Clean Architecture with four layers. Dependencies point inward only. - **DI via `AppState`**: All services are `Arc`-wrapped and assembled in `common/di.rs`. `AppState` is wrapped in `Arc` and passed as Axum state. Many services are `Option>` because they depend on features being enabled (auth, WOPI, trash, etc.). -- **Content-addressable storage**: Files use SHA-256 blob dedup. `storage.file_blobs` stores content; `storage.file_metadata` references blobs with ref-counting. See `file_blob_write_repository.rs` and `file_blob_read_repository.rs`. +- **Content-addressable storage**: Files use BLAKE3 blob dedup. `storage.file_blobs` stores content; `storage.file_metadata` references blobs with ref-counting. See `file_blob_write_repository.rs` and `file_blob_read_repository.rs`. - **ltree paths**: Folder hierarchy uses PostgreSQL `ltree` for efficient subtree queries (recursive copies, moves, searches). diff --git a/docs/architecture/caching.md b/docs/architecture/caching.md index be1776cc..ed93aae6 100644 --- a/docs/architecture/caching.md +++ b/docs/architecture/caching.md @@ -10,7 +10,7 @@ OxiCloud uses **moka** (a lock-free, concurrent cache) for write-behind caching | Directory listings | 120 s | 10 000 | Frequently accessed folder contents | | Thumbnail cache | configurable | 1 000 | Generated WebP/AVIF thumbnails | | Image transcode | configurable | 500 | On-the-fly image transcoding results | -| Blob hash | 30 s TTI | 5 000 | SHA-256 hashes for dedup lookups | +| Blob hash | 30 s TTI | 5 000 | BLAKE3 hashes for dedup lookups | | Audio metadata | — | 2 000 | ID3 tags and duration | ## How It Works diff --git a/docs/guide/index.md b/docs/guide/index.md index 25fd1e0e..15127f77 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -14,7 +14,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal | **Cold start** | < 1 s | 5–15 s | | **CPU at idle** | ~0 % | 1–5 % (cron, background jobs) | | **Min. hardware** | 1 vCPU / 512 MB RAM | 2 vCPU / 2 GB RAM | -| **File dedup** | SHA-256 content-addressable | None | +| **File dedup** | BLAKE3 content-addressable | None | | **Dependencies** | Single binary + PostgreSQL | PHP, Apache/Nginx, Redis, Cron, … | | **WebDAV** | Built-in (RFC 4918) | Built-in | | **CalDAV / CardDAV** | Built-in | Via apps | @@ -28,7 +28,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal ### Storage & Files - Drag-and-drop upload, multi-file, grid & list views - Chunked uploads (TUS-like, parallel, resumable, MD5 integrity) -- SHA-256 content-addressable file deduplication with ref-counting +- BLAKE3 content-addressable file deduplication with ref-counting - Adaptive compression (zstd / gzip per MIME type) - Trash bin with soft-delete and auto-purge - Favourites, recent files, full-text search diff --git a/docs/index.md b/docs/index.md index e65f730c..b7889a62 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ features: details: Single Rust binary, ~40 MB Docker image, <1s cold start, 30–50 MB idle RAM. - icon: 📁 title: Full File Management - details: Chunked uploads, SHA-256 deduplication, trash, favourites, full-text search, thumbnails. + details: Chunked uploads, BLAKE3 deduplication, trash, favourites, full-text search, thumbnails. - icon: 🔗 title: WebDAV / CalDAV / CardDAV details: RFC-compliant protocols for files, calendars, and contacts. Works with all major clients. diff --git a/justfile b/justfile index 8dbc1b11..b630a1d2 100644 --- a/justfile +++ b/justfile @@ -75,3 +75,4 @@ front-test-update-snapshot: # Hurl API functional tests (starts postgres + server, tears down after) api-test: bash tests/api/run.sh + bash tests/webdav/run.sh diff --git a/src/application/ports/blob_lifecycle.rs b/src/application/ports/blob_lifecycle.rs new file mode 100644 index 00000000..021f89cf --- /dev/null +++ b/src/application/ports/blob_lifecycle.rs @@ -0,0 +1,35 @@ +use std::future::Future; +use std::pin::Pin; + +/// Observer notified by [`DedupService`] when a genuinely new blob is stored +/// for the first time (no dedup hit). +/// +/// Register with [`DedupService::add_blob_creation_hook`] during DI wiring. +pub trait BlobCreationHook: Send + Sync { + /// Called after the new blob's chunks and manifest have been written. + /// `blob_hash` is the BLAKE3 hex, `content_type` is the MIME type if known. + /// Must be best-effort — must not propagate errors. + fn on_blob_created<'a>( + &'a self, + blob_hash: &'a str, + content_type: Option<&'a str>, + ) -> Pin + Send + 'a>>; +} +/// Observer notified by [`DedupService`] when a blob's ref_count reaches zero +/// and it is permanently removed from storage. +/// +/// Implement this trait on any service that needs to react to blob deletion +/// (e.g. thumbnail cleanup, CDN invalidation, audit logging). Register with +/// [`DedupService::add_blob_hook`] during DI wiring. +/// +/// The boxed-future return keeps the trait dyn-compatible so multiple +/// implementations can be stored as `Vec>`. +pub trait BlobDeletionHook: Send + Sync { + /// Called after the blob file has been removed from disk. + /// `blob_hash` is the BLAKE3 hex string identifying the blob. + /// Must be best-effort — must not propagate errors. + fn on_blob_deleted<'a>( + &'a self, + blob_hash: &'a str, + ) -> Pin + Send + 'a>>; +} diff --git a/src/application/ports/file_lifecycle.rs b/src/application/ports/file_lifecycle.rs new file mode 100644 index 00000000..fe416e28 --- /dev/null +++ b/src/application/ports/file_lifecycle.rs @@ -0,0 +1,57 @@ +use std::future::Future; +use std::pin::Pin; + +/// Observer notified by [`FileUploadService`] when a new file record is created +/// (including dedup hits where the blob already exists). +/// +/// Register with [`FileUploadService::with_file_created_hook`] during DI wiring. +pub trait FileCreatedHook: Send + Sync { + /// Called after the file record has been persisted. + /// `file_id` — opaque file UUID string. + /// `blob_hash` — BLAKE3 hex of the blob (may already exist on disk for dedup hits). + /// `content_type` — MIME type of the content. + /// Must be best-effort — must not propagate errors. + fn on_file_created<'a>( + &'a self, + file_id: &'a str, + blob_hash: &'a str, + content_type: &'a str, + ) -> Pin + Send + 'a>>; +} + +/// Observer notified by [`FileUploadService`] when an existing file's blob is +/// replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload). +/// +/// Implement this trait on any service that needs to react to a content swap +/// (e.g. thumbnail invalidation + regeneration, search index update). +/// Register with [`FileUploadService::with_file_updated_hook`] during DI wiring. +/// +/// The boxed-future return keeps the trait dyn-compatible so multiple +/// implementations can be stored as `Vec>`. +pub trait FileUpdatedHook: Send + Sync { + /// Called after the new blob has been stored and the file record updated. + /// + /// `file_id` is an opaque file UUID string, `blob_hash` is the BLAKE3 hex + /// of the new blob, `content_type` is the MIME type of the new content. + /// Must be best-effort — must not propagate errors. + fn on_file_updated<'a>( + &'a self, + file_id: &'a str, + blob_hash: &'a str, + content_type: &'a str, + ) -> Pin + Send + 'a>>; +} + +/// Observer notified by [`FileManagementService`] when a file is permanently +/// deleted (either directly or after being emptied from trash). +/// +/// Register with [`FileManagementService::with_file_deleted_hook`] during DI wiring. +pub trait FileDeletedHook: Send + Sync { + /// Called after the file record has been removed. + /// `file_id` — opaque file UUID string. + /// Must be best-effort — must not propagate errors. + fn on_file_deleted<'a>( + &'a self, + file_id: &'a str, + ) -> Pin + Send + 'a>>; +} diff --git a/src/application/ports/mod.rs b/src/application/ports/mod.rs index 22302921..d3250c1e 100644 --- a/src/application/ports/mod.rs +++ b/src/application/ports/mod.rs @@ -1,4 +1,5 @@ pub mod auth_ports; +pub mod blob_lifecycle; pub mod blob_storage_ports; pub mod cache_ports; pub mod calendar_ports; @@ -7,6 +8,7 @@ pub mod chunked_upload_ports; pub mod compression_ports; pub mod dedup_ports; pub mod favorites_ports; +pub mod file_lifecycle; pub mod file_ports; pub mod inbound; pub mod music_ports; diff --git a/src/application/services/file_management_service.rs b/src/application/services/file_management_service.rs index eb7a3d37..7bec7187 100644 --- a/src/application/services/file_management_service.rs +++ b/src/application/services/file_management_service.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use crate::application::dtos::file_dto::FileDto; +use crate::application::ports::file_lifecycle::FileDeletedHook; use crate::application::ports::file_ports::FileManagementUseCase; use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileReadPort, FileWritePort}; use crate::application::ports::trash_ports::TrashUseCase; @@ -11,7 +12,6 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository; use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository; use crate::infrastructure::services::file_content_cache::FileContentCache; -use crate::infrastructure::services::thumbnail_service::ThumbnailService; use tracing::{error, info, warn}; use uuid::Uuid; @@ -26,8 +26,9 @@ pub struct FileManagementService { file_read: Option>, folder_repo: Option>, trash_service: Option>, - thumbnail_service: Option>, content_cache: Option>, + /// Hooks fired after a file is permanently deleted. + file_deleted_hooks: Vec>, } impl FileManagementService { @@ -38,8 +39,8 @@ impl FileManagementService { file_read: None, folder_repo: None, trash_service: None, - thumbnail_service: None, content_cache: None, + file_deleted_hooks: Vec::new(), } } @@ -49,7 +50,6 @@ impl FileManagementService { trash_service: Option>, file_read: Option>, folder_repo: Option>, - thumbnail_service: Option>, content_cache: Option>, ) -> Self { Self { @@ -57,11 +57,17 @@ impl FileManagementService { file_read, folder_repo, trash_service, - thumbnail_service, content_cache, + file_deleted_hooks: Vec::new(), } } + /// Registers a hook to fire after a file is permanently deleted. + pub fn with_file_deleted_hook(mut self, hook: Arc) -> Self { + self.file_deleted_hooks.push(hook); + self + } + /// Verifies ownership via the read repository. async fn verify_owner(&self, file_id: &str, caller_id: Uuid) -> Result<(), DomainError> { if let Some(read) = &self.file_read { @@ -230,15 +236,11 @@ impl FileManagementUseCase for FileManagementService { async fn delete_file(&self, id: &str) -> Result<(), DomainError> { self.file_repository.delete_file(id).await?; - // Invalidate content cache — file no longer exists. if let Some(cc) = &self.content_cache { cc.invalidate(id).await; } - // Best-effort thumbnail cleanup - if let Some(thumb) = &self.thumbnail_service - && let Err(e) = thumb.delete_thumbnails(id).await - { - warn!("Failed to delete thumbnails for file {}: {}", id, e); + for hook in &self.file_deleted_hooks { + hook.on_file_deleted(id).await; } Ok(()) } @@ -283,15 +285,11 @@ impl FileManagementUseCase for FileManagementService { // Step 2: Permanent delete — trigger handles blob ref_count warn!("Permanently deleting file: {}", id); self.file_repository.delete_file(id).await?; - // Invalidate content cache — file permanently removed. if let Some(cc) = &self.content_cache { cc.invalidate(id).await; } - // Best-effort thumbnail cleanup - if let Some(thumb) = &self.thumbnail_service - && let Err(e) = thumb.delete_thumbnails(id).await - { - warn!("Failed to delete thumbnails for file {}: {}", id, e); + for hook in &self.file_deleted_hooks { + hook.on_file_deleted(id).await; } info!("File permanently deleted: {}", id); diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 5249789d..e160f7ab 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -2,6 +2,7 @@ use std::path::Path; use std::sync::Arc; use crate::application::dtos::file_dto::FileDto; +use crate::application::ports::file_lifecycle::{FileCreatedHook, FileUpdatedHook}; use crate::application::ports::file_ports::FileUploadUseCase; use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; use crate::application::services::storage_usage_service::StorageUsageService; @@ -52,6 +53,10 @@ pub struct FileUploadService { storage_usage_service: Option>, /// Content cache — invalidated on file update so stale content is never served. content_cache: Option>, + /// Hooks fired after a new file record is created. + file_created_hooks: Vec>, + /// Hooks fired after a file's blob is replaced (e.g. thumbnail refresh). + file_updated_hooks: Vec>, } impl FileUploadService { @@ -62,6 +67,8 @@ impl FileUploadService { file_read: None, storage_usage_service: None, content_cache: None, + file_created_hooks: Vec::new(), + file_updated_hooks: Vec::new(), } } @@ -75,6 +82,8 @@ impl FileUploadService { file_read: Some(file_read), storage_usage_service: None, content_cache: None, + file_created_hooks: Vec::new(), + file_updated_hooks: Vec::new(), } } @@ -84,6 +93,18 @@ impl FileUploadService { self } + /// Registers a hook to fire after a new file record is created. + pub fn with_file_created_hook(mut self, hook: Arc) -> Self { + self.file_created_hooks.push(hook); + self + } + + /// Registers a hook to fire after a file's blob is replaced. + pub fn with_file_updated_hook(mut self, hook: Arc) -> Self { + self.file_updated_hooks.push(hook); + self + } + /// Configures the storage usage service pub fn with_storage_usage_service( mut self, @@ -149,6 +170,10 @@ impl FileUploadUseCase for FileUploadService { name, size, dto.id ); self.maybe_update_storage_usage(&dto); + for hook in &self.file_created_hooks { + hook.on_file_created(&dto.id, &dto.etag, &dto.mime_type) + .await; + } Ok(dto) } @@ -301,7 +326,12 @@ impl FileUploadUseCase for FileUploadService { } // Re-read to get fresh DTO with updated etag and timestamps. let updated = file_read.get_file(&file_id).await?; - return Ok(FileDto::from(updated)); + let dto = FileDto::from(updated); + for hook in &self.file_updated_hooks { + hook.on_file_updated(&file_id, &dto.etag, content_type) + .await; + } + return Ok(dto); } // File doesn't exist — create it via streaming upload diff --git a/src/application/services/trash_service.rs b/src/application/services/trash_service.rs index a4ba03fd..95249994 100644 --- a/src/application/services/trash_service.rs +++ b/src/application/services/trash_service.rs @@ -16,6 +16,7 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository; use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository; use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository; +use crate::infrastructure::services::dedup_service::DedupService; use crate::infrastructure::services::file_content_cache::FileContentCache; use crate::infrastructure::services::thumbnail_service::ThumbnailService; @@ -45,6 +46,10 @@ pub struct TrashService { /// Port for folder operations (get folder, trash, restore, delete) folder_storage_port: Arc, + /// Dedup service — garbage-collected after bulk trash empty to clean up + /// orphaned blob files and thumbnails that the PG trigger cannot reach. + dedup_service: Arc, + /// Thumbnail service for cleaning up thumbnails on permanent delete thumbnail_service: Option>, @@ -56,12 +61,14 @@ pub struct TrashService { } impl TrashService { + #[allow(clippy::too_many_arguments)] pub fn new( trash_repository: Arc, file_read_port: Arc, file_write_port: Arc, folder_storage_port: Arc, retention_days: u32, + dedup_service: Arc, thumbnail_service: Option>, content_cache: Option>, ) -> Self { @@ -70,6 +77,7 @@ impl TrashService { file_read_port, file_write_port, folder_storage_port, + dedup_service, thumbnail_service, content_cache, retention_days, @@ -713,18 +721,24 @@ impl TrashUseCase for TrashService { Vec::new() }; - // clear_trash() already performs bulk SQL DELETEs in 2 queries: + // clear_trash() performs bulk SQL DELETEs in 2 queries: // 1. DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE // 2. DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE // // Folder deletion cascades (FK ON DELETE CASCADE) to child folders and // their files. The PG trigger `trg_files_decrement_blob_ref` automatically - // decrements blob ref_counts for every deleted file row — no Rust-side - // remove_reference() call is needed. + // decrements blob ref_counts for every deleted file row. // // Finally it clears the trash_items index for the user. self.trash_repository.clear_trash(&user_id).await?; + // The PG trigger decremented ref_counts but cannot delete disk files or + // thumbnails. Run garbage_collect() to remove any blobs whose ref_count + // reached 0, along with their blob-keyed thumbnail files. + if let Err(e) = self.dedup_service.garbage_collect().await { + warn!("empty_trash: garbage_collect failed: {:?}", e); + } + // Invalidate content cache for all permanently deleted files. if let Some(cc) = &self.content_cache { for file_id in &trashed_file_ids { diff --git a/src/common/di.rs b/src/common/di.rs index 7fd5528e..0c371f58 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -62,7 +62,7 @@ use crate::infrastructure::services::image_transcode_service::ImageTranscodeServ use crate::infrastructure::services::jwt_service::JwtTokenService; use crate::infrastructure::services::password_hasher::Argon2PasswordHasher; use crate::infrastructure::services::path_resolver_service::PathResolverService; -use crate::infrastructure::services::thumbnail_service::ThumbnailService; +use crate::infrastructure::services::thumbnail_service::{ThumbnailRefreshHook, ThumbnailService}; use crate::infrastructure::services::wopi_discovery_service::WopiDiscoveryService; use crate::infrastructure::services::zip_service::ZipService; @@ -264,7 +264,8 @@ impl AppServiceFactory { blob_backend, db_pool.clone(), maintenance_pool.clone(), - ), + ) + .add_blob_hook(thumbnail_service.clone()), ); dedup_service.initialize().await?; @@ -355,12 +356,18 @@ impl AppServiceFactory { // Refactored services with all infrastructure ports // In blob model, dedup is handled by the repository — no separate write-behind needed + let thumbnail_refresh_hook = Arc::new(ThumbnailRefreshHook::new( + core.thumbnail_service.clone(), + core.dedup_service.clone(), + )); let file_upload_service = Arc::new( FileUploadService::new_with_read( repos.file_write_repository.clone(), repos.file_read_repository.clone(), ) - .with_content_cache(core.file_content_cache.clone()), + .with_content_cache(core.file_content_cache.clone()) + .with_file_created_hook(thumbnail_refresh_hook.clone()) + .with_file_updated_hook(thumbnail_refresh_hook), ); let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( @@ -370,14 +377,16 @@ impl AppServiceFactory { )); // FileManagementService — ref_count handled by PG trigger, no dedup port needed - let file_management_service = Arc::new(FileManagementService::with_trash( - repos.file_write_repository.clone(), - trash_service.clone(), - Some(repos.file_read_repository.clone()), - Some(repos.folder_repository.clone()), - Some(core.thumbnail_service.clone()), - Some(core.file_content_cache.clone()), - )); + let file_management_service = Arc::new( + FileManagementService::with_trash( + repos.file_write_repository.clone(), + trash_service.clone(), + Some(repos.file_read_repository.clone()), + Some(repos.folder_repository.clone()), + Some(core.file_content_cache.clone()), + ) + .with_file_deleted_hook(core.thumbnail_service.clone()), + ); let file_use_case_factory = Arc::new(AppFileUseCaseFactory::new( repos.file_read_repository.clone(), @@ -451,6 +460,7 @@ impl AppServiceFactory { repos.file_write_repository.clone(), repos.folder_repository.clone(), self.config.storage.trash_retention_days, + core.dedup_service.clone(), Some(core.thumbnail_service.clone()), Some(core.file_content_cache.clone()), )); diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs index 1562ea34..e0c1e7a5 100644 --- a/src/infrastructure/repositories/pg/file_blob_write_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -604,8 +604,26 @@ impl FileWritePort for FileBlobWriteRepository { } async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> { - // Same as delete_file — removes from DB and decrements blob ref - self.delete_file(file_id).await + // Read blob_hash before deletion so we can clean up disk after the + // PG trigger has decremented the ref_count. + let blob_hash: Option = + sqlx::query_scalar("SELECT blob_hash FROM storage.files WHERE id = $1::uuid") + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}")) + })?; + + // DELETE fires trg_files_decrement_blob_ref → storage.blobs.ref_count-- + self.delete_file(file_id).await?; + + // If the blob is now unreferenced, remove disk file + thumbnails. + if let Some(hash) = blob_hash { + self.dedup.cleanup_if_orphaned(&hash).await; + } + + Ok(()) } async fn copy_folder_tree( diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 55f9d5f0..082fdcf3 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -44,6 +44,7 @@ use std::sync::Arc; use tokio::fs; use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use crate::application::ports::blob_lifecycle::{BlobCreationHook, BlobDeletionHook}; use crate::application::ports::blob_storage_ports::BlobStorageBackend; use crate::application::ports::dedup_ports::{ BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, @@ -83,6 +84,10 @@ pub struct DedupService { /// Isolated maintenance pool for long-running operations /// (verify_integrity, garbage_collect) that must never starve the primary. maintenance_pool: Arc, + /// Hooks notified when a genuinely new blob is stored (no dedup hit). + blob_creation_hooks: Vec>, + /// Hooks notified when a blob's ref_count reaches zero and it is deleted. + blob_hooks: Vec>, } impl DedupService { @@ -100,6 +105,36 @@ impl DedupService { backend, pool, maintenance_pool, + blob_creation_hooks: vec![], + blob_hooks: vec![], + } + } + + /// Register a [`BlobCreationHook`] to be called whenever a genuinely new + /// blob is stored. Hooks are called in registration order. + pub fn add_blob_creation_hook(mut self, hook: Arc) -> Self { + self.blob_creation_hooks.push(hook); + self + } + + /// Register a [`BlobDeletionHook`] to be called whenever a blob's + /// ref_count reaches zero. Hooks are called in registration order. + pub fn add_blob_hook(mut self, hook: Arc) -> Self { + self.blob_hooks.push(hook); + self + } + + /// Fire all registered creation hooks for a new blob. + async fn fire_blob_creation_hooks(&self, hash: &str, content_type: Option<&str>) { + for hook in &self.blob_creation_hooks { + hook.on_blob_created(hash, content_type).await; + } + } + + /// Fire all registered hooks for a deleted blob. + async fn fire_blob_hooks(&self, hash: &str) { + for hook in &self.blob_hooks { + hook.on_blob_deleted(hash).await; } } @@ -117,6 +152,8 @@ impl DedupService { backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))), pool: stub_pool.clone(), maintenance_pool: stub_pool, + blob_creation_hooks: vec![], + blob_hooks: vec![], } } @@ -334,6 +371,9 @@ impl DedupService { chunk_hashes.len() ); + self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()) + .await; + Ok(DedupResultDto::NewBlob { hash: file_hash, size: file_size, @@ -587,7 +627,7 @@ impl DedupService { /// references the blob identified by `hash`. pub async fn user_owns_blob_reference(&self, hash: &str, user_id: &str) -> bool { sqlx::query_scalar::<_, bool>( - "SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2 AND NOT is_trashed)", + "SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2::uuid AND NOT is_trashed)", ) .bind(hash) .bind(user_id) @@ -772,6 +812,9 @@ impl DedupService { } } + // Bug 4 fix: notify hooks — e.g. thumbnail cleanup keyed by file_hash + self.fire_blob_hooks(file_hash).await; + tracing::info!( "MANIFEST DELETED: {} ({} chunks, {} orphan chunks removed)", &file_hash[..12], @@ -849,6 +892,9 @@ impl DedupService { tracing::warn!("Failed to delete blob file {}: {}", hash, e); } + // Bug 3 fix: notify hooks — e.g. thumbnail cleanup keyed by hash + self.fire_blob_hooks(hash).await; + tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]); Ok(true) } else { @@ -874,6 +920,91 @@ impl DedupService { } } + /// Targeted cleanup for a single blob after the PG trigger has already + /// decremented its ref_count. Deletes the blob row, disk file, and + /// blob-keyed thumbnails if ref_count has reached 0. + /// + /// Handles both the legacy whole-file blob path (storage.blobs) and the + /// CDC manifest path (storage.chunk_manifests). Best-effort: logs + /// warnings on failure rather than returning an error. + pub async fn cleanup_if_orphaned(&self, hash: &str) { + let short = &hash[..hash.len().min(12)]; + + // ── CDC manifest path (must run FIRST) ─────────────────── + // For single-chunk CDC files file_hash == chunk_hash, so the PG + // trigger on storage.files already decremented storage.blobs.ref_count + // when this function is called. try_dedup_hit increments + // chunk_manifests.ref_count but NOT storage.blobs.ref_count, so + // blobs.ref_count can reach 0 while the manifest still has ref_count > 1 + // (other files sharing the same blob). Checking the manifest first + // prevents premature blob + manifest deletion. + let manifest = sqlx::query_as::<_, (i32, Vec)>( + "SELECT ref_count, chunk_hashes \ + FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(hash) + .fetch_optional(self.pool.as_ref()) + .await + .unwrap_or(None); + + if let Some((ref_count, chunk_hashes)) = manifest { + if ref_count <= 1 { + // Last reference — remove manifest and all its chunks. + if let Err(e) = self + .remove_manifest_reference(hash, ref_count, &chunk_hashes) + .await + { + tracing::warn!("cleanup_if_orphaned: manifest cleanup failed for {short}: {e}"); + } + } else { + // Other files still share this blob: just decrement the manifest + // counter and undo the PG trigger's premature chunk ref_count + // decrement (blobs.ref_count is chunk-level; the manifest is the + // authoritative file-level counter). + sqlx::query( + "UPDATE storage.chunk_manifests \ + SET ref_count = ref_count - 1 WHERE file_hash = $1", + ) + .bind(hash) + .execute(self.pool.as_ref()) + .await + .ok(); + // Undo the PG trigger's decrement of storage.blobs.ref_count. + // The trigger fired with blob_hash = file_hash, so only the row + // WHERE hash = file_hash is affected. For single-chunk files + // file_hash == chunk_hash and that row exists; for multi-chunk + // files file_hash is not in storage.blobs, making this a no-op. + sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1") + .bind(hash) + .execute(self.pool.as_ref()) + .await + .ok(); + tracing::debug!( + "cleanup_if_orphaned: manifest {short} ref_count {ref_count}→{}", + ref_count - 1 + ); + } + return; + } + + // ── Legacy blob path (no manifest) ─────────────────────── + let deleted_blob = sqlx::query_scalar::<_, String>( + "DELETE FROM storage.blobs WHERE hash = $1 AND ref_count <= 0 RETURNING hash", + ) + .bind(hash) + .fetch_optional(self.pool.as_ref()) + .await + .unwrap_or(None); + + if deleted_blob.is_some() { + if let Err(e) = self.backend.delete_blob(hash).await { + tracing::warn!("cleanup_if_orphaned: disk delete failed for {short}: {e}"); + } + self.fire_blob_hooks(hash).await; + tracing::info!("cleanup_if_orphaned: removed orphaned blob {short}"); + } + } + // ── Read operations ────────────────────────────────────────── /// Stream blob content — CDC-aware with legacy fallback. @@ -1324,16 +1455,7 @@ impl DedupService { if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete orphan blob {hash}: {e}"); } - // Clean up thumbnails (best-effort, only local backends) - if let Some(blob_path) = self.backend.local_blob_path(hash) - && let Some(storage_root) = blob_path.ancestors().nth(3) - { - let thumbnails_root = storage_root.join(".thumbnails"); - for dir in &["icon", "preview", "large"] { - let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg")); - let _ = fs::remove_file(&thumb).await; - } - } + self.fire_blob_hooks(hash).await; total_bytes += *size as u64; } total_deleted += batch.len() as u64; diff --git a/src/infrastructure/services/thumbnail_service.rs b/src/infrastructure/services/thumbnail_service.rs index 7c3622f1..5f934aa3 100644 --- a/src/infrastructure/services/thumbnail_service.rs +++ b/src/infrastructure/services/thumbnail_service.rs @@ -26,6 +26,7 @@ use crate::application::ports::thumbnail_ports::{ ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto, }; use crate::domain::errors::{DomainError, ErrorKind}; +use crate::infrastructure::services::dedup_service::DedupService; /// Thumbnail sizes supported by the system #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -831,10 +832,22 @@ impl ThumbnailService { file_id: String, blob_hash: String, original_data: Bytes, + dedup: Arc, ) { tokio::spawn(async move { tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id); + // Guard: if the blob was deleted before this task ran, cleanup_if_orphaned + // already fired with no thumbnails on disk — writing them now would leak them. + // Use the DB check (manifest + blobs tables) as the authoritative source. + if !dedup.blob_exists(&blob_hash).await { + tracing::debug!( + "Blob {}… deleted before thumbnail task ran, skipping", + &blob_hash[..blob_hash.len().min(12)] + ); + return; + } + let all_exist = { let mut ok = true; for size in ThumbnailSize::all() { @@ -971,6 +984,122 @@ impl ThumbnailService { } } +// ─── BlobDeletionHook ──────────────────────────────────────────────────────── + +impl crate::application::ports::blob_lifecycle::BlobDeletionHook for ThumbnailService { + fn on_blob_deleted<'a>( + &'a self, + blob_hash: &'a str, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { self.delete_blob_thumbnails(blob_hash).await }) + } +} + +// ─── FileUpdatedHook ───────────────────────────────────────────────────────── + +/// Wires thumbnail invalidation + regeneration into the file-update lifecycle. +/// +/// Registered on [`FileUploadService`] during DI. Fires whenever a file's blob +/// is replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload). +pub struct ThumbnailRefreshHook { + thumbnail: Arc, + dedup: Arc, +} + +impl ThumbnailRefreshHook { + pub fn new(thumbnail: Arc, dedup: Arc) -> Self { + Self { thumbnail, dedup } + } +} + +impl crate::application::ports::file_lifecycle::FileUpdatedHook for ThumbnailRefreshHook { + fn on_file_updated<'a>( + &'a self, + file_id: &'a str, + blob_hash: &'a str, + content_type: &'a str, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { + if !ThumbnailService::is_supported_image(content_type) { + return; + } + if let Err(e) = self.thumbnail.delete_thumbnails(file_id).await { + tracing::warn!( + "Failed to invalidate thumbnail cache for {}: {}", + file_id, + e + ); + } + Self::spawn_thumbnail_generation( + self.thumbnail.clone(), + self.dedup.clone(), + file_id.to_string(), + blob_hash.to_string(), + ); + }) + } +} + +impl crate::application::ports::file_lifecycle::FileCreatedHook for ThumbnailRefreshHook { + fn on_file_created<'a>( + &'a self, + file_id: &'a str, + blob_hash: &'a str, + content_type: &'a str, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { + if !ThumbnailService::is_supported_image(content_type) { + return; + } + Self::spawn_thumbnail_generation( + self.thumbnail.clone(), + self.dedup.clone(), + file_id.to_string(), + blob_hash.to_string(), + ); + }) + } +} + +impl ThumbnailRefreshHook { + fn spawn_thumbnail_generation( + ts: Arc, + ds: Arc, + file_id: String, + hash: String, + ) { + tokio::spawn(async move { + match ds.read_blob_bytes(&hash).await { + Ok(bytes) => { + ts.generate_all_sizes_background_from_bytes(file_id, hash, bytes, ds.clone()); + } + Err(e) => { + tracing::warn!( + "Failed to read blob for thumbnail generation {}: {}", + file_id, + e + ); + } + } + }); + } +} + +// ─── FileDeletedHook ───────────────────────────────────────────────────────── + +impl crate::application::ports::file_lifecycle::FileDeletedHook for ThumbnailService { + fn on_file_deleted<'a>( + &'a self, + file_id: &'a str, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { + if let Err(e) = self.delete_thumbnails(file_id).await { + tracing::warn!("Failed to delete thumbnails for file {}: {}", file_id, e); + } + }) + } +} + // ─── Port implementation ───────────────────────────────────────────────────── /// Convert port ThumbnailSize to infra ThumbnailSize. diff --git a/src/interfaces/api/handlers/dedup_handler.rs b/src/interfaces/api/handlers/dedup_handler.rs index 97dbfa6c..3defb869 100644 --- a/src/interfaces/api/handlers/dedup_handler.rs +++ b/src/interfaces/api/handlers/dedup_handler.rs @@ -21,12 +21,14 @@ type GlobalState = Arc; pub struct HashCheckResponse { /// Whether a blob with this hash already exists pub exists: bool, - /// The SHA-256 hash that was checked + /// The BLAKE3 hash that was checked pub hash: String, /// If exists, the size of the existing blob #[serde(skip_serializing_if = "Option::is_none")] pub existing_size: Option, - /// If exists, the number of references to this blob + /// Global reference count for this blob across all users. + /// Only populated when the authenticated user has the `admin` role; + /// omitted for regular users to prevent cross-user content inference. #[serde(skip_serializing_if = "Option::is_none")] pub ref_count: Option, } @@ -36,7 +38,7 @@ pub struct HashCheckResponse { pub struct DedupUploadResponse { /// Whether this was a new file or an existing one pub is_new: bool, - /// The SHA-256 hash of the content + /// The BLAKE3 hash of the content pub hash: String, /// The size of the content in bytes pub size: u64, @@ -85,7 +87,8 @@ impl DedupHandler { /// Check if the authenticated user already has a file with the given hash. /// /// User-scoped: only reveals whether **this user** owns a file that - /// references the blob — never exposes global existence or ref_count. + /// references the blob — never exposes global existence to non-admins. + /// Admins additionally receive the global `ref_count` in the response. /// /// GET /api/dedup/check/{hash} pub(super) async fn check_hash_impl( @@ -95,13 +98,13 @@ impl DedupHandler { ) -> impl IntoResponse { let dedup = &state.core.dedup_service; - // Validate hash format (SHA-256 = 64 hex chars) + // Validate hash format (BLAKE3 = 64 hex chars) if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) { return Response::builder() .status(StatusCode::BAD_REQUEST) .header(header::CONTENT_TYPE, "application/json") .body(Body::from( - r#"{"error": "Invalid hash format. Expected SHA-256 (64 hex characters)"}"#, + r#"{"error": "Invalid hash format. Expected BLAKE3 (64 hex characters)"}"#, )) .unwrap() .into_response(); @@ -113,13 +116,20 @@ impl DedupHandler { .await; if user_has_it { - // Fetch size from metadata (safe — user owns a reference) - let size = dedup.get_blob_metadata(&hash).await.map(|m| m.size); + // Fetch size from metadata (safe — user owns a reference). + // Admins also get the global ref_count for dedup accounting tests. + let metadata = dedup.get_blob_metadata(&hash).await; + let size = metadata.as_ref().map(|m| m.size); + let ref_count = if auth_user.role == "admin" { + metadata.map(|m| m.ref_count) + } else { + None // Never expose global ref_count to regular users + }; let response = HashCheckResponse { exists: true, hash, existing_size: size, - ref_count: None, // Never expose global ref_count + ref_count, }; Response::builder() .status(StatusCode::OK) @@ -508,10 +518,10 @@ impl DedupHandler { get, path = "/api/dedup/check/{hash}", params( - ("hash" = String, Path, description = "SHA-256 hash (64 hex characters)"), + ("hash" = String, Path, description = "BLAKE3 hash (64 hex characters)"), ), responses( - (status = 200, description = "Hash check result (user-scoped)", body = HashCheckResponse), + (status = 200, description = "Hash check result. `ref_count` is only present for admin users.", body = HashCheckResponse), (status = 400, description = "Invalid hash format"), ), tag = "dedup", @@ -564,7 +574,7 @@ pub async fn get_stats(state: State, auth_user: AuthUser) -> impl I get, path = "/api/dedup/blob/{hash}", params( - ("hash" = String, Path, description = "SHA-256 hash of the blob (64 hex characters)"), + ("hash" = String, Path, description = "BLAKE3 hash of the blob (64 hex characters)"), ), responses( (status = 200, description = "Raw blob content (user-scoped)"), diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index 5e68a609..5821e3af 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -783,6 +783,7 @@ impl FileHandler { file_id, blob_hash_owned, original_bytes, + dedup_service.clone(), ); } Err(err) => { diff --git a/src/interfaces/api/handlers/wopi_handler.rs b/src/interfaces/api/handlers/wopi_handler.rs index a2c1efa5..fc5cd595 100644 --- a/src/interfaces/api/handlers/wopi_handler.rs +++ b/src/interfaces/api/handlers/wopi_handler.rs @@ -288,7 +288,7 @@ async fn put_file( let _ = tokio::fs::remove_file(&temp_path).await; match result { - Ok(_) => StatusCode::OK.into_response(), + Ok(_file_dto) => StatusCode::OK.into_response(), Err(e) => { tracing::error!("WOPI PutFile failed: {}", e); StatusCode::INTERNAL_SERVER_ERROR.into_response() diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index e5227ce6..e49522a9 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -163,6 +163,7 @@ async fn handle_assemble( ) .await .map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?; + Some(dto.etag) } else { // For new files we still need to read the temp file since create_file takes &[u8]. diff --git a/tests/api/dedup_blob_cleanup.hurl b/tests/api/dedup_blob_cleanup.hurl new file mode 100644 index 00000000..17c6424e --- /dev/null +++ b/tests/api/dedup_blob_cleanup.hurl @@ -0,0 +1,271 @@ +# ============================================================= +# OxiCloud – Dedup blob lifecycle (bugs 3 & 4) +# ============================================================= +# Verifies that when two files share the same blob (dedup hit) +# and both are permanently deleted, the blob lifecycle is correct. +# +# Bug 3: blob not deleted when last file reference is removed +# Bug 4: blob-keyed thumbnail not cleaned up with the blob +# +# Sequence: +# 1. Upload dedup-test.jpg twice → two file records, one blob +# 2. Both thumbnails return identical bytes → proves shared blob +# 3. Permanently delete file 1 → file 2 thumbnail still 200 +# (proves blob NOT prematurely deleted — bug 3 detection) +# 4. Permanently delete file 2 → blob and thumbnail cleaned up +# +# NOTE: The /api/dedup/stats endpoint counts CDC chunk rows in +# storage.blobs and derives bytes_saved from chunk_manifests. +# Both tables may be 0 when the CDC path is disabled or the +# server uses the legacy blob path — so we avoid stats-based +# assertions and rely on observable thumbnail behaviour instead. +# +# BLAKE3 hash of fixtures/dedup-test.jpg (= dedup-test-2.jpg content): +# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +# Used in /api/dedup/check/{hash} calls below to track ref_count lifecycle. +# ref_count is only returned for admin users; setup.hurl creates an admin. +# +# Prerequisites: setup.hurl must have run (admin user exists). +# +# Run: +# hurl --variables-file tests/api/test.env --test tests/api/dedup_blob_cleanup.hurl +# ============================================================= + + +# ───────────────────────────────────────────────────────────── +# Step 1 – Login as admin +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/auth/login +Content-Type: application/json +{ + "username": "{{username}}", + "password": "{{password}}" +} + +HTTP 200 +[Captures] +token: jsonpath "$.access_token" +[Asserts] +jsonpath "$.access_token" isString + + +# ───────────────────────────────────────────────────────────── +# Step 2 – Create a folder for this test +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/folders +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +home_folder_id: jsonpath "$[0].id" + + +POST {{base_url}}/api/folders +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "name": "hurl-dedup-blob-test", + "parent_id": "{{home_folder_id}}" +} + +HTTP 201 +[Captures] +test_folder_id: jsonpath "$.id" +[Asserts] +jsonpath "$.name" == "hurl-dedup-blob-test" + + +# ───────────────────────────────────────────────────────────── +# Step 3 – Upload dedup-test.jpg (file 1) +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{token}} +[MultipartFormData] +folder_id: {{test_folder_id}} +file: file,fixtures/dedup-test.jpg; image/jpeg + +HTTP 201 +[Captures] +file1_id: jsonpath "$.id" +[Asserts] +jsonpath "$.name" == "dedup-test.jpg" +jsonpath "$.folder_id" == {{test_folder_id}} + + +# ref_count == 1: blob has exactly one file reference after first upload +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + +# ───────────────────────────────────────────────────────────── +# Step 4 – Upload identical content again as dedup-test-2.jpg +# Dedup: same blob, new file record, different file ID +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{token}} +[MultipartFormData] +folder_id: {{test_folder_id}} +file: file,fixtures/dedup-test-2.jpg; image/jpeg + +HTTP 201 +[Captures] +file2_id: jsonpath "$.id" +[Asserts] +jsonpath "$.name" == "dedup-test-2.jpg" +jsonpath "$.id" != "{{file1_id}}" + + +# ref_count == 2: dedup hit — same blob now referenced by two file records +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 2 + + +# ───────────────────────────────────────────────────────────── +# Step 5 – Dedup proof: thumbnails are byte-identical +# Thumbnail generation reads blob bytes and is keyed by +# blob_hash on disk. If both files share the same blob, +# GET /thumbnail returns the same bytes for both. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/files/{{file1_id}}/thumbnail/icon +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +thumb1: bytes + + +GET {{base_url}}/api/files/{{file2_id}}/thumbnail/icon +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +bytes == {{thumb1}} + + +# ───────────────────────────────────────────────────────────── +# Step 6 – Move file 1 to trash +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/files/{{file1_id}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 7 – Permanently delete file 1 from trash +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/trash +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +trash_item1_id: jsonpath "$[?(@.original_id == '{{file1_id}}')].id" +[Asserts] +jsonpath "$[?(@.original_id == '{{file1_id}}')].id" isString +jsonpath "$[?(@.original_id == '{{file1_id}}')].item_type" == "file" + + +DELETE {{base_url}}/api/trash/{{trash_item1_id}} +Authorization: Bearer {{token}} + +HTTP 200 + + +# ref_count == 1: blob survives — file2 still holds a reference +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + +# ───────────────────────────────────────────────────────────── +# Step 8 – Blob still alive: file 2 thumbnail is accessible +# After file 1 is permanently deleted the blob ref_count +# drops to 1 but the blob must NOT be removed yet. +# Thumbnail generation reads blob bytes live — a 200 here +# proves the blob is still present. +# If bug 3 is present the blob is deleted prematurely and +# this request returns a 5xx error. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/files/{{file2_id}}/thumbnail/icon +Authorization: Bearer {{token}} + +HTTP 200 + + +# ───────────────────────────────────────────────────────────── +# Step 9 – Move file 2 to trash +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/files/{{file2_id}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 10 – Permanently delete file 2 from trash +# ref_count hits 0 → blob and its disk thumbnail deleted +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/trash +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +trash_item2_id: jsonpath "$[?(@.original_id == '{{file2_id}}')].id" +[Asserts] +jsonpath "$[?(@.original_id == '{{file2_id}}')].id" isString +jsonpath "$[?(@.original_id == '{{file2_id}}')].item_type" == "file" + + +DELETE {{base_url}}/api/trash/{{trash_item2_id}} +Authorization: Bearer {{token}} + +HTTP 200 + + +# ref_count hits 0 → blob and manifest deleted; user no longer owns this hash +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == false + + +# ───────────────────────────────────────────────────────────── +# Step 11 – Cleanup: delete the (now empty) test folder +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/folders/{{test_folder_id}} +Authorization: Bearer {{token}} + +HTTP 204 + + +GET {{base_url}}/api/trash +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +trash_folder_id: jsonpath "$[?(@.original_id == '{{test_folder_id}}')].id" +[Asserts] +jsonpath "$[?(@.original_id == '{{test_folder_id}}')].id" isString +jsonpath "$[?(@.original_id == '{{test_folder_id}}')].item_type" == "folder" + + +DELETE {{base_url}}/api/trash/{{trash_folder_id}} +Authorization: Bearer {{token}} + +HTTP 200 diff --git a/tests/api/run.sh b/tests/api/run.sh index 400d2567..5f660b71 100755 --- a/tests/api/run.sh +++ b/tests/api/run.sh @@ -62,6 +62,9 @@ OXICLOUD_SERVER_PORT=$SERVER_PORT OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage" set +a +# ensure storage is empty before starting +echo "Wipe $OXICLOUD_STORAGE_PATH to ensure clean startup" +rm -rf "$OXICLOUD_STORAGE_PATH" mkdir -p "$OXICLOUD_STORAGE_PATH" # ── 3. Start OxiCloud server ────────────────────────────────────────────────── @@ -92,6 +95,11 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test "$API_DIR/trash.hurl" \ "$API_DIR/recent.hurl" \ "$API_DIR/batch_folder_copy.hurl" \ + "$API_DIR/dedup_blob_cleanup.hurl" \ "$API_DIR/contacts.hurl" +#bash "$API_DIR/dedup_bulk_upload.sh" + +bash "$API_DIR/storage_cleanup_check.sh" + log "All tests passed." diff --git a/tests/api/storage_cleanup_check.sh b/tests/api/storage_cleanup_check.sh new file mode 100755 index 00000000..1a421cac --- /dev/null +++ b/tests/api/storage_cleanup_check.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Storage disk-cleanup verification +# ============================================================= +# 1. Moves every live file and folder to trash via the REST API. +# 2. Calls DELETE /api/trash/empty to permanently delete all +# remaining trash items (including any left by previous tests). +# 3. Asserts that no regular files remain under +# $OXICLOUD_STORAGE_PATH/.thumbnails or .blobs. +# +# Called by run.sh after all Hurl tests have passed. +# Can also be run standalone (server must already be up): +# bash tests/api/storage_cleanup_check.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +STORAGE_PATH="${OXICLOUD_STORAGE_PATH:-$REPO_ROOT/tests/api/storage}" + +# shellcheck source=test.env +source "$SCRIPT_DIR/test.env" + +log() { echo "[storage-check] $*"; } +fail() { echo $'\e[31m'"[storage-check] FAIL: $*"$'\e[0m' >&2; exit 1; } + +# ── 1. Login ────────────────────────────────────────────────────────────────── + +TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"$username\",\"password\":\"$password\"}" \ + | jq -r '.access_token') + +[[ -z "$TOKEN" || "$TOKEN" == "null" ]] && fail "login failed" +log "Logged in." + +AUTH="Authorization: Bearer $TOKEN" + +# ── 1b. Upload a probe image and verify its blob + thumbnail exist on disk ───── + +# shellcheck source=../common/internal_storage_helper.sh +source "$REPO_ROOT/tests/common/internal_storage_helper.sh" + +FIXTURE="$REPO_ROOT/tests/fixtures/blue-image.png" + +HOME_FOLDER_ID=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[0].id') +[[ -z "$HOME_FOLDER_ID" || "$HOME_FOLDER_ID" == "null" ]] && fail "could not get home folder id" + +PROBE_FILE_ID=$(curl -sf -X POST -H "$AUTH" \ + -F "folder_id=$HOME_FOLDER_ID" \ + -F "file=@$FIXTURE;type=image/png" \ + "$base_url/api/files/upload" | jq -r '.id') +[[ -z "$PROBE_FILE_ID" || "$PROBE_FILE_ID" == "null" ]] && fail "probe file upload failed" +log "Probe file uploaded (id=$PROBE_FILE_ID)." + +# GET thumbnail to trigger on-demand generation +HTTP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" -H "$AUTH" \ + "$base_url/api/files/$PROBE_FILE_ID/thumbnail/icon") +[[ "$HTTP_STATUS" != "200" ]] && fail "thumbnail GET returned HTTP $HTTP_STATUS (expected 200)" +log "Thumbnail fetched (HTTP 200)." + +assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not found on disk" +assert_preview_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk" +log "Probe blob and thumbnail confirmed present on disk." + +# ── 2. Move all live files and folders to trash ─────────────────────────────── +# +# For each root folder, list its direct children and soft-delete them. +# The server cascades folder deletion to all nested contents, so we only +# need to iterate one level deep. + +ROOT_FOLDERS=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[].id') + +for folder_id in $ROOT_FOLDERS; do + CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing") + + while IFS= read -r sub_id; do + [[ -z "$sub_id" ]] && continue + curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$sub_id" >/dev/null + done < <(echo "$CONTENTS" | jq -r '.folders[].id') + + while IFS= read -r file_id; do + [[ -z "$file_id" ]] && continue + curl -sf -X DELETE -H "$AUTH" "$base_url/api/files/$file_id" >/dev/null + done < <(echo "$CONTENTS" | jq -r '.files[].id') +done + +log "All live objects moved to trash." + +# ── 2b. Verify all root folders are empty according to the API ──────────────── + +for folder_id in $ROOT_FOLDERS; do + CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing") + SUB_COUNT=$(echo "$CONTENTS" | jq '.folders | length') + FILE_COUNT=$(echo "$CONTENTS" | jq '.files | length') + if [[ "$SUB_COUNT" -ne 0 || "$FILE_COUNT" -ne 0 ]]; then + fail "folder $folder_id still has $SUB_COUNT subfolder(s) and $FILE_COUNT file(s)" + fi +done + +log "API confirms all root folders are empty." + +# ── 3. Permanently delete everything in trash ───────────────────────────────── + +curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/empty" >/dev/null +log "Trash emptied." + +# ── 3b. Verify trash is empty according to the API ─────────────────────────── + +TRASH_COUNT=$(curl -sf -H "$AUTH" "$base_url/api/trash" | jq 'length') +if [[ "$TRASH_COUNT" -ne 0 ]]; then + fail "trash still contains $TRASH_COUNT item(s) after empty" +fi + +log "API confirms trash is empty." + +# ── 4. Disk verification ────────────────────────────────────────────────────── + +THUMB_FILES=$(find "$STORAGE_PATH/.thumbnails" -type f 2>/dev/null || true) +BLOB_FILES=$(find "$STORAGE_PATH/.blobs" -type f 2>/dev/null || true) + +if [[ -n "$THUMB_FILES" ]]; then + THUMB_COUNT=$(echo "$THUMB_FILES" | wc -l | tr -d ' ') + log "Leftover thumbnail files ($THUMB_COUNT):" + echo "$THUMB_FILES" + fail "$THUMB_COUNT thumbnail file(s) remain on disk after full cleanup" +fi + +if [[ -n "$BLOB_FILES" ]]; then + BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ') + log "Leftover blob files ($BLOB_COUNT):" + echo "$BLOB_FILES" + fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup" +fi + +log "OK — no blobs or thumbnails remain on disk." diff --git a/tests/common/internal_storage_helper.sh b/tests/common/internal_storage_helper.sh new file mode 100755 index 00000000..0fe70822 --- /dev/null +++ b/tests/common/internal_storage_helper.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +if ! which b3sum >/dev/null 2>/dev/null +then + echo "please install b3sum (brew install b3sum on Mac, apt installb3sum on Debian, etc)" >&2 + exit 1 +fi + +HASH="" +FILE_CACHE="" + +# return the hash of a file (stores into HASH variable) +oxi_hash() { + if [[ -z "$HASH" || "$FILE_CACHE" != "$1" ]] + then + HASH=$(b3sum --no-names "$1") + FILE_CACHE="$1" + fi + echo "$HASH" +} + +# returns the local blob localisation +local_blob_path() { + local BLOB_PREFIX + oxi_hash "$1" >/dev/null + BLOB_PREFIX=${HASH:0:2} + echo ".blobs/$BLOB_PREFIX/$HASH.blob" +} + +# returns the preview localisation without it's extension +preview_path() { + local SIZE + oxi_hash "$1" >/dev/null + # default size: icon + SIZE="${3:-icon}" + echo ".thumbnails/$SIZE/$HASH" +} + +assert_local_blob_existsy() { + BLOB_PATH=$(local_blob_path "$1") + STORAGE="$2" + if [[ -e $STORAGE/$BLOB_PATH ]] + then + echo "$BLOB_PATH exists" + return 0 + else + echo $'\e[31m'"$BLOB_PATH does not exist"$'\e[0m' >&2 + return 1 + fi +} + +assert_preview_existsy() { + THUMBNAIL_PATH=$(preview_path "$1") + STORAGE="$2" + if [[ -e "$STORAGE/$THUMBNAIL_PATH.jpg" || -e "$STORAGE/$THUMBNAIL_PATH.webp" ]] + then + echo "thumbnail $THUMBNAIL_PATH.(jpg|webp) exists" + return 0 + else + echo $'\e[31m'"thumbnail $THUMBNAIL_PATH.(jpg|webp) does not exist"$'\e[0m' >&2 + echo $STORAGE + find $STORAGE/.thumbnails + return 1 + fi +} + diff --git a/tests/common/server.env b/tests/common/server.env index 65e1a78b..f60ccef3 100644 --- a/tests/common/server.env +++ b/tests/common/server.env @@ -16,3 +16,4 @@ OXICLOUD_EXPOSE_SYSTEM_USERS=true OXICLOUD_WOPI_ENABLED=false OXICLOUD_OIDC_ENABLED=false RUST_LOG=warn +#RUST_LOG=debug diff --git a/tests/fixtures/blue-image.png b/tests/fixtures/blue-image.png new file mode 100644 index 00000000..ac6fca15 Binary files /dev/null and b/tests/fixtures/blue-image.png differ diff --git a/tests/fixtures/dedup-test-2.jpg b/tests/fixtures/dedup-test-2.jpg new file mode 100644 index 00000000..07fa4fe2 Binary files /dev/null and b/tests/fixtures/dedup-test-2.jpg differ diff --git a/tests/fixtures/dedup-test.jpg b/tests/fixtures/dedup-test.jpg new file mode 100644 index 00000000..07fa4fe2 Binary files /dev/null and b/tests/fixtures/dedup-test.jpg differ diff --git a/tests/fixtures/free_video_over_1MB.mp4 b/tests/fixtures/free_video_over_1MB.mp4 new file mode 100644 index 00000000..21e28777 Binary files /dev/null and b/tests/fixtures/free_video_over_1MB.mp4 differ diff --git a/tests/fixtures/green-image.png b/tests/fixtures/green-image.png new file mode 100644 index 00000000..e689f584 Binary files /dev/null and b/tests/fixtures/green-image.png differ diff --git a/tests/fixtures/hello-copy.txt b/tests/fixtures/hello-copy.txt new file mode 100644 index 00000000..d95f2d01 --- /dev/null +++ b/tests/fixtures/hello-copy.txt @@ -0,0 +1 @@ +Hello from OxiCloud Hurl tests. diff --git a/tests/fixtures/red-image.png b/tests/fixtures/red-image.png new file mode 100644 index 00000000..da137d1c Binary files /dev/null and b/tests/fixtures/red-image.png differ diff --git a/tests/webdav/README.md b/tests/webdav/README.md new file mode 100644 index 00000000..ff4f68d6 --- /dev/null +++ b/tests/webdav/README.md @@ -0,0 +1 @@ +# purpose is to test webdav coverage + different scenarios diff --git a/tests/webdav/common.sh b/tests/webdav/common.sh new file mode 100644 index 00000000..7dada61c --- /dev/null +++ b/tests/webdav/common.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +source test.env + +err() { + echo "$*" >&2 +} + +warn() { + echo "$*" >&2 +} + +success() { + echo "ok"; +} + +# Create the first admin account if the server is freshly initialised. +# Silently succeeds when the account already exists (403 = already set up). +oxicloud_setup() { + SETUP_DATA='{"username":"'$username'","email":"'$email'","password":"'$password'"}' + SETUP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST -H "Content-Type: application/json" \ + -d "$SETUP_DATA" "$base_url/api/setup") + + case "$SETUP_STATUS" in + 201) echo "setup: admin account created" ;; + 403) ;; # already initialised — normal on second run + *) err "setup: unexpected status $SETUP_STATUS"; exit 1 ;; + esac +} + +# returns TOKEN variable +oxicloud_login() { + + oxicloud_setup + + LOGIN_DATA='{"username":"'$username'","password":"'$password'"}' + + LOGIN_RESPONSE=$(curl -s -X POST -H "Content-Type: application/json" -d "$LOGIN_DATA" $base_url/api/auth/login) + + echo $? + jq -r '.error' <<<$LOGIN_RESPONSE + + if [[ "$(jq -r '.error' <<<$LOGIN_RESPONSE)" != "null" ]] + then + err "Login Error: $LOGIN_RESPONSE" + exit 1 + fi + + TOKEN=$(jq -r '.access_token' <<<$LOGIN_RESPONSE) + + if [[ -z "$TOKEN" || "$TOKEN" == "null" ]] + then + echo access_token missing in response: $LOGIN_RESPONSE + exit 1 + fi + + echo "login successful, Got JWT token containing informations:" $(cut -d . -f 2 <<<$TOKEN | base64 -d) +} + +# remove trailing / +base_url="${base_url%/}" + +echo starting $0 tests on server $base_url diff --git a/tests/webdav/run.sh b/tests/webdav/run.sh new file mode 100755 index 00000000..62a88363 --- /dev/null +++ b/tests/webdav/run.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Full Hurl API test runner. +# Starts postgres + OxiCloud server, runs Hurl tests, tears everything down. +# +# Usage (from repo root): +# bash tests/api/run.sh +# +# Prerequisites: docker, cargo, hurl ≥ 4.0 + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +COMMON="$REPO_ROOT/tests/common" +WEBDAV_DIR="$REPO_ROOT/tests/webdav" + +# test.env is the single source of truth for connection details and credentials. +# shellcheck source=test.env +source "$WEBDAV_DIR/test.env" + +# Derive server port from base_url (e.g. http://localhost:8087 → 8087) +SERVER_PORT="${base_url##*:}" + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +log() { echo "[api-test] $*"; } +die() { echo "[api-test] ERROR: $*" >&2; exit 1; } + +wait_for_http() { + local url="$1" timeout="${2:-60}" + local deadline=$(( $(date +%s) + timeout )) + until curl -sf "$url" >/dev/null 2>&1; do + [[ $(date +%s) -ge $deadline ]] && die "Timeout waiting for $url" + sleep 1 + done +} + +# ── Teardown (always runs on exit) ──────────────────────────────────────────── + +SERVER_PID="" + +cleanup() { + if [[ -n "$SERVER_PID" ]]; then + log "Stopping OxiCloud server (pid $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + bash "$COMMON/stop-db.sh" +} + +trap cleanup EXIT + +# ── 1. Start postgres ───────────────────────────────────────────────────────── + +bash "$COMMON/spawn-db.sh" + +# ── 2. Load shared server env + port from .env ─────────────────────────────── + +set -a +# shellcheck source=../common/server.env +source "$COMMON/server.env" +OXICLOUD_SERVER_PORT=$SERVER_PORT +OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage" +set +a + +mkdir -p "$OXICLOUD_STORAGE_PATH" + +# ── 3. Start OxiCloud server ────────────────────────────────────────────────── + +BUILD_TARGET="${BUILD_TARGET:-debug}" +OXICLOUD_BIN="$REPO_ROOT/target/$BUILD_TARGET/oxicloud" + +if [[ -x "$OXICLOUD_BIN" ]]; then + log "Starting pre-built OxiCloud server ($BUILD_TARGET) on port $SERVER_PORT..." + "$OXICLOUD_BIN" & +else + log "Building and starting OxiCloud server on port $SERVER_PORT..." + cd "$REPO_ROOT" + cargo run & +fi +SERVER_PID=$! +log "Waiting for server at $base_url..." +wait_for_http "$base_url/ready" 120 +log "Server is ready." + +# ── 4. Run Hurl tests ───────────────────────────────────────────────────────── + +log "Running Hurl tests..." +for T in "$WEBDAV_DIR"/test_*.sh; do + if bash "$T" + then + echo $'\e[32m'"Success $T"$'\e[0m' >&2 + else + echo $'\e[31m'"Failure $T"$'\e[0m' >&2 + false + fi +done + +log "All tests passed." diff --git a/tests/webdav/test.env b/tests/webdav/test.env new file mode 100644 index 00000000..9e12d698 --- /dev/null +++ b/tests/webdav/test.env @@ -0,0 +1,6 @@ +# Test credentials for local/CI API tests — NOT real secrets. +base_url=http://localhost:8087 +username=admin +email=admin@example.com +# gitguardian:ignore +password=TestPassword1! diff --git a/tests/webdav/test_chunked_upload_dedup.sh b/tests/webdav/test_chunked_upload_dedup.sh new file mode 100755 index 00000000..51094cf1 --- /dev/null +++ b/tests/webdav/test_chunked_upload_dedup.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Chunked upload API + dedup check +# ============================================================= +# Uploads free_video_over_1MB.mp4 (2760653 bytes) in 3 chunks +# of 1 MiB each via the TUS-like chunked upload API, then: +# 1. Verifies the file appears in folder listing with video/mp4 MIME type +# 2. Checks GET /api/dedup/check/{hash} → ref_count == 1 +# +# BLAKE3 hash of free_video_over_1MB.mp4: +# 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80 +# +# Prerequisites: +# - Server running at base_url with credentials from test.env +# - OXICLOUD_ENABLE_AUTH=true +# - jq, dd in PATH +# +# Run (from repo root): +# bash tests/webdav/test_chunked_upload_dedup.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# ── fixture ─────────────────────────────────────────────────────────────────── + +BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80" +FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4" +[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; } + +REMOTE_NAME="chunked-upload-test.mp4" +FILE_SIZE=2760653 +CHUNK_SIZE=1048576 # 1 MiB — minimum accepted by the server +TOTAL_CHUNKS=3 # ceil(2760653 / 1048576) = 3 + +echo +echo "=== Chunked upload API + dedup check ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# ── home folder ─────────────────────────────────────────────────────────────── + +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \ + || fail "Could not retrieve home folder ID" +echo " home folder id: $HOME_FOLDER_ID" + +# ── idempotent pre-test cleanup ─────────────────────────────────────────────── + +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleanup: deleting existing $REMOTE_NAME (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null +fi +purge_from_trash "$REMOTE_NAME" + +# ── Step 1: Create upload session ───────────────────────────────────────────── + +echo " step 1: POST /api/uploads (create session, $TOTAL_CHUNKS chunks of ${CHUNK_SIZE}B)..." +SESSION=$(curl -s \ + -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"filename\":\"$REMOTE_NAME\",\"folder_id\":\"$HOME_FOLDER_ID\",\"content_type\":\"video/mp4\",\"total_size\":$FILE_SIZE,\"chunk_size\":$CHUNK_SIZE}" \ + "$base_url/api/uploads") + +UPLOAD_ID=$(jq -r '.upload_id' <<< "$SESSION") +[[ -n "$UPLOAD_ID" && "$UPLOAD_ID" != "null" ]] \ + || fail "POST /api/uploads: could not get upload_id (response: $SESSION)" +pass "Upload session created: upload_id=$UPLOAD_ID" + +# ── Step 2: Upload chunks ───────────────────────────────────────────────────── + +echo " step 2: PATCH chunks 0..$(( TOTAL_CHUNKS - 1 ))..." +for (( i=0; i/dev/null \ + | curl -s \ + -X PATCH \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @- \ + "$base_url/api/uploads/$UPLOAD_ID?chunk_index=$i") + + IS_COMPLETE=$(jq -r '.is_complete' <<< "$PATCH_RESP") + BYTES=$(jq -r '.bytes_received' <<< "$PATCH_RESP") + [[ -n "$BYTES" && "$BYTES" != "null" ]] \ + || fail "PATCH chunk $i: unexpected response: $PATCH_RESP" + echo " chunk $i: bytes_received=$BYTES is_complete=$IS_COMPLETE" +done +pass "All $TOTAL_CHUNKS chunks uploaded" + +# ── Step 3: Complete the upload ─────────────────────────────────────────────── + +echo " step 3: POST /api/uploads/$UPLOAD_ID/complete..." +COMPLETE_RESP=$(curl -s \ + -X POST \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/api/uploads/$UPLOAD_ID/complete") + +FILE_ID=$(jq -r '.file_id' <<< "$COMPLETE_RESP") +[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \ + || fail "POST complete: could not get file_id (response: $COMPLETE_RESP)" +pass "Upload complete: file_id=$FILE_ID" + +# ── Step 4: Verify file appears in folder listing with correct MIME type ────── + +echo " step 4: verify file in folder listing with video/mp4 MIME type..." +LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID") +LISTED_FILE=$(jq -r --arg id "$FILE_ID" 'first(.[] | select(.id == $id))' <<< "$LISTING") + +[[ -n "$LISTED_FILE" && "$LISTED_FILE" != "null" ]] \ + || fail "File $FILE_ID not found in folder listing" + +MIME=$(jq -r '.mime_type' <<< "$LISTED_FILE") +[[ "$MIME" == "video/mp4" ]] \ + || fail "Expected MIME type video/mp4, got: $MIME" +pass "File listed with MIME type: $MIME" + +# ── Step 5: Dedup check → ref_count == 1 ───────────────────────────────────── + +echo " step 5: GET /api/dedup/check/$BLOB_HASH..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)" +[[ "$RC" == "1" ]] \ + || fail "dedup/check: expected ref_count=1, got $RC" +pass "ref_count == 1: blob registered after chunked upload" + +# ── cleanup ─────────────────────────────────────────────────────────────────── + +echo " cleanup..." +ST=$(rest_delete "/api/files/$FILE_ID") +[[ "$ST" == "204" ]] || fail "DELETE file expected 204, got $ST" +purge_from_trash "$REMOTE_NAME" +pass "Cleanup complete" + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo +echo "Results: $PASS passed, $FAIL failed." +[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1 diff --git a/tests/webdav/test_dedup_webdav_multichunk.sh b/tests/webdav/test_dedup_webdav_multichunk.sh new file mode 100755 index 00000000..967124a8 --- /dev/null +++ b/tests/webdav/test_dedup_webdav_multichunk.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Dedup ref_count: multi-chunk CDC file via WebDAV +# ============================================================= +# Case A (files < 64kB = 1 chunk) is treated in tests/api/dedup_blob_cleanup.hurl +# Validates Case B of the cleanup_if_orphaned fix: +# file_hash ≠ chunk_hashes (file split into 8 CDC chunks) +# +# free_video_over_1MB.mp4 — 2.6 MB, 8 CDC chunks +# BLAKE3: 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80 +# +# Sequence: +# 1. PUT video as file A → new manifest (ref_count=1, 8 chunk blobs) +# 2. PUT video as file B → dedup hit (ref_count=2, chunks unchanged) +# 3. /api/dedup/check → ref_count == 2 +# 4. Permanently delete file A +# 5. /api/dedup/check → ref_count == 1 (chunks must NOT be freed) +# 6. Permanently delete file B +# 7. /api/dedup/check → exists == false (manifest + chunks cleaned up) +# +# Case B regression: cleanup_if_orphaned must decrement chunk_manifests +# ref_count without touching chunk blobs (the PG trigger is a no-op for +# multi-chunk files because file_hash is not stored in storage.blobs). +# +# Prerequisites: +# - Server running at base_url with credentials from test.env +# - OXICLOUD_ENABLE_AUTH=true +# - jq in PATH +# +# Run (from repo root): +# bash tests/webdav/test_dedup_webdav_multichunk.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +webdav_put() { + local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}" + curl -s -o /dev/null -w "%{http_code}" \ + -X PUT \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: $mime" \ + --data-binary "@$local_file" \ + "$base_url/webdav/$remote_name" +} + +webdav_delete() { + local remote_name="$1" + curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/webdav/$remote_name" +} + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# ── fixture ─────────────────────────────────────────────────────────────────── + +# free_video_over_1MB.mp4 → 2.6 MB → 8 CDC chunks → confirmed Case B +BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80" +FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4" +[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; } + +FILE_A="webdav-dedup-mc-a.mp4" +FILE_B="webdav-dedup-mc-b.mp4" + +echo +echo "=== Dedup ref_count: multi-chunk CDC (Case B) via WebDAV ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# ── home folder ─────────────────────────────────────────────────────────────── + +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \ + || fail "Could not retrieve home folder ID" +echo " home folder id: $HOME_FOLDER_ID" + +# ── idempotent pre-test cleanup ─────────────────────────────────────────────── + +for REMOTE in "$FILE_A" "$FILE_B"; do + EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') + if [[ -n "$EXISTING_ID" ]]; then + echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + fi + purge_from_trash "$REMOTE" +done + +# ── Step 1: Upload file A ───────────────────────────────────────────────────── + +echo " step 1: PUT $FILE_A..." +STATUS=$(webdav_put "$FILE_A" "$FIXTURE" "video/mp4") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS" +pass "PUT $FILE_A → 204 (new manifest, 8 chunk blobs created)" + +# ── Step 2: Upload file B (same content, different name → dedup hit) ────────── + +echo " step 2: PUT $FILE_B (same bytes → dedup hit)..." +STATUS=$(webdav_put "$FILE_B" "$FIXTURE" "video/mp4") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS" +pass "PUT $FILE_B → 204 (dedup hit: manifest ref_count → 2, chunks unchanged)" + +# ── Resolve file IDs ────────────────────────────────────────────────────────── + +LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID") +FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$LISTING") +FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$LISTING") + +[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing" +[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing" +[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \ + || fail "File A and B share the same ID — dedup must create two distinct records" +pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID" + +# ── Step 3: ref_count == 2 ──────────────────────────────────────────────────── + +echo " step 3: dedup/check → expect ref_count=2..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)" +[[ "$RC" == "2" ]] \ + || fail "dedup/check: expected ref_count=2, got $RC" +pass "ref_count == 2: both files reference the same 8-chunk blob" + +# ── Step 4: Permanently delete file A ──────────────────────────────────────── +# For multi-chunk files: PG trigger is a no-op (file_hash not in storage.blobs). +# cleanup_if_orphaned must decrement chunk_manifests.ref_count only (2→1) +# and leave the 8 chunk blobs untouched. + +echo " step 4: trash + permanently delete $FILE_A..." +ST=$(rest_delete "/api/files/$FILE_A_ID") +[[ "$ST" == "204" ]] || fail "DELETE $FILE_A expected 204, got $ST" + +TRASH_A=$(rest_get "/api/trash" \ + | jq -r --arg n "$FILE_A" 'first(.[] | select(.name == $n) | .id) // empty') +[[ -n "$TRASH_A" ]] || fail "File A not found in trash" +ST=$(rest_delete "/api/trash/$TRASH_A") +[[ "$ST" == "200" ]] || fail "Permanent delete file A expected 200, got $ST" +pass "File A permanently deleted" + +# ── Step 5: ref_count == 1 — chunk blobs must still be alive ───────────────── + +echo " step 5: dedup/check → expect ref_count=1 (chunks must survive)..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true (file B still references blob), got $EXISTS" +[[ "$RC" == "1" ]] \ + || fail "dedup/check: expected ref_count=1, got $RC (chunk blobs may have been freed prematurely)" +pass "ref_count == 1: manifest decremented, all 8 chunk blobs still alive" + +# ── Step 6: Permanently delete file B ──────────────────────────────────────── +# ref_count hits 0 → manifest deleted, all 8 chunk blobs freed. + +echo " step 6: trash + permanently delete $FILE_B..." +ST=$(rest_delete "/api/files/$FILE_B_ID") +[[ "$ST" == "204" ]] || fail "DELETE $FILE_B expected 204, got $ST" + +TRASH_B=$(rest_get "/api/trash" \ + | jq -r --arg n "$FILE_B" 'first(.[] | select(.name == $n) | .id) // empty') +[[ -n "$TRASH_B" ]] || fail "File B not found in trash" +ST=$(rest_delete "/api/trash/$TRASH_B") +[[ "$ST" == "200" ]] || fail "Permanent delete file B expected 200, got $ST" +pass "File B permanently deleted" + +# ── Step 7: blob gone ───────────────────────────────────────────────────────── + +echo " step 7: dedup/check → expect exists=false (manifest + chunks freed)..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") + +[[ "$EXISTS" == "false" ]] \ + || fail "dedup/check: expected exists=false after both files deleted, got $EXISTS" +pass "exists == false: manifest and all 8 chunk blobs cleaned up" + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo +echo "Results: $PASS passed, $FAIL failed." +[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1 diff --git a/tests/webdav/test_dedup_webdav_ref_count.sh b/tests/webdav/test_dedup_webdav_ref_count.sh new file mode 100755 index 00000000..7a87762d --- /dev/null +++ b/tests/webdav/test_dedup_webdav_ref_count.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Dedup ref_count via WebDAV (two uploads + overwrite) +# ============================================================= +# Scenario: +# 1. PUT dedup-test.jpg via WebDAV as file A +# 2. PUT dedup-test-2.jpg (identical content) via WebDAV as file B +# → same blob, two distinct file records, ref_count == 2 +# 3. GET /api/dedup/check/{hash} → assert ref_count == 2 +# 4. Overwrite file B via WebDAV PUT with different content +# → file B now references a new blob; original ref_count drops +# 5. GET /api/dedup/check/{hash} → assert ref_count == 1 +# +# BLAKE3 hash of dedup-test.jpg (== dedup-test-2.jpg — same content): +# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +# +# Prerequisites: +# - Server running at base_url with credentials from test.env +# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth) +# - jq in PATH +# +# Run (from repo root): +# bash tests/webdav/test_dedup_webdav_ref_count.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +webdav_put() { + local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}" + curl -s -o /dev/null -w "%{http_code}" \ + -X PUT \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: $mime" \ + --data-binary "@$local_file" \ + "$base_url/webdav/$remote_name" +} + +webdav_delete() { + local remote_name="$1" + curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/webdav/$remote_name" +} + +rest_get() { + curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1" +} + +rest_delete() { + curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url$1" +} + +dedup_check() { + curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1" +} + +# ── fixtures ────────────────────────────────────────────────────────────────── + +# BLAKE3 hash of dedup-test.jpg (= dedup-test-2.jpg — byte-identical content) +BLOB_HASH="cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066" + +FIXTURE_A="$REPO_ROOT/tests/fixtures/dedup-test.jpg" +FIXTURE_B="$REPO_ROOT/tests/fixtures/dedup-test-2.jpg" +FIXTURE_OTHER="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg" + +[[ -f "$FIXTURE_A" ]] || { echo "Missing fixture: $FIXTURE_A" >&2; exit 1; } +[[ -f "$FIXTURE_B" ]] || { echo "Missing fixture: $FIXTURE_B" >&2; exit 1; } +[[ -f "$FIXTURE_OTHER" ]] || { echo "Missing fixture: $FIXTURE_OTHER" >&2; exit 1; } + +FILE_A="webdav-dedup-ref-a.jpg" +FILE_B="webdav-dedup-ref-b.jpg" + +echo +echo "=== Dedup ref_count: two WebDAV uploads + overwrite ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# ── home folder ─────────────────────────────────────────────────────────────── + +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \ + || fail "Could not retrieve home folder ID" +echo " home folder id: $HOME_FOLDER_ID" + +# ── idempotent pre-test cleanup ─────────────────────────────────────────────── + +for REMOTE in "$FILE_A" "$FILE_B"; do + EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') + if [[ -n "$EXISTING_ID" ]]; then + echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + fi + STALE=$(rest_get "/api/trash" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') + if [[ -n "$STALE" ]]; then + echo " cleanup: purging $REMOTE from trash (id=$STALE)" + rest_delete "/api/trash/$STALE" > /dev/null + fi +done + +# ── Step 1: Upload file A ───────────────────────────────────────────────────── + +echo " step 1: PUT $FILE_A (dedup-test.jpg)..." +STATUS=$(webdav_put "$FILE_A" "$FIXTURE_A" "image/jpeg") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS" +pass "PUT $FILE_A → 204" + +# ── Step 2: Upload file B (identical content, different name) ───────────────── + +echo " step 2: PUT $FILE_B (dedup-test-2.jpg, same bytes)..." +STATUS=$(webdav_put "$FILE_B" "$FIXTURE_B" "image/jpeg") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS" +pass "PUT $FILE_B → 204" + +# ── Step 3: Resolve file IDs and assert two distinct records ────────────────── + +FILE_LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID") +FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING") +FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING") + +[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing" +[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing" +[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \ + || fail "File A and B share the same ID — dedup must produce two distinct records" +pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID" + +# ── Step 4: Dedup check → ref_count == 2 ───────────────────────────────────── + +echo " step 4: GET /api/dedup/check/$BLOB_HASH..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true, got $EXISTS (full response: $RESP)" +[[ "$RC" == "2" ]] \ + || fail "dedup/check: expected ref_count=2 after two identical uploads, got $RC" +pass "ref_count == 2: both files reference the same blob" + +# ── Step 5: Overwrite file B with different content ─────────────────────────── +# swap_blob_hash calls remove_reference on the old hash → manifest ref_count 2→1 + +echo " step 5: PUT $FILE_B (oxicloud-logo.jpg, new content)..." +STATUS=$(webdav_put "$FILE_B" "$FIXTURE_OTHER" "image/jpeg") +[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B (overwrite) expected 204, got $STATUS" +pass "PUT $FILE_B overwrite → 204" + +# ── Step 6: Dedup check → ref_count == 1 ───────────────────────────────────── +# File B now references a different blob; file A still holds the original. + +echo " step 6: GET /api/dedup/check/$BLOB_HASH..." +RESP=$(dedup_check "$BLOB_HASH") +EXISTS=$(jq -r '.exists' <<< "$RESP") +RC=$( jq -r '.ref_count' <<< "$RESP") + +[[ "$EXISTS" == "true" ]] \ + || fail "dedup/check: expected exists=true (file A still references blob), got $EXISTS" +[[ "$RC" == "1" ]] \ + || fail "dedup/check: expected ref_count=1 after overwriting file B, got $RC" +pass "ref_count == 1: only file A still references the original blob" + +# ── cleanup ─────────────────────────────────────────────────────────────────── + +echo " cleanup..." +for REMOTE in "$FILE_A" "$FILE_B"; do + ST=$(webdav_delete "$REMOTE") + [[ "$ST" == "204" ]] || fail "WebDAV DELETE $REMOTE expected 204, got $ST" + TRASH_ITEM=$(rest_get "/api/trash" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') + if [[ -n "$TRASH_ITEM" ]]; then + rest_delete "/api/trash/$TRASH_ITEM" > /dev/null + fi +done +pass "Cleanup complete" + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo +echo "Results: $PASS passed, $FAIL failed." +[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1 diff --git a/tests/webdav/test_thumbnail_update.sh b/tests/webdav/test_thumbnail_update.sh new file mode 100755 index 00000000..5273507a --- /dev/null +++ b/tests/webdav/test_thumbnail_update.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Bug 1 & 2: thumbnail refresh after WebDAV PUT +# ============================================================= +# Bug 1: stale moka cache served after blob swap (overwrite) +# Bug 2: no background thumbnail generation after update +# +# Fix (webdav_handler.rs handle_put): after update branch, +# refresh_thumbnails_after_update() calls delete_thumbnails() +# (evicts moka) then spawns background regen from new blob. +# +# Test strategy: +# 1. PUT dedup-test.jpg via /webdav → thumbnail generated +# 2. Prime the moka cache with GET /thumbnail +# 3. PUT oxicloud-logo.jpg to the same path (overwrite) +# 4. GET /thumbnail again → must return different bytes +# +# Bug 1 detection: if moka is not evicted, step 4 returns the +# cached dedup-test thumbnail → SHA-256 matches step 2 → test fails. +# +# Prerequisites: +# - Server running at base_url with credentials from test.env +# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth) +# - jq in PATH +# +# Run (from repo root): +# bash tests/webdav/test_thumbnail_update.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + + +# ── helpers ────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +# WebDAV PUT: returns HTTP status code +webdav_put() { + local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}" + curl -s -o /dev/null -w "%{http_code}" \ + -X PUT \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: $mime" \ + --data-binary "@$local_file" \ + "$base_url/webdav/$remote_name" +} + +# WebDAV DELETE: returns HTTP status code +webdav_delete() { + local remote_name="$1" + curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/webdav/$remote_name" +} + +# REST GET with JWT bearer +rest_get() { + curl -s \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url$1" +} + +# REST DELETE with JWT bearer, returns HTTP status code +rest_delete() { + curl -s -o /dev/null -w "%{http_code}" \ + -X DELETE \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url$1" +} + +# Download thumbnail and return its SHA-256 checksum +thumbnail_sha256() { + local file_id="$1" + curl -s \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/api/files/$file_id/thumbnail/icon" \ + | sha256sum | cut -d' ' -f1 +} + +# SHA-256 of an empty stream (thumbnail missing = empty body) +EMPTY_SHA="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +FIXTURE_V1="$REPO_ROOT/tests/fixtures/dedup-test.jpg" +FIXTURE_V2="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg" + +[[ -f "$FIXTURE_V1" ]] || { echo "Missing fixture: $FIXTURE_V1" >&2; exit 1; } +[[ -f "$FIXTURE_V2" ]] || { echo "Missing fixture: $FIXTURE_V2" >&2; exit 1; } + +echo +echo "=== Bug 1 & 2: thumbnail refresh after WebDAV PUT overwrite ===" +echo + +# ── authenticate ───────────────────────────────────────────── + +oxicloud_login + +# ── home folder ID ──────────────────────────────────────────── + +echo " home folder..." +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \ + || fail "Could not retrieve home folder ID" +echo " home folder id: $HOME_FOLDER_ID" + +REMOTE="webdav-thumb-bug12.jpg" + +# ── Pre-test cleanup (idempotent) ───────────────────────────── +# Remove any leftover from a previous run via the REST API so that +# the first WebDAV PUT below is guaranteed to be a CREATE (201). + +echo " cleanup: checking regular listing for '$REMOTE'..." +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleanup: found existing file id=$EXISTING_ID — deleting..." + ST=$(rest_delete "/api/files/$EXISTING_ID") + echo " cleanup: DELETE /api/files/$EXISTING_ID → $ST" +else + echo " cleanup: no existing file in regular listing" +fi + +echo " cleanup: checking trash for '$REMOTE'..." +STALE=$(rest_get "/api/trash" \ + | jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$STALE" ]]; then + echo " cleanup: found trash item id=$STALE — purging..." + ST=$(rest_delete "/api/trash/$STALE") + echo " cleanup: DELETE /api/trash/$STALE → $ST" +else + echo " cleanup: trash is clean" +fi + +# ── Step 1: PUT dedup-test.jpg ─────────────────────────────── +# /webdav always returns 204 (update_file_streaming handles create+update) + +echo " step 1: PUT $REMOTE..." +STATUS=$(webdav_put "$REMOTE" "$FIXTURE_V1" "image/jpeg") +echo " step 1: WebDAV PUT → $STATUS" +[[ "$STATUS" == "204" ]] || fail "WebDAV PUT expected 204, got $STATUS" +pass "WebDAV PUT dedup-test.jpg → 204" + +# ── find file_id from REST listing ─────────────────────────── + +echo " step 1: resolving file_id..." +FILE_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE" '.[] | select(.name == $n) | .id') +[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \ + || fail "File '$REMOTE' not found in folder listing after WebDAV PUT" +pass "File found via REST API — id=$FILE_ID" + +# ── Step 2: GET thumbnail to prime moka cache ──────────────── +# Background generation may still be running; wait briefly. + +echo " step 2: waiting 1s for background thumbnail generation..." +sleep 1 + +echo " step 2: GET /thumbnail/icon..." +HTTP=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/api/files/$FILE_ID/thumbnail/icon") +echo " step 2: GET /thumbnail → $HTTP" +[[ "$HTTP" == "200" ]] \ + || fail "GET /thumbnail after initial upload expected 200, got $HTTP" + +THUMB_V1=$(thumbnail_sha256 "$FILE_ID") +echo " step 2: thumbnail sha256=$THUMB_V1" +[[ -n "$THUMB_V1" && "$THUMB_V1" != "$EMPTY_SHA" ]] \ + || fail "Thumbnail after initial upload is empty (sha256=$THUMB_V1)" +pass "Initial thumbnail present and non-empty (sha256=$THUMB_V1)" + +# ── Step 3: PUT oxicloud-logo.jpg (overwrite) ──────────────── + +echo " step 3: PUT $REMOTE (overwrite)..." +STATUS=$(webdav_put "$REMOTE" "$FIXTURE_V2" "image/jpeg") +echo " step 3: WebDAV PUT → $STATUS" +[[ "$STATUS" == "204" ]] || fail "WebDAV PUT (overwrite) expected 204, got $STATUS" +pass "WebDAV PUT oxicloud-logo.jpg overwrite → 204 No Content" + +# ── Step 4: GET thumbnail after overwrite ──────────────────── +# Fix: delete_thumbnails() evicts moka (bug 1), +# background regen populates new blob thumbnail (bug 2). +# Without the fix the stale dedup-test thumbnail is served from moka. + +echo " step 4: waiting 1s for background thumbnail regeneration..." +sleep 1 + +echo " step 4: GET /thumbnail/icon after overwrite..." +HTTP=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $TOKEN" \ + "$base_url/api/files/$FILE_ID/thumbnail/icon") +echo " step 4: GET /thumbnail → $HTTP" +[[ "$HTTP" == "200" ]] \ + || fail "GET /thumbnail after overwrite expected 200, got $HTTP" + +THUMB_V2=$(thumbnail_sha256 "$FILE_ID") +echo " step 4: thumbnail sha256=$THUMB_V2" +[[ -n "$THUMB_V2" && "$THUMB_V2" != "$EMPTY_SHA" ]] \ + || fail "Thumbnail after overwrite is empty" + +[[ "$THUMB_V1" != "$THUMB_V2" ]] \ + || fail "Bug 1 present: moka cache not evicted — thumbnail unchanged after overwrite (sha256=$THUMB_V1)" +pass "Thumbnail changed after overwrite — bugs 1 & 2 fixed (sha256=$THUMB_V2)" + +# ── cleanup ─────────────────────────────────────────────────── + +echo " cleanup: WebDAV DELETE $REMOTE..." +STATUS=$(webdav_delete "$REMOTE") +echo " cleanup: WebDAV DELETE → $STATUS" +[[ "$STATUS" == "204" ]] || fail "WebDAV DELETE expected 204, got $STATUS" +pass "WebDAV DELETE → 204" + +TRASH_ITEM=$(rest_get "/api/trash" \ + | jq -r --arg n "$REMOTE" '.[] | select(.name == $n) | .id // empty') +if [[ -n "$TRASH_ITEM" ]]; then + ST=$(rest_delete "/api/trash/$TRASH_ITEM") + echo " cleanup: DELETE /api/trash/$TRASH_ITEM → $ST" + pass "Permanently deleted from trash" +fi + +# ── summary ─────────────────────────────────────────────────── + +echo +echo "Results: $PASS passed, $FAIL failed." +[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1