diff --git a/src/application/ports/mod.rs b/src/application/ports/mod.rs index 22302921..2cd8e061 100644 --- a/src/application/ports/mod.rs +++ b/src/application/ports/mod.rs @@ -1,4 +1,5 @@ pub mod auth_ports; +pub mod blob_lifecycle; pub mod blob_storage_ports; pub mod cache_ports; pub mod calendar_ports; diff --git a/src/application/services/trash_service.rs b/src/application/services/trash_service.rs index a4ba03fd..fedb5262 100644 --- a/src/application/services/trash_service.rs +++ b/src/application/services/trash_service.rs @@ -16,6 +16,7 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository; use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository; use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository; +use crate::infrastructure::services::dedup_service::DedupService; use crate::infrastructure::services::file_content_cache::FileContentCache; use crate::infrastructure::services::thumbnail_service::ThumbnailService; @@ -45,6 +46,10 @@ pub struct TrashService { /// Port for folder operations (get folder, trash, restore, delete) folder_storage_port: Arc, + /// Dedup service — garbage-collected after bulk trash empty to clean up + /// orphaned blob files and thumbnails that the PG trigger cannot reach. + dedup_service: Arc, + /// Thumbnail service for cleaning up thumbnails on permanent delete thumbnail_service: Option>, @@ -62,6 +67,7 @@ impl TrashService { file_write_port: Arc, folder_storage_port: Arc, retention_days: u32, + dedup_service: Arc, thumbnail_service: Option>, content_cache: Option>, ) -> Self { @@ -70,6 +76,7 @@ impl TrashService { file_read_port, file_write_port, folder_storage_port, + dedup_service, thumbnail_service, content_cache, retention_days, @@ -713,18 +720,24 @@ impl TrashUseCase for TrashService { Vec::new() }; - // clear_trash() already performs bulk SQL DELETEs in 2 queries: + // clear_trash() performs bulk SQL DELETEs in 2 queries: // 1. DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE // 2. DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE // // Folder deletion cascades (FK ON DELETE CASCADE) to child folders and // their files. The PG trigger `trg_files_decrement_blob_ref` automatically - // decrements blob ref_counts for every deleted file row — no Rust-side - // remove_reference() call is needed. + // decrements blob ref_counts for every deleted file row. // // Finally it clears the trash_items index for the user. self.trash_repository.clear_trash(&user_id).await?; + // The PG trigger decremented ref_counts but cannot delete disk files or + // thumbnails. Run garbage_collect() to remove any blobs whose ref_count + // reached 0, along with their blob-keyed thumbnail files. + if let Err(e) = self.dedup_service.garbage_collect().await { + warn!("empty_trash: garbage_collect failed: {:?}", e); + } + // Invalidate content cache for all permanently deleted files. if let Some(cc) = &self.content_cache { for file_id in &trashed_file_ids { diff --git a/src/common/di.rs b/src/common/di.rs index 28a6a8e4..b6acc872 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -264,7 +264,7 @@ impl AppServiceFactory { db_pool.clone(), maintenance_pool.clone(), ) - .with_thumbnail_service(thumbnail_service.clone()), + .add_blob_hook(thumbnail_service.clone()), ); dedup_service.initialize().await?; @@ -451,10 +451,12 @@ impl AppServiceFactory { repos.file_write_repository.clone(), repos.folder_repository.clone(), self.config.storage.trash_retention_days, + core.dedup_service.clone(), Some(core.thumbnail_service.clone()), Some(core.file_content_cache.clone()), )); + // Initialize cleanup service (bulk-deletes expired items in 2 SQL queries) let cleanup_service = TrashCleanupService::new( trash_repo.clone(), @@ -1010,7 +1012,7 @@ impl CoreServices { tokio::spawn(async move { match ds.read_blob_bytes(&hash).await { Ok(bytes) => { - ts.generate_all_sizes_background_from_bytes(file_id, hash, bytes); + ts.generate_all_sizes_background_from_bytes(file_id, hash, bytes, ds.clone()); } Err(e) => { tracing::warn!( diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs index 1562ea34..7e6c94ae 100644 --- a/src/infrastructure/repositories/pg/file_blob_write_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -604,8 +604,27 @@ impl FileWritePort for FileBlobWriteRepository { } async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> { - // Same as delete_file — removes from DB and decrements blob ref - self.delete_file(file_id).await + // Read blob_hash before deletion so we can clean up disk after the + // PG trigger has decremented the ref_count. + let blob_hash: Option = sqlx::query_scalar( + "SELECT blob_hash FROM storage.files WHERE id = $1::uuid", + ) + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}")) + })?; + + // DELETE fires trg_files_decrement_blob_ref → storage.blobs.ref_count-- + self.delete_file(file_id).await?; + + // If the blob is now unreferenced, remove disk file + thumbnails. + if let Some(hash) = blob_hash { + self.dedup.cleanup_if_orphaned(&hash).await; + } + + Ok(()) } async fn copy_folder_tree( diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 99c07d36..378fcbf0 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -45,11 +45,11 @@ use tokio::fs; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use crate::application::ports::blob_storage_ports::BlobStorageBackend; +use crate::application::ports::blob_lifecycle::BlobDeletionHook; use crate::application::ports::dedup_ports::{ BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, }; use crate::domain::errors::{DomainError, ErrorKind}; -use crate::infrastructure::services::thumbnail_service::ThumbnailService; // ── CDC Constants ──────────────────────────────────────────────────────────── @@ -84,9 +84,8 @@ pub struct DedupService { /// Isolated maintenance pool for long-running operations /// (verify_integrity, garbage_collect) that must never starve the primary. maintenance_pool: Arc, - /// Optional thumbnail service — when set, blob-hash thumbnails are deleted - /// from disk whenever a blob's ref_count reaches zero. - thumbnail_service: Option>, + /// Hooks notified when a blob's ref_count reaches zero and it is deleted. + blob_hooks: Vec>, } impl DedupService { @@ -104,17 +103,24 @@ impl DedupService { backend, pool, maintenance_pool, - thumbnail_service: None, + blob_hooks: vec![], } } - /// Attach a thumbnail service so that disk thumbnails are cleaned up when - /// a blob's ref_count drops to zero. - pub fn with_thumbnail_service(mut self, svc: Arc) -> Self { - self.thumbnail_service = Some(svc); + /// Register a [`BlobDeletionHook`] to be called whenever a blob's + /// ref_count reaches zero. Hooks are called in registration order. + pub fn add_blob_hook(mut self, hook: Arc) -> Self { + self.blob_hooks.push(hook); self } + /// Fire all registered hooks for a deleted blob. + async fn fire_blob_hooks(&self, hash: &str) { + for hook in &self.blob_hooks { + hook.on_blob_deleted(hash).await; + } + } + /// Creates a stub instance for testing — never hits PG or the filesystem. #[cfg(any(test, feature = "integration_tests"))] pub fn new_stub() -> Self { @@ -129,7 +135,7 @@ impl DedupService { backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))), pool: stub_pool.clone(), maintenance_pool: stub_pool, - thumbnail_service: None, + blob_hooks: vec![], } } @@ -600,7 +606,7 @@ impl DedupService { /// references the blob identified by `hash`. pub async fn user_owns_blob_reference(&self, hash: &str, user_id: &str) -> bool { sqlx::query_scalar::<_, bool>( - "SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2 AND NOT is_trashed)", + "SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2::uuid AND NOT is_trashed)", ) .bind(hash) .bind(user_id) @@ -785,10 +791,8 @@ impl DedupService { } } - // Bug 4 fix: delete disk thumbnails keyed by file_hash (last reference gone) - if let Some(ts) = &self.thumbnail_service { - ts.delete_blob_thumbnails(file_hash).await; - } + // Bug 4 fix: notify hooks — e.g. thumbnail cleanup keyed by file_hash + self.fire_blob_hooks(file_hash).await; tracing::info!( "MANIFEST DELETED: {} ({} chunks, {} orphan chunks removed)", @@ -867,10 +871,8 @@ impl DedupService { tracing::warn!("Failed to delete blob file {}: {}", hash, e); } - // Bug 3 fix: delete disk thumbnails keyed by hash (last reference gone) - if let Some(ts) = &self.thumbnail_service { - ts.delete_blob_thumbnails(hash).await; - } + // Bug 3 fix: notify hooks — e.g. thumbnail cleanup keyed by hash + self.fire_blob_hooks(hash).await; tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]); Ok(true) @@ -897,6 +899,91 @@ impl DedupService { } } + /// Targeted cleanup for a single blob after the PG trigger has already + /// decremented its ref_count. Deletes the blob row, disk file, and + /// blob-keyed thumbnails if ref_count has reached 0. + /// + /// Handles both the legacy whole-file blob path (storage.blobs) and the + /// CDC manifest path (storage.chunk_manifests). Best-effort: logs + /// warnings on failure rather than returning an error. + pub async fn cleanup_if_orphaned(&self, hash: &str) { + let short = &hash[..hash.len().min(12)]; + + // ── CDC manifest path (must run FIRST) ─────────────────── + // For single-chunk CDC files file_hash == chunk_hash, so the PG + // trigger on storage.files already decremented storage.blobs.ref_count + // when this function is called. try_dedup_hit increments + // chunk_manifests.ref_count but NOT storage.blobs.ref_count, so + // blobs.ref_count can reach 0 while the manifest still has ref_count > 1 + // (other files sharing the same blob). Checking the manifest first + // prevents premature blob + manifest deletion. + let manifest = sqlx::query_as::<_, (i32, Vec)>( + "SELECT ref_count, chunk_hashes \ + FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(hash) + .fetch_optional(self.pool.as_ref()) + .await + .unwrap_or(None); + + if let Some((ref_count, chunk_hashes)) = manifest { + if ref_count <= 1 { + // Last reference — remove manifest and all its chunks. + if let Err(e) = self + .remove_manifest_reference(hash, ref_count, &chunk_hashes) + .await + { + tracing::warn!("cleanup_if_orphaned: manifest cleanup failed for {short}: {e}"); + } + } else { + // Other files still share this blob: just decrement the manifest + // counter and undo the PG trigger's premature chunk ref_count + // decrement (blobs.ref_count is chunk-level; the manifest is the + // authoritative file-level counter). + sqlx::query( + "UPDATE storage.chunk_manifests \ + SET ref_count = ref_count - 1 WHERE file_hash = $1", + ) + .bind(hash) + .execute(self.pool.as_ref()) + .await + .ok(); + // Undo the PG trigger's decrement of storage.blobs.ref_count. + // The trigger fired with blob_hash = file_hash, so only the row + // WHERE hash = file_hash is affected. For single-chunk files + // file_hash == chunk_hash and that row exists; for multi-chunk + // files file_hash is not in storage.blobs, making this a no-op. + sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1") + .bind(hash) + .execute(self.pool.as_ref()) + .await + .ok(); + tracing::debug!( + "cleanup_if_orphaned: manifest {short} ref_count {ref_count}→{}", + ref_count - 1 + ); + } + return; + } + + // ── Legacy blob path (no manifest) ─────────────────────── + let deleted_blob = sqlx::query_scalar::<_, String>( + "DELETE FROM storage.blobs WHERE hash = $1 AND ref_count <= 0 RETURNING hash", + ) + .bind(hash) + .fetch_optional(self.pool.as_ref()) + .await + .unwrap_or(None); + + if deleted_blob.is_some() { + if let Err(e) = self.backend.delete_blob(hash).await { + tracing::warn!("cleanup_if_orphaned: disk delete failed for {short}: {e}"); + } + self.fire_blob_hooks(hash).await; + tracing::info!("cleanup_if_orphaned: removed orphaned blob {short}"); + } + } + // ── Read operations ────────────────────────────────────────── /// Stream blob content — CDC-aware with legacy fallback. @@ -1347,16 +1434,7 @@ impl DedupService { if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete orphan blob {hash}: {e}"); } - // Clean up thumbnails (best-effort, only local backends) - if let Some(blob_path) = self.backend.local_blob_path(hash) - && let Some(storage_root) = blob_path.ancestors().nth(3) - { - let thumbnails_root = storage_root.join(".thumbnails"); - for dir in &["icon", "preview", "large"] { - let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg")); - let _ = fs::remove_file(&thumb).await; - } - } + self.fire_blob_hooks(hash).await; total_bytes += *size as u64; } total_deleted += batch.len() as u64; diff --git a/src/infrastructure/services/thumbnail_service.rs b/src/infrastructure/services/thumbnail_service.rs index 7c3622f1..814df6d6 100644 --- a/src/infrastructure/services/thumbnail_service.rs +++ b/src/infrastructure/services/thumbnail_service.rs @@ -25,6 +25,7 @@ use tokio::time::timeout; use crate::application::ports::thumbnail_ports::{ ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto, }; +use crate::infrastructure::services::dedup_service::DedupService; use crate::domain::errors::{DomainError, ErrorKind}; /// Thumbnail sizes supported by the system @@ -831,10 +832,22 @@ impl ThumbnailService { file_id: String, blob_hash: String, original_data: Bytes, + dedup: Arc, ) { tokio::spawn(async move { tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id); + // Guard: if the blob was deleted before this task ran, cleanup_if_orphaned + // already fired with no thumbnails on disk — writing them now would leak them. + // Use the DB check (manifest + blobs tables) as the authoritative source. + if !dedup.blob_exists(&blob_hash).await { + tracing::debug!( + "Blob {}… deleted before thumbnail task ran, skipping", + &blob_hash[..blob_hash.len().min(12)] + ); + return; + } + let all_exist = { let mut ok = true; for size in ThumbnailSize::all() { @@ -971,6 +984,17 @@ impl ThumbnailService { } } +// ─── BlobDeletionHook ──────────────────────────────────────────────────────── + +impl crate::application::ports::blob_lifecycle::BlobDeletionHook for ThumbnailService { + fn on_blob_deleted<'a>( + &'a self, + blob_hash: &'a str, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { self.delete_blob_thumbnails(blob_hash).await }) + } +} + // ─── Port implementation ───────────────────────────────────────────────────── /// Convert port ThumbnailSize to infra ThumbnailSize. diff --git a/src/interfaces/api/handlers/dedup_handler.rs b/src/interfaces/api/handlers/dedup_handler.rs index 9ed5b746..3cab36ba 100644 --- a/src/interfaces/api/handlers/dedup_handler.rs +++ b/src/interfaces/api/handlers/dedup_handler.rs @@ -26,7 +26,9 @@ pub struct HashCheckResponse { /// If exists, the size of the existing blob #[serde(skip_serializing_if = "Option::is_none")] pub existing_size: Option, - /// If exists, the number of references to this blob + /// Global reference count for this blob across all users. + /// Only populated when the authenticated user has the `admin` role; + /// omitted for regular users to prevent cross-user content inference. #[serde(skip_serializing_if = "Option::is_none")] pub ref_count: Option, } @@ -85,7 +87,8 @@ impl DedupHandler { /// Check if the authenticated user already has a file with the given hash. /// /// User-scoped: only reveals whether **this user** owns a file that - /// references the blob — never exposes global existence or ref_count. + /// references the blob — never exposes global existence to non-admins. + /// Admins additionally receive the global `ref_count` in the response. /// /// GET /api/dedup/check/{hash} pub(super) async fn check_hash_impl( @@ -113,13 +116,20 @@ impl DedupHandler { .await; if user_has_it { - // Fetch size from metadata (safe — user owns a reference) - let size = dedup.get_blob_metadata(&hash).await.map(|m| m.size); + // Fetch size from metadata (safe — user owns a reference). + // Admins also get the global ref_count for dedup accounting tests. + let metadata = dedup.get_blob_metadata(&hash).await; + let size = metadata.as_ref().map(|m| m.size); + let ref_count = if auth_user.role == "admin" { + metadata.map(|m| m.ref_count) + } else { + None // Never expose global ref_count to regular users + }; let response = HashCheckResponse { exists: true, hash, existing_size: size, - ref_count: None, // Never expose global ref_count + ref_count, }; Response::builder() .status(StatusCode::OK) @@ -492,6 +502,7 @@ impl DedupHandler { .unwrap() .into_response() } + } // ── Route handlers (free functions) ────────────────────────────────────────── @@ -511,7 +522,7 @@ impl DedupHandler { ("hash" = String, Path, description = "BLAKE3 hash (64 hex characters)"), ), responses( - (status = 200, description = "Hash check result (user-scoped)", body = HashCheckResponse), + (status = 200, description = "Hash check result. `ref_count` is only present for admin users.", body = HashCheckResponse), (status = 400, description = "Invalid hash format"), ), tag = "dedup", diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index 8ff0c570..e543047e 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -783,6 +783,7 @@ impl FileHandler { file_id, blob_hash_owned, original_bytes, + dedup_service.clone(), ); } Err(err) => { diff --git a/tests/api/dedup_blob_cleanup.hurl b/tests/api/dedup_blob_cleanup.hurl index 57a533af..17c6424e 100644 --- a/tests/api/dedup_blob_cleanup.hurl +++ b/tests/api/dedup_blob_cleanup.hurl @@ -20,6 +20,11 @@ # server uses the legacy blob path — so we avoid stats-based # assertions and rely on observable thumbnail behaviour instead. # +# BLAKE3 hash of fixtures/dedup-test.jpg (= dedup-test-2.jpg content): +# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +# Used in /api/dedup/check/{hash} calls below to track ref_count lifecycle. +# ref_count is only returned for admin users; setup.hurl creates an admin. +# # Prerequisites: setup.hurl must have run (admin user exists). # # Run: @@ -87,6 +92,16 @@ jsonpath "$.name" == "dedup-test.jpg" jsonpath "$.folder_id" == {{test_folder_id}} +# ref_count == 1: blob has exactly one file reference after first upload +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + # ───────────────────────────────────────────────────────────── # Step 4 – Upload identical content again as dedup-test-2.jpg # Dedup: same blob, new file record, different file ID @@ -105,6 +120,16 @@ jsonpath "$.name" == "dedup-test-2.jpg" jsonpath "$.id" != "{{file1_id}}" +# ref_count == 2: dedup hit — same blob now referenced by two file records +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 2 + + # ───────────────────────────────────────────────────────────── # Step 5 – Dedup proof: thumbnails are byte-identical # Thumbnail generation reads blob bytes and is keyed by @@ -156,6 +181,16 @@ Authorization: Bearer {{token}} HTTP 200 +# ref_count == 1: blob survives — file2 still holds a reference +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + # ───────────────────────────────────────────────────────────── # Step 8 – Blob still alive: file 2 thumbnail is accessible # After file 1 is permanently deleted the blob ref_count @@ -201,6 +236,15 @@ Authorization: Bearer {{token}} HTTP 200 +# ref_count hits 0 → blob and manifest deleted; user no longer owns this hash +GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == false + + # ───────────────────────────────────────────────────────────── # Step 11 – Cleanup: delete the (now empty) test folder # ───────────────────────────────────────────────────────────── diff --git a/tests/api/run.sh b/tests/api/run.sh index fa12b7af..5f660b71 100755 --- a/tests/api/run.sh +++ b/tests/api/run.sh @@ -62,6 +62,9 @@ OXICLOUD_SERVER_PORT=$SERVER_PORT OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage" set +a +# ensure storage is empty before starting +echo "Wipe $OXICLOUD_STORAGE_PATH to ensure clean startup" +rm -rf "$OXICLOUD_STORAGE_PATH" mkdir -p "$OXICLOUD_STORAGE_PATH" # ── 3. Start OxiCloud server ────────────────────────────────────────────────── @@ -97,4 +100,6 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test #bash "$API_DIR/dedup_bulk_upload.sh" +bash "$API_DIR/storage_cleanup_check.sh" + log "All tests passed." diff --git a/tests/api/storage_cleanup_check.sh b/tests/api/storage_cleanup_check.sh new file mode 100755 index 00000000..1a421cac --- /dev/null +++ b/tests/api/storage_cleanup_check.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud – Storage disk-cleanup verification +# ============================================================= +# 1. Moves every live file and folder to trash via the REST API. +# 2. Calls DELETE /api/trash/empty to permanently delete all +# remaining trash items (including any left by previous tests). +# 3. Asserts that no regular files remain under +# $OXICLOUD_STORAGE_PATH/.thumbnails or .blobs. +# +# Called by run.sh after all Hurl tests have passed. +# Can also be run standalone (server must already be up): +# bash tests/api/storage_cleanup_check.sh +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +STORAGE_PATH="${OXICLOUD_STORAGE_PATH:-$REPO_ROOT/tests/api/storage}" + +# shellcheck source=test.env +source "$SCRIPT_DIR/test.env" + +log() { echo "[storage-check] $*"; } +fail() { echo $'\e[31m'"[storage-check] FAIL: $*"$'\e[0m' >&2; exit 1; } + +# ── 1. Login ────────────────────────────────────────────────────────────────── + +TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"$username\",\"password\":\"$password\"}" \ + | jq -r '.access_token') + +[[ -z "$TOKEN" || "$TOKEN" == "null" ]] && fail "login failed" +log "Logged in." + +AUTH="Authorization: Bearer $TOKEN" + +# ── 1b. Upload a probe image and verify its blob + thumbnail exist on disk ───── + +# shellcheck source=../common/internal_storage_helper.sh +source "$REPO_ROOT/tests/common/internal_storage_helper.sh" + +FIXTURE="$REPO_ROOT/tests/fixtures/blue-image.png" + +HOME_FOLDER_ID=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[0].id') +[[ -z "$HOME_FOLDER_ID" || "$HOME_FOLDER_ID" == "null" ]] && fail "could not get home folder id" + +PROBE_FILE_ID=$(curl -sf -X POST -H "$AUTH" \ + -F "folder_id=$HOME_FOLDER_ID" \ + -F "file=@$FIXTURE;type=image/png" \ + "$base_url/api/files/upload" | jq -r '.id') +[[ -z "$PROBE_FILE_ID" || "$PROBE_FILE_ID" == "null" ]] && fail "probe file upload failed" +log "Probe file uploaded (id=$PROBE_FILE_ID)." + +# GET thumbnail to trigger on-demand generation +HTTP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" -H "$AUTH" \ + "$base_url/api/files/$PROBE_FILE_ID/thumbnail/icon") +[[ "$HTTP_STATUS" != "200" ]] && fail "thumbnail GET returned HTTP $HTTP_STATUS (expected 200)" +log "Thumbnail fetched (HTTP 200)." + +assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not found on disk" +assert_preview_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk" +log "Probe blob and thumbnail confirmed present on disk." + +# ── 2. Move all live files and folders to trash ─────────────────────────────── +# +# For each root folder, list its direct children and soft-delete them. +# The server cascades folder deletion to all nested contents, so we only +# need to iterate one level deep. + +ROOT_FOLDERS=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[].id') + +for folder_id in $ROOT_FOLDERS; do + CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing") + + while IFS= read -r sub_id; do + [[ -z "$sub_id" ]] && continue + curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$sub_id" >/dev/null + done < <(echo "$CONTENTS" | jq -r '.folders[].id') + + while IFS= read -r file_id; do + [[ -z "$file_id" ]] && continue + curl -sf -X DELETE -H "$AUTH" "$base_url/api/files/$file_id" >/dev/null + done < <(echo "$CONTENTS" | jq -r '.files[].id') +done + +log "All live objects moved to trash." + +# ── 2b. Verify all root folders are empty according to the API ──────────────── + +for folder_id in $ROOT_FOLDERS; do + CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing") + SUB_COUNT=$(echo "$CONTENTS" | jq '.folders | length') + FILE_COUNT=$(echo "$CONTENTS" | jq '.files | length') + if [[ "$SUB_COUNT" -ne 0 || "$FILE_COUNT" -ne 0 ]]; then + fail "folder $folder_id still has $SUB_COUNT subfolder(s) and $FILE_COUNT file(s)" + fi +done + +log "API confirms all root folders are empty." + +# ── 3. Permanently delete everything in trash ───────────────────────────────── + +curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/empty" >/dev/null +log "Trash emptied." + +# ── 3b. Verify trash is empty according to the API ─────────────────────────── + +TRASH_COUNT=$(curl -sf -H "$AUTH" "$base_url/api/trash" | jq 'length') +if [[ "$TRASH_COUNT" -ne 0 ]]; then + fail "trash still contains $TRASH_COUNT item(s) after empty" +fi + +log "API confirms trash is empty." + +# ── 4. Disk verification ────────────────────────────────────────────────────── + +THUMB_FILES=$(find "$STORAGE_PATH/.thumbnails" -type f 2>/dev/null || true) +BLOB_FILES=$(find "$STORAGE_PATH/.blobs" -type f 2>/dev/null || true) + +if [[ -n "$THUMB_FILES" ]]; then + THUMB_COUNT=$(echo "$THUMB_FILES" | wc -l | tr -d ' ') + log "Leftover thumbnail files ($THUMB_COUNT):" + echo "$THUMB_FILES" + fail "$THUMB_COUNT thumbnail file(s) remain on disk after full cleanup" +fi + +if [[ -n "$BLOB_FILES" ]]; then + BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ') + log "Leftover blob files ($BLOB_COUNT):" + echo "$BLOB_FILES" + fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup" +fi + +log "OK — no blobs or thumbnails remain on disk." diff --git a/tests/common/internal_storage_helper.sh b/tests/common/internal_storage_helper.sh new file mode 100755 index 00000000..0fe70822 --- /dev/null +++ b/tests/common/internal_storage_helper.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +if ! which b3sum >/dev/null 2>/dev/null +then + echo "please install b3sum (brew install b3sum on Mac, apt installb3sum on Debian, etc)" >&2 + exit 1 +fi + +HASH="" +FILE_CACHE="" + +# return the hash of a file (stores into HASH variable) +oxi_hash() { + if [[ -z "$HASH" || "$FILE_CACHE" != "$1" ]] + then + HASH=$(b3sum --no-names "$1") + FILE_CACHE="$1" + fi + echo "$HASH" +} + +# returns the local blob localisation +local_blob_path() { + local BLOB_PREFIX + oxi_hash "$1" >/dev/null + BLOB_PREFIX=${HASH:0:2} + echo ".blobs/$BLOB_PREFIX/$HASH.blob" +} + +# returns the preview localisation without it's extension +preview_path() { + local SIZE + oxi_hash "$1" >/dev/null + # default size: icon + SIZE="${3:-icon}" + echo ".thumbnails/$SIZE/$HASH" +} + +assert_local_blob_existsy() { + BLOB_PATH=$(local_blob_path "$1") + STORAGE="$2" + if [[ -e $STORAGE/$BLOB_PATH ]] + then + echo "$BLOB_PATH exists" + return 0 + else + echo $'\e[31m'"$BLOB_PATH does not exist"$'\e[0m' >&2 + return 1 + fi +} + +assert_preview_existsy() { + THUMBNAIL_PATH=$(preview_path "$1") + STORAGE="$2" + if [[ -e "$STORAGE/$THUMBNAIL_PATH.jpg" || -e "$STORAGE/$THUMBNAIL_PATH.webp" ]] + then + echo "thumbnail $THUMBNAIL_PATH.(jpg|webp) exists" + return 0 + else + echo $'\e[31m'"thumbnail $THUMBNAIL_PATH.(jpg|webp) does not exist"$'\e[0m' >&2 + echo $STORAGE + find $STORAGE/.thumbnails + return 1 + fi +} + diff --git a/tests/common/server.env b/tests/common/server.env index 65e1a78b..f60ccef3 100644 --- a/tests/common/server.env +++ b/tests/common/server.env @@ -16,3 +16,4 @@ OXICLOUD_EXPOSE_SYSTEM_USERS=true OXICLOUD_WOPI_ENABLED=false OXICLOUD_OIDC_ENABLED=false RUST_LOG=warn +#RUST_LOG=debug diff --git a/tests/fixtures/blue-image.png b/tests/fixtures/blue-image.png new file mode 100644 index 00000000..ac6fca15 Binary files /dev/null and b/tests/fixtures/blue-image.png differ diff --git a/tests/fixtures/green-image.png b/tests/fixtures/green-image.png new file mode 100644 index 00000000..e689f584 Binary files /dev/null and b/tests/fixtures/green-image.png differ diff --git a/tests/fixtures/red-image.png b/tests/fixtures/red-image.png new file mode 100644 index 00000000..da137d1c Binary files /dev/null and b/tests/fixtures/red-image.png differ