Merge pull request #330 from EdouardVanbelle/fix/thumbnail-on-update
bugfix/thumbnails on update
This commit is contained in:
@@ -157,7 +157,7 @@ jobs:
|
|||||||
retention-days: 1
|
retention-days: 1
|
||||||
|
|
||||||
api-test:
|
api-test:
|
||||||
name: API tests (via Hurl)
|
name: API & Webdav tests
|
||||||
needs: build
|
needs: build
|
||||||
if: github.event_name == 'pull_request'
|
if: github.event_name == 'pull_request'
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
@@ -173,7 +173,7 @@ jobs:
|
|||||||
- name: Set execute bit on pre-built binary
|
- name: Set execute bit on pre-built binary
|
||||||
run: chmod +x target/release/oxicloud
|
run: chmod +x target/release/oxicloud
|
||||||
|
|
||||||
- name: Install Hurl
|
- name: Install Hurl and b3sum
|
||||||
env:
|
env:
|
||||||
HURL_MAJOR: "8"
|
HURL_MAJOR: "8"
|
||||||
run: |
|
run: |
|
||||||
@@ -181,13 +181,18 @@ jobs:
|
|||||||
https://api.github.com/repos/Orange-OpenSource/hurl/releases \
|
https://api.github.com/repos/Orange-OpenSource/hurl/releases \
|
||||||
| jq -r "map(select(.tag_name | startswith(\"${HURL_MAJOR}.\"))) | first | .tag_name")
|
| jq -r "map(select(.tag_name | startswith(\"${HURL_MAJOR}.\"))) | first | .tag_name")
|
||||||
curl -fLO "https://github.com/Orange-OpenSource/hurl/releases/download/${HURL_VERSION}/hurl_${HURL_VERSION}_amd64.deb"
|
curl -fLO "https://github.com/Orange-OpenSource/hurl/releases/download/${HURL_VERSION}/hurl_${HURL_VERSION}_amd64.deb"
|
||||||
sudo apt-get install -y "./hurl_${HURL_VERSION}_amd64.deb"
|
sudo apt-get install -y "./hurl_${HURL_VERSION}_amd64.deb" b3sum
|
||||||
|
|
||||||
- name: Run Hurl API tests
|
- name: Run Hurl API tests
|
||||||
run: bash tests/api/run.sh
|
run: bash tests/api/run.sh
|
||||||
env:
|
env:
|
||||||
BUILD_TARGET: release
|
BUILD_TARGET: release
|
||||||
|
|
||||||
|
- name: Run Webdav tests
|
||||||
|
run: bash tests/webdav/run.sh
|
||||||
|
env:
|
||||||
|
BUILD_TARGET: release
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
if: ${{ !cancelled() }}
|
if: ${{ !cancelled() }}
|
||||||
with:
|
with:
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ Hexagonal / Clean Architecture with four layers. Dependencies point inward only.
|
|||||||
|
|
||||||
- **DI via `AppState`**: All services are `Arc`-wrapped and assembled in `common/di.rs`. `AppState` is wrapped in `Arc` and passed as Axum state. Many services are `Option<Arc<T>>` because they depend on features being enabled (auth, WOPI, trash, etc.).
|
- **DI via `AppState`**: All services are `Arc`-wrapped and assembled in `common/di.rs`. `AppState` is wrapped in `Arc` and passed as Axum state. Many services are `Option<Arc<T>>` because they depend on features being enabled (auth, WOPI, trash, etc.).
|
||||||
|
|
||||||
- **Content-addressable storage**: Files use SHA-256 blob dedup. `storage.file_blobs` stores content; `storage.file_metadata` references blobs with ref-counting. See `file_blob_write_repository.rs` and `file_blob_read_repository.rs`.
|
- **Content-addressable storage**: Files use BLAKE3 blob dedup. `storage.file_blobs` stores content; `storage.file_metadata` references blobs with ref-counting. See `file_blob_write_repository.rs` and `file_blob_read_repository.rs`.
|
||||||
|
|
||||||
- **ltree paths**: Folder hierarchy uses PostgreSQL `ltree` for efficient subtree queries (recursive copies, moves, searches).
|
- **ltree paths**: Folder hierarchy uses PostgreSQL `ltree` for efficient subtree queries (recursive copies, moves, searches).
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ OxiCloud uses **moka** (a lock-free, concurrent cache) for write-behind caching
|
|||||||
| Directory listings | 120 s | 10 000 | Frequently accessed folder contents |
|
| Directory listings | 120 s | 10 000 | Frequently accessed folder contents |
|
||||||
| Thumbnail cache | configurable | 1 000 | Generated WebP/AVIF thumbnails |
|
| Thumbnail cache | configurable | 1 000 | Generated WebP/AVIF thumbnails |
|
||||||
| Image transcode | configurable | 500 | On-the-fly image transcoding results |
|
| Image transcode | configurable | 500 | On-the-fly image transcoding results |
|
||||||
| Blob hash | 30 s TTI | 5 000 | SHA-256 hashes for dedup lookups |
|
| Blob hash | 30 s TTI | 5 000 | BLAKE3 hashes for dedup lookups |
|
||||||
| Audio metadata | — | 2 000 | ID3 tags and duration |
|
| Audio metadata | — | 2 000 | ID3 tags and duration |
|
||||||
|
|
||||||
## How It Works
|
## How It Works
|
||||||
|
|||||||
+2
-2
@@ -14,7 +14,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal
|
|||||||
| **Cold start** | < 1 s | 5–15 s |
|
| **Cold start** | < 1 s | 5–15 s |
|
||||||
| **CPU at idle** | ~0 % | 1–5 % (cron, background jobs) |
|
| **CPU at idle** | ~0 % | 1–5 % (cron, background jobs) |
|
||||||
| **Min. hardware** | 1 vCPU / 512 MB RAM | 2 vCPU / 2 GB RAM |
|
| **Min. hardware** | 1 vCPU / 512 MB RAM | 2 vCPU / 2 GB RAM |
|
||||||
| **File dedup** | SHA-256 content-addressable | None |
|
| **File dedup** | BLAKE3 content-addressable | None |
|
||||||
| **Dependencies** | Single binary + PostgreSQL | PHP, Apache/Nginx, Redis, Cron, … |
|
| **Dependencies** | Single binary + PostgreSQL | PHP, Apache/Nginx, Redis, Cron, … |
|
||||||
| **WebDAV** | Built-in (RFC 4918) | Built-in |
|
| **WebDAV** | Built-in (RFC 4918) | Built-in |
|
||||||
| **CalDAV / CardDAV** | Built-in | Via apps |
|
| **CalDAV / CardDAV** | Built-in | Via apps |
|
||||||
@@ -28,7 +28,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal
|
|||||||
### Storage & Files
|
### Storage & Files
|
||||||
- Drag-and-drop upload, multi-file, grid & list views
|
- Drag-and-drop upload, multi-file, grid & list views
|
||||||
- Chunked uploads (TUS-like, parallel, resumable, MD5 integrity)
|
- Chunked uploads (TUS-like, parallel, resumable, MD5 integrity)
|
||||||
- SHA-256 content-addressable file deduplication with ref-counting
|
- BLAKE3 content-addressable file deduplication with ref-counting
|
||||||
- Adaptive compression (zstd / gzip per MIME type)
|
- Adaptive compression (zstd / gzip per MIME type)
|
||||||
- Trash bin with soft-delete and auto-purge
|
- Trash bin with soft-delete and auto-purge
|
||||||
- Favourites, recent files, full-text search
|
- Favourites, recent files, full-text search
|
||||||
|
|||||||
+1
-1
@@ -25,7 +25,7 @@ features:
|
|||||||
details: Single Rust binary, ~40 MB Docker image, <1s cold start, 30–50 MB idle RAM.
|
details: Single Rust binary, ~40 MB Docker image, <1s cold start, 30–50 MB idle RAM.
|
||||||
- icon: 📁
|
- icon: 📁
|
||||||
title: Full File Management
|
title: Full File Management
|
||||||
details: Chunked uploads, SHA-256 deduplication, trash, favourites, full-text search, thumbnails.
|
details: Chunked uploads, BLAKE3 deduplication, trash, favourites, full-text search, thumbnails.
|
||||||
- icon: 🔗
|
- icon: 🔗
|
||||||
title: WebDAV / CalDAV / CardDAV
|
title: WebDAV / CalDAV / CardDAV
|
||||||
details: RFC-compliant protocols for files, calendars, and contacts. Works with all major clients.
|
details: RFC-compliant protocols for files, calendars, and contacts. Works with all major clients.
|
||||||
|
|||||||
@@ -75,3 +75,4 @@ front-test-update-snapshot:
|
|||||||
# Hurl API functional tests (starts postgres + server, tears down after)
|
# Hurl API functional tests (starts postgres + server, tears down after)
|
||||||
api-test:
|
api-test:
|
||||||
bash tests/api/run.sh
|
bash tests/api/run.sh
|
||||||
|
bash tests/webdav/run.sh
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
use std::future::Future;
|
||||||
|
use std::pin::Pin;
|
||||||
|
|
||||||
|
/// Observer notified by [`DedupService`] when a genuinely new blob is stored
|
||||||
|
/// for the first time (no dedup hit).
|
||||||
|
///
|
||||||
|
/// Register with [`DedupService::add_blob_creation_hook`] during DI wiring.
|
||||||
|
pub trait BlobCreationHook: Send + Sync {
|
||||||
|
/// Called after the new blob's chunks and manifest have been written.
|
||||||
|
/// `blob_hash` is the BLAKE3 hex, `content_type` is the MIME type if known.
|
||||||
|
/// Must be best-effort — must not propagate errors.
|
||||||
|
fn on_blob_created<'a>(
|
||||||
|
&'a self,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
content_type: Option<&'a str>,
|
||||||
|
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||||
|
}
|
||||||
|
/// Observer notified by [`DedupService`] when a blob's ref_count reaches zero
|
||||||
|
/// and it is permanently removed from storage.
|
||||||
|
///
|
||||||
|
/// Implement this trait on any service that needs to react to blob deletion
|
||||||
|
/// (e.g. thumbnail cleanup, CDN invalidation, audit logging). Register with
|
||||||
|
/// [`DedupService::add_blob_hook`] during DI wiring.
|
||||||
|
///
|
||||||
|
/// The boxed-future return keeps the trait dyn-compatible so multiple
|
||||||
|
/// implementations can be stored as `Vec<Arc<dyn BlobDeletionHook>>`.
|
||||||
|
pub trait BlobDeletionHook: Send + Sync {
|
||||||
|
/// Called after the blob file has been removed from disk.
|
||||||
|
/// `blob_hash` is the BLAKE3 hex string identifying the blob.
|
||||||
|
/// Must be best-effort — must not propagate errors.
|
||||||
|
fn on_blob_deleted<'a>(
|
||||||
|
&'a self,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
use std::future::Future;
|
||||||
|
use std::pin::Pin;
|
||||||
|
|
||||||
|
/// Observer notified by [`FileUploadService`] when a new file record is created
|
||||||
|
/// (including dedup hits where the blob already exists).
|
||||||
|
///
|
||||||
|
/// Register with [`FileUploadService::with_file_created_hook`] during DI wiring.
|
||||||
|
pub trait FileCreatedHook: Send + Sync {
|
||||||
|
/// Called after the file record has been persisted.
|
||||||
|
/// `file_id` — opaque file UUID string.
|
||||||
|
/// `blob_hash` — BLAKE3 hex of the blob (may already exist on disk for dedup hits).
|
||||||
|
/// `content_type` — MIME type of the content.
|
||||||
|
/// Must be best-effort — must not propagate errors.
|
||||||
|
fn on_file_created<'a>(
|
||||||
|
&'a self,
|
||||||
|
file_id: &'a str,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
content_type: &'a str,
|
||||||
|
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Observer notified by [`FileUploadService`] when an existing file's blob is
|
||||||
|
/// replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload).
|
||||||
|
///
|
||||||
|
/// Implement this trait on any service that needs to react to a content swap
|
||||||
|
/// (e.g. thumbnail invalidation + regeneration, search index update).
|
||||||
|
/// Register with [`FileUploadService::with_file_updated_hook`] during DI wiring.
|
||||||
|
///
|
||||||
|
/// The boxed-future return keeps the trait dyn-compatible so multiple
|
||||||
|
/// implementations can be stored as `Vec<Arc<dyn FileUpdatedHook>>`.
|
||||||
|
pub trait FileUpdatedHook: Send + Sync {
|
||||||
|
/// Called after the new blob has been stored and the file record updated.
|
||||||
|
///
|
||||||
|
/// `file_id` is an opaque file UUID string, `blob_hash` is the BLAKE3 hex
|
||||||
|
/// of the new blob, `content_type` is the MIME type of the new content.
|
||||||
|
/// Must be best-effort — must not propagate errors.
|
||||||
|
fn on_file_updated<'a>(
|
||||||
|
&'a self,
|
||||||
|
file_id: &'a str,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
content_type: &'a str,
|
||||||
|
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Observer notified by [`FileManagementService`] when a file is permanently
|
||||||
|
/// deleted (either directly or after being emptied from trash).
|
||||||
|
///
|
||||||
|
/// Register with [`FileManagementService::with_file_deleted_hook`] during DI wiring.
|
||||||
|
pub trait FileDeletedHook: Send + Sync {
|
||||||
|
/// Called after the file record has been removed.
|
||||||
|
/// `file_id` — opaque file UUID string.
|
||||||
|
/// Must be best-effort — must not propagate errors.
|
||||||
|
fn on_file_deleted<'a>(
|
||||||
|
&'a self,
|
||||||
|
file_id: &'a str,
|
||||||
|
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
pub mod auth_ports;
|
pub mod auth_ports;
|
||||||
|
pub mod blob_lifecycle;
|
||||||
pub mod blob_storage_ports;
|
pub mod blob_storage_ports;
|
||||||
pub mod cache_ports;
|
pub mod cache_ports;
|
||||||
pub mod calendar_ports;
|
pub mod calendar_ports;
|
||||||
@@ -7,6 +8,7 @@ pub mod chunked_upload_ports;
|
|||||||
pub mod compression_ports;
|
pub mod compression_ports;
|
||||||
pub mod dedup_ports;
|
pub mod dedup_ports;
|
||||||
pub mod favorites_ports;
|
pub mod favorites_ports;
|
||||||
|
pub mod file_lifecycle;
|
||||||
pub mod file_ports;
|
pub mod file_ports;
|
||||||
pub mod inbound;
|
pub mod inbound;
|
||||||
pub mod music_ports;
|
pub mod music_ports;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::application::dtos::file_dto::FileDto;
|
use crate::application::dtos::file_dto::FileDto;
|
||||||
|
use crate::application::ports::file_lifecycle::FileDeletedHook;
|
||||||
use crate::application::ports::file_ports::FileManagementUseCase;
|
use crate::application::ports::file_ports::FileManagementUseCase;
|
||||||
use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileReadPort, FileWritePort};
|
use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileReadPort, FileWritePort};
|
||||||
use crate::application::ports::trash_ports::TrashUseCase;
|
use crate::application::ports::trash_ports::TrashUseCase;
|
||||||
@@ -11,7 +12,6 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob
|
|||||||
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
||||||
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
||||||
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
||||||
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
|
||||||
use tracing::{error, info, warn};
|
use tracing::{error, info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -26,8 +26,9 @@ pub struct FileManagementService {
|
|||||||
file_read: Option<Arc<FileBlobReadRepository>>,
|
file_read: Option<Arc<FileBlobReadRepository>>,
|
||||||
folder_repo: Option<Arc<FolderDbRepository>>,
|
folder_repo: Option<Arc<FolderDbRepository>>,
|
||||||
trash_service: Option<Arc<TrashService>>,
|
trash_service: Option<Arc<TrashService>>,
|
||||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
|
||||||
content_cache: Option<Arc<FileContentCache>>,
|
content_cache: Option<Arc<FileContentCache>>,
|
||||||
|
/// Hooks fired after a file is permanently deleted.
|
||||||
|
file_deleted_hooks: Vec<Arc<dyn FileDeletedHook>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FileManagementService {
|
impl FileManagementService {
|
||||||
@@ -38,8 +39,8 @@ impl FileManagementService {
|
|||||||
file_read: None,
|
file_read: None,
|
||||||
folder_repo: None,
|
folder_repo: None,
|
||||||
trash_service: None,
|
trash_service: None,
|
||||||
thumbnail_service: None,
|
|
||||||
content_cache: None,
|
content_cache: None,
|
||||||
|
file_deleted_hooks: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,7 +50,6 @@ impl FileManagementService {
|
|||||||
trash_service: Option<Arc<TrashService>>,
|
trash_service: Option<Arc<TrashService>>,
|
||||||
file_read: Option<Arc<FileBlobReadRepository>>,
|
file_read: Option<Arc<FileBlobReadRepository>>,
|
||||||
folder_repo: Option<Arc<FolderDbRepository>>,
|
folder_repo: Option<Arc<FolderDbRepository>>,
|
||||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
|
||||||
content_cache: Option<Arc<FileContentCache>>,
|
content_cache: Option<Arc<FileContentCache>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -57,11 +57,17 @@ impl FileManagementService {
|
|||||||
file_read,
|
file_read,
|
||||||
folder_repo,
|
folder_repo,
|
||||||
trash_service,
|
trash_service,
|
||||||
thumbnail_service,
|
|
||||||
content_cache,
|
content_cache,
|
||||||
|
file_deleted_hooks: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Registers a hook to fire after a file is permanently deleted.
|
||||||
|
pub fn with_file_deleted_hook(mut self, hook: Arc<dyn FileDeletedHook>) -> Self {
|
||||||
|
self.file_deleted_hooks.push(hook);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Verifies ownership via the read repository.
|
/// Verifies ownership via the read repository.
|
||||||
async fn verify_owner(&self, file_id: &str, caller_id: Uuid) -> Result<(), DomainError> {
|
async fn verify_owner(&self, file_id: &str, caller_id: Uuid) -> Result<(), DomainError> {
|
||||||
if let Some(read) = &self.file_read {
|
if let Some(read) = &self.file_read {
|
||||||
@@ -230,15 +236,11 @@ impl FileManagementUseCase for FileManagementService {
|
|||||||
|
|
||||||
async fn delete_file(&self, id: &str) -> Result<(), DomainError> {
|
async fn delete_file(&self, id: &str) -> Result<(), DomainError> {
|
||||||
self.file_repository.delete_file(id).await?;
|
self.file_repository.delete_file(id).await?;
|
||||||
// Invalidate content cache — file no longer exists.
|
|
||||||
if let Some(cc) = &self.content_cache {
|
if let Some(cc) = &self.content_cache {
|
||||||
cc.invalidate(id).await;
|
cc.invalidate(id).await;
|
||||||
}
|
}
|
||||||
// Best-effort thumbnail cleanup
|
for hook in &self.file_deleted_hooks {
|
||||||
if let Some(thumb) = &self.thumbnail_service
|
hook.on_file_deleted(id).await;
|
||||||
&& let Err(e) = thumb.delete_thumbnails(id).await
|
|
||||||
{
|
|
||||||
warn!("Failed to delete thumbnails for file {}: {}", id, e);
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -283,15 +285,11 @@ impl FileManagementUseCase for FileManagementService {
|
|||||||
// Step 2: Permanent delete — trigger handles blob ref_count
|
// Step 2: Permanent delete — trigger handles blob ref_count
|
||||||
warn!("Permanently deleting file: {}", id);
|
warn!("Permanently deleting file: {}", id);
|
||||||
self.file_repository.delete_file(id).await?;
|
self.file_repository.delete_file(id).await?;
|
||||||
// Invalidate content cache — file permanently removed.
|
|
||||||
if let Some(cc) = &self.content_cache {
|
if let Some(cc) = &self.content_cache {
|
||||||
cc.invalidate(id).await;
|
cc.invalidate(id).await;
|
||||||
}
|
}
|
||||||
// Best-effort thumbnail cleanup
|
for hook in &self.file_deleted_hooks {
|
||||||
if let Some(thumb) = &self.thumbnail_service
|
hook.on_file_deleted(id).await;
|
||||||
&& let Err(e) = thumb.delete_thumbnails(id).await
|
|
||||||
{
|
|
||||||
warn!("Failed to delete thumbnails for file {}: {}", id, e);
|
|
||||||
}
|
}
|
||||||
info!("File permanently deleted: {}", id);
|
info!("File permanently deleted: {}", id);
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use std::path::Path;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::application::dtos::file_dto::FileDto;
|
use crate::application::dtos::file_dto::FileDto;
|
||||||
|
use crate::application::ports::file_lifecycle::{FileCreatedHook, FileUpdatedHook};
|
||||||
use crate::application::ports::file_ports::FileUploadUseCase;
|
use crate::application::ports::file_ports::FileUploadUseCase;
|
||||||
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
|
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
|
||||||
use crate::application::services::storage_usage_service::StorageUsageService;
|
use crate::application::services::storage_usage_service::StorageUsageService;
|
||||||
@@ -52,6 +53,10 @@ pub struct FileUploadService {
|
|||||||
storage_usage_service: Option<Arc<StorageUsageService>>,
|
storage_usage_service: Option<Arc<StorageUsageService>>,
|
||||||
/// Content cache — invalidated on file update so stale content is never served.
|
/// Content cache — invalidated on file update so stale content is never served.
|
||||||
content_cache: Option<Arc<FileContentCache>>,
|
content_cache: Option<Arc<FileContentCache>>,
|
||||||
|
/// Hooks fired after a new file record is created.
|
||||||
|
file_created_hooks: Vec<Arc<dyn FileCreatedHook>>,
|
||||||
|
/// Hooks fired after a file's blob is replaced (e.g. thumbnail refresh).
|
||||||
|
file_updated_hooks: Vec<Arc<dyn FileUpdatedHook>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FileUploadService {
|
impl FileUploadService {
|
||||||
@@ -62,6 +67,8 @@ impl FileUploadService {
|
|||||||
file_read: None,
|
file_read: None,
|
||||||
storage_usage_service: None,
|
storage_usage_service: None,
|
||||||
content_cache: None,
|
content_cache: None,
|
||||||
|
file_created_hooks: Vec::new(),
|
||||||
|
file_updated_hooks: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,6 +82,8 @@ impl FileUploadService {
|
|||||||
file_read: Some(file_read),
|
file_read: Some(file_read),
|
||||||
storage_usage_service: None,
|
storage_usage_service: None,
|
||||||
content_cache: None,
|
content_cache: None,
|
||||||
|
file_created_hooks: Vec::new(),
|
||||||
|
file_updated_hooks: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,6 +93,18 @@ impl FileUploadService {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Registers a hook to fire after a new file record is created.
|
||||||
|
pub fn with_file_created_hook(mut self, hook: Arc<dyn FileCreatedHook>) -> Self {
|
||||||
|
self.file_created_hooks.push(hook);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Registers a hook to fire after a file's blob is replaced.
|
||||||
|
pub fn with_file_updated_hook(mut self, hook: Arc<dyn FileUpdatedHook>) -> Self {
|
||||||
|
self.file_updated_hooks.push(hook);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Configures the storage usage service
|
/// Configures the storage usage service
|
||||||
pub fn with_storage_usage_service(
|
pub fn with_storage_usage_service(
|
||||||
mut self,
|
mut self,
|
||||||
@@ -149,6 +170,10 @@ impl FileUploadUseCase for FileUploadService {
|
|||||||
name, size, dto.id
|
name, size, dto.id
|
||||||
);
|
);
|
||||||
self.maybe_update_storage_usage(&dto);
|
self.maybe_update_storage_usage(&dto);
|
||||||
|
for hook in &self.file_created_hooks {
|
||||||
|
hook.on_file_created(&dto.id, &dto.etag, &dto.mime_type)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
Ok(dto)
|
Ok(dto)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,7 +326,12 @@ impl FileUploadUseCase for FileUploadService {
|
|||||||
}
|
}
|
||||||
// Re-read to get fresh DTO with updated etag and timestamps.
|
// Re-read to get fresh DTO with updated etag and timestamps.
|
||||||
let updated = file_read.get_file(&file_id).await?;
|
let updated = file_read.get_file(&file_id).await?;
|
||||||
return Ok(FileDto::from(updated));
|
let dto = FileDto::from(updated);
|
||||||
|
for hook in &self.file_updated_hooks {
|
||||||
|
hook.on_file_updated(&file_id, &dto.etag, content_type)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
return Ok(dto);
|
||||||
}
|
}
|
||||||
|
|
||||||
// File doesn't exist — create it via streaming upload
|
// File doesn't exist — create it via streaming upload
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlob
|
|||||||
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository;
|
||||||
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository;
|
||||||
use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository;
|
use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository;
|
||||||
|
use crate::infrastructure::services::dedup_service::DedupService;
|
||||||
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
||||||
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
||||||
|
|
||||||
@@ -45,6 +46,10 @@ pub struct TrashService {
|
|||||||
/// Port for folder operations (get folder, trash, restore, delete)
|
/// Port for folder operations (get folder, trash, restore, delete)
|
||||||
folder_storage_port: Arc<FolderDbRepository>,
|
folder_storage_port: Arc<FolderDbRepository>,
|
||||||
|
|
||||||
|
/// Dedup service — garbage-collected after bulk trash empty to clean up
|
||||||
|
/// orphaned blob files and thumbnails that the PG trigger cannot reach.
|
||||||
|
dedup_service: Arc<DedupService>,
|
||||||
|
|
||||||
/// Thumbnail service for cleaning up thumbnails on permanent delete
|
/// Thumbnail service for cleaning up thumbnails on permanent delete
|
||||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
thumbnail_service: Option<Arc<ThumbnailService>>,
|
||||||
|
|
||||||
@@ -56,12 +61,14 @@ pub struct TrashService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TrashService {
|
impl TrashService {
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn new(
|
pub fn new(
|
||||||
trash_repository: Arc<TrashDbRepository>,
|
trash_repository: Arc<TrashDbRepository>,
|
||||||
file_read_port: Arc<FileBlobReadRepository>,
|
file_read_port: Arc<FileBlobReadRepository>,
|
||||||
file_write_port: Arc<FileBlobWriteRepository>,
|
file_write_port: Arc<FileBlobWriteRepository>,
|
||||||
folder_storage_port: Arc<FolderDbRepository>,
|
folder_storage_port: Arc<FolderDbRepository>,
|
||||||
retention_days: u32,
|
retention_days: u32,
|
||||||
|
dedup_service: Arc<DedupService>,
|
||||||
thumbnail_service: Option<Arc<ThumbnailService>>,
|
thumbnail_service: Option<Arc<ThumbnailService>>,
|
||||||
content_cache: Option<Arc<FileContentCache>>,
|
content_cache: Option<Arc<FileContentCache>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
@@ -70,6 +77,7 @@ impl TrashService {
|
|||||||
file_read_port,
|
file_read_port,
|
||||||
file_write_port,
|
file_write_port,
|
||||||
folder_storage_port,
|
folder_storage_port,
|
||||||
|
dedup_service,
|
||||||
thumbnail_service,
|
thumbnail_service,
|
||||||
content_cache,
|
content_cache,
|
||||||
retention_days,
|
retention_days,
|
||||||
@@ -713,18 +721,24 @@ impl TrashUseCase for TrashService {
|
|||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
|
|
||||||
// clear_trash() already performs bulk SQL DELETEs in 2 queries:
|
// clear_trash() performs bulk SQL DELETEs in 2 queries:
|
||||||
// 1. DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE
|
// 1. DELETE FROM storage.files WHERE user_id = $1 AND is_trashed = TRUE
|
||||||
// 2. DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE
|
// 2. DELETE FROM storage.folders WHERE user_id = $1 AND is_trashed = TRUE
|
||||||
//
|
//
|
||||||
// Folder deletion cascades (FK ON DELETE CASCADE) to child folders and
|
// Folder deletion cascades (FK ON DELETE CASCADE) to child folders and
|
||||||
// their files. The PG trigger `trg_files_decrement_blob_ref` automatically
|
// their files. The PG trigger `trg_files_decrement_blob_ref` automatically
|
||||||
// decrements blob ref_counts for every deleted file row — no Rust-side
|
// decrements blob ref_counts for every deleted file row.
|
||||||
// remove_reference() call is needed.
|
|
||||||
//
|
//
|
||||||
// Finally it clears the trash_items index for the user.
|
// Finally it clears the trash_items index for the user.
|
||||||
self.trash_repository.clear_trash(&user_id).await?;
|
self.trash_repository.clear_trash(&user_id).await?;
|
||||||
|
|
||||||
|
// The PG trigger decremented ref_counts but cannot delete disk files or
|
||||||
|
// thumbnails. Run garbage_collect() to remove any blobs whose ref_count
|
||||||
|
// reached 0, along with their blob-keyed thumbnail files.
|
||||||
|
if let Err(e) = self.dedup_service.garbage_collect().await {
|
||||||
|
warn!("empty_trash: garbage_collect failed: {:?}", e);
|
||||||
|
}
|
||||||
|
|
||||||
// Invalidate content cache for all permanently deleted files.
|
// Invalidate content cache for all permanently deleted files.
|
||||||
if let Some(cc) = &self.content_cache {
|
if let Some(cc) = &self.content_cache {
|
||||||
for file_id in &trashed_file_ids {
|
for file_id in &trashed_file_ids {
|
||||||
|
|||||||
+21
-11
@@ -62,7 +62,7 @@ use crate::infrastructure::services::image_transcode_service::ImageTranscodeServ
|
|||||||
use crate::infrastructure::services::jwt_service::JwtTokenService;
|
use crate::infrastructure::services::jwt_service::JwtTokenService;
|
||||||
use crate::infrastructure::services::password_hasher::Argon2PasswordHasher;
|
use crate::infrastructure::services::password_hasher::Argon2PasswordHasher;
|
||||||
use crate::infrastructure::services::path_resolver_service::PathResolverService;
|
use crate::infrastructure::services::path_resolver_service::PathResolverService;
|
||||||
use crate::infrastructure::services::thumbnail_service::ThumbnailService;
|
use crate::infrastructure::services::thumbnail_service::{ThumbnailRefreshHook, ThumbnailService};
|
||||||
use crate::infrastructure::services::wopi_discovery_service::WopiDiscoveryService;
|
use crate::infrastructure::services::wopi_discovery_service::WopiDiscoveryService;
|
||||||
use crate::infrastructure::services::zip_service::ZipService;
|
use crate::infrastructure::services::zip_service::ZipService;
|
||||||
|
|
||||||
@@ -264,7 +264,8 @@ impl AppServiceFactory {
|
|||||||
blob_backend,
|
blob_backend,
|
||||||
db_pool.clone(),
|
db_pool.clone(),
|
||||||
maintenance_pool.clone(),
|
maintenance_pool.clone(),
|
||||||
),
|
)
|
||||||
|
.add_blob_hook(thumbnail_service.clone()),
|
||||||
);
|
);
|
||||||
dedup_service.initialize().await?;
|
dedup_service.initialize().await?;
|
||||||
|
|
||||||
@@ -355,12 +356,18 @@ impl AppServiceFactory {
|
|||||||
|
|
||||||
// Refactored services with all infrastructure ports
|
// Refactored services with all infrastructure ports
|
||||||
// In blob model, dedup is handled by the repository — no separate write-behind needed
|
// In blob model, dedup is handled by the repository — no separate write-behind needed
|
||||||
|
let thumbnail_refresh_hook = Arc::new(ThumbnailRefreshHook::new(
|
||||||
|
core.thumbnail_service.clone(),
|
||||||
|
core.dedup_service.clone(),
|
||||||
|
));
|
||||||
let file_upload_service = Arc::new(
|
let file_upload_service = Arc::new(
|
||||||
FileUploadService::new_with_read(
|
FileUploadService::new_with_read(
|
||||||
repos.file_write_repository.clone(),
|
repos.file_write_repository.clone(),
|
||||||
repos.file_read_repository.clone(),
|
repos.file_read_repository.clone(),
|
||||||
)
|
)
|
||||||
.with_content_cache(core.file_content_cache.clone()),
|
.with_content_cache(core.file_content_cache.clone())
|
||||||
|
.with_file_created_hook(thumbnail_refresh_hook.clone())
|
||||||
|
.with_file_updated_hook(thumbnail_refresh_hook),
|
||||||
);
|
);
|
||||||
|
|
||||||
let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache(
|
let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache(
|
||||||
@@ -370,14 +377,16 @@ impl AppServiceFactory {
|
|||||||
));
|
));
|
||||||
|
|
||||||
// FileManagementService — ref_count handled by PG trigger, no dedup port needed
|
// FileManagementService — ref_count handled by PG trigger, no dedup port needed
|
||||||
let file_management_service = Arc::new(FileManagementService::with_trash(
|
let file_management_service = Arc::new(
|
||||||
repos.file_write_repository.clone(),
|
FileManagementService::with_trash(
|
||||||
trash_service.clone(),
|
repos.file_write_repository.clone(),
|
||||||
Some(repos.file_read_repository.clone()),
|
trash_service.clone(),
|
||||||
Some(repos.folder_repository.clone()),
|
Some(repos.file_read_repository.clone()),
|
||||||
Some(core.thumbnail_service.clone()),
|
Some(repos.folder_repository.clone()),
|
||||||
Some(core.file_content_cache.clone()),
|
Some(core.file_content_cache.clone()),
|
||||||
));
|
)
|
||||||
|
.with_file_deleted_hook(core.thumbnail_service.clone()),
|
||||||
|
);
|
||||||
|
|
||||||
let file_use_case_factory = Arc::new(AppFileUseCaseFactory::new(
|
let file_use_case_factory = Arc::new(AppFileUseCaseFactory::new(
|
||||||
repos.file_read_repository.clone(),
|
repos.file_read_repository.clone(),
|
||||||
@@ -451,6 +460,7 @@ impl AppServiceFactory {
|
|||||||
repos.file_write_repository.clone(),
|
repos.file_write_repository.clone(),
|
||||||
repos.folder_repository.clone(),
|
repos.folder_repository.clone(),
|
||||||
self.config.storage.trash_retention_days,
|
self.config.storage.trash_retention_days,
|
||||||
|
core.dedup_service.clone(),
|
||||||
Some(core.thumbnail_service.clone()),
|
Some(core.thumbnail_service.clone()),
|
||||||
Some(core.file_content_cache.clone()),
|
Some(core.file_content_cache.clone()),
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -604,8 +604,26 @@ impl FileWritePort for FileBlobWriteRepository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
|
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
|
||||||
// Same as delete_file — removes from DB and decrements blob ref
|
// Read blob_hash before deletion so we can clean up disk after the
|
||||||
self.delete_file(file_id).await
|
// PG trigger has decremented the ref_count.
|
||||||
|
let blob_hash: Option<String> =
|
||||||
|
sqlx::query_scalar("SELECT blob_hash FROM storage.files WHERE id = $1::uuid")
|
||||||
|
.bind(file_id)
|
||||||
|
.fetch_optional(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// DELETE fires trg_files_decrement_blob_ref → storage.blobs.ref_count--
|
||||||
|
self.delete_file(file_id).await?;
|
||||||
|
|
||||||
|
// If the blob is now unreferenced, remove disk file + thumbnails.
|
||||||
|
if let Some(hash) = blob_hash {
|
||||||
|
self.dedup.cleanup_if_orphaned(&hash).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn copy_folder_tree(
|
async fn copy_folder_tree(
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ use std::sync::Arc;
|
|||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||||
|
|
||||||
|
use crate::application::ports::blob_lifecycle::{BlobCreationHook, BlobDeletionHook};
|
||||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||||
use crate::application::ports::dedup_ports::{
|
use crate::application::ports::dedup_ports::{
|
||||||
BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto,
|
BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto,
|
||||||
@@ -83,6 +84,10 @@ pub struct DedupService {
|
|||||||
/// Isolated maintenance pool for long-running operations
|
/// Isolated maintenance pool for long-running operations
|
||||||
/// (verify_integrity, garbage_collect) that must never starve the primary.
|
/// (verify_integrity, garbage_collect) that must never starve the primary.
|
||||||
maintenance_pool: Arc<PgPool>,
|
maintenance_pool: Arc<PgPool>,
|
||||||
|
/// Hooks notified when a genuinely new blob is stored (no dedup hit).
|
||||||
|
blob_creation_hooks: Vec<Arc<dyn BlobCreationHook>>,
|
||||||
|
/// Hooks notified when a blob's ref_count reaches zero and it is deleted.
|
||||||
|
blob_hooks: Vec<Arc<dyn BlobDeletionHook>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DedupService {
|
impl DedupService {
|
||||||
@@ -100,6 +105,36 @@ impl DedupService {
|
|||||||
backend,
|
backend,
|
||||||
pool,
|
pool,
|
||||||
maintenance_pool,
|
maintenance_pool,
|
||||||
|
blob_creation_hooks: vec![],
|
||||||
|
blob_hooks: vec![],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Register a [`BlobCreationHook`] to be called whenever a genuinely new
|
||||||
|
/// blob is stored. Hooks are called in registration order.
|
||||||
|
pub fn add_blob_creation_hook(mut self, hook: Arc<dyn BlobCreationHook>) -> Self {
|
||||||
|
self.blob_creation_hooks.push(hook);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Register a [`BlobDeletionHook`] to be called whenever a blob's
|
||||||
|
/// ref_count reaches zero. Hooks are called in registration order.
|
||||||
|
pub fn add_blob_hook(mut self, hook: Arc<dyn BlobDeletionHook>) -> Self {
|
||||||
|
self.blob_hooks.push(hook);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fire all registered creation hooks for a new blob.
|
||||||
|
async fn fire_blob_creation_hooks(&self, hash: &str, content_type: Option<&str>) {
|
||||||
|
for hook in &self.blob_creation_hooks {
|
||||||
|
hook.on_blob_created(hash, content_type).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fire all registered hooks for a deleted blob.
|
||||||
|
async fn fire_blob_hooks(&self, hash: &str) {
|
||||||
|
for hook in &self.blob_hooks {
|
||||||
|
hook.on_blob_deleted(hash).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,6 +152,8 @@ impl DedupService {
|
|||||||
backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))),
|
backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))),
|
||||||
pool: stub_pool.clone(),
|
pool: stub_pool.clone(),
|
||||||
maintenance_pool: stub_pool,
|
maintenance_pool: stub_pool,
|
||||||
|
blob_creation_hooks: vec![],
|
||||||
|
blob_hooks: vec![],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -334,6 +371,9 @@ impl DedupService {
|
|||||||
chunk_hashes.len()
|
chunk_hashes.len()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
self.fire_blob_creation_hooks(&file_hash, content_type.as_deref())
|
||||||
|
.await;
|
||||||
|
|
||||||
Ok(DedupResultDto::NewBlob {
|
Ok(DedupResultDto::NewBlob {
|
||||||
hash: file_hash,
|
hash: file_hash,
|
||||||
size: file_size,
|
size: file_size,
|
||||||
@@ -587,7 +627,7 @@ impl DedupService {
|
|||||||
/// references the blob identified by `hash`.
|
/// references the blob identified by `hash`.
|
||||||
pub async fn user_owns_blob_reference(&self, hash: &str, user_id: &str) -> bool {
|
pub async fn user_owns_blob_reference(&self, hash: &str, user_id: &str) -> bool {
|
||||||
sqlx::query_scalar::<_, bool>(
|
sqlx::query_scalar::<_, bool>(
|
||||||
"SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2 AND NOT is_trashed)",
|
"SELECT EXISTS(SELECT 1 FROM storage.files WHERE blob_hash = $1 AND user_id = $2::uuid AND NOT is_trashed)",
|
||||||
)
|
)
|
||||||
.bind(hash)
|
.bind(hash)
|
||||||
.bind(user_id)
|
.bind(user_id)
|
||||||
@@ -772,6 +812,9 @@ impl DedupService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Bug 4 fix: notify hooks — e.g. thumbnail cleanup keyed by file_hash
|
||||||
|
self.fire_blob_hooks(file_hash).await;
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
"MANIFEST DELETED: {} ({} chunks, {} orphan chunks removed)",
|
"MANIFEST DELETED: {} ({} chunks, {} orphan chunks removed)",
|
||||||
&file_hash[..12],
|
&file_hash[..12],
|
||||||
@@ -849,6 +892,9 @@ impl DedupService {
|
|||||||
tracing::warn!("Failed to delete blob file {}: {}", hash, e);
|
tracing::warn!("Failed to delete blob file {}: {}", hash, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Bug 3 fix: notify hooks — e.g. thumbnail cleanup keyed by hash
|
||||||
|
self.fire_blob_hooks(hash).await;
|
||||||
|
|
||||||
tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]);
|
tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]);
|
||||||
Ok(true)
|
Ok(true)
|
||||||
} else {
|
} else {
|
||||||
@@ -874,6 +920,91 @@ impl DedupService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Targeted cleanup for a single blob after the PG trigger has already
|
||||||
|
/// decremented its ref_count. Deletes the blob row, disk file, and
|
||||||
|
/// blob-keyed thumbnails if ref_count has reached 0.
|
||||||
|
///
|
||||||
|
/// Handles both the legacy whole-file blob path (storage.blobs) and the
|
||||||
|
/// CDC manifest path (storage.chunk_manifests). Best-effort: logs
|
||||||
|
/// warnings on failure rather than returning an error.
|
||||||
|
pub async fn cleanup_if_orphaned(&self, hash: &str) {
|
||||||
|
let short = &hash[..hash.len().min(12)];
|
||||||
|
|
||||||
|
// ── CDC manifest path (must run FIRST) ───────────────────
|
||||||
|
// For single-chunk CDC files file_hash == chunk_hash, so the PG
|
||||||
|
// trigger on storage.files already decremented storage.blobs.ref_count
|
||||||
|
// when this function is called. try_dedup_hit increments
|
||||||
|
// chunk_manifests.ref_count but NOT storage.blobs.ref_count, so
|
||||||
|
// blobs.ref_count can reach 0 while the manifest still has ref_count > 1
|
||||||
|
// (other files sharing the same blob). Checking the manifest first
|
||||||
|
// prevents premature blob + manifest deletion.
|
||||||
|
let manifest = sqlx::query_as::<_, (i32, Vec<String>)>(
|
||||||
|
"SELECT ref_count, chunk_hashes \
|
||||||
|
FROM storage.chunk_manifests WHERE file_hash = $1",
|
||||||
|
)
|
||||||
|
.bind(hash)
|
||||||
|
.fetch_optional(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.unwrap_or(None);
|
||||||
|
|
||||||
|
if let Some((ref_count, chunk_hashes)) = manifest {
|
||||||
|
if ref_count <= 1 {
|
||||||
|
// Last reference — remove manifest and all its chunks.
|
||||||
|
if let Err(e) = self
|
||||||
|
.remove_manifest_reference(hash, ref_count, &chunk_hashes)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::warn!("cleanup_if_orphaned: manifest cleanup failed for {short}: {e}");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Other files still share this blob: just decrement the manifest
|
||||||
|
// counter and undo the PG trigger's premature chunk ref_count
|
||||||
|
// decrement (blobs.ref_count is chunk-level; the manifest is the
|
||||||
|
// authoritative file-level counter).
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE storage.chunk_manifests \
|
||||||
|
SET ref_count = ref_count - 1 WHERE file_hash = $1",
|
||||||
|
)
|
||||||
|
.bind(hash)
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
// Undo the PG trigger's decrement of storage.blobs.ref_count.
|
||||||
|
// The trigger fired with blob_hash = file_hash, so only the row
|
||||||
|
// WHERE hash = file_hash is affected. For single-chunk files
|
||||||
|
// file_hash == chunk_hash and that row exists; for multi-chunk
|
||||||
|
// files file_hash is not in storage.blobs, making this a no-op.
|
||||||
|
sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1")
|
||||||
|
.bind(hash)
|
||||||
|
.execute(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
tracing::debug!(
|
||||||
|
"cleanup_if_orphaned: manifest {short} ref_count {ref_count}→{}",
|
||||||
|
ref_count - 1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Legacy blob path (no manifest) ───────────────────────
|
||||||
|
let deleted_blob = sqlx::query_scalar::<_, String>(
|
||||||
|
"DELETE FROM storage.blobs WHERE hash = $1 AND ref_count <= 0 RETURNING hash",
|
||||||
|
)
|
||||||
|
.bind(hash)
|
||||||
|
.fetch_optional(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.unwrap_or(None);
|
||||||
|
|
||||||
|
if deleted_blob.is_some() {
|
||||||
|
if let Err(e) = self.backend.delete_blob(hash).await {
|
||||||
|
tracing::warn!("cleanup_if_orphaned: disk delete failed for {short}: {e}");
|
||||||
|
}
|
||||||
|
self.fire_blob_hooks(hash).await;
|
||||||
|
tracing::info!("cleanup_if_orphaned: removed orphaned blob {short}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Read operations ──────────────────────────────────────────
|
// ── Read operations ──────────────────────────────────────────
|
||||||
|
|
||||||
/// Stream blob content — CDC-aware with legacy fallback.
|
/// Stream blob content — CDC-aware with legacy fallback.
|
||||||
@@ -1324,16 +1455,7 @@ impl DedupService {
|
|||||||
if let Err(e) = self.backend.delete_blob(hash).await {
|
if let Err(e) = self.backend.delete_blob(hash).await {
|
||||||
tracing::warn!("Failed to delete orphan blob {hash}: {e}");
|
tracing::warn!("Failed to delete orphan blob {hash}: {e}");
|
||||||
}
|
}
|
||||||
// Clean up thumbnails (best-effort, only local backends)
|
self.fire_blob_hooks(hash).await;
|
||||||
if let Some(blob_path) = self.backend.local_blob_path(hash)
|
|
||||||
&& let Some(storage_root) = blob_path.ancestors().nth(3)
|
|
||||||
{
|
|
||||||
let thumbnails_root = storage_root.join(".thumbnails");
|
|
||||||
for dir in &["icon", "preview", "large"] {
|
|
||||||
let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg"));
|
|
||||||
let _ = fs::remove_file(&thumb).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
total_bytes += *size as u64;
|
total_bytes += *size as u64;
|
||||||
}
|
}
|
||||||
total_deleted += batch.len() as u64;
|
total_deleted += batch.len() as u64;
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ use crate::application::ports::thumbnail_ports::{
|
|||||||
ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto,
|
ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto,
|
||||||
};
|
};
|
||||||
use crate::domain::errors::{DomainError, ErrorKind};
|
use crate::domain::errors::{DomainError, ErrorKind};
|
||||||
|
use crate::infrastructure::services::dedup_service::DedupService;
|
||||||
|
|
||||||
/// Thumbnail sizes supported by the system
|
/// Thumbnail sizes supported by the system
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
@@ -831,10 +832,22 @@ impl ThumbnailService {
|
|||||||
file_id: String,
|
file_id: String,
|
||||||
blob_hash: String,
|
blob_hash: String,
|
||||||
original_data: Bytes,
|
original_data: Bytes,
|
||||||
|
dedup: Arc<DedupService>,
|
||||||
) {
|
) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id);
|
tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id);
|
||||||
|
|
||||||
|
// Guard: if the blob was deleted before this task ran, cleanup_if_orphaned
|
||||||
|
// already fired with no thumbnails on disk — writing them now would leak them.
|
||||||
|
// Use the DB check (manifest + blobs tables) as the authoritative source.
|
||||||
|
if !dedup.blob_exists(&blob_hash).await {
|
||||||
|
tracing::debug!(
|
||||||
|
"Blob {}… deleted before thumbnail task ran, skipping",
|
||||||
|
&blob_hash[..blob_hash.len().min(12)]
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let all_exist = {
|
let all_exist = {
|
||||||
let mut ok = true;
|
let mut ok = true;
|
||||||
for size in ThumbnailSize::all() {
|
for size in ThumbnailSize::all() {
|
||||||
@@ -971,6 +984,122 @@ impl ThumbnailService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── BlobDeletionHook ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
impl crate::application::ports::blob_lifecycle::BlobDeletionHook for ThumbnailService {
|
||||||
|
fn on_blob_deleted<'a>(
|
||||||
|
&'a self,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||||
|
Box::pin(async move { self.delete_blob_thumbnails(blob_hash).await })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── FileUpdatedHook ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Wires thumbnail invalidation + regeneration into the file-update lifecycle.
|
||||||
|
///
|
||||||
|
/// Registered on [`FileUploadService`] during DI. Fires whenever a file's blob
|
||||||
|
/// is replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload).
|
||||||
|
pub struct ThumbnailRefreshHook {
|
||||||
|
thumbnail: Arc<ThumbnailService>,
|
||||||
|
dedup: Arc<DedupService>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ThumbnailRefreshHook {
|
||||||
|
pub fn new(thumbnail: Arc<ThumbnailService>, dedup: Arc<DedupService>) -> Self {
|
||||||
|
Self { thumbnail, dedup }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::application::ports::file_lifecycle::FileUpdatedHook for ThumbnailRefreshHook {
|
||||||
|
fn on_file_updated<'a>(
|
||||||
|
&'a self,
|
||||||
|
file_id: &'a str,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
content_type: &'a str,
|
||||||
|
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
if !ThumbnailService::is_supported_image(content_type) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Err(e) = self.thumbnail.delete_thumbnails(file_id).await {
|
||||||
|
tracing::warn!(
|
||||||
|
"Failed to invalidate thumbnail cache for {}: {}",
|
||||||
|
file_id,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Self::spawn_thumbnail_generation(
|
||||||
|
self.thumbnail.clone(),
|
||||||
|
self.dedup.clone(),
|
||||||
|
file_id.to_string(),
|
||||||
|
blob_hash.to_string(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::application::ports::file_lifecycle::FileCreatedHook for ThumbnailRefreshHook {
|
||||||
|
fn on_file_created<'a>(
|
||||||
|
&'a self,
|
||||||
|
file_id: &'a str,
|
||||||
|
blob_hash: &'a str,
|
||||||
|
content_type: &'a str,
|
||||||
|
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
if !ThumbnailService::is_supported_image(content_type) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Self::spawn_thumbnail_generation(
|
||||||
|
self.thumbnail.clone(),
|
||||||
|
self.dedup.clone(),
|
||||||
|
file_id.to_string(),
|
||||||
|
blob_hash.to_string(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ThumbnailRefreshHook {
|
||||||
|
fn spawn_thumbnail_generation(
|
||||||
|
ts: Arc<ThumbnailService>,
|
||||||
|
ds: Arc<DedupService>,
|
||||||
|
file_id: String,
|
||||||
|
hash: String,
|
||||||
|
) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
match ds.read_blob_bytes(&hash).await {
|
||||||
|
Ok(bytes) => {
|
||||||
|
ts.generate_all_sizes_background_from_bytes(file_id, hash, bytes, ds.clone());
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Failed to read blob for thumbnail generation {}: {}",
|
||||||
|
file_id,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── FileDeletedHook ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
impl crate::application::ports::file_lifecycle::FileDeletedHook for ThumbnailService {
|
||||||
|
fn on_file_deleted<'a>(
|
||||||
|
&'a self,
|
||||||
|
file_id: &'a str,
|
||||||
|
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
if let Err(e) = self.delete_thumbnails(file_id).await {
|
||||||
|
tracing::warn!("Failed to delete thumbnails for file {}: {}", file_id, e);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ─── Port implementation ─────────────────────────────────────────────────────
|
// ─── Port implementation ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Convert port ThumbnailSize to infra ThumbnailSize.
|
/// Convert port ThumbnailSize to infra ThumbnailSize.
|
||||||
|
|||||||
@@ -21,12 +21,14 @@ type GlobalState = Arc<AppState>;
|
|||||||
pub struct HashCheckResponse {
|
pub struct HashCheckResponse {
|
||||||
/// Whether a blob with this hash already exists
|
/// Whether a blob with this hash already exists
|
||||||
pub exists: bool,
|
pub exists: bool,
|
||||||
/// The SHA-256 hash that was checked
|
/// The BLAKE3 hash that was checked
|
||||||
pub hash: String,
|
pub hash: String,
|
||||||
/// If exists, the size of the existing blob
|
/// If exists, the size of the existing blob
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub existing_size: Option<u64>,
|
pub existing_size: Option<u64>,
|
||||||
/// If exists, the number of references to this blob
|
/// Global reference count for this blob across all users.
|
||||||
|
/// Only populated when the authenticated user has the `admin` role;
|
||||||
|
/// omitted for regular users to prevent cross-user content inference.
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub ref_count: Option<u32>,
|
pub ref_count: Option<u32>,
|
||||||
}
|
}
|
||||||
@@ -36,7 +38,7 @@ pub struct HashCheckResponse {
|
|||||||
pub struct DedupUploadResponse {
|
pub struct DedupUploadResponse {
|
||||||
/// Whether this was a new file or an existing one
|
/// Whether this was a new file or an existing one
|
||||||
pub is_new: bool,
|
pub is_new: bool,
|
||||||
/// The SHA-256 hash of the content
|
/// The BLAKE3 hash of the content
|
||||||
pub hash: String,
|
pub hash: String,
|
||||||
/// The size of the content in bytes
|
/// The size of the content in bytes
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
@@ -85,7 +87,8 @@ impl DedupHandler {
|
|||||||
/// Check if the authenticated user already has a file with the given hash.
|
/// Check if the authenticated user already has a file with the given hash.
|
||||||
///
|
///
|
||||||
/// User-scoped: only reveals whether **this user** owns a file that
|
/// User-scoped: only reveals whether **this user** owns a file that
|
||||||
/// references the blob — never exposes global existence or ref_count.
|
/// references the blob — never exposes global existence to non-admins.
|
||||||
|
/// Admins additionally receive the global `ref_count` in the response.
|
||||||
///
|
///
|
||||||
/// GET /api/dedup/check/{hash}
|
/// GET /api/dedup/check/{hash}
|
||||||
pub(super) async fn check_hash_impl(
|
pub(super) async fn check_hash_impl(
|
||||||
@@ -95,13 +98,13 @@ impl DedupHandler {
|
|||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
let dedup = &state.core.dedup_service;
|
let dedup = &state.core.dedup_service;
|
||||||
|
|
||||||
// Validate hash format (SHA-256 = 64 hex chars)
|
// Validate hash format (BLAKE3 = 64 hex chars)
|
||||||
if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) {
|
if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||||
return Response::builder()
|
return Response::builder()
|
||||||
.status(StatusCode::BAD_REQUEST)
|
.status(StatusCode::BAD_REQUEST)
|
||||||
.header(header::CONTENT_TYPE, "application/json")
|
.header(header::CONTENT_TYPE, "application/json")
|
||||||
.body(Body::from(
|
.body(Body::from(
|
||||||
r#"{"error": "Invalid hash format. Expected SHA-256 (64 hex characters)"}"#,
|
r#"{"error": "Invalid hash format. Expected BLAKE3 (64 hex characters)"}"#,
|
||||||
))
|
))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.into_response();
|
.into_response();
|
||||||
@@ -113,13 +116,20 @@ impl DedupHandler {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
if user_has_it {
|
if user_has_it {
|
||||||
// Fetch size from metadata (safe — user owns a reference)
|
// Fetch size from metadata (safe — user owns a reference).
|
||||||
let size = dedup.get_blob_metadata(&hash).await.map(|m| m.size);
|
// Admins also get the global ref_count for dedup accounting tests.
|
||||||
|
let metadata = dedup.get_blob_metadata(&hash).await;
|
||||||
|
let size = metadata.as_ref().map(|m| m.size);
|
||||||
|
let ref_count = if auth_user.role == "admin" {
|
||||||
|
metadata.map(|m| m.ref_count)
|
||||||
|
} else {
|
||||||
|
None // Never expose global ref_count to regular users
|
||||||
|
};
|
||||||
let response = HashCheckResponse {
|
let response = HashCheckResponse {
|
||||||
exists: true,
|
exists: true,
|
||||||
hash,
|
hash,
|
||||||
existing_size: size,
|
existing_size: size,
|
||||||
ref_count: None, // Never expose global ref_count
|
ref_count,
|
||||||
};
|
};
|
||||||
Response::builder()
|
Response::builder()
|
||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
@@ -508,10 +518,10 @@ impl DedupHandler {
|
|||||||
get,
|
get,
|
||||||
path = "/api/dedup/check/{hash}",
|
path = "/api/dedup/check/{hash}",
|
||||||
params(
|
params(
|
||||||
("hash" = String, Path, description = "SHA-256 hash (64 hex characters)"),
|
("hash" = String, Path, description = "BLAKE3 hash (64 hex characters)"),
|
||||||
),
|
),
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "Hash check result (user-scoped)", body = HashCheckResponse),
|
(status = 200, description = "Hash check result. `ref_count` is only present for admin users.", body = HashCheckResponse),
|
||||||
(status = 400, description = "Invalid hash format"),
|
(status = 400, description = "Invalid hash format"),
|
||||||
),
|
),
|
||||||
tag = "dedup",
|
tag = "dedup",
|
||||||
@@ -564,7 +574,7 @@ pub async fn get_stats(state: State<GlobalState>, auth_user: AuthUser) -> impl I
|
|||||||
get,
|
get,
|
||||||
path = "/api/dedup/blob/{hash}",
|
path = "/api/dedup/blob/{hash}",
|
||||||
params(
|
params(
|
||||||
("hash" = String, Path, description = "SHA-256 hash of the blob (64 hex characters)"),
|
("hash" = String, Path, description = "BLAKE3 hash of the blob (64 hex characters)"),
|
||||||
),
|
),
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "Raw blob content (user-scoped)"),
|
(status = 200, description = "Raw blob content (user-scoped)"),
|
||||||
|
|||||||
@@ -783,6 +783,7 @@ impl FileHandler {
|
|||||||
file_id,
|
file_id,
|
||||||
blob_hash_owned,
|
blob_hash_owned,
|
||||||
original_bytes,
|
original_bytes,
|
||||||
|
dedup_service.clone(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
|||||||
@@ -288,7 +288,7 @@ async fn put_file(
|
|||||||
let _ = tokio::fs::remove_file(&temp_path).await;
|
let _ = tokio::fs::remove_file(&temp_path).await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => StatusCode::OK.into_response(),
|
Ok(_file_dto) => StatusCode::OK.into_response(),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("WOPI PutFile failed: {}", e);
|
tracing::error!("WOPI PutFile failed: {}", e);
|
||||||
StatusCode::INTERNAL_SERVER_ERROR.into_response()
|
StatusCode::INTERNAL_SERVER_ERROR.into_response()
|
||||||
|
|||||||
@@ -163,6 +163,7 @@ async fn handle_assemble(
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?;
|
.map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?;
|
||||||
|
|
||||||
Some(dto.etag)
|
Some(dto.etag)
|
||||||
} else {
|
} else {
|
||||||
// For new files we still need to read the temp file since create_file takes &[u8].
|
// For new files we still need to read the temp file since create_file takes &[u8].
|
||||||
|
|||||||
@@ -0,0 +1,271 @@
|
|||||||
|
# =============================================================
|
||||||
|
# OxiCloud – Dedup blob lifecycle (bugs 3 & 4)
|
||||||
|
# =============================================================
|
||||||
|
# Verifies that when two files share the same blob (dedup hit)
|
||||||
|
# and both are permanently deleted, the blob lifecycle is correct.
|
||||||
|
#
|
||||||
|
# Bug 3: blob not deleted when last file reference is removed
|
||||||
|
# Bug 4: blob-keyed thumbnail not cleaned up with the blob
|
||||||
|
#
|
||||||
|
# Sequence:
|
||||||
|
# 1. Upload dedup-test.jpg twice → two file records, one blob
|
||||||
|
# 2. Both thumbnails return identical bytes → proves shared blob
|
||||||
|
# 3. Permanently delete file 1 → file 2 thumbnail still 200
|
||||||
|
# (proves blob NOT prematurely deleted — bug 3 detection)
|
||||||
|
# 4. Permanently delete file 2 → blob and thumbnail cleaned up
|
||||||
|
#
|
||||||
|
# NOTE: The /api/dedup/stats endpoint counts CDC chunk rows in
|
||||||
|
# storage.blobs and derives bytes_saved from chunk_manifests.
|
||||||
|
# Both tables may be 0 when the CDC path is disabled or the
|
||||||
|
# server uses the legacy blob path — so we avoid stats-based
|
||||||
|
# assertions and rely on observable thumbnail behaviour instead.
|
||||||
|
#
|
||||||
|
# BLAKE3 hash of fixtures/dedup-test.jpg (= dedup-test-2.jpg content):
|
||||||
|
# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||||
|
# Used in /api/dedup/check/{hash} calls below to track ref_count lifecycle.
|
||||||
|
# ref_count is only returned for admin users; setup.hurl creates an admin.
|
||||||
|
#
|
||||||
|
# Prerequisites: setup.hurl must have run (admin user exists).
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# hurl --variables-file tests/api/test.env --test tests/api/dedup_blob_cleanup.hurl
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 1 – Login as admin
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
POST {{base_url}}/api/auth/login
|
||||||
|
Content-Type: application/json
|
||||||
|
{
|
||||||
|
"username": "{{username}}",
|
||||||
|
"password": "{{password}}"
|
||||||
|
}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Captures]
|
||||||
|
token: jsonpath "$.access_token"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.access_token" isString
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 2 – Create a folder for this test
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
GET {{base_url}}/api/folders
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Captures]
|
||||||
|
home_folder_id: jsonpath "$[0].id"
|
||||||
|
|
||||||
|
|
||||||
|
POST {{base_url}}/api/folders
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
Content-Type: application/json
|
||||||
|
{
|
||||||
|
"name": "hurl-dedup-blob-test",
|
||||||
|
"parent_id": "{{home_folder_id}}"
|
||||||
|
}
|
||||||
|
|
||||||
|
HTTP 201
|
||||||
|
[Captures]
|
||||||
|
test_folder_id: jsonpath "$.id"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.name" == "hurl-dedup-blob-test"
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 3 – Upload dedup-test.jpg (file 1)
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
POST {{base_url}}/api/files/upload
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
[MultipartFormData]
|
||||||
|
folder_id: {{test_folder_id}}
|
||||||
|
file: file,fixtures/dedup-test.jpg; image/jpeg
|
||||||
|
|
||||||
|
HTTP 201
|
||||||
|
[Captures]
|
||||||
|
file1_id: jsonpath "$.id"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.name" == "dedup-test.jpg"
|
||||||
|
jsonpath "$.folder_id" == {{test_folder_id}}
|
||||||
|
|
||||||
|
|
||||||
|
# ref_count == 1: blob has exactly one file reference after first upload
|
||||||
|
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.exists" == true
|
||||||
|
jsonpath "$.ref_count" == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 4 – Upload identical content again as dedup-test-2.jpg
|
||||||
|
# Dedup: same blob, new file record, different file ID
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
POST {{base_url}}/api/files/upload
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
[MultipartFormData]
|
||||||
|
folder_id: {{test_folder_id}}
|
||||||
|
file: file,fixtures/dedup-test-2.jpg; image/jpeg
|
||||||
|
|
||||||
|
HTTP 201
|
||||||
|
[Captures]
|
||||||
|
file2_id: jsonpath "$.id"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.name" == "dedup-test-2.jpg"
|
||||||
|
jsonpath "$.id" != "{{file1_id}}"
|
||||||
|
|
||||||
|
|
||||||
|
# ref_count == 2: dedup hit — same blob now referenced by two file records
|
||||||
|
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.exists" == true
|
||||||
|
jsonpath "$.ref_count" == 2
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 5 – Dedup proof: thumbnails are byte-identical
|
||||||
|
# Thumbnail generation reads blob bytes and is keyed by
|
||||||
|
# blob_hash on disk. If both files share the same blob,
|
||||||
|
# GET /thumbnail returns the same bytes for both.
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
GET {{base_url}}/api/files/{{file1_id}}/thumbnail/icon
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Captures]
|
||||||
|
thumb1: bytes
|
||||||
|
|
||||||
|
|
||||||
|
GET {{base_url}}/api/files/{{file2_id}}/thumbnail/icon
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Asserts]
|
||||||
|
bytes == {{thumb1}}
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 6 – Move file 1 to trash
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
DELETE {{base_url}}/api/files/{{file1_id}}
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 204
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 7 – Permanently delete file 1 from trash
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
GET {{base_url}}/api/trash
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Captures]
|
||||||
|
trash_item1_id: jsonpath "$[?(@.original_id == '{{file1_id}}')].id"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$[?(@.original_id == '{{file1_id}}')].id" isString
|
||||||
|
jsonpath "$[?(@.original_id == '{{file1_id}}')].item_type" == "file"
|
||||||
|
|
||||||
|
|
||||||
|
DELETE {{base_url}}/api/trash/{{trash_item1_id}}
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
|
||||||
|
|
||||||
|
# ref_count == 1: blob survives — file2 still holds a reference
|
||||||
|
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.exists" == true
|
||||||
|
jsonpath "$.ref_count" == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 8 – Blob still alive: file 2 thumbnail is accessible
|
||||||
|
# After file 1 is permanently deleted the blob ref_count
|
||||||
|
# drops to 1 but the blob must NOT be removed yet.
|
||||||
|
# Thumbnail generation reads blob bytes live — a 200 here
|
||||||
|
# proves the blob is still present.
|
||||||
|
# If bug 3 is present the blob is deleted prematurely and
|
||||||
|
# this request returns a 5xx error.
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
GET {{base_url}}/api/files/{{file2_id}}/thumbnail/icon
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 9 – Move file 2 to trash
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
DELETE {{base_url}}/api/files/{{file2_id}}
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 204
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 10 – Permanently delete file 2 from trash
|
||||||
|
# ref_count hits 0 → blob and its disk thumbnail deleted
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
GET {{base_url}}/api/trash
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Captures]
|
||||||
|
trash_item2_id: jsonpath "$[?(@.original_id == '{{file2_id}}')].id"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$[?(@.original_id == '{{file2_id}}')].id" isString
|
||||||
|
jsonpath "$[?(@.original_id == '{{file2_id}}')].item_type" == "file"
|
||||||
|
|
||||||
|
|
||||||
|
DELETE {{base_url}}/api/trash/{{trash_item2_id}}
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
|
||||||
|
|
||||||
|
# ref_count hits 0 → blob and manifest deleted; user no longer owns this hash
|
||||||
|
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$.exists" == false
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Step 11 – Cleanup: delete the (now empty) test folder
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
DELETE {{base_url}}/api/folders/{{test_folder_id}}
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 204
|
||||||
|
|
||||||
|
|
||||||
|
GET {{base_url}}/api/trash
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
|
[Captures]
|
||||||
|
trash_folder_id: jsonpath "$[?(@.original_id == '{{test_folder_id}}')].id"
|
||||||
|
[Asserts]
|
||||||
|
jsonpath "$[?(@.original_id == '{{test_folder_id}}')].id" isString
|
||||||
|
jsonpath "$[?(@.original_id == '{{test_folder_id}}')].item_type" == "folder"
|
||||||
|
|
||||||
|
|
||||||
|
DELETE {{base_url}}/api/trash/{{trash_folder_id}}
|
||||||
|
Authorization: Bearer {{token}}
|
||||||
|
|
||||||
|
HTTP 200
|
||||||
@@ -62,6 +62,9 @@ OXICLOUD_SERVER_PORT=$SERVER_PORT
|
|||||||
OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
|
OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
|
||||||
set +a
|
set +a
|
||||||
|
|
||||||
|
# ensure storage is empty before starting
|
||||||
|
echo "Wipe $OXICLOUD_STORAGE_PATH to ensure clean startup"
|
||||||
|
rm -rf "$OXICLOUD_STORAGE_PATH"
|
||||||
mkdir -p "$OXICLOUD_STORAGE_PATH"
|
mkdir -p "$OXICLOUD_STORAGE_PATH"
|
||||||
|
|
||||||
# ── 3. Start OxiCloud server ──────────────────────────────────────────────────
|
# ── 3. Start OxiCloud server ──────────────────────────────────────────────────
|
||||||
@@ -92,6 +95,11 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test
|
|||||||
"$API_DIR/trash.hurl" \
|
"$API_DIR/trash.hurl" \
|
||||||
"$API_DIR/recent.hurl" \
|
"$API_DIR/recent.hurl" \
|
||||||
"$API_DIR/batch_folder_copy.hurl" \
|
"$API_DIR/batch_folder_copy.hurl" \
|
||||||
|
"$API_DIR/dedup_blob_cleanup.hurl" \
|
||||||
"$API_DIR/contacts.hurl"
|
"$API_DIR/contacts.hurl"
|
||||||
|
|
||||||
|
#bash "$API_DIR/dedup_bulk_upload.sh"
|
||||||
|
|
||||||
|
bash "$API_DIR/storage_cleanup_check.sh"
|
||||||
|
|
||||||
log "All tests passed."
|
log "All tests passed."
|
||||||
|
|||||||
Executable
+137
@@ -0,0 +1,137 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================
|
||||||
|
# OxiCloud – Storage disk-cleanup verification
|
||||||
|
# =============================================================
|
||||||
|
# 1. Moves every live file and folder to trash via the REST API.
|
||||||
|
# 2. Calls DELETE /api/trash/empty to permanently delete all
|
||||||
|
# remaining trash items (including any left by previous tests).
|
||||||
|
# 3. Asserts that no regular files remain under
|
||||||
|
# $OXICLOUD_STORAGE_PATH/.thumbnails or .blobs.
|
||||||
|
#
|
||||||
|
# Called by run.sh after all Hurl tests have passed.
|
||||||
|
# Can also be run standalone (server must already be up):
|
||||||
|
# bash tests/api/storage_cleanup_check.sh
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
STORAGE_PATH="${OXICLOUD_STORAGE_PATH:-$REPO_ROOT/tests/api/storage}"
|
||||||
|
|
||||||
|
# shellcheck source=test.env
|
||||||
|
source "$SCRIPT_DIR/test.env"
|
||||||
|
|
||||||
|
log() { echo "[storage-check] $*"; }
|
||||||
|
fail() { echo $'\e[31m'"[storage-check] FAIL: $*"$'\e[0m' >&2; exit 1; }
|
||||||
|
|
||||||
|
# ── 1. Login ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{\"username\":\"$username\",\"password\":\"$password\"}" \
|
||||||
|
| jq -r '.access_token')
|
||||||
|
|
||||||
|
[[ -z "$TOKEN" || "$TOKEN" == "null" ]] && fail "login failed"
|
||||||
|
log "Logged in."
|
||||||
|
|
||||||
|
AUTH="Authorization: Bearer $TOKEN"
|
||||||
|
|
||||||
|
# ── 1b. Upload a probe image and verify its blob + thumbnail exist on disk ─────
|
||||||
|
|
||||||
|
# shellcheck source=../common/internal_storage_helper.sh
|
||||||
|
source "$REPO_ROOT/tests/common/internal_storage_helper.sh"
|
||||||
|
|
||||||
|
FIXTURE="$REPO_ROOT/tests/fixtures/blue-image.png"
|
||||||
|
|
||||||
|
HOME_FOLDER_ID=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[0].id')
|
||||||
|
[[ -z "$HOME_FOLDER_ID" || "$HOME_FOLDER_ID" == "null" ]] && fail "could not get home folder id"
|
||||||
|
|
||||||
|
PROBE_FILE_ID=$(curl -sf -X POST -H "$AUTH" \
|
||||||
|
-F "folder_id=$HOME_FOLDER_ID" \
|
||||||
|
-F "file=@$FIXTURE;type=image/png" \
|
||||||
|
"$base_url/api/files/upload" | jq -r '.id')
|
||||||
|
[[ -z "$PROBE_FILE_ID" || "$PROBE_FILE_ID" == "null" ]] && fail "probe file upload failed"
|
||||||
|
log "Probe file uploaded (id=$PROBE_FILE_ID)."
|
||||||
|
|
||||||
|
# GET thumbnail to trigger on-demand generation
|
||||||
|
HTTP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" -H "$AUTH" \
|
||||||
|
"$base_url/api/files/$PROBE_FILE_ID/thumbnail/icon")
|
||||||
|
[[ "$HTTP_STATUS" != "200" ]] && fail "thumbnail GET returned HTTP $HTTP_STATUS (expected 200)"
|
||||||
|
log "Thumbnail fetched (HTTP 200)."
|
||||||
|
|
||||||
|
assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not found on disk"
|
||||||
|
assert_preview_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk"
|
||||||
|
log "Probe blob and thumbnail confirmed present on disk."
|
||||||
|
|
||||||
|
# ── 2. Move all live files and folders to trash ───────────────────────────────
|
||||||
|
#
|
||||||
|
# For each root folder, list its direct children and soft-delete them.
|
||||||
|
# The server cascades folder deletion to all nested contents, so we only
|
||||||
|
# need to iterate one level deep.
|
||||||
|
|
||||||
|
ROOT_FOLDERS=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[].id')
|
||||||
|
|
||||||
|
for folder_id in $ROOT_FOLDERS; do
|
||||||
|
CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
|
||||||
|
|
||||||
|
while IFS= read -r sub_id; do
|
||||||
|
[[ -z "$sub_id" ]] && continue
|
||||||
|
curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$sub_id" >/dev/null
|
||||||
|
done < <(echo "$CONTENTS" | jq -r '.folders[].id')
|
||||||
|
|
||||||
|
while IFS= read -r file_id; do
|
||||||
|
[[ -z "$file_id" ]] && continue
|
||||||
|
curl -sf -X DELETE -H "$AUTH" "$base_url/api/files/$file_id" >/dev/null
|
||||||
|
done < <(echo "$CONTENTS" | jq -r '.files[].id')
|
||||||
|
done
|
||||||
|
|
||||||
|
log "All live objects moved to trash."
|
||||||
|
|
||||||
|
# ── 2b. Verify all root folders are empty according to the API ────────────────
|
||||||
|
|
||||||
|
for folder_id in $ROOT_FOLDERS; do
|
||||||
|
CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
|
||||||
|
SUB_COUNT=$(echo "$CONTENTS" | jq '.folders | length')
|
||||||
|
FILE_COUNT=$(echo "$CONTENTS" | jq '.files | length')
|
||||||
|
if [[ "$SUB_COUNT" -ne 0 || "$FILE_COUNT" -ne 0 ]]; then
|
||||||
|
fail "folder $folder_id still has $SUB_COUNT subfolder(s) and $FILE_COUNT file(s)"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
log "API confirms all root folders are empty."
|
||||||
|
|
||||||
|
# ── 3. Permanently delete everything in trash ─────────────────────────────────
|
||||||
|
|
||||||
|
curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/empty" >/dev/null
|
||||||
|
log "Trash emptied."
|
||||||
|
|
||||||
|
# ── 3b. Verify trash is empty according to the API ───────────────────────────
|
||||||
|
|
||||||
|
TRASH_COUNT=$(curl -sf -H "$AUTH" "$base_url/api/trash" | jq 'length')
|
||||||
|
if [[ "$TRASH_COUNT" -ne 0 ]]; then
|
||||||
|
fail "trash still contains $TRASH_COUNT item(s) after empty"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "API confirms trash is empty."
|
||||||
|
|
||||||
|
# ── 4. Disk verification ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
THUMB_FILES=$(find "$STORAGE_PATH/.thumbnails" -type f 2>/dev/null || true)
|
||||||
|
BLOB_FILES=$(find "$STORAGE_PATH/.blobs" -type f 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [[ -n "$THUMB_FILES" ]]; then
|
||||||
|
THUMB_COUNT=$(echo "$THUMB_FILES" | wc -l | tr -d ' ')
|
||||||
|
log "Leftover thumbnail files ($THUMB_COUNT):"
|
||||||
|
echo "$THUMB_FILES"
|
||||||
|
fail "$THUMB_COUNT thumbnail file(s) remain on disk after full cleanup"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$BLOB_FILES" ]]; then
|
||||||
|
BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ')
|
||||||
|
log "Leftover blob files ($BLOB_COUNT):"
|
||||||
|
echo "$BLOB_FILES"
|
||||||
|
fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "OK — no blobs or thumbnails remain on disk."
|
||||||
Executable
+66
@@ -0,0 +1,66 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if ! which b3sum >/dev/null 2>/dev/null
|
||||||
|
then
|
||||||
|
echo "please install b3sum (brew install b3sum on Mac, apt installb3sum on Debian, etc)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
HASH=""
|
||||||
|
FILE_CACHE=""
|
||||||
|
|
||||||
|
# return the hash of a file (stores into HASH variable)
|
||||||
|
oxi_hash() {
|
||||||
|
if [[ -z "$HASH" || "$FILE_CACHE" != "$1" ]]
|
||||||
|
then
|
||||||
|
HASH=$(b3sum --no-names "$1")
|
||||||
|
FILE_CACHE="$1"
|
||||||
|
fi
|
||||||
|
echo "$HASH"
|
||||||
|
}
|
||||||
|
|
||||||
|
# returns the local blob localisation
|
||||||
|
local_blob_path() {
|
||||||
|
local BLOB_PREFIX
|
||||||
|
oxi_hash "$1" >/dev/null
|
||||||
|
BLOB_PREFIX=${HASH:0:2}
|
||||||
|
echo ".blobs/$BLOB_PREFIX/$HASH.blob"
|
||||||
|
}
|
||||||
|
|
||||||
|
# returns the preview localisation without it's extension
|
||||||
|
preview_path() {
|
||||||
|
local SIZE
|
||||||
|
oxi_hash "$1" >/dev/null
|
||||||
|
# default size: icon
|
||||||
|
SIZE="${3:-icon}"
|
||||||
|
echo ".thumbnails/$SIZE/$HASH"
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_local_blob_existsy() {
|
||||||
|
BLOB_PATH=$(local_blob_path "$1")
|
||||||
|
STORAGE="$2"
|
||||||
|
if [[ -e $STORAGE/$BLOB_PATH ]]
|
||||||
|
then
|
||||||
|
echo "$BLOB_PATH exists"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
echo $'\e[31m'"$BLOB_PATH does not exist"$'\e[0m' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_preview_existsy() {
|
||||||
|
THUMBNAIL_PATH=$(preview_path "$1")
|
||||||
|
STORAGE="$2"
|
||||||
|
if [[ -e "$STORAGE/$THUMBNAIL_PATH.jpg" || -e "$STORAGE/$THUMBNAIL_PATH.webp" ]]
|
||||||
|
then
|
||||||
|
echo "thumbnail $THUMBNAIL_PATH.(jpg|webp) exists"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
echo $'\e[31m'"thumbnail $THUMBNAIL_PATH.(jpg|webp) does not exist"$'\e[0m' >&2
|
||||||
|
echo $STORAGE
|
||||||
|
find $STORAGE/.thumbnails
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
@@ -16,3 +16,4 @@ OXICLOUD_EXPOSE_SYSTEM_USERS=true
|
|||||||
OXICLOUD_WOPI_ENABLED=false
|
OXICLOUD_WOPI_ENABLED=false
|
||||||
OXICLOUD_OIDC_ENABLED=false
|
OXICLOUD_OIDC_ENABLED=false
|
||||||
RUST_LOG=warn
|
RUST_LOG=warn
|
||||||
|
#RUST_LOG=debug
|
||||||
|
|||||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
Hello from OxiCloud Hurl tests.
|
||||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
@@ -0,0 +1 @@
|
|||||||
|
# purpose is to test webdav coverage + different scenarios
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
source test.env
|
||||||
|
|
||||||
|
err() {
|
||||||
|
echo "$*" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
warn() {
|
||||||
|
echo "$*" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
success() {
|
||||||
|
echo "ok";
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create the first admin account if the server is freshly initialised.
|
||||||
|
# Silently succeeds when the account already exists (403 = already set up).
|
||||||
|
oxicloud_setup() {
|
||||||
|
SETUP_DATA='{"username":"'$username'","email":"'$email'","password":"'$password'"}'
|
||||||
|
SETUP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X POST -H "Content-Type: application/json" \
|
||||||
|
-d "$SETUP_DATA" "$base_url/api/setup")
|
||||||
|
|
||||||
|
case "$SETUP_STATUS" in
|
||||||
|
201) echo "setup: admin account created" ;;
|
||||||
|
403) ;; # already initialised — normal on second run
|
||||||
|
*) err "setup: unexpected status $SETUP_STATUS"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# returns TOKEN variable
|
||||||
|
oxicloud_login() {
|
||||||
|
|
||||||
|
oxicloud_setup
|
||||||
|
|
||||||
|
LOGIN_DATA='{"username":"'$username'","password":"'$password'"}'
|
||||||
|
|
||||||
|
LOGIN_RESPONSE=$(curl -s -X POST -H "Content-Type: application/json" -d "$LOGIN_DATA" $base_url/api/auth/login)
|
||||||
|
|
||||||
|
echo $?
|
||||||
|
jq -r '.error' <<<$LOGIN_RESPONSE
|
||||||
|
|
||||||
|
if [[ "$(jq -r '.error' <<<$LOGIN_RESPONSE)" != "null" ]]
|
||||||
|
then
|
||||||
|
err "Login Error: $LOGIN_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
TOKEN=$(jq -r '.access_token' <<<$LOGIN_RESPONSE)
|
||||||
|
|
||||||
|
if [[ -z "$TOKEN" || "$TOKEN" == "null" ]]
|
||||||
|
then
|
||||||
|
echo access_token missing in response: $LOGIN_RESPONSE
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "login successful, Got JWT token containing informations:" $(cut -d . -f 2 <<<$TOKEN | base64 -d)
|
||||||
|
}
|
||||||
|
|
||||||
|
# remove trailing /
|
||||||
|
base_url="${base_url%/}"
|
||||||
|
|
||||||
|
echo starting $0 tests on server $base_url
|
||||||
Executable
+98
@@ -0,0 +1,98 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Full Hurl API test runner.
|
||||||
|
# Starts postgres + OxiCloud server, runs Hurl tests, tears everything down.
|
||||||
|
#
|
||||||
|
# Usage (from repo root):
|
||||||
|
# bash tests/api/run.sh
|
||||||
|
#
|
||||||
|
# Prerequisites: docker, cargo, hurl ≥ 4.0
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
COMMON="$REPO_ROOT/tests/common"
|
||||||
|
WEBDAV_DIR="$REPO_ROOT/tests/webdav"
|
||||||
|
|
||||||
|
# test.env is the single source of truth for connection details and credentials.
|
||||||
|
# shellcheck source=test.env
|
||||||
|
source "$WEBDAV_DIR/test.env"
|
||||||
|
|
||||||
|
# Derive server port from base_url (e.g. http://localhost:8087 → 8087)
|
||||||
|
SERVER_PORT="${base_url##*:}"
|
||||||
|
|
||||||
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
log() { echo "[api-test] $*"; }
|
||||||
|
die() { echo "[api-test] ERROR: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
wait_for_http() {
|
||||||
|
local url="$1" timeout="${2:-60}"
|
||||||
|
local deadline=$(( $(date +%s) + timeout ))
|
||||||
|
until curl -sf "$url" >/dev/null 2>&1; do
|
||||||
|
[[ $(date +%s) -ge $deadline ]] && die "Timeout waiting for $url"
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Teardown (always runs on exit) ────────────────────────────────────────────
|
||||||
|
|
||||||
|
SERVER_PID=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [[ -n "$SERVER_PID" ]]; then
|
||||||
|
log "Stopping OxiCloud server (pid $SERVER_PID)..."
|
||||||
|
kill "$SERVER_PID" 2>/dev/null || true
|
||||||
|
wait "$SERVER_PID" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
bash "$COMMON/stop-db.sh"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
# ── 1. Start postgres ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
bash "$COMMON/spawn-db.sh"
|
||||||
|
|
||||||
|
# ── 2. Load shared server env + port from .env ───────────────────────────────
|
||||||
|
|
||||||
|
set -a
|
||||||
|
# shellcheck source=../common/server.env
|
||||||
|
source "$COMMON/server.env"
|
||||||
|
OXICLOUD_SERVER_PORT=$SERVER_PORT
|
||||||
|
OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
|
||||||
|
set +a
|
||||||
|
|
||||||
|
mkdir -p "$OXICLOUD_STORAGE_PATH"
|
||||||
|
|
||||||
|
# ── 3. Start OxiCloud server ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
BUILD_TARGET="${BUILD_TARGET:-debug}"
|
||||||
|
OXICLOUD_BIN="$REPO_ROOT/target/$BUILD_TARGET/oxicloud"
|
||||||
|
|
||||||
|
if [[ -x "$OXICLOUD_BIN" ]]; then
|
||||||
|
log "Starting pre-built OxiCloud server ($BUILD_TARGET) on port $SERVER_PORT..."
|
||||||
|
"$OXICLOUD_BIN" &
|
||||||
|
else
|
||||||
|
log "Building and starting OxiCloud server on port $SERVER_PORT..."
|
||||||
|
cd "$REPO_ROOT"
|
||||||
|
cargo run &
|
||||||
|
fi
|
||||||
|
SERVER_PID=$!
|
||||||
|
log "Waiting for server at $base_url..."
|
||||||
|
wait_for_http "$base_url/ready" 120
|
||||||
|
log "Server is ready."
|
||||||
|
|
||||||
|
# ── 4. Run Hurl tests ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
log "Running Hurl tests..."
|
||||||
|
for T in "$WEBDAV_DIR"/test_*.sh; do
|
||||||
|
if bash "$T"
|
||||||
|
then
|
||||||
|
echo $'\e[32m'"Success $T"$'\e[0m' >&2
|
||||||
|
else
|
||||||
|
echo $'\e[31m'"Failure $T"$'\e[0m' >&2
|
||||||
|
false
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
log "All tests passed."
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Test credentials for local/CI API tests — NOT real secrets.
|
||||||
|
base_url=http://localhost:8087
|
||||||
|
username=admin
|
||||||
|
email=admin@example.com
|
||||||
|
# gitguardian:ignore
|
||||||
|
password=TestPassword1!
|
||||||
Executable
+174
@@ -0,0 +1,174 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================
|
||||||
|
# OxiCloud – Chunked upload API + dedup check
|
||||||
|
# =============================================================
|
||||||
|
# Uploads free_video_over_1MB.mp4 (2760653 bytes) in 3 chunks
|
||||||
|
# of 1 MiB each via the TUS-like chunked upload API, then:
|
||||||
|
# 1. Verifies the file appears in folder listing with video/mp4 MIME type
|
||||||
|
# 2. Checks GET /api/dedup/check/{hash} → ref_count == 1
|
||||||
|
#
|
||||||
|
# BLAKE3 hash of free_video_over_1MB.mp4:
|
||||||
|
# 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80
|
||||||
|
#
|
||||||
|
# Prerequisites:
|
||||||
|
# - Server running at base_url with credentials from test.env
|
||||||
|
# - OXICLOUD_ENABLE_AUTH=true
|
||||||
|
# - jq, dd in PATH
|
||||||
|
#
|
||||||
|
# Run (from repo root):
|
||||||
|
# bash tests/webdav/test_chunked_upload_dedup.sh
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
cd "$SCRIPT_DIR"
|
||||||
|
|
||||||
|
source test.env
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||||
|
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||||
|
rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||||
|
dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; }
|
||||||
|
|
||||||
|
purge_from_trash() {
|
||||||
|
local name="$1"
|
||||||
|
local tid
|
||||||
|
tid=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
[[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── fixture ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80"
|
||||||
|
FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4"
|
||||||
|
[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; }
|
||||||
|
|
||||||
|
REMOTE_NAME="chunked-upload-test.mp4"
|
||||||
|
FILE_SIZE=2760653
|
||||||
|
CHUNK_SIZE=1048576 # 1 MiB — minimum accepted by the server
|
||||||
|
TOTAL_CHUNKS=3 # ceil(2760653 / 1048576) = 3
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "=== Chunked upload API + dedup check ==="
|
||||||
|
echo
|
||||||
|
|
||||||
|
# ── authenticate ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
oxicloud_login
|
||||||
|
|
||||||
|
# ── home folder ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||||
|
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||||
|
|| fail "Could not retrieve home folder ID"
|
||||||
|
echo " home folder id: $HOME_FOLDER_ID"
|
||||||
|
|
||||||
|
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||||
|
| jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$EXISTING_ID" ]]; then
|
||||||
|
echo " cleanup: deleting existing $REMOTE_NAME (id=$EXISTING_ID)"
|
||||||
|
rest_delete "/api/files/$EXISTING_ID" > /dev/null
|
||||||
|
fi
|
||||||
|
purge_from_trash "$REMOTE_NAME"
|
||||||
|
|
||||||
|
# ── Step 1: Create upload session ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 1: POST /api/uploads (create session, $TOTAL_CHUNKS chunks of ${CHUNK_SIZE}B)..."
|
||||||
|
SESSION=$(curl -s \
|
||||||
|
-X POST \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{\"filename\":\"$REMOTE_NAME\",\"folder_id\":\"$HOME_FOLDER_ID\",\"content_type\":\"video/mp4\",\"total_size\":$FILE_SIZE,\"chunk_size\":$CHUNK_SIZE}" \
|
||||||
|
"$base_url/api/uploads")
|
||||||
|
|
||||||
|
UPLOAD_ID=$(jq -r '.upload_id' <<< "$SESSION")
|
||||||
|
[[ -n "$UPLOAD_ID" && "$UPLOAD_ID" != "null" ]] \
|
||||||
|
|| fail "POST /api/uploads: could not get upload_id (response: $SESSION)"
|
||||||
|
pass "Upload session created: upload_id=$UPLOAD_ID"
|
||||||
|
|
||||||
|
# ── Step 2: Upload chunks ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 2: PATCH chunks 0..$(( TOTAL_CHUNKS - 1 ))..."
|
||||||
|
for (( i=0; i<TOTAL_CHUNKS; i++ )); do
|
||||||
|
PATCH_RESP=$(dd if="$FIXTURE" bs="$CHUNK_SIZE" skip="$i" count=1 2>/dev/null \
|
||||||
|
| curl -s \
|
||||||
|
-X PATCH \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-H "Content-Type: application/octet-stream" \
|
||||||
|
--data-binary @- \
|
||||||
|
"$base_url/api/uploads/$UPLOAD_ID?chunk_index=$i")
|
||||||
|
|
||||||
|
IS_COMPLETE=$(jq -r '.is_complete' <<< "$PATCH_RESP")
|
||||||
|
BYTES=$(jq -r '.bytes_received' <<< "$PATCH_RESP")
|
||||||
|
[[ -n "$BYTES" && "$BYTES" != "null" ]] \
|
||||||
|
|| fail "PATCH chunk $i: unexpected response: $PATCH_RESP"
|
||||||
|
echo " chunk $i: bytes_received=$BYTES is_complete=$IS_COMPLETE"
|
||||||
|
done
|
||||||
|
pass "All $TOTAL_CHUNKS chunks uploaded"
|
||||||
|
|
||||||
|
# ── Step 3: Complete the upload ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 3: POST /api/uploads/$UPLOAD_ID/complete..."
|
||||||
|
COMPLETE_RESP=$(curl -s \
|
||||||
|
-X POST \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/api/uploads/$UPLOAD_ID/complete")
|
||||||
|
|
||||||
|
FILE_ID=$(jq -r '.file_id' <<< "$COMPLETE_RESP")
|
||||||
|
[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \
|
||||||
|
|| fail "POST complete: could not get file_id (response: $COMPLETE_RESP)"
|
||||||
|
pass "Upload complete: file_id=$FILE_ID"
|
||||||
|
|
||||||
|
# ── Step 4: Verify file appears in folder listing with correct MIME type ──────
|
||||||
|
|
||||||
|
echo " step 4: verify file in folder listing with video/mp4 MIME type..."
|
||||||
|
LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
|
||||||
|
LISTED_FILE=$(jq -r --arg id "$FILE_ID" 'first(.[] | select(.id == $id))' <<< "$LISTING")
|
||||||
|
|
||||||
|
[[ -n "$LISTED_FILE" && "$LISTED_FILE" != "null" ]] \
|
||||||
|
|| fail "File $FILE_ID not found in folder listing"
|
||||||
|
|
||||||
|
MIME=$(jq -r '.mime_type' <<< "$LISTED_FILE")
|
||||||
|
[[ "$MIME" == "video/mp4" ]] \
|
||||||
|
|| fail "Expected MIME type video/mp4, got: $MIME"
|
||||||
|
pass "File listed with MIME type: $MIME"
|
||||||
|
|
||||||
|
# ── Step 5: Dedup check → ref_count == 1 ─────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 5: GET /api/dedup/check/$BLOB_HASH..."
|
||||||
|
RESP=$(dedup_check "$BLOB_HASH")
|
||||||
|
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||||
|
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||||
|
|
||||||
|
[[ "$EXISTS" == "true" ]] \
|
||||||
|
|| fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)"
|
||||||
|
[[ "$RC" == "1" ]] \
|
||||||
|
|| fail "dedup/check: expected ref_count=1, got $RC"
|
||||||
|
pass "ref_count == 1: blob registered after chunked upload"
|
||||||
|
|
||||||
|
# ── cleanup ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " cleanup..."
|
||||||
|
ST=$(rest_delete "/api/files/$FILE_ID")
|
||||||
|
[[ "$ST" == "204" ]] || fail "DELETE file expected 204, got $ST"
|
||||||
|
purge_from_trash "$REMOTE_NAME"
|
||||||
|
pass "Cleanup complete"
|
||||||
|
|
||||||
|
# ── summary ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Results: $PASS passed, $FAIL failed."
|
||||||
|
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||||
Executable
+214
@@ -0,0 +1,214 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================
|
||||||
|
# OxiCloud – Dedup ref_count: multi-chunk CDC file via WebDAV
|
||||||
|
# =============================================================
|
||||||
|
# Case A (files < 64kB = 1 chunk) is treated in tests/api/dedup_blob_cleanup.hurl
|
||||||
|
# Validates Case B of the cleanup_if_orphaned fix:
|
||||||
|
# file_hash ≠ chunk_hashes (file split into 8 CDC chunks)
|
||||||
|
#
|
||||||
|
# free_video_over_1MB.mp4 — 2.6 MB, 8 CDC chunks
|
||||||
|
# BLAKE3: 95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80
|
||||||
|
#
|
||||||
|
# Sequence:
|
||||||
|
# 1. PUT video as file A → new manifest (ref_count=1, 8 chunk blobs)
|
||||||
|
# 2. PUT video as file B → dedup hit (ref_count=2, chunks unchanged)
|
||||||
|
# 3. /api/dedup/check → ref_count == 2
|
||||||
|
# 4. Permanently delete file A
|
||||||
|
# 5. /api/dedup/check → ref_count == 1 (chunks must NOT be freed)
|
||||||
|
# 6. Permanently delete file B
|
||||||
|
# 7. /api/dedup/check → exists == false (manifest + chunks cleaned up)
|
||||||
|
#
|
||||||
|
# Case B regression: cleanup_if_orphaned must decrement chunk_manifests
|
||||||
|
# ref_count without touching chunk blobs (the PG trigger is a no-op for
|
||||||
|
# multi-chunk files because file_hash is not stored in storage.blobs).
|
||||||
|
#
|
||||||
|
# Prerequisites:
|
||||||
|
# - Server running at base_url with credentials from test.env
|
||||||
|
# - OXICLOUD_ENABLE_AUTH=true
|
||||||
|
# - jq in PATH
|
||||||
|
#
|
||||||
|
# Run (from repo root):
|
||||||
|
# bash tests/webdav/test_dedup_webdav_multichunk.sh
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
cd "$SCRIPT_DIR"
|
||||||
|
|
||||||
|
source test.env
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||||
|
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
webdav_put() {
|
||||||
|
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X PUT \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-H "Content-Type: $mime" \
|
||||||
|
--data-binary "@$local_file" \
|
||||||
|
"$base_url/webdav/$remote_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
webdav_delete() {
|
||||||
|
local remote_name="$1"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X DELETE \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/webdav/$remote_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||||
|
rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
|
||||||
|
dedup_check() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"; }
|
||||||
|
|
||||||
|
purge_from_trash() {
|
||||||
|
local name="$1"
|
||||||
|
local tid
|
||||||
|
tid=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
[[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── fixture ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# free_video_over_1MB.mp4 → 2.6 MB → 8 CDC chunks → confirmed Case B
|
||||||
|
BLOB_HASH="95d42b25a2d39f24f1b2f38bf1b947d4ec74201271a98ea0e76a9cea421eff80"
|
||||||
|
FIXTURE="$REPO_ROOT/tests/fixtures/free_video_over_1MB.mp4"
|
||||||
|
[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; }
|
||||||
|
|
||||||
|
FILE_A="webdav-dedup-mc-a.mp4"
|
||||||
|
FILE_B="webdav-dedup-mc-b.mp4"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "=== Dedup ref_count: multi-chunk CDC (Case B) via WebDAV ==="
|
||||||
|
echo
|
||||||
|
|
||||||
|
# ── authenticate ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
oxicloud_login
|
||||||
|
|
||||||
|
# ── home folder ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||||
|
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||||
|
|| fail "Could not retrieve home folder ID"
|
||||||
|
echo " home folder id: $HOME_FOLDER_ID"
|
||||||
|
|
||||||
|
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
for REMOTE in "$FILE_A" "$FILE_B"; do
|
||||||
|
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||||
|
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$EXISTING_ID" ]]; then
|
||||||
|
echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)"
|
||||||
|
rest_delete "/api/files/$EXISTING_ID" > /dev/null
|
||||||
|
fi
|
||||||
|
purge_from_trash "$REMOTE"
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── Step 1: Upload file A ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 1: PUT $FILE_A..."
|
||||||
|
STATUS=$(webdav_put "$FILE_A" "$FIXTURE" "video/mp4")
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS"
|
||||||
|
pass "PUT $FILE_A → 204 (new manifest, 8 chunk blobs created)"
|
||||||
|
|
||||||
|
# ── Step 2: Upload file B (same content, different name → dedup hit) ──────────
|
||||||
|
|
||||||
|
echo " step 2: PUT $FILE_B (same bytes → dedup hit)..."
|
||||||
|
STATUS=$(webdav_put "$FILE_B" "$FIXTURE" "video/mp4")
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS"
|
||||||
|
pass "PUT $FILE_B → 204 (dedup hit: manifest ref_count → 2, chunks unchanged)"
|
||||||
|
|
||||||
|
# ── Resolve file IDs ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
|
||||||
|
FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$LISTING")
|
||||||
|
FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$LISTING")
|
||||||
|
|
||||||
|
[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing"
|
||||||
|
[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing"
|
||||||
|
[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \
|
||||||
|
|| fail "File A and B share the same ID — dedup must create two distinct records"
|
||||||
|
pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID"
|
||||||
|
|
||||||
|
# ── Step 3: ref_count == 2 ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 3: dedup/check → expect ref_count=2..."
|
||||||
|
RESP=$(dedup_check "$BLOB_HASH")
|
||||||
|
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||||
|
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||||
|
|
||||||
|
[[ "$EXISTS" == "true" ]] \
|
||||||
|
|| fail "dedup/check: expected exists=true, got $EXISTS (response: $RESP)"
|
||||||
|
[[ "$RC" == "2" ]] \
|
||||||
|
|| fail "dedup/check: expected ref_count=2, got $RC"
|
||||||
|
pass "ref_count == 2: both files reference the same 8-chunk blob"
|
||||||
|
|
||||||
|
# ── Step 4: Permanently delete file A ────────────────────────────────────────
|
||||||
|
# For multi-chunk files: PG trigger is a no-op (file_hash not in storage.blobs).
|
||||||
|
# cleanup_if_orphaned must decrement chunk_manifests.ref_count only (2→1)
|
||||||
|
# and leave the 8 chunk blobs untouched.
|
||||||
|
|
||||||
|
echo " step 4: trash + permanently delete $FILE_A..."
|
||||||
|
ST=$(rest_delete "/api/files/$FILE_A_ID")
|
||||||
|
[[ "$ST" == "204" ]] || fail "DELETE $FILE_A expected 204, got $ST"
|
||||||
|
|
||||||
|
TRASH_A=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$FILE_A" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
[[ -n "$TRASH_A" ]] || fail "File A not found in trash"
|
||||||
|
ST=$(rest_delete "/api/trash/$TRASH_A")
|
||||||
|
[[ "$ST" == "200" ]] || fail "Permanent delete file A expected 200, got $ST"
|
||||||
|
pass "File A permanently deleted"
|
||||||
|
|
||||||
|
# ── Step 5: ref_count == 1 — chunk blobs must still be alive ─────────────────
|
||||||
|
|
||||||
|
echo " step 5: dedup/check → expect ref_count=1 (chunks must survive)..."
|
||||||
|
RESP=$(dedup_check "$BLOB_HASH")
|
||||||
|
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||||
|
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||||
|
|
||||||
|
[[ "$EXISTS" == "true" ]] \
|
||||||
|
|| fail "dedup/check: expected exists=true (file B still references blob), got $EXISTS"
|
||||||
|
[[ "$RC" == "1" ]] \
|
||||||
|
|| fail "dedup/check: expected ref_count=1, got $RC (chunk blobs may have been freed prematurely)"
|
||||||
|
pass "ref_count == 1: manifest decremented, all 8 chunk blobs still alive"
|
||||||
|
|
||||||
|
# ── Step 6: Permanently delete file B ────────────────────────────────────────
|
||||||
|
# ref_count hits 0 → manifest deleted, all 8 chunk blobs freed.
|
||||||
|
|
||||||
|
echo " step 6: trash + permanently delete $FILE_B..."
|
||||||
|
ST=$(rest_delete "/api/files/$FILE_B_ID")
|
||||||
|
[[ "$ST" == "204" ]] || fail "DELETE $FILE_B expected 204, got $ST"
|
||||||
|
|
||||||
|
TRASH_B=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$FILE_B" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
[[ -n "$TRASH_B" ]] || fail "File B not found in trash"
|
||||||
|
ST=$(rest_delete "/api/trash/$TRASH_B")
|
||||||
|
[[ "$ST" == "200" ]] || fail "Permanent delete file B expected 200, got $ST"
|
||||||
|
pass "File B permanently deleted"
|
||||||
|
|
||||||
|
# ── Step 7: blob gone ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 7: dedup/check → expect exists=false (manifest + chunks freed)..."
|
||||||
|
RESP=$(dedup_check "$BLOB_HASH")
|
||||||
|
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||||
|
|
||||||
|
[[ "$EXISTS" == "false" ]] \
|
||||||
|
|| fail "dedup/check: expected exists=false after both files deleted, got $EXISTS"
|
||||||
|
pass "exists == false: manifest and all 8 chunk blobs cleaned up"
|
||||||
|
|
||||||
|
# ── summary ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Results: $PASS passed, $FAIL failed."
|
||||||
|
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||||
Executable
+203
@@ -0,0 +1,203 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================
|
||||||
|
# OxiCloud – Dedup ref_count via WebDAV (two uploads + overwrite)
|
||||||
|
# =============================================================
|
||||||
|
# Scenario:
|
||||||
|
# 1. PUT dedup-test.jpg via WebDAV as file A
|
||||||
|
# 2. PUT dedup-test-2.jpg (identical content) via WebDAV as file B
|
||||||
|
# → same blob, two distinct file records, ref_count == 2
|
||||||
|
# 3. GET /api/dedup/check/{hash} → assert ref_count == 2
|
||||||
|
# 4. Overwrite file B via WebDAV PUT with different content
|
||||||
|
# → file B now references a new blob; original ref_count drops
|
||||||
|
# 5. GET /api/dedup/check/{hash} → assert ref_count == 1
|
||||||
|
#
|
||||||
|
# BLAKE3 hash of dedup-test.jpg (== dedup-test-2.jpg — same content):
|
||||||
|
# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||||
|
#
|
||||||
|
# Prerequisites:
|
||||||
|
# - Server running at base_url with credentials from test.env
|
||||||
|
# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth)
|
||||||
|
# - jq in PATH
|
||||||
|
#
|
||||||
|
# Run (from repo root):
|
||||||
|
# bash tests/webdav/test_dedup_webdav_ref_count.sh
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
cd "$SCRIPT_DIR"
|
||||||
|
|
||||||
|
source test.env
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||||
|
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
webdav_put() {
|
||||||
|
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X PUT \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-H "Content-Type: $mime" \
|
||||||
|
--data-binary "@$local_file" \
|
||||||
|
"$base_url/webdav/$remote_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
webdav_delete() {
|
||||||
|
local remote_name="$1"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X DELETE \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/webdav/$remote_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
rest_get() {
|
||||||
|
curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
rest_delete() {
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X DELETE \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
dedup_check() {
|
||||||
|
curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── fixtures ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# BLAKE3 hash of dedup-test.jpg (= dedup-test-2.jpg — byte-identical content)
|
||||||
|
BLOB_HASH="cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066"
|
||||||
|
|
||||||
|
FIXTURE_A="$REPO_ROOT/tests/fixtures/dedup-test.jpg"
|
||||||
|
FIXTURE_B="$REPO_ROOT/tests/fixtures/dedup-test-2.jpg"
|
||||||
|
FIXTURE_OTHER="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg"
|
||||||
|
|
||||||
|
[[ -f "$FIXTURE_A" ]] || { echo "Missing fixture: $FIXTURE_A" >&2; exit 1; }
|
||||||
|
[[ -f "$FIXTURE_B" ]] || { echo "Missing fixture: $FIXTURE_B" >&2; exit 1; }
|
||||||
|
[[ -f "$FIXTURE_OTHER" ]] || { echo "Missing fixture: $FIXTURE_OTHER" >&2; exit 1; }
|
||||||
|
|
||||||
|
FILE_A="webdav-dedup-ref-a.jpg"
|
||||||
|
FILE_B="webdav-dedup-ref-b.jpg"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "=== Dedup ref_count: two WebDAV uploads + overwrite ==="
|
||||||
|
echo
|
||||||
|
|
||||||
|
# ── authenticate ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
oxicloud_login
|
||||||
|
|
||||||
|
# ── home folder ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||||
|
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||||
|
|| fail "Could not retrieve home folder ID"
|
||||||
|
echo " home folder id: $HOME_FOLDER_ID"
|
||||||
|
|
||||||
|
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
for REMOTE in "$FILE_A" "$FILE_B"; do
|
||||||
|
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||||
|
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$EXISTING_ID" ]]; then
|
||||||
|
echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)"
|
||||||
|
rest_delete "/api/files/$EXISTING_ID" > /dev/null
|
||||||
|
fi
|
||||||
|
STALE=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$STALE" ]]; then
|
||||||
|
echo " cleanup: purging $REMOTE from trash (id=$STALE)"
|
||||||
|
rest_delete "/api/trash/$STALE" > /dev/null
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── Step 1: Upload file A ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 1: PUT $FILE_A (dedup-test.jpg)..."
|
||||||
|
STATUS=$(webdav_put "$FILE_A" "$FIXTURE_A" "image/jpeg")
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS"
|
||||||
|
pass "PUT $FILE_A → 204"
|
||||||
|
|
||||||
|
# ── Step 2: Upload file B (identical content, different name) ─────────────────
|
||||||
|
|
||||||
|
echo " step 2: PUT $FILE_B (dedup-test-2.jpg, same bytes)..."
|
||||||
|
STATUS=$(webdav_put "$FILE_B" "$FIXTURE_B" "image/jpeg")
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS"
|
||||||
|
pass "PUT $FILE_B → 204"
|
||||||
|
|
||||||
|
# ── Step 3: Resolve file IDs and assert two distinct records ──────────────────
|
||||||
|
|
||||||
|
FILE_LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
|
||||||
|
FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING")
|
||||||
|
FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING")
|
||||||
|
|
||||||
|
[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing"
|
||||||
|
[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing"
|
||||||
|
[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \
|
||||||
|
|| fail "File A and B share the same ID — dedup must produce two distinct records"
|
||||||
|
pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID"
|
||||||
|
|
||||||
|
# ── Step 4: Dedup check → ref_count == 2 ─────────────────────────────────────
|
||||||
|
|
||||||
|
echo " step 4: GET /api/dedup/check/$BLOB_HASH..."
|
||||||
|
RESP=$(dedup_check "$BLOB_HASH")
|
||||||
|
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||||
|
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||||
|
|
||||||
|
[[ "$EXISTS" == "true" ]] \
|
||||||
|
|| fail "dedup/check: expected exists=true, got $EXISTS (full response: $RESP)"
|
||||||
|
[[ "$RC" == "2" ]] \
|
||||||
|
|| fail "dedup/check: expected ref_count=2 after two identical uploads, got $RC"
|
||||||
|
pass "ref_count == 2: both files reference the same blob"
|
||||||
|
|
||||||
|
# ── Step 5: Overwrite file B with different content ───────────────────────────
|
||||||
|
# swap_blob_hash calls remove_reference on the old hash → manifest ref_count 2→1
|
||||||
|
|
||||||
|
echo " step 5: PUT $FILE_B (oxicloud-logo.jpg, new content)..."
|
||||||
|
STATUS=$(webdav_put "$FILE_B" "$FIXTURE_OTHER" "image/jpeg")
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B (overwrite) expected 204, got $STATUS"
|
||||||
|
pass "PUT $FILE_B overwrite → 204"
|
||||||
|
|
||||||
|
# ── Step 6: Dedup check → ref_count == 1 ─────────────────────────────────────
|
||||||
|
# File B now references a different blob; file A still holds the original.
|
||||||
|
|
||||||
|
echo " step 6: GET /api/dedup/check/$BLOB_HASH..."
|
||||||
|
RESP=$(dedup_check "$BLOB_HASH")
|
||||||
|
EXISTS=$(jq -r '.exists' <<< "$RESP")
|
||||||
|
RC=$( jq -r '.ref_count' <<< "$RESP")
|
||||||
|
|
||||||
|
[[ "$EXISTS" == "true" ]] \
|
||||||
|
|| fail "dedup/check: expected exists=true (file A still references blob), got $EXISTS"
|
||||||
|
[[ "$RC" == "1" ]] \
|
||||||
|
|| fail "dedup/check: expected ref_count=1 after overwriting file B, got $RC"
|
||||||
|
pass "ref_count == 1: only file A still references the original blob"
|
||||||
|
|
||||||
|
# ── cleanup ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " cleanup..."
|
||||||
|
for REMOTE in "$FILE_A" "$FILE_B"; do
|
||||||
|
ST=$(webdav_delete "$REMOTE")
|
||||||
|
[[ "$ST" == "204" ]] || fail "WebDAV DELETE $REMOTE expected 204, got $ST"
|
||||||
|
TRASH_ITEM=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$TRASH_ITEM" ]]; then
|
||||||
|
rest_delete "/api/trash/$TRASH_ITEM" > /dev/null
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
pass "Cleanup complete"
|
||||||
|
|
||||||
|
# ── summary ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Results: $PASS passed, $FAIL failed."
|
||||||
|
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||||
Executable
+236
@@ -0,0 +1,236 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================
|
||||||
|
# OxiCloud – Bug 1 & 2: thumbnail refresh after WebDAV PUT
|
||||||
|
# =============================================================
|
||||||
|
# Bug 1: stale moka cache served after blob swap (overwrite)
|
||||||
|
# Bug 2: no background thumbnail generation after update
|
||||||
|
#
|
||||||
|
# Fix (webdav_handler.rs handle_put): after update branch,
|
||||||
|
# refresh_thumbnails_after_update() calls delete_thumbnails()
|
||||||
|
# (evicts moka) then spawns background regen from new blob.
|
||||||
|
#
|
||||||
|
# Test strategy:
|
||||||
|
# 1. PUT dedup-test.jpg via /webdav → thumbnail generated
|
||||||
|
# 2. Prime the moka cache with GET /thumbnail
|
||||||
|
# 3. PUT oxicloud-logo.jpg to the same path (overwrite)
|
||||||
|
# 4. GET /thumbnail again → must return different bytes
|
||||||
|
#
|
||||||
|
# Bug 1 detection: if moka is not evicted, step 4 returns the
|
||||||
|
# cached dedup-test thumbnail → SHA-256 matches step 2 → test fails.
|
||||||
|
#
|
||||||
|
# Prerequisites:
|
||||||
|
# - Server running at base_url with credentials from test.env
|
||||||
|
# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth)
|
||||||
|
# - jq in PATH
|
||||||
|
#
|
||||||
|
# Run (from repo root):
|
||||||
|
# bash tests/webdav/test_thumbnail_update.sh
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
cd "$SCRIPT_DIR"
|
||||||
|
|
||||||
|
source test.env
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
|
||||||
|
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
# WebDAV PUT: returns HTTP status code
|
||||||
|
webdav_put() {
|
||||||
|
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X PUT \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-H "Content-Type: $mime" \
|
||||||
|
--data-binary "@$local_file" \
|
||||||
|
"$base_url/webdav/$remote_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
# WebDAV DELETE: returns HTTP status code
|
||||||
|
webdav_delete() {
|
||||||
|
local remote_name="$1"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X DELETE \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/webdav/$remote_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
# REST GET with JWT bearer
|
||||||
|
rest_get() {
|
||||||
|
curl -s \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# REST DELETE with JWT bearer, returns HTTP status code
|
||||||
|
rest_delete() {
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X DELETE \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Download thumbnail and return its SHA-256 checksum
|
||||||
|
thumbnail_sha256() {
|
||||||
|
local file_id="$1"
|
||||||
|
curl -s \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/api/files/$file_id/thumbnail/icon" \
|
||||||
|
| sha256sum | cut -d' ' -f1
|
||||||
|
}
|
||||||
|
|
||||||
|
# SHA-256 of an empty stream (thumbnail missing = empty body)
|
||||||
|
EMPTY_SHA="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||||
|
|
||||||
|
FIXTURE_V1="$REPO_ROOT/tests/fixtures/dedup-test.jpg"
|
||||||
|
FIXTURE_V2="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg"
|
||||||
|
|
||||||
|
[[ -f "$FIXTURE_V1" ]] || { echo "Missing fixture: $FIXTURE_V1" >&2; exit 1; }
|
||||||
|
[[ -f "$FIXTURE_V2" ]] || { echo "Missing fixture: $FIXTURE_V2" >&2; exit 1; }
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "=== Bug 1 & 2: thumbnail refresh after WebDAV PUT overwrite ==="
|
||||||
|
echo
|
||||||
|
|
||||||
|
# ── authenticate ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
oxicloud_login
|
||||||
|
|
||||||
|
# ── home folder ID ────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " home folder..."
|
||||||
|
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
|
||||||
|
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|
||||||
|
|| fail "Could not retrieve home folder ID"
|
||||||
|
echo " home folder id: $HOME_FOLDER_ID"
|
||||||
|
|
||||||
|
REMOTE="webdav-thumb-bug12.jpg"
|
||||||
|
|
||||||
|
# ── Pre-test cleanup (idempotent) ─────────────────────────────
|
||||||
|
# Remove any leftover from a previous run via the REST API so that
|
||||||
|
# the first WebDAV PUT below is guaranteed to be a CREATE (201).
|
||||||
|
|
||||||
|
echo " cleanup: checking regular listing for '$REMOTE'..."
|
||||||
|
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||||
|
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$EXISTING_ID" ]]; then
|
||||||
|
echo " cleanup: found existing file id=$EXISTING_ID — deleting..."
|
||||||
|
ST=$(rest_delete "/api/files/$EXISTING_ID")
|
||||||
|
echo " cleanup: DELETE /api/files/$EXISTING_ID → $ST"
|
||||||
|
else
|
||||||
|
echo " cleanup: no existing file in regular listing"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " cleanup: checking trash for '$REMOTE'..."
|
||||||
|
STALE=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
|
||||||
|
if [[ -n "$STALE" ]]; then
|
||||||
|
echo " cleanup: found trash item id=$STALE — purging..."
|
||||||
|
ST=$(rest_delete "/api/trash/$STALE")
|
||||||
|
echo " cleanup: DELETE /api/trash/$STALE → $ST"
|
||||||
|
else
|
||||||
|
echo " cleanup: trash is clean"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Step 1: PUT dedup-test.jpg ───────────────────────────────
|
||||||
|
# /webdav always returns 204 (update_file_streaming handles create+update)
|
||||||
|
|
||||||
|
echo " step 1: PUT $REMOTE..."
|
||||||
|
STATUS=$(webdav_put "$REMOTE" "$FIXTURE_V1" "image/jpeg")
|
||||||
|
echo " step 1: WebDAV PUT → $STATUS"
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "WebDAV PUT expected 204, got $STATUS"
|
||||||
|
pass "WebDAV PUT dedup-test.jpg → 204"
|
||||||
|
|
||||||
|
# ── find file_id from REST listing ───────────────────────────
|
||||||
|
|
||||||
|
echo " step 1: resolving file_id..."
|
||||||
|
FILE_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
|
||||||
|
| jq -r --arg n "$REMOTE" '.[] | select(.name == $n) | .id')
|
||||||
|
[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] \
|
||||||
|
|| fail "File '$REMOTE' not found in folder listing after WebDAV PUT"
|
||||||
|
pass "File found via REST API — id=$FILE_ID"
|
||||||
|
|
||||||
|
# ── Step 2: GET thumbnail to prime moka cache ────────────────
|
||||||
|
# Background generation may still be running; wait briefly.
|
||||||
|
|
||||||
|
echo " step 2: waiting 1s for background thumbnail generation..."
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
echo " step 2: GET /thumbnail/icon..."
|
||||||
|
HTTP=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/api/files/$FILE_ID/thumbnail/icon")
|
||||||
|
echo " step 2: GET /thumbnail → $HTTP"
|
||||||
|
[[ "$HTTP" == "200" ]] \
|
||||||
|
|| fail "GET /thumbnail after initial upload expected 200, got $HTTP"
|
||||||
|
|
||||||
|
THUMB_V1=$(thumbnail_sha256 "$FILE_ID")
|
||||||
|
echo " step 2: thumbnail sha256=$THUMB_V1"
|
||||||
|
[[ -n "$THUMB_V1" && "$THUMB_V1" != "$EMPTY_SHA" ]] \
|
||||||
|
|| fail "Thumbnail after initial upload is empty (sha256=$THUMB_V1)"
|
||||||
|
pass "Initial thumbnail present and non-empty (sha256=$THUMB_V1)"
|
||||||
|
|
||||||
|
# ── Step 3: PUT oxicloud-logo.jpg (overwrite) ────────────────
|
||||||
|
|
||||||
|
echo " step 3: PUT $REMOTE (overwrite)..."
|
||||||
|
STATUS=$(webdav_put "$REMOTE" "$FIXTURE_V2" "image/jpeg")
|
||||||
|
echo " step 3: WebDAV PUT → $STATUS"
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "WebDAV PUT (overwrite) expected 204, got $STATUS"
|
||||||
|
pass "WebDAV PUT oxicloud-logo.jpg overwrite → 204 No Content"
|
||||||
|
|
||||||
|
# ── Step 4: GET thumbnail after overwrite ────────────────────
|
||||||
|
# Fix: delete_thumbnails() evicts moka (bug 1),
|
||||||
|
# background regen populates new blob thumbnail (bug 2).
|
||||||
|
# Without the fix the stale dedup-test thumbnail is served from moka.
|
||||||
|
|
||||||
|
echo " step 4: waiting 1s for background thumbnail regeneration..."
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
echo " step 4: GET /thumbnail/icon after overwrite..."
|
||||||
|
HTTP=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$base_url/api/files/$FILE_ID/thumbnail/icon")
|
||||||
|
echo " step 4: GET /thumbnail → $HTTP"
|
||||||
|
[[ "$HTTP" == "200" ]] \
|
||||||
|
|| fail "GET /thumbnail after overwrite expected 200, got $HTTP"
|
||||||
|
|
||||||
|
THUMB_V2=$(thumbnail_sha256 "$FILE_ID")
|
||||||
|
echo " step 4: thumbnail sha256=$THUMB_V2"
|
||||||
|
[[ -n "$THUMB_V2" && "$THUMB_V2" != "$EMPTY_SHA" ]] \
|
||||||
|
|| fail "Thumbnail after overwrite is empty"
|
||||||
|
|
||||||
|
[[ "$THUMB_V1" != "$THUMB_V2" ]] \
|
||||||
|
|| fail "Bug 1 present: moka cache not evicted — thumbnail unchanged after overwrite (sha256=$THUMB_V1)"
|
||||||
|
pass "Thumbnail changed after overwrite — bugs 1 & 2 fixed (sha256=$THUMB_V2)"
|
||||||
|
|
||||||
|
# ── cleanup ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo " cleanup: WebDAV DELETE $REMOTE..."
|
||||||
|
STATUS=$(webdav_delete "$REMOTE")
|
||||||
|
echo " cleanup: WebDAV DELETE → $STATUS"
|
||||||
|
[[ "$STATUS" == "204" ]] || fail "WebDAV DELETE expected 204, got $STATUS"
|
||||||
|
pass "WebDAV DELETE → 204"
|
||||||
|
|
||||||
|
TRASH_ITEM=$(rest_get "/api/trash" \
|
||||||
|
| jq -r --arg n "$REMOTE" '.[] | select(.name == $n) | .id // empty')
|
||||||
|
if [[ -n "$TRASH_ITEM" ]]; then
|
||||||
|
ST=$(rest_delete "/api/trash/$TRASH_ITEM")
|
||||||
|
echo " cleanup: DELETE /api/trash/$TRASH_ITEM → $ST"
|
||||||
|
pass "Permanently deleted from trash"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── summary ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Results: $PASS passed, $FAIL failed."
|
||||||
|
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1
|
||||||
Reference in New Issue
Block a user