cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix+test: check hash ref count on copy-on-write (a duplicated beeing updated)

Browse Source
This commit is contained in:
Edouard Vanbelle
2026-05-13 13:27:33 +02:00
parent 78cb37b311
commit 28e25f9d16
8 changed files with 235 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/ports/blob_lifecycle.rs
+21
View File
@@ -0,0 +1,21 @@
use std::future::Future;
use std::pin::Pin;
/// Observer notified by [`DedupService`] when a blob's ref_count reaches zero
/// and it is permanently removed from storage.
///
/// Implement this trait on any service that needs to react to blob deletion
/// (e.g. thumbnail cleanup, CDN invalidation, audit logging). Register with
/// [`DedupService::add_blob_hook`] during DI wiring.
///
/// The boxed-future return keeps the trait dyn-compatible so multiple
/// implementations can be stored as `Vec<Arc<dyn BlobDeletionHook>>`.
pub trait BlobDeletionHook: Send + Sync {
/// Called after the blob file has been removed from disk.
/// `blob_hash` is the BLAKE3 hex string identifying the blob.
/// Must be best-effort — must not propagate errors.
fn on_blob_deleted<'a>(
&'a self,
blob_hash: &'a str,
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
}
src/application/services/trash_service.rs
+1
View File
@@ -61,6 +61,7 @@ pub struct TrashService {
}
impl TrashService {
#[allow(clippy::too_many_arguments)]
pub fn new(
trash_repository: Arc<TrashDbRepository>,
file_read_port: Arc<FileBlobReadRepository>,
src/common/di.rs
-1
View File
@@ -456,7 +456,6 @@ impl AppServiceFactory {
Some(core.file_content_cache.clone()),
));
// Initialize cleanup service (bulk-deletes expired items in 2 SQL queries)
let cleanup_service = TrashCleanupService::new(
trash_repo.clone(),
src/infrastructure/repositories/pg/file_blob_write_repository.rs
+8 -9
View File
@@ -606,15 +606,14 @@ impl FileWritePort for FileBlobWriteRepository {
async fn delete_file_permanently(&self, file_id: &str) -> Result<(), DomainError> {
// Read blob_hash before deletion so we can clean up disk after the
// PG trigger has decremented the ref_count.
let blob_hash: Option<String> = sqlx::query_scalar(
"SELECT blob_hash FROM storage.files WHERE id = $1::uuid",
)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}"))
})?;
let blob_hash: Option<String> =
sqlx::query_scalar("SELECT blob_hash FROM storage.files WHERE id = $1::uuid")
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}"))
})?;
// DELETE fires trg_files_decrement_blob_ref → storage.blobs.ref_count--
self.delete_file(file_id).await?;
src/infrastructure/services/dedup_service.rs
+1 -1
View File
@@ -44,8 +44,8 @@ use std::sync::Arc;
use tokio::fs;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
use crate::application::ports::blob_lifecycle::BlobDeletionHook;
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
use crate::application::ports::dedup_ports::{
BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto,
};
src/infrastructure/services/thumbnail_service.rs
+1 -1
View File
@@ -25,8 +25,8 @@ use tokio::time::timeout;
use crate::application::ports::thumbnail_ports::{
ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto,
};
use crate::infrastructure::services::dedup_service::DedupService;
use crate::domain::errors::{DomainError, ErrorKind};
use crate::infrastructure::services::dedup_service::DedupService;
/// Thumbnail sizes supported by the system
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
src/interfaces/api/handlers/dedup_handler.rs
-1
View File
@@ -502,7 +502,6 @@ impl DedupHandler {
.unwrap()
.into_response()
}
}
// ── Route handlers (free functions) ──────────────────────────────────────────
tests/webdav/test_dedup_webdav_ref_count.sh
Executable
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env bash
# =============================================================
# OxiCloud – Dedup ref_count via WebDAV (two uploads + overwrite)
# =============================================================
# Scenario:
# 1. PUT dedup-test.jpg via WebDAV as file A
# 2. PUT dedup-test-2.jpg (identical content) via WebDAV as file B
# → same blob, two distinct file records, ref_count == 2
# 3. GET /api/dedup/check/{hash} → assert ref_count == 2
# 4. Overwrite file B via WebDAV PUT with different content
# → file B now references a new blob; original ref_count drops
# 5. GET /api/dedup/check/{hash} → assert ref_count == 1
#
# BLAKE3 hash of dedup-test.jpg (== dedup-test-2.jpg — same content):
# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
#
# Prerequisites:
# - Server running at base_url with credentials from test.env
# - OXICLOUD_ENABLE_AUTH=true (/webdav uses JWT Bearer auth)
# - jq in PATH
#
# Run (from repo root):
# bash tests/webdav/test_dedup_webdav_ref_count.sh
# =============================================================
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
cd "$SCRIPT_DIR"
source test.env
source common.sh
# ── helpers ──────────────────────────────────────────────────────────────────
PASS=0
FAIL=0
pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; }
fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; }
webdav_put() {
local remote_name="$1" local_file="$2" mime="${3:-application/octet-stream}"
curl -s -o /dev/null -w "%{http_code}" \
-X PUT \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: $mime" \
--data-binary "@$local_file" \
"$base_url/webdav/$remote_name"
}
webdav_delete() {
local remote_name="$1"
curl -s -o /dev/null -w "%{http_code}" \
-X DELETE \
-H "Authorization: Bearer $TOKEN" \
"$base_url/webdav/$remote_name"
}
rest_get() {
curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"
}
rest_delete() {
curl -s -o /dev/null -w "%{http_code}" \
-X DELETE \
-H "Authorization: Bearer $TOKEN" \
"$base_url$1"
}
dedup_check() {
curl -s -H "Authorization: Bearer $TOKEN" "$base_url/api/dedup/check/$1"
}
# ── fixtures ──────────────────────────────────────────────────────────────────
# BLAKE3 hash of dedup-test.jpg (= dedup-test-2.jpg — byte-identical content)
BLOB_HASH="cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066"
FIXTURE_A="$REPO_ROOT/tests/fixtures/dedup-test.jpg"
FIXTURE_B="$REPO_ROOT/tests/fixtures/dedup-test-2.jpg"
FIXTURE_OTHER="$REPO_ROOT/tests/fixtures/oxicloud-logo.jpg"
[[ -f "$FIXTURE_A" ]] || { echo "Missing fixture: $FIXTURE_A" >&2; exit 1; }
[[ -f "$FIXTURE_B" ]] || { echo "Missing fixture: $FIXTURE_B" >&2; exit 1; }
[[ -f "$FIXTURE_OTHER" ]] || { echo "Missing fixture: $FIXTURE_OTHER" >&2; exit 1; }
FILE_A="webdav-dedup-ref-a.jpg"
FILE_B="webdav-dedup-ref-b.jpg"
echo
echo "=== Dedup ref_count: two WebDAV uploads + overwrite ==="
echo
# ── authenticate ──────────────────────────────────────────────────────────────
oxicloud_login
# ── home folder ───────────────────────────────────────────────────────────────
HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
[[ -n "$HOME_FOLDER_ID" && "$HOME_FOLDER_ID" != "null" ]] \
|| fail "Could not retrieve home folder ID"
echo " home folder id: $HOME_FOLDER_ID"
# ── idempotent pre-test cleanup ───────────────────────────────────────────────
for REMOTE in "$FILE_A" "$FILE_B"; do
EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
if [[ -n "$EXISTING_ID" ]]; then
echo " cleanup: deleting existing $REMOTE (id=$EXISTING_ID)"
rest_delete "/api/files/$EXISTING_ID" > /dev/null
fi
STALE=$(rest_get "/api/trash" \
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
if [[ -n "$STALE" ]]; then
echo " cleanup: purging $REMOTE from trash (id=$STALE)"
rest_delete "/api/trash/$STALE" > /dev/null
fi
done
# ── Step 1: Upload file A ─────────────────────────────────────────────────────
echo " step 1: PUT $FILE_A (dedup-test.jpg)..."
STATUS=$(webdav_put "$FILE_A" "$FIXTURE_A" "image/jpeg")
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_A expected 204, got $STATUS"
pass "PUT $FILE_A → 204"
# ── Step 2: Upload file B (identical content, different name) ─────────────────
echo " step 2: PUT $FILE_B (dedup-test-2.jpg, same bytes)..."
STATUS=$(webdav_put "$FILE_B" "$FIXTURE_B" "image/jpeg")
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B expected 204, got $STATUS"
pass "PUT $FILE_B → 204"
# ── Step 3: Resolve file IDs and assert two distinct records ──────────────────
FILE_LISTING=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID")
FILE_A_ID=$(jq -r --arg n "$FILE_A" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING")
FILE_B_ID=$(jq -r --arg n "$FILE_B" '.[] | select(.name == $n) | .id' <<< "$FILE_LISTING")
[[ -n "$FILE_A_ID" && "$FILE_A_ID" != "null" ]] || fail "File A not found in listing"
[[ -n "$FILE_B_ID" && "$FILE_B_ID" != "null" ]] || fail "File B not found in listing"
[[ "$FILE_A_ID" != "$FILE_B_ID" ]] \
|| fail "File A and B share the same ID — dedup must produce two distinct records"
pass "Two distinct file records: A=$FILE_A_ID B=$FILE_B_ID"
# ── Step 4: Dedup check → ref_count == 2 ─────────────────────────────────────
echo " step 4: GET /api/dedup/check/$BLOB_HASH..."
RESP=$(dedup_check "$BLOB_HASH")
EXISTS=$(jq -r '.exists' <<< "$RESP")
RC=$( jq -r '.ref_count' <<< "$RESP")
[[ "$EXISTS" == "true" ]] \
|| fail "dedup/check: expected exists=true, got $EXISTS (full response: $RESP)"
[[ "$RC" == "2" ]] \
|| fail "dedup/check: expected ref_count=2 after two identical uploads, got $RC"
pass "ref_count == 2: both files reference the same blob"
# ── Step 5: Overwrite file B with different content ───────────────────────────
# swap_blob_hash calls remove_reference on the old hash → manifest ref_count 2→1
echo " step 5: PUT $FILE_B (oxicloud-logo.jpg, new content)..."
STATUS=$(webdav_put "$FILE_B" "$FIXTURE_OTHER" "image/jpeg")
[[ "$STATUS" == "204" ]] || fail "PUT $FILE_B (overwrite) expected 204, got $STATUS"
pass "PUT $FILE_B overwrite → 204"
# ── Step 6: Dedup check → ref_count == 1 ─────────────────────────────────────
# File B now references a different blob; file A still holds the original.
echo " step 6: GET /api/dedup/check/$BLOB_HASH..."
RESP=$(dedup_check "$BLOB_HASH")
EXISTS=$(jq -r '.exists' <<< "$RESP")
RC=$( jq -r '.ref_count' <<< "$RESP")
[[ "$EXISTS" == "true" ]] \
|| fail "dedup/check: expected exists=true (file A still references blob), got $EXISTS"
[[ "$RC" == "1" ]] \
|| fail "dedup/check: expected ref_count=1 after overwriting file B, got $RC"
pass "ref_count == 1: only file A still references the original blob"
# ── cleanup ───────────────────────────────────────────────────────────────────
echo " cleanup..."
for REMOTE in "$FILE_A" "$FILE_B"; do
ST=$(webdav_delete "$REMOTE")
[[ "$ST" == "204" ]] || fail "WebDAV DELETE $REMOTE expected 204, got $ST"
TRASH_ITEM=$(rest_get "/api/trash" \
| jq -r --arg n "$REMOTE" 'first(.[] | select(.name == $n) | .id) // empty')
if [[ -n "$TRASH_ITEM" ]]; then
rest_delete "/api/trash/$TRASH_ITEM" > /dev/null
fi
done
pass "Cleanup complete"
# ── summary ───────────────────────────────────────────────────────────────────
echo
echo "Results: $PASS passed, $FAIL failed."
[[ "$FAIL" -eq 0 ]] && echo "All tests passed." || exit 1