feat: check thumbnail cleanup on files deletion + correct ref counter
This commit is contained in:
Edouard Vanbelle
@@ -20,6 +20,11 @@
|
||||
# server uses the legacy blob path — so we avoid stats-based
|
||||
# assertions and rely on observable thumbnail behaviour instead.
|
||||
#
|
||||
# BLAKE3 hash of fixtures/dedup-test.jpg (= dedup-test-2.jpg content):
|
||||
# cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
# Used in /api/dedup/check/{hash} calls below to track ref_count lifecycle.
|
||||
# ref_count is only returned for admin users; setup.hurl creates an admin.
|
||||
#
|
||||
# Prerequisites: setup.hurl must have run (admin user exists).
|
||||
#
|
||||
# Run:
|
||||
@@ -87,6 +92,16 @@ jsonpath "$.name" == "dedup-test.jpg"
|
||||
jsonpath "$.folder_id" == {{test_folder_id}}
|
||||
|
||||
|
||||
# ref_count == 1: blob has exactly one file reference after first upload
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == true
|
||||
jsonpath "$.ref_count" == 1
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4 – Upload identical content again as dedup-test-2.jpg
|
||||
# Dedup: same blob, new file record, different file ID
|
||||
@@ -105,6 +120,16 @@ jsonpath "$.name" == "dedup-test-2.jpg"
|
||||
jsonpath "$.id" != "{{file1_id}}"
|
||||
|
||||
|
||||
# ref_count == 2: dedup hit — same blob now referenced by two file records
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == true
|
||||
jsonpath "$.ref_count" == 2
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 5 – Dedup proof: thumbnails are byte-identical
|
||||
# Thumbnail generation reads blob bytes and is keyed by
|
||||
@@ -156,6 +181,16 @@ Authorization: Bearer {{token}}
|
||||
HTTP 200
|
||||
|
||||
|
||||
# ref_count == 1: blob survives — file2 still holds a reference
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == true
|
||||
jsonpath "$.ref_count" == 1
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 8 – Blob still alive: file 2 thumbnail is accessible
|
||||
# After file 1 is permanently deleted the blob ref_count
|
||||
@@ -201,6 +236,15 @@ Authorization: Bearer {{token}}
|
||||
HTTP 200
|
||||
|
||||
|
||||
# ref_count hits 0 → blob and manifest deleted; user no longer owns this hash
|
||||
GET {{base_url}}/api/dedup/check/cde1ca663a2e62e0dadb41c3194e11ecb7d971d84c7451db17063b55c09e8066
|
||||
Authorization: Bearer {{token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.exists" == false
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 11 – Cleanup: delete the (now empty) test folder
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -62,6 +62,9 @@ OXICLOUD_SERVER_PORT=$SERVER_PORT
|
||||
OXICLOUD_STORAGE_PATH="$REPO_ROOT/tests/api/storage"
|
||||
set +a
|
||||
|
||||
# ensure storage is empty before starting
|
||||
echo "Wipe $OXICLOUD_STORAGE_PATH to ensure clean startup"
|
||||
rm -rf "$OXICLOUD_STORAGE_PATH"
|
||||
mkdir -p "$OXICLOUD_STORAGE_PATH"
|
||||
|
||||
# ── 3. Start OxiCloud server ──────────────────────────────────────────────────
|
||||
@@ -97,4 +100,6 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test
|
||||
|
||||
#bash "$API_DIR/dedup_bulk_upload.sh"
|
||||
|
||||
bash "$API_DIR/storage_cleanup_check.sh"
|
||||
|
||||
log "All tests passed."
|
||||
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# OxiCloud – Storage disk-cleanup verification
|
||||
# =============================================================
|
||||
# 1. Moves every live file and folder to trash via the REST API.
|
||||
# 2. Calls DELETE /api/trash/empty to permanently delete all
|
||||
# remaining trash items (including any left by previous tests).
|
||||
# 3. Asserts that no regular files remain under
|
||||
# $OXICLOUD_STORAGE_PATH/.thumbnails or .blobs.
|
||||
#
|
||||
# Called by run.sh after all Hurl tests have passed.
|
||||
# Can also be run standalone (server must already be up):
|
||||
# bash tests/api/storage_cleanup_check.sh
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
STORAGE_PATH="${OXICLOUD_STORAGE_PATH:-$REPO_ROOT/tests/api/storage}"
|
||||
|
||||
# shellcheck source=test.env
|
||||
source "$SCRIPT_DIR/test.env"
|
||||
|
||||
log() { echo "[storage-check] $*"; }
|
||||
fail() { echo $'\e[31m'"[storage-check] FAIL: $*"$'\e[0m' >&2; exit 1; }
|
||||
|
||||
# ── 1. Login ──────────────────────────────────────────────────────────────────
|
||||
|
||||
TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\":\"$username\",\"password\":\"$password\"}" \
|
||||
| jq -r '.access_token')
|
||||
|
||||
[[ -z "$TOKEN" || "$TOKEN" == "null" ]] && fail "login failed"
|
||||
log "Logged in."
|
||||
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
# ── 1b. Upload a probe image and verify its blob + thumbnail exist on disk ─────
|
||||
|
||||
# shellcheck source=../common/internal_storage_helper.sh
|
||||
source "$REPO_ROOT/tests/common/internal_storage_helper.sh"
|
||||
|
||||
FIXTURE="$REPO_ROOT/tests/fixtures/blue-image.png"
|
||||
|
||||
HOME_FOLDER_ID=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[0].id')
|
||||
[[ -z "$HOME_FOLDER_ID" || "$HOME_FOLDER_ID" == "null" ]] && fail "could not get home folder id"
|
||||
|
||||
PROBE_FILE_ID=$(curl -sf -X POST -H "$AUTH" \
|
||||
-F "folder_id=$HOME_FOLDER_ID" \
|
||||
-F "file=@$FIXTURE;type=image/png" \
|
||||
"$base_url/api/files/upload" | jq -r '.id')
|
||||
[[ -z "$PROBE_FILE_ID" || "$PROBE_FILE_ID" == "null" ]] && fail "probe file upload failed"
|
||||
log "Probe file uploaded (id=$PROBE_FILE_ID)."
|
||||
|
||||
# GET thumbnail to trigger on-demand generation
|
||||
HTTP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" -H "$AUTH" \
|
||||
"$base_url/api/files/$PROBE_FILE_ID/thumbnail/icon")
|
||||
[[ "$HTTP_STATUS" != "200" ]] && fail "thumbnail GET returned HTTP $HTTP_STATUS (expected 200)"
|
||||
log "Thumbnail fetched (HTTP 200)."
|
||||
|
||||
assert_local_blob_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe blob not found on disk"
|
||||
assert_preview_existsy "$FIXTURE" "$STORAGE_PATH" || fail "probe thumbnail not found on disk"
|
||||
log "Probe blob and thumbnail confirmed present on disk."
|
||||
|
||||
# ── 2. Move all live files and folders to trash ───────────────────────────────
|
||||
#
|
||||
# For each root folder, list its direct children and soft-delete them.
|
||||
# The server cascades folder deletion to all nested contents, so we only
|
||||
# need to iterate one level deep.
|
||||
|
||||
ROOT_FOLDERS=$(curl -sf -H "$AUTH" "$base_url/api/folders" | jq -r '.[].id')
|
||||
|
||||
for folder_id in $ROOT_FOLDERS; do
|
||||
CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
|
||||
|
||||
while IFS= read -r sub_id; do
|
||||
[[ -z "$sub_id" ]] && continue
|
||||
curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$sub_id" >/dev/null
|
||||
done < <(echo "$CONTENTS" | jq -r '.folders[].id')
|
||||
|
||||
while IFS= read -r file_id; do
|
||||
[[ -z "$file_id" ]] && continue
|
||||
curl -sf -X DELETE -H "$AUTH" "$base_url/api/files/$file_id" >/dev/null
|
||||
done < <(echo "$CONTENTS" | jq -r '.files[].id')
|
||||
done
|
||||
|
||||
log "All live objects moved to trash."
|
||||
|
||||
# ── 2b. Verify all root folders are empty according to the API ────────────────
|
||||
|
||||
for folder_id in $ROOT_FOLDERS; do
|
||||
CONTENTS=$(curl -sf -H "$AUTH" "$base_url/api/folders/$folder_id/listing")
|
||||
SUB_COUNT=$(echo "$CONTENTS" | jq '.folders | length')
|
||||
FILE_COUNT=$(echo "$CONTENTS" | jq '.files | length')
|
||||
if [[ "$SUB_COUNT" -ne 0 || "$FILE_COUNT" -ne 0 ]]; then
|
||||
fail "folder $folder_id still has $SUB_COUNT subfolder(s) and $FILE_COUNT file(s)"
|
||||
fi
|
||||
done
|
||||
|
||||
log "API confirms all root folders are empty."
|
||||
|
||||
# ── 3. Permanently delete everything in trash ─────────────────────────────────
|
||||
|
||||
curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/empty" >/dev/null
|
||||
log "Trash emptied."
|
||||
|
||||
# ── 3b. Verify trash is empty according to the API ───────────────────────────
|
||||
|
||||
TRASH_COUNT=$(curl -sf -H "$AUTH" "$base_url/api/trash" | jq 'length')
|
||||
if [[ "$TRASH_COUNT" -ne 0 ]]; then
|
||||
fail "trash still contains $TRASH_COUNT item(s) after empty"
|
||||
fi
|
||||
|
||||
log "API confirms trash is empty."
|
||||
|
||||
# ── 4. Disk verification ──────────────────────────────────────────────────────
|
||||
|
||||
THUMB_FILES=$(find "$STORAGE_PATH/.thumbnails" -type f 2>/dev/null || true)
|
||||
BLOB_FILES=$(find "$STORAGE_PATH/.blobs" -type f 2>/dev/null || true)
|
||||
|
||||
if [[ -n "$THUMB_FILES" ]]; then
|
||||
THUMB_COUNT=$(echo "$THUMB_FILES" | wc -l | tr -d ' ')
|
||||
log "Leftover thumbnail files ($THUMB_COUNT):"
|
||||
echo "$THUMB_FILES"
|
||||
fail "$THUMB_COUNT thumbnail file(s) remain on disk after full cleanup"
|
||||
fi
|
||||
|
||||
if [[ -n "$BLOB_FILES" ]]; then
|
||||
BLOB_COUNT=$(echo "$BLOB_FILES" | wc -l | tr -d ' ')
|
||||
log "Leftover blob files ($BLOB_COUNT):"
|
||||
echo "$BLOB_FILES"
|
||||
fail "$BLOB_COUNT blob file(s) remain on disk after full cleanup"
|
||||
fi
|
||||
|
||||
log "OK — no blobs or thumbnails remain on disk."
|
||||
Reference in New Issue
Block a user