diff --git a/tests/api/copy_folder_ref_count.hurl b/tests/api/copy_folder_ref_count.hurl new file mode 100644 index 00000000..d0be196c --- /dev/null +++ b/tests/api/copy_folder_ref_count.hurl @@ -0,0 +1,514 @@ +# ============================================================= +# OxiCloud – Copy-folder ref_count regression +# ============================================================= +# Regression test for the ref_count drift found on Ed's sandbox +# (2026-08-22) and traced to the copy-folder path. When a folder +# is copied, every file inside gets duplicated as a NEW file row +# pointing at the SAME blob(s) — dedup wins bytes-on-disk, but +# `storage.blobs.ref_count` MUST bump by the number of new refs. +# If it doesn't, dedup GC will reap a blob that a live file row +# still references → dangling reference → user gets 404 on +# download of the copied file. +# +# Covers two paths so the CDC boundary can't hide a regression: +# +# 1. Small file (`copy-ref-small.txt`) → single legacy whole- +# file blob. `storage.blobs.ref_count` counted directly on +# the file's content hash. +# 2. 2 MB file (`copy-ref-cdc.bin`) → FastCDC produces multiple +# distinct chunks. Whole-file `content_hash` still resolves +# through the dedup API. +# +# Both fixtures are DEDICATED — unique content so ref_count +# assertions are absolute (== 1, == 2). Do NOT reuse these +# fixtures in other hurl files or absolute assertions here will +# flake. +# +# Deletion is TWO STEPS in OxiCloud: +# `DELETE /api/folders/{id}` → moves to trash (ref_count +# unchanged; children still +# reference the blob). +# `DELETE /api/trash/{id}` → permanent purge; NOW +# ref_count decrements. If it +# hits 0, blob row is deleted +# synchronously (`exists=false`). +# So the test purges trash after every folder-delete step — +# skipping that would make the assertions wrong regardless of +# whether the copy-side bug is present. +# +# `/api/dedup/check/{hash}` returns `ref_count` only for admin +# callers (regular users get `null` for anti-enumeration). This +# suite requires the admin token; setup.hurl seeds it. +# +# Run: +# hurl --variables-file tests/api/test.env --test \ +# tests/api/copy_folder_ref_count.hurl +# ============================================================= + + +# ───────────────────────────────────────────────────────────── +# Step 1 — Login (admin, required for ref_count in dedup API) +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/auth/login +Content-Type: application/json +{ + "username": "{{username}}", + "password": "{{password}}" +} + +HTTP 200 +[Captures] +token: jsonpath "$.access_token" + + +# ───────────────────────────────────────────────────────────── +# Step 2 — Baseline sweep: run `blobs_consistency` before we +# touch anything and capture the finding count. Later +# sweeps assert equality with this baseline instead of +# `== 0` — so a stale finding from a previous test's +# leftover state doesn't flunk this test, only NEW +# drift introduced by our copy/delete does. +# +# Trigger returns `outcome.count = stats.finding_count` +# for recoverable tenants (see +# `scheduler/recoverable.rs::JobOutcome::ok_with` in +# the Completed branch). `outcome.outcome == "ok"` +# means the run walked the whole subject; it does NOT +# mean zero findings. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/blobs_consistency/trigger +Authorization: Bearer {{token}} + +HTTP 200 +[Captures] +baseline_findings: jsonpath "$.outcome.count" +[Asserts] +jsonpath "$.ok" == true +jsonpath "$.outcome.outcome" == "ok" + + +# ============================================================= +# Scenario A — small file (single legacy whole-file blob) +# ============================================================= + +# ───────────────────────────────────────────────────────────── +# A1 — Create source folder +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/folders +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "name": "hurl-ref-source-small" +} + +HTTP 201 +[Captures] +src_small_id: jsonpath "$.id" + + +# ───────────────────────────────────────────────────────────── +# A2 — Upload the small fixture. Content is unique to this +# test, so ref_count starts at exactly 1 (no dedup +# collision with any other fixture in the suite). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{token}} +[MultipartFormData] +folder_id: {{src_small_id}} +file: file,fixtures/copy-ref-small.txt; text/plain + +HTTP 201 +[Captures] +small_file_id: jsonpath "$.id" +[Asserts] +jsonpath "$.content_hash" == "2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3" + + +# ───────────────────────────────────────────────────────────── +# A3 — Baseline: exactly one live reference. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/dedup/check/2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + +# ───────────────────────────────────────────────────────────── +# A4 — Create target folder + copy source into it. The batch +# endpoint is the single entry point every FE / WebDAV +# code path funnels through. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/folders +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "name": "hurl-ref-target-small" +} + +HTTP 201 +[Captures] +tgt_small_id: jsonpath "$.id" + +POST {{base_url}}/api/batch/folders/copy +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "folder_ids": ["{{src_small_id}}"], + "target_folder_id": "{{tgt_small_id}}" +} + +HTTP 200 +[Captures] +copy_small_root: jsonpath "$.successful[0].new_root_folder_id" +[Asserts] +jsonpath "$.stats.successful" == 1 +jsonpath "$.stats.failed" == 0 +jsonpath "$.successful[0].files_copied" == 1 + + +# ───────────────────────────────────────────────────────────── +# A5 — COPY-SIDE ASSERTION: ref_count must be exactly 2. The +# pre-fix bug left this at 1 — a subsequent GC would then +# reap the blob out from under the copied file. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/dedup/check/2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 2 + + +# ───────────────────────────────────────────────────────────── +# A5b — Sweep the ENTIRE blob table via `blobs_consistency`. +# Complements the single-hash probe above: if the copy +# path miscounted some OTHER blob shared by an unrelated +# row (e.g. a global thumbnail blob, an OS icon dedup +# hit), the single-hash probe wouldn't catch it. Delta +# vs `baseline_findings` isolates NEW drift from ambient. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/blobs_consistency/trigger +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.ok" == true +jsonpath "$.outcome.outcome" == "ok" +jsonpath "$.outcome.count" == {{baseline_findings}} + + +# ───────────────────────────────────────────────────────────── +# A6 — Soft-delete the copied folder tree (moves to trash). +# ref_count is EXPECTED to stay at 2 — trashed files +# still reference the blob per the `NOT is_trashed` gap +# the auditor deliberately closed (see +# `[[project_by_hash_drop_is_trashed_filter]]`). Asserting +# == 2 here makes the trash-vs-permanent boundary explicit +# so a future refactor that changed the semantics would +# surface at THIS line, not several steps downstream. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/folders/{{copy_small_root}} +Authorization: Bearer {{token}} + +HTTP 204 + +GET {{base_url}}/api/dedup/check/2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 2 + + +# ───────────────────────────────────────────────────────────── +# A7 — Purge the copy folder permanently. +# `DELETE /api/trash/{id}` accepts the ORIGINAL resource +# id as the path param (verified in `trash_handler.rs:: +# delete_permanently`) — no need to GET+filter the trash +# listing to translate. If soft-delete silently failed, +# the ref_count assertion two lines below catches it. +# ref_count must drop to exactly 1 (the source folder +# still holds its file). Guards the DECREMENT half of +# the invariant: a double-decrement here would go to 0 +# and the next GC wipes the still-live original's blob. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/trash/{{copy_small_root}} +Authorization: Bearer {{token}} + +HTTP 200 + + +GET {{base_url}}/api/dedup/check/2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + +# ───────────────────────────────────────────────────────────── +# A8 — Soft-delete + purge the SOURCE folder (last holder). +# ref_count hits 0 → blob row deleted synchronously → +# `exists == false` on the next probe. Mirror pattern to +# `dedup_blob_cleanup.hurl` step 10, applied to folder- +# scoped delete. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/folders/{{src_small_id}} +Authorization: Bearer {{token}} + +HTTP 204 + + +DELETE {{base_url}}/api/trash/{{src_small_id}} +Authorization: Bearer {{token}} + +HTTP 200 + + +GET {{base_url}}/api/dedup/check/2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == false + + +# ───────────────────────────────────────────────────────────── +# A9 — End-of-Scenario-A sweep. Scenario A introduced two file +# rows (source + copy), then deleted both. Net effect on +# the DB is zero — so the drift count must be exactly the +# baseline, no more, no less. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/blobs_consistency/trigger +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.ok" == true +jsonpath "$.outcome.outcome" == "ok" +jsonpath "$.outcome.count" == {{baseline_findings}} + + +# ============================================================= +# Scenario B — 2 MB multi-chunk file (CDC manifest path) +# +# Coverage note: `blobs_consistency` iterates every blob row +# (whole-file AND per-chunk) and checks each ref_count against +# the auditor SQL, so a chunk-level under-count is caught by +# the sweep in B5b. But the DEDICATED tenant for the CDC path +# is `manifest_consistency` (queued on a separate branch as of +# 2026-08-23) — when it lands, replace the `blobs_consistency` +# trigger in B5b/B9 with `manifest_consistency` (or run both +# in the batch) so the assertion is scoped to what actually +# describes the CDC invariant. +# ============================================================= + +# ───────────────────────────────────────────────────────────── +# B1 — Source folder for the multi-chunk scenario. Isolated +# from Scenario A so deletion order can't mask a bug (e.g. +# a shared blob whose counter goes negative). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/folders +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "name": "hurl-ref-source-cdc" +} + +HTTP 201 +[Captures] +src_cdc_id: jsonpath "$.id" + + +# ───────────────────────────────────────────────────────────── +# B2 — Upload the 2 MB dedicated fixture. Deterministic per- +# position content → FastCDC produces multiple distinct +# chunks (no chunk-level dedup with any other fixture). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{token}} +[MultipartFormData] +folder_id: {{src_cdc_id}} +file: file,fixtures/copy-ref-cdc.bin; application/octet-stream + +HTTP 201 +[Captures] +cdc_file_id: jsonpath "$.id" +[Asserts] +jsonpath "$.content_hash" == "fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83" + + +# ───────────────────────────────────────────────────────────── +# B3 — Baseline. If exists=false here, the whole-file hash +# isn't registered in `storage.blobs` for CDC uploads on +# this build — swap to a chunk-hash probe or a +# blobs_consistency-driven assertion. See +# `docs/plan/recovery.md`. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/dedup/check/fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + +# ───────────────────────────────────────────────────────────── +# B4 — Target folder + folder copy. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/folders +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "name": "hurl-ref-target-cdc" +} + +HTTP 201 +[Captures] +tgt_cdc_id: jsonpath "$.id" + +POST {{base_url}}/api/batch/folders/copy +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "folder_ids": ["{{src_cdc_id}}"], + "target_folder_id": "{{tgt_cdc_id}}" +} + +HTTP 200 +[Captures] +copy_cdc_root: jsonpath "$.successful[0].new_root_folder_id" +[Asserts] +jsonpath "$.stats.successful" == 1 +jsonpath "$.successful[0].files_copied" == 1 + + +# ───────────────────────────────────────────────────────────── +# B5 — CDC copy-side assertion: ref_count == 2. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/dedup/check/fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 2 + + +# ───────────────────────────────────────────────────────────── +# B5b — Full-DB sweep after CDC copy. Multi-chunk path +# exercises the manifest side of the ref-count invariant +# — a bug that skips one chunk out of N would leak that +# chunk without touching the single whole-file assertion +# above. Sweep catches it. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/blobs_consistency/trigger +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.ok" == true +jsonpath "$.outcome.outcome" == "ok" +jsonpath "$.outcome.count" == {{baseline_findings}} + + +# ───────────────────────────────────────────────────────────── +# B6 — Soft-delete copy: ref_count stays at 2. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/folders/{{copy_cdc_root}} +Authorization: Bearer {{token}} + +HTTP 204 + +GET {{base_url}}/api/dedup/check/fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 2 + + +# ───────────────────────────────────────────────────────────── +# B7 — Purge copy from trash directly by folder id: ref_count → 1. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/trash/{{copy_cdc_root}} +Authorization: Bearer {{token}} + +HTTP 200 + + +GET {{base_url}}/api/dedup/check/fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == true +jsonpath "$.ref_count" == 1 + + +# ───────────────────────────────────────────────────────────── +# B8 — Soft-delete + purge SOURCE: ref_count → 0, blob purged. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/folders/{{src_cdc_id}} +Authorization: Bearer {{token}} + +HTTP 204 + + +DELETE {{base_url}}/api/trash/{{src_cdc_id}} +Authorization: Bearer {{token}} + +HTTP 200 + + +GET {{base_url}}/api/dedup/check/fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83 +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.exists" == false + + +# ───────────────────────────────────────────────────────────── +# B9 — End-of-Scenario-B sweep. All Scenario B rows gone; +# finding count must be back at baseline. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/blobs_consistency/trigger +Authorization: Bearer {{token}} + +HTTP 200 +[Asserts] +jsonpath "$.ok" == true +jsonpath "$.outcome.outcome" == "ok" +jsonpath "$.outcome.count" == {{baseline_findings}} + + +# ============================================================= +# Cleanup — soft-delete + purge the two empty target folders so +# the run leaves nothing behind. Same direct-by-id pattern as +# above; no trash-listing filter needed. +# ============================================================= +DELETE {{base_url}}/api/folders/{{tgt_small_id}} +Authorization: Bearer {{token}} +HTTP 204 + +DELETE {{base_url}}/api/trash/{{tgt_small_id}} +Authorization: Bearer {{token}} +HTTP 200 + + +DELETE {{base_url}}/api/folders/{{tgt_cdc_id}} +Authorization: Bearer {{token}} +HTTP 204 + +DELETE {{base_url}}/api/trash/{{tgt_cdc_id}} +Authorization: Bearer {{token}} +HTTP 200 diff --git a/tests/api/copy_folder_ref_count_diag.sh b/tests/api/copy_folder_ref_count_diag.sh new file mode 100755 index 00000000..b3d1fe0d --- /dev/null +++ b/tests/api/copy_folder_ref_count_diag.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# ============================================================= +# copy_folder_ref_count.hurl — post-failure diagnostic +# ============================================================= +# When `copy_folder_ref_count.hurl` asserts a specific +# `ref_count` value and the API returns something else, this +# script inspects the two DB tables the API surface consults +# to distinguish which side is broken: +# +# 1. `storage.chunk_manifests.ref_count` — queried FIRST by +# `dedup_service::get_blob_metadata` (`dedup_service.rs` +# :1543-1552). If a manifest row exists for the hash, +# the API returns THIS ref_count. +# 2. `storage.blobs.ref_count` — legacy whole-file fallback, +# returned only when NO manifest row exists. +# +# Small files (< CDC min chunk size) still get a manifest row +# — one degenerate chunk with `chunk_hashes = [file_hash]` — +# so BOTH tables carry a ref_count for the same hash. When the +# copy or purge path only updates one of the two, the counters +# diverge. +# +# The `actual_auditor` column is the source of truth: the +# `blobs_consistency` auditor formula counting live references +# from `storage.files` + `chunk_manifests.chunk_hashes[]` +# (`blobs_consistency_service.rs:395-408`). Both stored values +# should equal this. +# +# Bug interpretation matrix (S=stored, M=manifest, A=auditor): +# +# S == M == A → consistent (test bug, unlikely) +# S < A and M == A → blob decrement over-fires +# S == A and M > A → manifest decrement missed +# S > A and M > A → decrement missed both sides +# S < A and M < A → double-decrement both sides +# S > A and M == A → increment missed on blob side +# S == A and M < A → increment missed on manifest +# (S=0, M=2, A=1) [seen 8/23]→ blob double-decrement + +# manifest never decremented +# +# The 2026-08-22 sandbox drift showed the raw shape +# (`stored=0, actual=1`) that this diagnostic now separates +# per-table. +# +# Called from `run.sh` immediately on hurl failure — see the +# dedicated `if ! hurl …; then bash …_diag.sh; fi` block. +# ============================================================= + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" + +# Same connection string every other test-time script uses. +export PGPASSWORD=oxicloud_test +PSQL=(psql -h 127.0.0.1 -p 5433 -U oxicloud_test -d oxicloud_test + --set ON_ERROR_STOP=1 --pset pager=off) + +SMALL_HASH='2d8eb13178cff0036a22e0c3c42446061e86579f9d59ea73c8343bccc2df0fd3' +CDC_HASH='fb1e63c28bb792e0f69cd16cd7595989f83c218cf70894e07d1f811ab1dc6f83' + +log() { echo "[ref_count-diag] $*"; } + +log "─────────────────────────────────────────────────────────" +log "copy_folder_ref_count.hurl failed — running diagnostic." +log "Shows blob.ref_count AND manifest.ref_count side by side —" +log "the API queries manifest first (dedup_service.rs:1543), so" +log "if the two diverge, the API surface + auditor + on-disk" +log "state all report different numbers. See script header." +log "─────────────────────────────────────────────────────────" + +# Full picture for each fixture hash — one row per hash. +# LEFT JOINs so a hash present in only one table still surfaces +# (the other column comes back NULL, which is itself diagnostic). +"${PSQL[@]}" <A → manifest decrement missed" +log " blobA → double-decrement on blob +" +log " no-decrement on manifest" +log " (matches the 2026-08-23 case:" +log " blob=0, manifest=2, actual=1)" +log " blob>A, manifest=A → blob increment missed" +log " blob=A, manifest