From 671e6ac0e78e4e614f4484fdd26abbebdbb27904 Mon Sep 17 00:00:00 2001
From: Edouard Vanbelle <opensource@edouard.vanbelle.fr>
Date: Wed, 26 Aug 2026 00:49:49 +0200
Subject: [PATCH] test(api): end-to-end check of both sidecar import jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Nothing exercised these jobs. Their unit tests cover the directory walk
— which files each claims — but neither had ever executed a run.

The test environment always starts fresh, so there is no pre-migration
data to import. This creates it, and the reconstruction is EXACT rather
than an imitation: the on-disk layout did not change in this work. A
rendered thumbnail has always been written to {size}/{hash}.webp and an
uploaded preview to {size}/ext-{id}.jpg; the only new thing is the row.
So upload through the real API, then delete the row, and what remains on
disk is byte-for-byte what a pre-migration install has.

Deleting the row must also release the reference it held, or the
manufactured state would carry a reference no legacy install ever had
and storage_cleanup_check.sh would report a leak this script caused.
file_attached_blobs has an ON DELETE trigger for that;
content_derived_blobs does not — its Rust purge path releases explicitly
— so the strip decrements it directly.

Three assertions, in increasing order of what they catch:

  1. Both rows come back, and the imported attached row carries the nil
     uploader sentinel rather than a fabricated one.
  2. A COPY inherits the imported preview. This is the user-visible
     point and was impossible before the row existed: the ext- sidecar
     is keyed by file_id, no copy path duplicates it, so the copy
     silently fell back to a render.
  3. Re-running imports nothing and changes no refcount. The likeliest
     silent defect — store_attached_blob is ON CONFLICT DO UPDATE, so an
     import that skipped its existence check would release and retake a
     reference every run, invisible except as drift.

psql runs inside the compose container rather than depending on a host
binary, matching how spawn-db.sh probes readiness. Ordered before
storage_cleanup_check.sh, which deletes everything it needs.
---
 tests/api/run.sh                |   4 +
 tests/api/thumb_import_check.sh | 213 ++++++++++++++++++++++++++++++++
 2 files changed, 217 insertions(+)
 create mode 100755 tests/api/thumb_import_check.sh

diff --git a/tests/api/run.sh b/tests/api/run.sh
index 3d17c303..75d89c34 100755
--- a/tests/api/run.sh
+++ b/tests/api/run.sh
@@ -243,6 +243,10 @@ if ! hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --
   exit 1
 fi
 
+# Migration check runs BEFORE the cleanup sweep, which deletes everything
+# it would otherwise need.
+bash "$API_DIR/thumb_import_check.sh"
+
 bash "$API_DIR/storage_cleanup_check.sh"
 
 # ── 5. OPAQUE crypto handshake — the parts Hurl can't drive ─────────────
diff --git a/tests/api/thumb_import_check.sh b/tests/api/thumb_import_check.sh
new file mode 100755
index 00000000..6b42420f
--- /dev/null
+++ b/tests/api/thumb_import_check.sh
@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+# =============================================================
+# OxiCloud – legacy sidecar → blob migration (both import jobs)
+# =============================================================
+# Exercises `thumb_derived_import` and `thumb_attached_import` end to end,
+# which nothing else does: their unit tests cover only the directory walk,
+# never a run.
+#
+# ── How legacy state is manufactured ─────────────────────────────────────
+#
+# The test environment always starts fresh, so there is no pre-migration
+# data to import. We create it, and the reconstruction is EXACT rather than
+# an imitation: the on-disk layout did not change in this work. A
+# server-rendered thumbnail has always been written to
+# `{size}/{hash}.webp`, and an uploaded preview to `{size}/ext-{id}.jpg`.
+# The only thing that is new is the DB row.
+#
+# So: upload through the real API (which writes both the file and the row),
+# then delete the row. What remains on disk is byte-for-byte what a
+# pre-migration install has.
+#
+# Deleting the row must also release the reference it held, or the
+# manufactured state would carry a reference no legacy install ever had and
+# the end-of-suite registry check would report a leak that this script
+# caused. `file_attached_blobs` has an ON DELETE trigger that does it;
+# `content_derived_blobs` does not, so we decrement explicitly.
+#
+# ── What is asserted ─────────────────────────────────────────────────────
+#
+#   1. Both rows come back after the import.
+#   2. The uploaded preview survives a COPY — the user-visible point of
+#      `file_attached_blobs`, and impossible before the row existed.
+#   3. Re-running imports nothing and changes no refcount. This is the
+#      defect most likely to be silent: `store_attached_blob` is
+#      ON CONFLICT DO UPDATE, so an import that skipped its existence
+#      check would release and retake a reference on every run.
+#
+# Runs BEFORE storage_cleanup_check.sh, which deletes everything.
+#
+# Prerequisites: setup.hurl has run (admin exists); docker compose db up.
+# =============================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+COMPOSE_FILE="$REPO_ROOT/tests/common/docker-compose.test.yml"
+
+# shellcheck source=test.env
+source "$SCRIPT_DIR/test.env"
+
+log()  { echo "[thumb-import] $*"; }
+fail() { echo $'\e[31m'"[thumb-import] FAIL: $*"$'\e[0m' >&2; exit 1; }
+
+# psql inside the compose container — no host psql dependency, matching
+# how spawn-db.sh probes readiness.
+sql() {
+  docker compose -f "$COMPOSE_FILE" exec -T postgres-test \
+    psql -U oxicloud_test -d oxicloud_test -tAqc "$1"
+}
+
+TOKEN=$(curl -sf -X POST "$base_url/api/auth/login" \
+  -H "Content-Type: application/json" \
+  -d "{\"username\":\"$username\",\"password\":\"$password\"}" \
+  | jq -r '.access_token')
+[[ -n "$TOKEN" && "$TOKEN" != "null" ]] || fail "login failed"
+AUTH="Authorization: Bearer $TOKEN"
+
+# ── 1. Create a file with BOTH sidecar shapes ────────────────────────────
+
+SRC_FOLDER=$(curl -sf -X POST "$base_url/api/folders" -H "$AUTH" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"hurl-import-src"}' | jq -r '.id')
+DST_FOLDER=$(curl -sf -X POST "$base_url/api/folders" -H "$AUTH" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"hurl-import-dst"}' | jq -r '.id')
+[[ -n "$SRC_FOLDER" && "$SRC_FOLDER" != "null" ]] || fail "folder create failed"
+
+UPLOAD=$(curl -sf -X POST "$base_url/api/files/upload" -H "$AUTH" \
+  -F "folder_id=$SRC_FOLDER" \
+  -F "file=@$REPO_ROOT/tests/fixtures/red-image.png;type=image/png")
+FILE_ID=$(echo "$UPLOAD" | jq -r '.id')
+BLOB_HASH=$(echo "$UPLOAD" | jq -r '.content_hash')
+[[ -n "$FILE_ID" && "$FILE_ID" != "null" ]] || fail "upload failed: $UPLOAD"
+log "uploaded file=$FILE_ID hash=${BLOB_HASH:0:12}"
+
+# Render → writes {size}/{hash}.webp AND the content_derived_blobs row.
+curl -sf -H "$AUTH" "$base_url/api/files/$FILE_ID/thumbnail/preview" -o /dev/null \
+  || fail "render thumbnail failed"
+
+# Upload → writes ext-{file_id}.jpg AND the file_attached_blobs row.
+curl -sf -X PUT -H "$AUTH" -H "Content-Type: image/png" \
+  --data-binary "@$REPO_ROOT/tests/fixtures/green-image.png" \
+  "$base_url/api/files/$FILE_ID/thumbnail/preview" -o /dev/null \
+  || fail "upload thumbnail failed"
+
+UPLOADED_THUMB=$(mktemp)
+curl -sf -H "$AUTH" "$base_url/api/files/$FILE_ID/thumbnail/preview" -o "$UPLOADED_THUMB"
+
+DERIVED_BEFORE=$(sql "SELECT count(*) FROM storage.content_derived_blobs WHERE source_hash='$BLOB_HASH';")
+ATTACHED_BEFORE=$(sql "SELECT count(*) FROM storage.file_attached_blobs WHERE file_id='$FILE_ID';")
+[[ "$DERIVED_BEFORE"  -ge 1 ]] || fail "expected a content_derived_blobs row before stripping"
+[[ "$ATTACHED_BEFORE" -ge 1 ]] || fail "expected a file_attached_blobs row before stripping"
+log "rows present before stripping: derived=$DERIVED_BEFORE attached=$ATTACHED_BEFORE"
+
+# ── 2. Strip the rows → this IS the legacy state ─────────────────────────
+#
+# Release each reference as the row goes, so the manufactured state matches
+# a pre-migration install rather than carrying references it never had.
+# file_attached_blobs does this via its ON DELETE trigger; the derived table
+# has no trigger (its Rust purge path releases explicitly), so do it here.
+
+sql "WITH gone AS (
+       DELETE FROM storage.content_derived_blobs
+        WHERE source_hash='$BLOB_HASH'
+       RETURNING blob_hash
+     )
+     UPDATE storage.chunk_manifests m
+        SET ref_count = GREATEST(m.ref_count - 1, 0)
+       FROM gone WHERE m.file_hash = gone.blob_hash;" >/dev/null
+
+sql "DELETE FROM storage.file_attached_blobs WHERE file_id='$FILE_ID';" >/dev/null
+
+[[ "$(sql "SELECT count(*) FROM storage.content_derived_blobs WHERE source_hash='$BLOB_HASH';")" == "0" ]] \
+  || fail "derived row survived the strip"
+[[ "$(sql "SELECT count(*) FROM storage.file_attached_blobs WHERE file_id='$FILE_ID';")" == "0" ]] \
+  || fail "attached row survived the strip"
+log "legacy state manufactured: files on disk, no rows."
+
+# ── 3. Run the imports ───────────────────────────────────────────────────
+
+for job in thumb_derived_import thumb_attached_import; do
+  curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/$job/trigger" >/dev/null \
+    || fail "$job trigger failed"
+  log "$job triggered."
+done
+
+DERIVED_AFTER=$(sql "SELECT count(*) FROM storage.content_derived_blobs WHERE source_hash='$BLOB_HASH';")
+ATTACHED_AFTER=$(sql "SELECT count(*) FROM storage.file_attached_blobs WHERE file_id='$FILE_ID';")
+[[ "$DERIVED_AFTER"  -ge 1 ]] || fail "thumb_derived_import did not restore the row"
+[[ "$ATTACHED_AFTER" -ge 1 ]] || fail "thumb_attached_import did not restore the row"
+log "rows restored: derived=$DERIVED_AFTER attached=$ATTACHED_AFTER"
+
+# Provenance: imported rows carry the sentinel, which is how an operator
+# tells them from previews with a real uploader.
+UPLOADER=$(sql "SELECT uploaded_by FROM storage.file_attached_blobs WHERE file_id='$FILE_ID';")
+[[ "$UPLOADER" == "00000000-0000-0000-0000-000000000000" ]] \
+  || fail "imported row should carry the nil uploader sentinel, got '$UPLOADER'"
+
+# ── 4. The user-visible point: a COPY inherits the preview ───────────────
+#
+# Impossible before the row existed — the ext- sidecar is keyed by file_id
+# and no copy path duplicates it, so the copy fell back to a render.
+
+COPY_ID=$(curl -sf -X POST "$base_url/api/batch/files/copy" -H "$AUTH" \
+  -H "Content-Type: application/json" \
+  -d "{\"file_ids\":[\"$FILE_ID\"],\"target_folder_id\":\"$DST_FOLDER\"}" \
+  | jq -r '.successful[0].id')
+[[ -n "$COPY_ID" && "$COPY_ID" != "null" ]] || fail "copy failed"
+
+COPY_THUMB=$(mktemp)
+curl -sf -H "$AUTH" "$base_url/api/files/$COPY_ID/thumbnail/preview" -o "$COPY_THUMB"
+cmp -s "$UPLOADED_THUMB" "$COPY_THUMB" \
+  || fail "copy did not inherit the imported preview"
+log "copy inherits the imported preview."
+
+# ── 5. Idempotence: a second run imports nothing and churns nothing ──────
+#
+# The likely silent defect. store_attached_blob is ON CONFLICT DO UPDATE, so
+# an import that skipped its existence check would release and retake a
+# reference every run — invisible except as refcount drift.
+
+ATTACHED_HASH=$(sql "SELECT blob_hash FROM storage.file_attached_blobs WHERE file_id='$FILE_ID' LIMIT 1;")
+[[ -n "$ATTACHED_HASH" ]] || fail "no attached blob_hash to check refcounts against"
+# A single-chunk blob has a manifest whose file_hash equals its own hash, so
+# this is the counter add_reference actually touches. `-` rather than empty
+# keeps the later comparison meaningful if the manifest is unexpectedly absent.
+REFS_BEFORE=$(sql "SELECT ref_count FROM storage.chunk_manifests WHERE file_hash='$ATTACHED_HASH';")
+REFS_BEFORE=${REFS_BEFORE:--}
+
+for job in thumb_derived_import thumb_attached_import; do
+  curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/$job/trigger" >/dev/null \
+    || fail "$job re-trigger failed"
+done
+
+REFS_AFTER=$(sql "SELECT ref_count FROM storage.chunk_manifests WHERE file_hash='$ATTACHED_HASH';")
+REFS_AFTER=${REFS_AFTER:--}
+DERIVED_2=$(sql "SELECT count(*) FROM storage.content_derived_blobs WHERE source_hash='$BLOB_HASH';")
+ATTACHED_2=$(sql "SELECT count(*) FROM storage.file_attached_blobs WHERE file_id='$FILE_ID';")
+
+[[ "$REFS_AFTER" == "$REFS_BEFORE" ]] \
+  || fail "re-run changed the attached blob refcount: $REFS_BEFORE → $REFS_AFTER"
+[[ "$DERIVED_2"  == "$DERIVED_AFTER"  ]] || fail "re-run duplicated derived rows"
+[[ "$ATTACHED_2" == "$ATTACHED_AFTER" ]] || fail "re-run duplicated attached rows"
+log "re-run is a no-op: rows and refcounts unchanged."
+
+# ── 6. Teardown ──────────────────────────────────────────────────────────
+# Everything created here must go — one database serves the whole suite,
+# and storage_cleanup_check.sh afterwards asserts the registry drains to
+# zero.
+
+rm -f "$UPLOADED_THUMB" "$COPY_THUMB"
+
+for folder in "$SRC_FOLDER" "$DST_FOLDER"; do
+  curl -sf -X DELETE -H "$AUTH" "$base_url/api/folders/$folder" -o /dev/null || true
+done
+TRASH=$(curl -sf -H "$AUTH" "$base_url/api/trash/resources" || echo '{}')
+for folder in "$SRC_FOLDER" "$DST_FOLDER"; do
+  tid=$(echo "$TRASH" | jq -r --arg id "$folder" '.items[]? | select(.resource.id == $id) | .resource.id')
+  [[ -n "$tid" ]] && curl -sf -X DELETE -H "$AUTH" "$base_url/api/trash/$tid" -o /dev/null || true
+done
+
+log "OK — both imports restore their rows, the copy inherits the preview, and re-running is a no-op."