feat(upload): cover chunk upload + add support of different digest hash

Prefer stream storage rather using buffered (in memory) note: on many unix like tmpfs are in-memory, sungle PUT are sized limited Storage map (NC stands for Nextcloud gateway) ┌───────────────────────────────────────────────────────┬────────────────────────────────────────────────────────────────────┬─────────────────────────────────────────────────┐ │ Streaming surface │ Destination │ Configurable via │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST chunked PUT /api/uploads/{id} chunk │ {storage_path}/.uploads/{upload_id}/chunk_{NNNNNN} │ OXICLOUD_STORAGE_PATH (the .uploads subdir is │ │ │ │ hard-wired) │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST chunked assemble (during /complete) │ {storage_path}/.uploads/{upload_id}/assembled │ same │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ NC chunked PUT /dav/uploads/.../{chunk} │ {storage_path}/.uploads/nextcloud/{user}/{upload_id}/{chunk_name} │ same │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ NC chunked assemble (during MOVE) │ {storage_path}/.uploads/nextcloud/{user}/{upload_id}/.assembled │ same │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ NC single-file PUT /dav/files/.../{path} (via │ OXICLOUD_UPLOAD_TMPDIR if set, else OS default temp (/tmp on │ OXICLOUD_UPLOAD_TMPDIR │ │ spool_body_to_temp) │ Linux) │ │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST WebDAV PUT /webdav/{path} (via │ same as above │ OXICLOUD_UPLOAD_TMPDIR │ │ spool_body_to_temp) │ │ │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST multipart upload /api/files/upload │ {storage_path}/.dedup_temp/upload-{uuid} │ OXICLOUD_STORAGE_PATH (hard-wired subdir) │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ WOPI PutFile │ OS default temp via NamedTempFile::new() (no override) │ (none — bug worth tracking) │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ Final blob storage (after fsync + rename) │ {storage_path}/.blobs/{ab}/{abc…}.blob │ OXICLOUD_STORAGE_PATH │ └───────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────┴─────────────────────────────────────────────────┘ one caveat: a malicious user can create many chunked upload and saturate local storage
2026-06-08 23:17:07 +02:00
parent 5e638691ad
commit 41aad26702
9 changed files with 1048 additions and 91 deletions
@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+# =============================================================
+# OxiCloud — NextCloud chunked upload cap + streaming check
+# =============================================================
+# Validates the per-chunk `storage.chunk_max_bytes` cap on the
+# NextCloud-compat chunked-upload surface (`/remote.php/dav/uploads/`),
+# and round-trips a small file through MKCOL → PUT → MOVE to
+# prove the streaming write at `handle_put_chunk` produces a
+# byte-exact blob on disk.
+#
+# Sister test of `tests/api/chunked_upload_cap.hurl` (REST chunked).
+# Both surfaces share the same `chunk_max_bytes` cap and the same
+# `stream_body_to_path` helper; this test exercises the NC half:
+# Basic Auth via app-password, WebDAV verbs, fewer protocol
+# affordances than the REST API.
+#
+# Cases covered:
+#   1. SUCCESS — MKCOL → PUT a single chunk (hello.txt, 32 B) →
+#      MOVE → verify the assembled file's BLAKE3 over REST.
+#   2. CAP REJECTION — MKCOL a fresh session → PUT a 5 MiB chunk →
+#      413 Payload Too Large.
+#
+# Prerequisites:
+#   - Server running at $base_url with admin credentials (test.env).
+#   - OXICLOUD_ENABLE_AUTH=true, OXICLOUD_NEXTCLOUD_ENABLED=true.
+#   - tests/fixtures/chunk-over-cap-5mb.bin generated by
+#     tests/api/run.sh (or by hand: dd if=/dev/zero of=… bs=1024 count=5120).
+#   - jq, dd, curl in PATH.
+# =============================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+cd "$SCRIPT_DIR"
+
+source test.env
+source common.sh
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+PASS=0
+FAIL=0
+
+pass() { PASS=$(( PASS + 1 )); echo "  PASS: $*"; }
+fail() { FAIL=$(( FAIL + 1 )); echo "  FAIL: $*" >&2; exit 1; }
+
+rest_get()    { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
+rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
+
+purge_from_trash() {
+    local name="$1"
+    local tid
+    tid=$(rest_get "/api/trash" \
+        | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty')
+    [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true
+}
+
+# Helper: NC WebDAV request with Basic Auth, prints HTTP status.
+nc_req() {
+    local method="$1" url="$2"
+    shift 2
+    curl -s -o /dev/null -w "%{http_code}" -X "$method" \
+        -u "$username:$APP_PASSWORD" \
+        "$@" \
+        "$base_url$url"
+}
+
+# ── fixtures ──────────────────────────────────────────────────────────────────
+
+FIXTURE_SMALL="$REPO_ROOT/tests/fixtures/hello.txt"
+FIXTURE_BIG="$REPO_ROOT/tests/fixtures/chunk-over-cap-5mb.bin"
+[[ -f "$FIXTURE_SMALL" ]] || { echo "Missing fixture: $FIXTURE_SMALL" >&2; exit 1; }
+# Self-generate the 5 MiB fixture if absent — the Hurl runner
+# (`tests/api/run.sh`) generates it for the REST cap test; this
+# script may run standalone, so we don't depend on that runner
+# having executed first.
+if [[ ! -s "$FIXTURE_BIG" ]]; then
+    echo "  Generating 5 MiB fixture → $FIXTURE_BIG"
+    dd if=/dev/zero of="$FIXTURE_BIG" bs=1024 count=5120 status=none
+fi
+
+# BLAKE3 of hello.txt (32 B). Asserted against `content_hash` in
+# the file DTO after the round trip — proves streaming wrote the
+# exact bytes with no truncation / off-by-one.
+EXPECTED_BLAKE3="b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a"
+
+REMOTE_NAME="nc-chunked-cap-test.txt"
+UPLOAD_ID_OK="oxi-cap-ok-$(date +%s)"
+UPLOAD_ID_BIG="oxi-cap-big-$(date +%s)"
+
+echo
+echo "=== NextCloud chunked upload: cap + streaming ==="
+echo
+
+# ── authenticate ──────────────────────────────────────────────────────────────
+
+oxicloud_login
+
+# Mint an NC app password — NC endpoints use Basic Auth, not JWT.
+APP_PASSWORD_RESPONSE=$(curl -s -X POST \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"label":"chunked-cap-test","scopes":"webdav"}' \
+    "$base_url/api/auth/app-passwords")
+
+APP_PASSWORD=$(jq -r '.password' <<<"$APP_PASSWORD_RESPONSE")
+APP_PASSWORD_ID=$(jq -r '.id' <<<"$APP_PASSWORD_RESPONSE")
+[[ -n "$APP_PASSWORD" && "$APP_PASSWORD" != "null" ]] \
+    || fail "Failed to mint NC app password: $APP_PASSWORD_RESPONSE"
+echo "  app password minted (id=$APP_PASSWORD_ID)"
+
+# Clean up the app password when the script exits (success or fail).
+trap '[[ -n "${APP_PASSWORD_ID:-}" ]] && rest_delete "/api/auth/app-passwords/$APP_PASSWORD_ID" > /dev/null || true' EXIT
+
+# Idempotent cleanup of any leftover file from a previous failed run.
+HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
+EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
+    | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty')
+if [[ -n "$EXISTING_ID" ]]; then
+    echo "  cleaning up leftover $REMOTE_NAME (id=$EXISTING_ID)"
+    rest_delete "/api/files/$EXISTING_ID" > /dev/null
+    purge_from_trash "$REMOTE_NAME"
+fi
+
+# ── Case 1: SUCCESS path ──────────────────────────────────────────────────────
+
+echo
+echo "[1/2] SUCCESS path — MKCOL → PUT → MOVE → verify BLAKE3"
+
+# 1a. Create chunked-upload session.
+STATUS=$(nc_req MKCOL "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK")
+[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MKCOL upload session: got $STATUS, expected 201/204"
+pass "MKCOL upload session (status=$STATUS)"
+
+# 1b. PUT a single chunk (the whole 32-byte file).
+STATUS=$(nc_req PUT \
+    "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK/00001" \
+    -H "Content-Type: application/octet-stream" \
+    --data-binary "@$FIXTURE_SMALL")
+[[ "$STATUS" == "201" ]] || fail "PUT chunk: got $STATUS, expected 201"
+pass "PUT chunk (status=$STATUS)"
+
+# 1c. MOVE to assemble into the final destination.
+STATUS=$(nc_req MOVE \
+    "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK/.file" \
+    -H "Destination: $base_url/remote.php/dav/files/$username/$REMOTE_NAME")
+[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MOVE assemble: got $STATUS, expected 201/204"
+pass "MOVE assemble (status=$STATUS)"
+
+# 1d. Verify the assembled file landed with the right BLAKE3.
+ASSEMBLED=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
+    | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n))')
+ACTUAL_HASH=$(jq -r '.content_hash' <<<"$ASSEMBLED")
+ACTUAL_SIZE=$(jq -r '.size' <<<"$ASSEMBLED")
+FILE_ID=$(jq -r '.id' <<<"$ASSEMBLED")
+
+[[ "$ACTUAL_SIZE" == "32" ]] || fail "Assembled file size: got $ACTUAL_SIZE, expected 32"
+[[ "$ACTUAL_HASH" == "$EXPECTED_BLAKE3" ]] \
+    || fail "BLAKE3 mismatch: got $ACTUAL_HASH, expected $EXPECTED_BLAKE3"
+pass "Assembled file size + BLAKE3 match fixture"
+
+# 1e. Cleanup the assembled file so a re-run finds a clean slate.
+rest_delete "/api/files/$FILE_ID" > /dev/null
+purge_from_trash "$REMOTE_NAME"
+
+# ── Case 2: CAP REJECTION ─────────────────────────────────────────────────────
+
+echo
+echo "[2/2] CAP REJECTION — 5 MiB chunk on a 4 MiB cap → 413"
+
+# 2a. Fresh session.
+STATUS=$(nc_req MKCOL "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG")
+[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MKCOL over-cap session: got $STATUS"
+pass "MKCOL over-cap session (status=$STATUS)"
+
+# 2b. PUT a 5 MiB chunk → expect 413. Pre-fix this would either OOM
+#     the server (the body was buffered up to `max_upload_size`,
+#     which is the *whole-file* cap, multi-GB) or — depending on the
+#     code path — succeed silently and assemble a corrupted file.
+STATUS=$(nc_req PUT \
+    "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG/00001" \
+    -H "Content-Type: application/octet-stream" \
+    --data-binary "@$FIXTURE_BIG")
+[[ "$STATUS" == "413" ]] || fail "PUT over-cap chunk: got $STATUS, expected 413"
+pass "PUT over-cap chunk rejected (status=$STATUS)"
+
+# 2c. Abort the leftover session — the cap-rejected PUT removed the
+#     partial chunk file, but the session metadata is still in
+#     `chunked_uploads/`. DELETE on the session dir cleans it.
+STATUS=$(nc_req DELETE "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG")
+[[ "$STATUS" =~ ^(204|404)$ ]] || fail "DELETE abandoned session: got $STATUS"
+pass "DELETE abandoned session (status=$STATUS)"
+
+# ── summary ──────────────────────────────────────────────────────────────────
+
+echo
+echo "=== NC chunked upload cap test: $PASS passed, $FAIL failed ==="
+[[ "$FAIL" == "0" ]] || exit 1
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+# =============================================================
+# OxiCloud — NextCloud single-file PUT BLAKE3 round-trip
+# =============================================================
+# Validates that the NextCloud single-file PUT surface
+# (`PUT /remote.php/dav/files/{user}/{path}`) writes byte-exact
+# content to blob storage. This is the spool-based streaming
+# path in `nextcloud/webdav_handler::handle_put` — it streams
+# the request body to a temp file via `spool_body_to_temp`
+# (which also computes BLAKE3 on the fly), then promotes the
+# blob into `.blobs/{prefix}/{hash}.blob`.
+#
+# Sister tests:
+#   - `test_nextcloud_chunked_upload_cap.sh` — NC chunked PUT
+#     (the `/dav/uploads/...` surface, which uses
+#     `stream_body_to_path` instead of `spool_body_to_temp`).
+#   - `test_dedup_webdav_multichunk.sh` — native `/webdav/...`
+#     PUT (different handler, same spool helper).
+#
+# Together these three pin BLAKE3 correctness on all three of
+# OxiCloud's first-class file-write surfaces.
+#
+# Prerequisites:
+#   - Server running at $base_url with admin credentials.
+#   - OXICLOUD_ENABLE_AUTH=true, OXICLOUD_NEXTCLOUD_ENABLED=true.
+#   - jq, curl in PATH.
+# =============================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+cd "$SCRIPT_DIR"
+
+source test.env
+source common.sh
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+PASS=0
+FAIL=0
+
+pass() { PASS=$(( PASS + 1 )); echo "  PASS: $*"; }
+fail() { FAIL=$(( FAIL + 1 )); echo "  FAIL: $*" >&2; exit 1; }
+
+rest_get()    { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
+rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; }
+
+purge_from_trash() {
+    local name="$1"
+    local tid
+    tid=$(rest_get "/api/trash" \
+        | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty')
+    [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true
+}
+
+# ── fixture ──────────────────────────────────────────────────────────────────
+
+FIXTURE="$REPO_ROOT/tests/fixtures/hello.txt"
+[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; }
+EXPECTED_BLAKE3="b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a"
+EXPECTED_SIZE=32
+REMOTE_NAME="nc-put-blake3-test.txt"
+
+echo
+echo "=== NC single-file PUT: BLAKE3 round-trip ==="
+echo
+
+# ── authenticate ─────────────────────────────────────────────────────────────
+
+oxicloud_login
+
+APP_PASSWORD_RESPONSE=$(curl -s -X POST \
+    -H "Authorization: Bearer $TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"label":"nc-put-blake3-test","scopes":"webdav"}' \
+    "$base_url/api/auth/app-passwords")
+APP_PASSWORD=$(jq -r '.password' <<<"$APP_PASSWORD_RESPONSE")
+APP_PASSWORD_ID=$(jq -r '.id' <<<"$APP_PASSWORD_RESPONSE")
+[[ -n "$APP_PASSWORD" && "$APP_PASSWORD" != "null" ]] \
+    || fail "Failed to mint NC app password: $APP_PASSWORD_RESPONSE"
+
+trap '[[ -n "${APP_PASSWORD_ID:-}" ]] && rest_delete "/api/auth/app-passwords/$APP_PASSWORD_ID" > /dev/null || true' EXIT
+
+# Idempotent cleanup of any leftover file.
+HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id')
+EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
+    | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty')
+if [[ -n "$EXISTING_ID" ]]; then
+    echo "  cleaning up leftover $REMOTE_NAME (id=$EXISTING_ID)"
+    rest_delete "/api/files/$EXISTING_ID" > /dev/null
+    purge_from_trash "$REMOTE_NAME"
+fi
+
+# ── PUT via the NC surface ───────────────────────────────────────────────────
+
+echo "  step 1: PUT /remote.php/dav/files/$username/$REMOTE_NAME"
+STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+    -X PUT \
+    -u "$username:$APP_PASSWORD" \
+    -H "Content-Type: text/plain" \
+    --data-binary "@$FIXTURE" \
+    "$base_url/remote.php/dav/files/$username/$REMOTE_NAME")
+# 201 = created (first PUT), 204 = updated (idempotent overwrite path).
+# Either signals a successful write to the spool + blob promotion.
+[[ "$STATUS" =~ ^(201|204)$ ]] || fail "PUT got $STATUS, expected 201/204"
+pass "PUT status=$STATUS"
+
+# ── Verify BLAKE3 + size via the REST listing ────────────────────────────────
+
+echo "  step 2: GET /api/files → assert content_hash + size"
+LISTED=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \
+    | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n))')
+ACTUAL_HASH=$(jq -r '.content_hash' <<<"$LISTED")
+ACTUAL_SIZE=$(jq -r '.size' <<<"$LISTED")
+FILE_ID=$(jq -r '.id' <<<"$LISTED")
+
+[[ "$ACTUAL_SIZE" == "$EXPECTED_SIZE" ]] \
+    || fail "size mismatch: got $ACTUAL_SIZE, expected $EXPECTED_SIZE"
+[[ "$ACTUAL_HASH" == "$EXPECTED_BLAKE3" ]] \
+    || fail "BLAKE3 mismatch: got $ACTUAL_HASH, expected $EXPECTED_BLAKE3"
+pass "content_hash + size match fixture ($EXPECTED_BLAKE3, $EXPECTED_SIZE B)"
+
+# ── Cleanup ──────────────────────────────────────────────────────────────────
+
+rest_delete "/api/files/$FILE_ID" > /dev/null
+purge_from_trash "$REMOTE_NAME"
+
+echo
+echo "=== NC single-file PUT BLAKE3 test: $PASS passed, $FAIL failed ==="
+[[ "$FAIL" == "0" ]] || exit 1