From 41aad2670201e69a6daca195f0af5f54b67b1161 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Mon, 8 Jun 2026 23:17:07 +0200 Subject: [PATCH] feat(upload): cover chunk upload + add support of different digest hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prefer stream storage rather using buffered (in memory) note: on many unix like tmpfs are in-memory, sungle PUT are sized limited Storage map (NC stands for Nextcloud gateway) ┌───────────────────────────────────────────────────────┬────────────────────────────────────────────────────────────────────┬─────────────────────────────────────────────────┐ │ Streaming surface │ Destination │ Configurable via │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST chunked PUT /api/uploads/{id} chunk │ {storage_path}/.uploads/{upload_id}/chunk_{NNNNNN} │ OXICLOUD_STORAGE_PATH (the .uploads subdir is │ │ │ │ hard-wired) │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST chunked assemble (during /complete) │ {storage_path}/.uploads/{upload_id}/assembled │ same │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ NC chunked PUT /dav/uploads/.../{chunk} │ {storage_path}/.uploads/nextcloud/{user}/{upload_id}/{chunk_name} │ same │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ NC chunked assemble (during MOVE) │ {storage_path}/.uploads/nextcloud/{user}/{upload_id}/.assembled │ same │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ NC single-file PUT /dav/files/.../{path} (via │ OXICLOUD_UPLOAD_TMPDIR if set, else OS default temp (/tmp on │ OXICLOUD_UPLOAD_TMPDIR │ │ spool_body_to_temp) │ Linux) │ │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST WebDAV PUT /webdav/{path} (via │ same as above │ OXICLOUD_UPLOAD_TMPDIR │ │ spool_body_to_temp) │ │ │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ REST multipart upload /api/files/upload │ {storage_path}/.dedup_temp/upload-{uuid} │ OXICLOUD_STORAGE_PATH (hard-wired subdir) │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ WOPI PutFile │ OS default temp via NamedTempFile::new() (no override) │ (none — bug worth tracking) │ ├───────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼─────────────────────────────────────────────────┤ │ Final blob storage (after fsync + rename) │ {storage_path}/.blobs/{ab}/{abc…}.blob │ OXICLOUD_STORAGE_PATH │ └───────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────┴─────────────────────────────────────────────────┘ one caveat: a malicious user can create many chunked upload and saturate local storage --- src/application/ports/chunked_upload_ports.rs | 49 ++++ .../services/chunked_upload_service.rs | 185 ++++++++++++++++ .../api/handlers/chunked_upload_handler.rs | 189 +++++++++------- src/interfaces/nextcloud/uploads_handler.rs | 6 +- src/interfaces/upload_spool.rs | 160 +++++++++++++- src/main.rs | 11 + tests/api/chunked_upload_cap.hurl | 209 +++++++++++++++++- .../test_nextcloud_chunked_upload_cap.sh | 199 +++++++++++++++++ tests/webdav/test_nextcloud_put_blake3.sh | 131 +++++++++++ 9 files changed, 1048 insertions(+), 91 deletions(-) create mode 100755 tests/webdav/test_nextcloud_chunked_upload_cap.sh create mode 100755 tests/webdav/test_nextcloud_put_blake3.sh diff --git a/src/application/ports/chunked_upload_ports.rs b/src/application/ports/chunked_upload_ports.rs index 93621a4d..5d7d5427 100644 --- a/src/application/ports/chunked_upload_ports.rs +++ b/src/application/ports/chunked_upload_ports.rs @@ -17,6 +17,55 @@ pub const DEFAULT_CHUNK_SIZE: usize = 5 * 1024 * 1024; /// Minimum file size to use chunked upload (10 MB). pub const CHUNKED_UPLOAD_THRESHOLD: usize = 10 * 1024 * 1024; +/// Algorithm used by the client-side chunk checksum. +/// +/// The wire format is `?checksum=&checksumalg=` (or the +/// equivalent header pair for older clients that send only `Content-MD5`). +/// Clients that omit `checksumalg` are assumed to mean MD5 — that's the +/// algorithm baked into the legacy `Content-MD5` header (RFC 1864), TUS- +/// like upload protocols, and S3 multipart ETags. +/// +/// Three supported variants, all from already-declared dependencies: +/// - `Md5` — legacy default; weak cryptographically but fine for +/// transport-integrity checks under TLS. +/// - `Sha256` — industry-standard, FIPS-compliant, widely supported by +/// sync clients (AWS S3 also accepts SHA-256 trailers). +/// - `Blake3` — fastest of the three; already used by the blob-storage +/// layer, so the chunk-level integrity check and the assembled-file +/// dedup hash use the same algorithm when clients opt in. +/// +/// Skipped intentionally: SHA-1 (deprecated, broken), CRC32 (too weak for +/// integrity claims). Both can be added if a real client need appears. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChecksumAlg { + Md5, + Sha256, + Blake3, +} + +impl ChecksumAlg { + /// Parse a client-supplied algorithm name. Case-insensitive. Accepts + /// `sha-256` as a synonym for `sha256` since both forms are common + /// in HTTP headers. Unknown names return `None` so the handler can + /// 400 with the offending value. + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "md5" => Some(Self::Md5), + "sha256" | "sha-256" => Some(Self::Sha256), + "blake3" => Some(Self::Blake3), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Md5 => "md5", + Self::Sha256 => "sha256", + Self::Blake3 => "blake3", + } + } +} + /// Response returned when a new upload session is created. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct CreateUploadResponseDto { diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index 4f007d81..ec6d8057 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -463,6 +463,191 @@ impl ChunkedUploadService { }) } + /// Prepare a chunk write — validates session ownership and chunk + /// index, returns the on-disk path the caller should stream the + /// HTTP body to plus the expected byte count for that chunk. + /// + /// Used by the streaming REST PUT path: the handler calls + /// `prepare_chunk` → streams body to disk via + /// `interfaces::upload_spool::stream_body_to_path` → calls + /// `commit_chunk` to finalise. This lets the body bypass the + /// in-memory `Bytes` allocation entirely (peak heap ~one HTTP + /// frame instead of "chunk size"). + /// + /// Returns `Err` if the session is unknown, owned by another user, + /// the chunk index is out of range, or the chunk is already complete. + pub async fn prepare_chunk( + &self, + upload_id: &str, + user_id: Uuid, + chunk_index: usize, + ) -> Result<(PathBuf, usize), DomainError> { + self.verify_session_owner(upload_id, &user_id.to_string()) + .map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?; + + let session = self.sessions.get(upload_id).ok_or_else(|| { + DomainError::new( + ErrorKind::NotFound, + "ChunkedUpload", + format!("Upload session not found: {}", upload_id), + ) + })?; + + if chunk_index >= session.chunks.len() { + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!( + "Invalid chunk index: {} (max: {})", + chunk_index, + session.chunks.len() - 1 + ), + )); + } + + let chunk = &session.chunks[chunk_index]; + if chunk.status == ChunkStatus::Complete { + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!("Chunk {} already uploaded", chunk_index), + )); + } + + Ok(( + session.temp_dir.join(format!("chunk_{:06}", chunk_index)), + chunk.size, + )) + } + + /// Finalise a chunk write — verifies the actually-written byte count + /// matches the chunk's declared size, validates an optional + /// algorithm-tagged checksum, and updates session state. The chunk + /// file at `{session.temp_dir}/chunk_{index:06}` must already have + /// been written by the caller (typically via + /// `stream_body_to_path`). + /// + /// `actual_size` is the byte count the streaming write reported; + /// `computed_checksum` is the hex digest computed during streaming + /// (or `None` if the client didn't request a checksum). When + /// `expected_checksum` is supplied the two are compared; a + /// mismatch removes the partial file and returns `ValidationError` + /// so a client retry against the same chunk index gets a clean + /// slot. A size mismatch does the same. + pub async fn commit_chunk( + &self, + upload_id: &str, + user_id: Uuid, + chunk_index: usize, + actual_size: u64, + computed_checksum: Option, + expected_checksum: Option, + ) -> Result { + self.verify_session_owner(upload_id, &user_id.to_string()) + .map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?; + + // Re-fetch chunk metadata under fresh lock — guards against the + // (vanishingly unlikely) case of a session expiry / cancellation + // racing with the write. + let (chunk_path, expected_size, persist_path) = { + let session = self.sessions.get(upload_id).ok_or_else(|| { + DomainError::new( + ErrorKind::NotFound, + "ChunkedUpload", + "Session disappeared".to_string(), + ) + })?; + if chunk_index >= session.chunks.len() { + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!("Invalid chunk index: {}", chunk_index), + )); + } + ( + session.temp_dir.join(format!("chunk_{:06}", chunk_index)), + session.chunks[chunk_index].size, + session.temp_dir.join(PROGRESS_FILE), + ) + }; + + // Size check — the streaming body may have been truncated by + // the client mid-flight or exceeded the chunk's declared + // length. Either way we don't want a partial chunk to count + // as complete; nuke it and ask the client to retry. + if actual_size != expected_size as u64 { + let _ = fs::remove_file(&chunk_path).await; + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!( + "Invalid chunk size: expected {} bytes, got {} bytes", + expected_size, actual_size + ), + )); + } + + // Checksum check — case-insensitive compare so clients that + // send uppercase hex still match. + if let Some(expected) = expected_checksum.as_ref() + && let Some(actual) = computed_checksum.as_ref() + && !expected.eq_ignore_ascii_case(actual) + { + let _ = fs::remove_file(&chunk_path).await; + return Err(DomainError::new( + ErrorKind::InvalidInput, + "ChunkedUpload", + format!( + "Checksum mismatch: expected {}, got {}", + expected, actual + ), + )); + } + + // Update session state — DashMap shard lock held only for the + // RAM updates (~µs). The bitmask write happens AFTER the ref + // is dropped so concurrent uploads to other sessions are never + // blocked. Mirrors the legacy `upload_chunk_inner` semantics. + let (bytes_received, progress, is_complete, persist_bitmask) = { + let mut session = self.sessions.get_mut(upload_id).ok_or_else(|| { + DomainError::new( + ErrorKind::NotFound, + "ChunkedUpload", + "Session disappeared".to_string(), + ) + })?; + session.chunks[chunk_index].status = ChunkStatus::Complete; + session.chunks[chunk_index].checksum = expected_checksum; + session.bytes_received += actual_size; + session.last_activity = Utc::now(); + let bitmask = session.build_progress_bitmask(); + ( + session.bytes_received, + session.progress(), + session.is_complete(), + bitmask, + ) + }; + + if let Err(e) = fs::write(&persist_path, &persist_bitmask).await { + tracing::warn!("Failed to persist progress for {upload_id}: {e}"); + } + + tracing::debug!( + "📦 Chunk {} committed for {} ({:.1}% complete)", + chunk_index, + upload_id, + progress * 100.0 + ); + + Ok(ChunkUploadResponseDto { + chunk_index, + bytes_received, + progress, + is_complete, + }) + } + /// Upload a single chunk (persists `progress.bin` after success) async fn upload_chunk_inner( &self, diff --git a/src/interfaces/api/handlers/chunked_upload_handler.rs b/src/interfaces/api/handlers/chunked_upload_handler.rs index 4c520123..6e1f1629 100644 --- a/src/interfaces/api/handlers/chunked_upload_handler.rs +++ b/src/interfaces/api/handlers/chunked_upload_handler.rs @@ -13,11 +13,11 @@ use axum::{ http::{HeaderMap, StatusCode, header}, response::{IntoResponse, Response}, }; -use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::sync::Arc; use utoipa::ToSchema; +use crate::application::ports::chunked_upload_ports::ChecksumAlg; use crate::application::ports::chunked_upload_ports::ChunkedUploadPort; use crate::application::ports::chunked_upload_ports::DEFAULT_CHUNK_SIZE; use crate::application::ports::file_ports::FileUploadUseCase; @@ -27,6 +27,7 @@ use crate::common::di::AppState; use crate::domain::services::authorization::Permission; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; +use crate::interfaces::upload_spool::stream_body_to_path; /// Request body for creating an upload session #[derive(Debug, Deserialize, ToSchema)] @@ -38,11 +39,17 @@ pub struct CreateUploadRequest { pub chunk_size: Option, } -/// Query params for chunk upload +/// Query params for chunk upload. +/// +/// `checksumalg` is parsed via [`ChecksumAlg::parse`] and defaults to +/// `Md5` when absent — matching the legacy `Content-MD5` contract that +/// older clients rely on. Unknown algorithm names produce a 400 with the +/// offending value echoed back. #[derive(Debug, Deserialize)] pub struct ChunkUploadParams { pub chunk_index: usize, pub checksum: Option, + pub checksumalg: Option, } /// Final response after completing upload @@ -200,58 +207,12 @@ impl ChunkedUploadHandler { } } - /// PATCH /api/uploads/:upload_id - Upload a chunk - /// - /// Query params: - /// - chunk_index: The index of the chunk (0-based) - /// - checksum: Optional MD5 checksum for verification - /// - /// Body: Raw bytes of the chunk - pub(super) async fn upload_chunk_impl( - State(state): State>, - auth_user: AuthUser, - Path(upload_id): Path, - Query(params): Query, - headers: HeaderMap, - body: Bytes, - ) -> impl IntoResponse { - let chunked_service = &state.core.chunked_upload_service; - - // Extract checksum from header or query param - let checksum = params.checksum.or_else(|| { - headers - .get("Content-MD5") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) - }); - - match chunked_service - .upload_chunk(&upload_id, auth_user.id, params.chunk_index, body, checksum) - .await - { - Ok(response) => { - let mut resp = Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "application/json") - .header("Upload-Offset", response.bytes_received.to_string()) - .header( - "Upload-Progress", - format!("{:.2}", response.progress * 100.0), - ); - - if response.is_complete { - resp = resp.header("Upload-Complete", "true"); - } - - resp.body(axum::body::Body::from( - serde_json::to_string(&response).unwrap(), - )) - .unwrap() - .into_response() - } - Err(e) => AppError::from(e).into_response(), - } - } + // PATCH /api/uploads/:upload_id — moved entirely to the free + // function `upload_chunk` below so the body can be streamed + // (axum::body::Body) instead of materialised as `Bytes` here. + // The port-level `ChunkedUploadPort::upload_chunk` (Bytes-based) + // remains for tests and any future caller that genuinely has the + // bytes already in memory. /// HEAD /api/uploads/:upload_id - Get upload status /// @@ -448,40 +409,114 @@ pub async fn create_upload( pub async fn upload_chunk( State(state): State>, auth_user: AuthUser, - path: Path, - query: Query, + Path(upload_id): Path, + Query(params): Query, headers: HeaderMap, request: Request, ) -> impl IntoResponse { - // Cap the chunk body at `storage.chunk_max_bytes` (env - // `OXICLOUD_CHUNK_MAX_BYTES`, default 100 MB). Previous code used - // `usize::MAX` and `unwrap_or_default()` — two compounding bugs: - // - No upper bound → an oversized chunk OOMs the server. - // - Silent fallback to an empty body on transport error → the - // inner size check would either reject (good case) or — if the - // declared chunk_size was 0 (illegal but conceivable) — accept - // an empty upload as success. Either way the client got no - // actionable error. + let chunked_service = &state.core.chunked_upload_service; let max_chunk = state.core.config.storage.chunk_max_bytes; - let body = match axum::body::to_bytes(request.into_body(), max_chunk).await { - Ok(b) => b, + + // ── Resolve the client's checksum + algorithm ──────────────────── + // Wire shape: `?checksum=&checksumalg=` (or `Content-MD5` + // header for older clients). When `checksumalg` is omitted we + // default to MD5, matching the legacy contract — switching the + // default would silently break any client still relying on + // `Content-MD5` semantics. + let expected_checksum = params.checksum.clone().or_else(|| { + headers + .get("Content-MD5") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }); + let alg = match params.checksumalg.as_deref() { + Some(name) => match ChecksumAlg::parse(name) { + Some(a) => a, + None => { + return AppError::bad_request(format!( + "Unsupported checksumalg: {name} (supported: md5, sha256, blake3)" + )) + .into_response(); + } + }, + None => ChecksumAlg::Md5, + }; + // Only compute the hash when the client supplied an `expected_checksum` + // to verify against — saves ~30 ms per chunk for clients that don't. + let alg_to_compute = expected_checksum.as_ref().map(|_| alg); + + // ── Phase 1: prepare ───────────────────────────────────────────── + // Validates session ownership + chunk index, returns the on-disk + // path and the chunk's declared size. The handler streams the body + // to that path; service finalises bookkeeping after the write. + let (chunk_path, _expected_size) = match chunked_service + .prepare_chunk(&upload_id, auth_user.id, params.chunk_index) + .await + { + Ok(p) => p, + Err(e) => return AppError::from(e).into_response(), + }; + + // ── Phase 2: stream the body straight to disk ──────────────────── + // Peak heap ~one HTTP frame (~64 KB) regardless of chunk size or + // `chunk_max_bytes`. Optional incremental hashing happens here so + // verification doesn't require reading the chunk file back. + let streamed = match stream_body_to_path( + request.into_body(), + &chunk_path, + max_chunk, + alg_to_compute, + ) + .await + { + Ok(s) => s, Err(e) => { tracing::warn!( - error = %e, - upload_id = %path.0, + error = ?e, + upload_id = %upload_id, + chunk_index = params.chunk_index, max_chunk, - "Chunked upload PATCH rejected — body read failed (size cap or transport error)" + "Chunked upload PATCH rejected — streaming write failed (cap, transport, or IO)" ); - return AppError::payload_too_large(format!( - "Chunk read failed (cap {} bytes): {}", - max_chunk, e - )) - .into_response(); + return e.into_response(); } }; - ChunkedUploadHandler::upload_chunk_impl(State(state), auth_user, path, query, headers, body) + + // ── Phase 3: commit ────────────────────────────────────────────── + // Size + checksum verification + session state update. Same RAM-only + // DashMap shard ownership pattern as the legacy `upload_chunk_inner` + // (held only for ~µs; bitmask persist done after release). + let response = match chunked_service + .commit_chunk( + &upload_id, + auth_user.id, + params.chunk_index, + streamed.bytes_written, + streamed.checksum_hex, + expected_checksum, + ) .await - .into_response() + { + Ok(r) => r, + Err(e) => return AppError::from(e).into_response(), + }; + + let mut resp = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/json") + .header("Upload-Offset", response.bytes_received.to_string()) + .header( + "Upload-Progress", + format!("{:.2}", response.progress * 100.0), + ); + if response.is_complete { + resp = resp.header("Upload-Complete", "true"); + } + resp.body(axum::body::Body::from( + serde_json::to_string(&response).unwrap(), + )) + .unwrap() + .into_response() } #[utoipa::path( diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index 192688ee..56737ada 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -196,7 +196,11 @@ async fn handle_put_chunk( .map_err(|e| AppError::bad_request(format!("Invalid chunk path: {}", e)))?; let max_chunk = state.core.config.storage.chunk_max_bytes; - stream_body_to_path(req.into_body(), &chunk_path, max_chunk).await?; + // No client-side integrity contract on the NC chunked surface — the + // NC desktop client validates the assembled-file ETag against the + // server-side `oc:checksums` after MOVE. So we skip per-chunk + // hashing here (peak heap stays at ~one HTTP frame). + stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?; Ok(Response::builder() .status(StatusCode::CREATED) diff --git a/src/interfaces/upload_spool.rs b/src/interfaces/upload_spool.rs index de6b6279..25878019 100644 --- a/src/interfaces/upload_spool.rs +++ b/src/interfaces/upload_spool.rs @@ -10,10 +10,17 @@ use std::path::{Path, PathBuf}; use axum::body::Body; use http_body_util::BodyStream; +// The `Digest` trait (re-exported by both `md5` and `sha2` from the +// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / +// `finalize` methods. Importing once via `sha2` covers both — +// otherwise every call site would need fully-qualified +// `::…` syntax. +use sha2::Digest as _; use tempfile::NamedTempFile; use tokio::io::AsyncWriteExt; use tokio_stream::StreamExt; +use crate::application::ports::chunked_upload_ports::ChecksumAlg; use crate::common::temp::new_spool_temp_file; use crate::interfaces::errors::AppError; @@ -85,32 +92,50 @@ pub async fn spool_body_to_temp( }) } +/// Result of a streamed write to a caller-supplied path. +pub struct StreamedToPath { + /// Total bytes written. + pub bytes_written: u64, + /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` + /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. + pub checksum_hex: Option, + /// Algorithm used to compute `checksum_hex`. Echoed back so the + /// caller can include it in audit logs or response headers. + pub alg: Option, +} + /// Stream an HTTP request body directly to a known destination file, /// enforcing `max_bytes` as a hard size limit. /// -/// Used by the chunked-upload PUT handlers — each chunk has a deterministic -/// on-disk path (computed by `NextcloudChunkedUploadService::safe_chunk_path` -/// or the equivalent REST helper), so there's no need for a spool/move -/// dance. Peak heap is ~one HTTP frame regardless of chunk size or `max_bytes`. +/// Used by the chunked-upload PUT handlers — each chunk has a +/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` +/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the +/// REST surface), so there's no spool/move dance. Peak heap is ~one +/// HTTP frame regardless of chunk size or `max_bytes`. /// -/// **No hashing** — chunked uploads dedup at the assembled-file level, not -/// the chunk level, so computing BLAKE3 here would be wasted work. +/// `checksum_alg` is the optional client-requested integrity check +/// (default `md5` per the legacy `Content-MD5` contract; `blake3` +/// available for forward-compat). When `Some`, the hash is computed +/// incrementally during streaming — no extra disk read for verification. /// -/// On size overflow the partial file is removed before the function returns, -/// so a client retry against the same chunk name starts from a clean slate. -/// On any other I/O error the partial file is also removed and the error -/// surfaces — callers can assume the path is either fully written or absent. +/// On size overflow the partial file is removed before the function +/// returns, so a client retry against the same chunk name starts from +/// a clean slate. On any other I/O error the partial file is also +/// removed and the error surfaces — callers can assume the path is +/// either fully written or absent. pub async fn stream_body_to_path( body: Body, path: &Path, max_bytes: usize, -) -> Result { + checksum_alg: Option, +) -> Result { let mut file = tokio::fs::File::create(path) .await .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; let mut total_bytes: usize = 0; let mut stream = BodyStream::new(body); + let mut hasher = checksum_alg.map(IncrementalHasher::new); while let Some(frame_result) = stream.next().await { let frame = match frame_result { @@ -132,6 +157,9 @@ pub async fn stream_body_to_path( "Chunk exceeds maximum size of {max_bytes} bytes" ))); } + if let Some(h) = hasher.as_mut() { + h.update(chunk); + } if let Err(e) = file.write_all(chunk).await { drop(file); let _ = tokio::fs::remove_file(path).await; @@ -146,5 +174,113 @@ pub async fn stream_body_to_path( .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; drop(file); - Ok(total_bytes as u64) + Ok(StreamedToPath { + bytes_written: total_bytes as u64, + checksum_hex: hasher.map(IncrementalHasher::finalize_hex), + alg: checksum_alg, + }) +} + +/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`]. +/// Per-frame `update` is sub-millisecond for all three algorithms at the +/// 64 KB frame sizes axum's body stream produces, so we don't need +/// `spawn_blocking` (which the old buffered path used because it hashed +/// the full multi-MB chunk in one shot). +enum IncrementalHasher { + Md5(md5::Md5), + Sha256(sha2::Sha256), + // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 + // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the + // outlier keeps the enum size proportional to the common case + // rather than the worst case. + Blake3(Box), +} + +impl IncrementalHasher { + fn new(alg: ChecksumAlg) -> Self { + match alg { + ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), + ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), + ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), + } + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Self::Md5(h) => h.update(bytes), + Self::Sha256(h) => h.update(bytes), + Self::Blake3(h) => { + h.update(bytes); + } + } + } + + fn finalize_hex(self) -> String { + match self { + Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Blake3(h) => h.finalize().to_hex().to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + + #[tokio::test] + async fn stream_body_to_path_caps_oversized() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + // 5 MiB body, 4 MiB cap → must reject. + let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + assert!( + result.is_err(), + "expected PayloadTooLarge, got Ok(bytes_written={})", + result.ok().map(|r| r.bytes_written).unwrap_or(0) + ); + // Partial file must be removed on rejection. + assert!( + !path.exists(), + "rejected chunk file should be removed, but {} still exists", + path.display() + ); + } + + #[tokio::test] + async fn stream_body_to_path_accepts_under_cap() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + let outcome = result.expect("should succeed"); + assert_eq!(outcome.bytes_written, 1024 * 1024); + assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); + assert!(path.exists()); + } + + #[tokio::test] + async fn stream_body_to_path_caps_at_exact_boundary() { + // Edge case: body exactly equal to cap should succeed; cap+1 must fail. + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 100])); + let outcome = stream_body_to_path(body, &path, 100, None) + .await + .expect("100 bytes at 100-byte cap should succeed"); + assert_eq!(outcome.bytes_written, 100); + + let path2 = temp_dir.path().join("chunk2"); + let body = Body::from(Bytes::from(vec![1u8; 101])); + assert!( + stream_body_to_path(body, &path2, 100, None).await.is_err(), + "101 bytes at 100-byte cap must reject" + ); + assert!(!path2.exists()); + } } diff --git a/src/main.rs b/src/main.rs index 192b4146..8f3463aa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -143,6 +143,17 @@ async fn main() -> Result<(), Box> { // Load configuration from environment variables let config = common::config::AppConfig::from_env(); + // Surface the upload-size limits at startup. Operators (and the + // CI runner) need to see what's actually in effect — a silent + // fallback to the 100 MB default when `OXICLOUD_CHUNK_MAX_BYTES` + // is mistyped or missing is the exact failure mode that's + // hardest to spot from chunked-upload tests. + tracing::info!( + max_upload_size_mb = config.storage.max_upload_size / (1024 * 1024), + chunk_max_bytes_mb = config.storage.chunk_max_bytes / (1024 * 1024), + "Upload limits loaded from config" + ); + // Ensure storage and locales directories exist let storage_path = config.storage_path.clone(); if !storage_path.exists() { diff --git a/tests/api/chunked_upload_cap.hurl b/tests/api/chunked_upload_cap.hurl index e4ced792..ffe0f06e 100644 --- a/tests/api/chunked_upload_cap.hurl +++ b/tests/api/chunked_upload_cap.hurl @@ -109,7 +109,7 @@ Authorization: Bearer {{token}} HTTP 200 [Asserts] -jsonpath "$[?(@.id == '{{file_id}}')].content_hash" includes "b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" +jsonpath "$[?(@.id == '{{file_id}}')].content_hash" == "b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" # ───────────────────────────────────────────────────────────── @@ -157,3 +157,210 @@ DELETE {{base_url}}/api/uploads/{{upload_id_big}} Authorization: Bearer {{token}} HTTP 204 + + +# ═════════════════════════════════════════════════════════════ +# Checksum verification scenarios +# ═════════════════════════════════════════════════════════════ +# `?checksum=&checksumalg=` (default md5 if alg +# omitted, preserving the legacy `Content-MD5` contract). All +# three algorithms compute incrementally during the streaming +# write — zero extra disk reads. Known hashes of hello.txt: +# md5: f02bc35b153756ad11e07885cd86cbcf +# sha256: 0237134783df857fd9634c004341dbfccd374be0a1dd3c08e257522fa4d44e20 +# blake3: b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a + + +# ───────────────────────────────────────────────────────────── +# Step 10 — MD5 (default alg, no `checksumalg` param). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-md5.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_md5: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_md5}}?chunk_index=0&checksum=f02bc35b153756ad11e07885cd86cbcf +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_md5}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 11 — SHA-256 (`?checksumalg=sha256`). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-sha256.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_sha: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_sha}}?chunk_index=0&checksum=0237134783df857fd9634c004341dbfccd374be0a1dd3c08e257522fa4d44e20&checksumalg=sha256 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_sha}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 12 — BLAKE3 (`?checksumalg=blake3`). Same algorithm the +# blob-storage layer uses for dedup, so the chunk-level +# hash and the assembled-file hash are comparable. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-blake3.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_blake3: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_blake3}}?chunk_index=0&checksum=b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a&checksumalg=blake3 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 200 + +POST {{base_url}}/api/uploads/{{upload_id_blake3}}/complete +Authorization: Bearer {{token}} + +HTTP 201 + + +# ───────────────────────────────────────────────────────────── +# Step 13 — Checksum MISMATCH: send a chunk with a deliberately +# wrong MD5. `commit_chunk` must reject (the chunk +# file is removed in the same path so a retry against +# the same index starts clean). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-badmd5.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_bad: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_bad}}?chunk_index=0&checksum=00000000000000000000000000000000 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_bad}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 14 — Unknown checksumalg → 400 BadRequest with the +# offending value echoed back. Guards against a typo +# silently disabling integrity verification. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-badalg.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_badalg: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_badalg}}?chunk_index=0&checksum=deadbeef&checksumalg=zoiberg +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +file,fixtures/hello.txt; + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_badalg}} +Authorization: Bearer {{token}} + +HTTP 204 + + +# ───────────────────────────────────────────────────────────── +# Step 15 — Size MISMATCH: declare chunk_size 32, send 4 bytes. +# Streaming write succeeds; `commit_chunk` rejects on +# the size check, removes the partial file. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/uploads +Authorization: Bearer {{token}} +Content-Type: application/json +{ + "filename": "chunked-cap-shortbody.txt", + "folder_id": "{{home_folder_id}}", + "content_type": "text/plain", + "total_size": 32, + "chunk_size": 1048576 +} + +HTTP 201 +[Captures] +upload_id_short: jsonpath "$.upload_id" + +PATCH {{base_url}}/api/uploads/{{upload_id_short}}?chunk_index=0 +Authorization: Bearer {{token}} +Content-Type: application/octet-stream +base64,aGFsdA==; + +HTTP 400 + +DELETE {{base_url}}/api/uploads/{{upload_id_short}} +Authorization: Bearer {{token}} + +HTTP 204 diff --git a/tests/webdav/test_nextcloud_chunked_upload_cap.sh b/tests/webdav/test_nextcloud_chunked_upload_cap.sh new file mode 100755 index 00000000..79c14b5a --- /dev/null +++ b/tests/webdav/test_nextcloud_chunked_upload_cap.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud — NextCloud chunked upload cap + streaming check +# ============================================================= +# Validates the per-chunk `storage.chunk_max_bytes` cap on the +# NextCloud-compat chunked-upload surface (`/remote.php/dav/uploads/`), +# and round-trips a small file through MKCOL → PUT → MOVE to +# prove the streaming write at `handle_put_chunk` produces a +# byte-exact blob on disk. +# +# Sister test of `tests/api/chunked_upload_cap.hurl` (REST chunked). +# Both surfaces share the same `chunk_max_bytes` cap and the same +# `stream_body_to_path` helper; this test exercises the NC half: +# Basic Auth via app-password, WebDAV verbs, fewer protocol +# affordances than the REST API. +# +# Cases covered: +# 1. SUCCESS — MKCOL → PUT a single chunk (hello.txt, 32 B) → +# MOVE → verify the assembled file's BLAKE3 over REST. +# 2. CAP REJECTION — MKCOL a fresh session → PUT a 5 MiB chunk → +# 413 Payload Too Large. +# +# Prerequisites: +# - Server running at $base_url with admin credentials (test.env). +# - OXICLOUD_ENABLE_AUTH=true, OXICLOUD_NEXTCLOUD_ENABLED=true. +# - tests/fixtures/chunk-over-cap-5mb.bin generated by +# tests/api/run.sh (or by hand: dd if=/dev/zero of=… bs=1024 count=5120). +# - jq, dd, curl in PATH. +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# Helper: NC WebDAV request with Basic Auth, prints HTTP status. +nc_req() { + local method="$1" url="$2" + shift 2 + curl -s -o /dev/null -w "%{http_code}" -X "$method" \ + -u "$username:$APP_PASSWORD" \ + "$@" \ + "$base_url$url" +} + +# ── fixtures ────────────────────────────────────────────────────────────────── + +FIXTURE_SMALL="$REPO_ROOT/tests/fixtures/hello.txt" +FIXTURE_BIG="$REPO_ROOT/tests/fixtures/chunk-over-cap-5mb.bin" +[[ -f "$FIXTURE_SMALL" ]] || { echo "Missing fixture: $FIXTURE_SMALL" >&2; exit 1; } +# Self-generate the 5 MiB fixture if absent — the Hurl runner +# (`tests/api/run.sh`) generates it for the REST cap test; this +# script may run standalone, so we don't depend on that runner +# having executed first. +if [[ ! -s "$FIXTURE_BIG" ]]; then + echo " Generating 5 MiB fixture → $FIXTURE_BIG" + dd if=/dev/zero of="$FIXTURE_BIG" bs=1024 count=5120 status=none +fi + +# BLAKE3 of hello.txt (32 B). Asserted against `content_hash` in +# the file DTO after the round trip — proves streaming wrote the +# exact bytes with no truncation / off-by-one. +EXPECTED_BLAKE3="b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" + +REMOTE_NAME="nc-chunked-cap-test.txt" +UPLOAD_ID_OK="oxi-cap-ok-$(date +%s)" +UPLOAD_ID_BIG="oxi-cap-big-$(date +%s)" + +echo +echo "=== NextCloud chunked upload: cap + streaming ===" +echo + +# ── authenticate ────────────────────────────────────────────────────────────── + +oxicloud_login + +# Mint an NC app password — NC endpoints use Basic Auth, not JWT. +APP_PASSWORD_RESPONSE=$(curl -s -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"label":"chunked-cap-test","scopes":"webdav"}' \ + "$base_url/api/auth/app-passwords") + +APP_PASSWORD=$(jq -r '.password' <<<"$APP_PASSWORD_RESPONSE") +APP_PASSWORD_ID=$(jq -r '.id' <<<"$APP_PASSWORD_RESPONSE") +[[ -n "$APP_PASSWORD" && "$APP_PASSWORD" != "null" ]] \ + || fail "Failed to mint NC app password: $APP_PASSWORD_RESPONSE" +echo " app password minted (id=$APP_PASSWORD_ID)" + +# Clean up the app password when the script exits (success or fail). +trap '[[ -n "${APP_PASSWORD_ID:-}" ]] && rest_delete "/api/auth/app-passwords/$APP_PASSWORD_ID" > /dev/null || true' EXIT + +# Idempotent cleanup of any leftover file from a previous failed run. +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleaning up leftover $REMOTE_NAME (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + purge_from_trash "$REMOTE_NAME" +fi + +# ── Case 1: SUCCESS path ────────────────────────────────────────────────────── + +echo +echo "[1/2] SUCCESS path — MKCOL → PUT → MOVE → verify BLAKE3" + +# 1a. Create chunked-upload session. +STATUS=$(nc_req MKCOL "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK") +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MKCOL upload session: got $STATUS, expected 201/204" +pass "MKCOL upload session (status=$STATUS)" + +# 1b. PUT a single chunk (the whole 32-byte file). +STATUS=$(nc_req PUT \ + "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK/00001" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@$FIXTURE_SMALL") +[[ "$STATUS" == "201" ]] || fail "PUT chunk: got $STATUS, expected 201" +pass "PUT chunk (status=$STATUS)" + +# 1c. MOVE to assemble into the final destination. +STATUS=$(nc_req MOVE \ + "/remote.php/dav/uploads/$username/$UPLOAD_ID_OK/.file" \ + -H "Destination: $base_url/remote.php/dav/files/$username/$REMOTE_NAME") +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MOVE assemble: got $STATUS, expected 201/204" +pass "MOVE assemble (status=$STATUS)" + +# 1d. Verify the assembled file landed with the right BLAKE3. +ASSEMBLED=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n))') +ACTUAL_HASH=$(jq -r '.content_hash' <<<"$ASSEMBLED") +ACTUAL_SIZE=$(jq -r '.size' <<<"$ASSEMBLED") +FILE_ID=$(jq -r '.id' <<<"$ASSEMBLED") + +[[ "$ACTUAL_SIZE" == "32" ]] || fail "Assembled file size: got $ACTUAL_SIZE, expected 32" +[[ "$ACTUAL_HASH" == "$EXPECTED_BLAKE3" ]] \ + || fail "BLAKE3 mismatch: got $ACTUAL_HASH, expected $EXPECTED_BLAKE3" +pass "Assembled file size + BLAKE3 match fixture" + +# 1e. Cleanup the assembled file so a re-run finds a clean slate. +rest_delete "/api/files/$FILE_ID" > /dev/null +purge_from_trash "$REMOTE_NAME" + +# ── Case 2: CAP REJECTION ───────────────────────────────────────────────────── + +echo +echo "[2/2] CAP REJECTION — 5 MiB chunk on a 4 MiB cap → 413" + +# 2a. Fresh session. +STATUS=$(nc_req MKCOL "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG") +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "MKCOL over-cap session: got $STATUS" +pass "MKCOL over-cap session (status=$STATUS)" + +# 2b. PUT a 5 MiB chunk → expect 413. Pre-fix this would either OOM +# the server (the body was buffered up to `max_upload_size`, +# which is the *whole-file* cap, multi-GB) or — depending on the +# code path — succeed silently and assemble a corrupted file. +STATUS=$(nc_req PUT \ + "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG/00001" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@$FIXTURE_BIG") +[[ "$STATUS" == "413" ]] || fail "PUT over-cap chunk: got $STATUS, expected 413" +pass "PUT over-cap chunk rejected (status=$STATUS)" + +# 2c. Abort the leftover session — the cap-rejected PUT removed the +# partial chunk file, but the session metadata is still in +# `chunked_uploads/`. DELETE on the session dir cleans it. +STATUS=$(nc_req DELETE "/remote.php/dav/uploads/$username/$UPLOAD_ID_BIG") +[[ "$STATUS" =~ ^(204|404)$ ]] || fail "DELETE abandoned session: got $STATUS" +pass "DELETE abandoned session (status=$STATUS)" + +# ── summary ────────────────────────────────────────────────────────────────── + +echo +echo "=== NC chunked upload cap test: $PASS passed, $FAIL failed ===" +[[ "$FAIL" == "0" ]] || exit 1 diff --git a/tests/webdav/test_nextcloud_put_blake3.sh b/tests/webdav/test_nextcloud_put_blake3.sh new file mode 100755 index 00000000..c5daf28c --- /dev/null +++ b/tests/webdav/test_nextcloud_put_blake3.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# ============================================================= +# OxiCloud — NextCloud single-file PUT BLAKE3 round-trip +# ============================================================= +# Validates that the NextCloud single-file PUT surface +# (`PUT /remote.php/dav/files/{user}/{path}`) writes byte-exact +# content to blob storage. This is the spool-based streaming +# path in `nextcloud/webdav_handler::handle_put` — it streams +# the request body to a temp file via `spool_body_to_temp` +# (which also computes BLAKE3 on the fly), then promotes the +# blob into `.blobs/{prefix}/{hash}.blob`. +# +# Sister tests: +# - `test_nextcloud_chunked_upload_cap.sh` — NC chunked PUT +# (the `/dav/uploads/...` surface, which uses +# `stream_body_to_path` instead of `spool_body_to_temp`). +# - `test_dedup_webdav_multichunk.sh` — native `/webdav/...` +# PUT (different handler, same spool helper). +# +# Together these three pin BLAKE3 correctness on all three of +# OxiCloud's first-class file-write surfaces. +# +# Prerequisites: +# - Server running at $base_url with admin credentials. +# - OXICLOUD_ENABLE_AUTH=true, OXICLOUD_NEXTCLOUD_ENABLED=true. +# - jq, curl in PATH. +# ============================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$SCRIPT_DIR" + +source test.env +source common.sh + +# ── helpers ────────────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 + +pass() { PASS=$(( PASS + 1 )); echo " PASS: $*"; } +fail() { FAIL=$(( FAIL + 1 )); echo " FAIL: $*" >&2; exit 1; } + +rest_get() { curl -s -H "Authorization: Bearer $TOKEN" "$base_url$1"; } +rest_delete() { curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer $TOKEN" "$base_url$1"; } + +purge_from_trash() { + local name="$1" + local tid + tid=$(rest_get "/api/trash" \ + | jq -r --arg n "$name" 'first(.[] | select(.name == $n) | .id) // empty') + [[ -n "$tid" ]] && rest_delete "/api/trash/$tid" > /dev/null || true +} + +# ── fixture ────────────────────────────────────────────────────────────────── + +FIXTURE="$REPO_ROOT/tests/fixtures/hello.txt" +[[ -f "$FIXTURE" ]] || { echo "Missing fixture: $FIXTURE" >&2; exit 1; } +EXPECTED_BLAKE3="b2208c5dc33ff951227bd0c139f5eccb04105d6da6a7519ee23f7bc00a17bb5a" +EXPECTED_SIZE=32 +REMOTE_NAME="nc-put-blake3-test.txt" + +echo +echo "=== NC single-file PUT: BLAKE3 round-trip ===" +echo + +# ── authenticate ───────────────────────────────────────────────────────────── + +oxicloud_login + +APP_PASSWORD_RESPONSE=$(curl -s -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"label":"nc-put-blake3-test","scopes":"webdav"}' \ + "$base_url/api/auth/app-passwords") +APP_PASSWORD=$(jq -r '.password' <<<"$APP_PASSWORD_RESPONSE") +APP_PASSWORD_ID=$(jq -r '.id' <<<"$APP_PASSWORD_RESPONSE") +[[ -n "$APP_PASSWORD" && "$APP_PASSWORD" != "null" ]] \ + || fail "Failed to mint NC app password: $APP_PASSWORD_RESPONSE" + +trap '[[ -n "${APP_PASSWORD_ID:-}" ]] && rest_delete "/api/auth/app-passwords/$APP_PASSWORD_ID" > /dev/null || true' EXIT + +# Idempotent cleanup of any leftover file. +HOME_FOLDER_ID=$(rest_get "/api/folders" | jq -r '.[0].id') +EXISTING_ID=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n) | .id) // empty') +if [[ -n "$EXISTING_ID" ]]; then + echo " cleaning up leftover $REMOTE_NAME (id=$EXISTING_ID)" + rest_delete "/api/files/$EXISTING_ID" > /dev/null + purge_from_trash "$REMOTE_NAME" +fi + +# ── PUT via the NC surface ─────────────────────────────────────────────────── + +echo " step 1: PUT /remote.php/dav/files/$username/$REMOTE_NAME" +STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X PUT \ + -u "$username:$APP_PASSWORD" \ + -H "Content-Type: text/plain" \ + --data-binary "@$FIXTURE" \ + "$base_url/remote.php/dav/files/$username/$REMOTE_NAME") +# 201 = created (first PUT), 204 = updated (idempotent overwrite path). +# Either signals a successful write to the spool + blob promotion. +[[ "$STATUS" =~ ^(201|204)$ ]] || fail "PUT got $STATUS, expected 201/204" +pass "PUT status=$STATUS" + +# ── Verify BLAKE3 + size via the REST listing ──────────────────────────────── + +echo " step 2: GET /api/files → assert content_hash + size" +LISTED=$(rest_get "/api/files?folder_id=$HOME_FOLDER_ID" \ + | jq -r --arg n "$REMOTE_NAME" 'first(.[] | select(.name == $n))') +ACTUAL_HASH=$(jq -r '.content_hash' <<<"$LISTED") +ACTUAL_SIZE=$(jq -r '.size' <<<"$LISTED") +FILE_ID=$(jq -r '.id' <<<"$LISTED") + +[[ "$ACTUAL_SIZE" == "$EXPECTED_SIZE" ]] \ + || fail "size mismatch: got $ACTUAL_SIZE, expected $EXPECTED_SIZE" +[[ "$ACTUAL_HASH" == "$EXPECTED_BLAKE3" ]] \ + || fail "BLAKE3 mismatch: got $ACTUAL_HASH, expected $EXPECTED_BLAKE3" +pass "content_hash + size match fixture ($EXPECTED_BLAKE3, $EXPECTED_SIZE B)" + +# ── Cleanup ────────────────────────────────────────────────────────────────── + +rest_delete "/api/files/$FILE_ID" > /dev/null +purge_from_trash "$REMOTE_NAME" + +echo +echo "=== NC single-file PUT BLAKE3 test: $PASS passed, $FAIL failed ===" +[[ "$FAIL" == "0" ]] || exit 1