//! Delta-upload protocol endpoints — "upload only what changed". //! //! Wire surface of [`DeltaUploadService`]; see that module (and //! `docs/delta-upload-protocol.md`) for the protocol and its security //! model. These handlers only authenticate, rate-limit and translate //! the wire formats — every decision lives in the application service. //! //! Chunk-frame wire format (`PUT /api/files/delta/chunks`): the body is a //! sequence of `[u32 big-endian length][length bytes]` frames, one per //! chunk, with `Content-Type: application/octet-stream`. Frames are capped //! at the CDC maximum chunk size (1 MiB) and the whole request at the same //! per-request ceiling as resumable chunk PUTs (`chunk_max_bytes`). use axum::{ Json, body::Body, extract::{Path, State}, http::{HeaderMap, StatusCode, header}, response::{IntoResponse, Response}, }; use bytes::{Buf, Bytes, BytesMut}; use futures::{Stream, TryStreamExt}; use std::sync::Arc; use tokio_stream::StreamExt; use crate::application::services::delta_upload_service::{ DeltaChunksResponse, DeltaCommitOutcome, DeltaCommitRequest, DeltaDownloadOutcome, DeltaDownloadRequest, DeltaManifestResponse, DeltaNegotiateRequest, DeltaNegotiateResponse, }; use crate::common::di::AppState; use crate::common::errors::DomainError; use crate::infrastructure::services::dedup_service::CDC_MAX_CHUNK; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; use http_body_util::BodyStream; use serde::Serialize; use utoipa::ToSchema; /// 409 body of `POST /api/files/delta/commit` when chunks vanished between /// negotiate and commit (GC race) or were never claimable: the client /// uploads exactly these hashes and retries the same commit. #[derive(Debug, Serialize, ToSchema)] pub struct DeltaStillMissingResponse { pub still_missing: Vec, } /// 404 body of `POST /api/files/delta/download` when some requested chunks /// are not reachable through the caller's files (or don't exist — the two /// are deliberately indistinguishable). #[derive(Debug, Serialize, ToSchema)] pub struct DeltaNotAvailableResponse { pub not_available: Vec, } /// Per-caller flood guard shared by the delta endpoints. fn check_rate_limit(state: &Arc, auth_user: &AuthUser) -> Result<(), AppError> { if state .delta_upload_rate_limiter .check_and_increment(&auth_user.id.to_string()) .is_err() { tracing::info!( target: "audit", event = "delta_upload.rejected", reason = "rate_limited", caller_id = %auth_user.id, "👮🏻‍♂️ Delta upload rejected: per-caller rate limit exceeded", ); return Err(AppError::new( StatusCode::TOO_MANY_REQUESTS, "Too many delta-upload requests; please retry shortly", "RateLimited", )); } Ok(()) } /// Parse a `[u32 BE length][bytes]` frame sequence from the request body. /// /// Streaming: peak RAM is one frame (≤ 1 MiB) plus one HTTP frame, /// regardless of how many chunks the request carries. `max_total` bounds /// the whole request body. fn parse_chunk_frames( body: Body, max_total: usize, ) -> impl Stream> + Send { async_stream::try_stream! { let mut body_stream = BodyStream::new(body); let mut buf = BytesMut::new(); let mut expecting: Option = None; let mut total: usize = 0; loop { // Drain every complete frame already buffered. loop { match expecting { None => { if buf.len() < 4 { break; } let len = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; buf.advance(4); if len == 0 || len > CDC_MAX_CHUNK { Err(DomainError::validation_error(format!( "Chunk frame of {len} bytes out of bounds (1 ..= {CDC_MAX_CHUNK})" )))?; } expecting = Some(len); } Some(len) => { if buf.len() < len { break; } let frame = buf.split_to(len).freeze(); expecting = None; yield frame; } } } match body_stream.next().await { Some(Ok(http_frame)) => { if let Some(data) = http_frame.data_ref() { total += data.len(); if total > max_total { Err(DomainError::validation_error(format!( "Request body exceeds the {max_total}-byte per-request cap; \ split the chunk upload into several requests" )))?; } buf.extend_from_slice(data); } } Some(Err(e)) => { Err(DomainError::validation_error(format!( "Failed to read request body: {e}" )))?; } None => { if expecting.is_some() || !buf.is_empty() { Err(DomainError::validation_error( "Truncated chunk frame at end of body", ))?; } break; } } } } } #[utoipa::path( post, path = "/api/files/delta/negotiate", request_body = DeltaNegotiateRequest, responses( (status = 200, description = "Chunks the caller must upload (the rest are claimable without bytes)", body = DeltaNegotiateResponse), (status = 400, description = "Malformed chunk list (hash format, size bounds, count ceiling)"), (status = 429, description = "Rate limited"), ), security(("bearerAuth" = [])), tag = "delta-upload" )] pub async fn delta_negotiate( State(state): State>, auth_user: AuthUser, Json(request): Json, ) -> Result { check_rate_limit(&state, &auth_user)?; let response = state .applications .delta_upload_service .negotiate_with_perms(auth_user.id, &request) .await .map_err(AppError::from)?; Ok(Json(response)) } #[utoipa::path( put, path = "/api/files/delta/chunks", request_body(content_type = "application/octet-stream", description = "Sequence of [u32 BE length][bytes] frames, one per chunk (each ≤ 1 MiB)"), responses( (status = 200, description = "Server-computed identity of every received frame, in order", body = DeltaChunksResponse), (status = 400, description = "Malformed framing, oversized frame, or oversized request"), (status = 429, description = "Rate limited"), ), security(("bearerAuth" = [])), tag = "delta-upload" )] pub async fn delta_upload_chunks( State(state): State>, auth_user: AuthUser, body: Body, ) -> Result { check_rate_limit(&state, &auth_user)?; let frames = parse_chunk_frames(body, state.core.config.storage.chunk_max_bytes); let response = state .applications .delta_upload_service .receive_chunks(frames) .await .map_err(AppError::from)?; Ok(Json(response)) } #[utoipa::path( post, path = "/api/files/delta/commit", request_body = DeltaCommitRequest, responses( (status = 201, description = "File created from the committed chunk sequence", body = crate::application::dtos::file_dto::FileDto), (status = 200, description = "Existing file's content replaced", body = crate::application::dtos::file_dto::FileDto), (status = 400, description = "Malformed request, or the declared file_hash does not match the chunk sequence"), (status = 404, description = "Target folder/file not found or not accessible"), (status = 409, description = "Chunks not claimable — upload them and retry", body = DeltaStillMissingResponse), (status = 429, description = "Rate limited"), (status = 507, description = "Storage quota exceeded"), ), security(("bearerAuth" = [])), tag = "delta-upload" )] pub async fn delta_commit( State(state): State>, auth_user: AuthUser, Json(request): Json, ) -> Result { check_rate_limit(&state, &auth_user)?; let outcome = state .applications .delta_upload_service .commit_with_perms(auth_user.id, request) .await .map_err(AppError::from)?; Ok(match outcome { DeltaCommitOutcome::Done { file, created } => { let status = if created { StatusCode::CREATED } else { StatusCode::OK }; (status, Json(file)).into_response() } DeltaCommitOutcome::StillMissing(still_missing) => ( StatusCode::CONFLICT, Json(DeltaStillMissingResponse { still_missing }), ) .into_response(), }) } #[utoipa::path( get, path = "/api/files/{id}/manifest", params(("id" = String, Path, description = "File ID")), responses( (status = 200, description = "Chunk recipe of the file (immutable per file_hash; served with ETag = file_hash)", body = DeltaManifestResponse), (status = 304, description = "Not modified (If-None-Match matched the current file_hash)"), (status = 404, description = "File not found, not accessible, or not owned by the caller"), (status = 429, description = "Rate limited"), ), security(("bearerAuth" = [])), tag = "delta-upload" )] pub async fn delta_file_manifest( State(state): State>, auth_user: AuthUser, Path(file_id): Path, headers: HeaderMap, ) -> Result { check_rate_limit(&state, &auth_user)?; let manifest = state .applications .delta_upload_service .file_manifest_with_perms(auth_user.id, &file_id) .await .map_err(AppError::from)?; // A manifest is immutable for a given file_hash, so the hash IS the // strong validator: sync clients polling a file revalidate for free. let etag = format!("\"{}\"", manifest.file_hash); if headers .get(header::IF_NONE_MATCH) .and_then(|v| v.to_str().ok()) .is_some_and(|inm| inm == etag) { return Ok(Response::builder() .status(StatusCode::NOT_MODIFIED) .header(header::ETAG, etag) .body(Body::empty()) .unwrap()); } Ok(( StatusCode::OK, [ (header::ETAG, etag), (header::CACHE_CONTROL, "private, no-cache".to_string()), ], Json(manifest), ) .into_response()) } #[utoipa::path( post, path = "/api/files/delta/download", request_body = DeltaDownloadRequest, responses( (status = 200, description = "Requested chunks as [u32 BE length][bytes] frames, in request order", content_type = "application/octet-stream"), (status = 400, description = "Malformed hashes, duplicates, or batch above the per-request ceiling"), (status = 404, description = "Some chunks are not available to this caller", body = DeltaNotAvailableResponse), (status = 429, description = "Rate limited"), ), security(("bearerAuth" = [])), tag = "delta-upload" )] pub async fn delta_download_chunks( State(state): State>, auth_user: AuthUser, Json(request): Json, ) -> Result { check_rate_limit(&state, &auth_user)?; let service = state.applications.delta_upload_service.clone(); let outcome = service .authorize_chunk_download_with_perms(auth_user.id, &request) .await .map_err(AppError::from)?; let ordered = match outcome { DeltaDownloadOutcome::NotAvailable(not_available) => { return Ok(( StatusCode::NOT_FOUND, Json(DeltaNotAvailableResponse { not_available }), ) .into_response()); } DeltaDownloadOutcome::Ready(ordered) => ordered, }; let total: u64 = ordered.iter().map(|(_, s)| 4 + s).sum(); // Stream the frames: 4-byte length headers come from the (entitled) // index sizes; bytes stream straight from the blob backend. Peak RAM // is bounded by `read_prefetch` open streams (their first frame), // independent of batch size. // // `buffered(read_prefetch)` overlaps the NEXT chunk's open with the // current chunk's drain — the same combinator/tuning as the main CDC // download path (benches/BLOB-PREFETCH.md). The old per-chunk await // paid every open's full round-trip serially: on an object-store // backend a 64-chunk batch at ~30 ms first-byte cost ~1.9 s of pure // latency. Frames still arrive strictly in request order. let prefetch = service.read_prefetch().max(1); let svc = service.clone(); // `futures::StreamExt` spelled out — this handler imports // `tokio_stream::StreamExt`, whose `map` adapter lacks `buffered`. let opened = futures::StreamExt::map(futures::stream::iter(ordered), move |(hash, size)| { let svc = svc.clone(); async move { let chunk = svc .chunk_stream(&hash) .await .map_err(std::io::Error::other)?; let header = futures::stream::once(async move { Ok::(Bytes::copy_from_slice(&(size as u32).to_be_bytes())) }); Ok::<_, std::io::Error>(futures::StreamExt::chain(header, chunk)) } }); let body_stream: std::pin::Pin> + Send>> = Box::pin(TryStreamExt::try_flatten(futures::StreamExt::buffered( opened, prefetch, ))); Ok(Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/octet-stream") .header(header::CONTENT_LENGTH, total.to_string()) .body(Body::from_stream(body_stream)) .unwrap()) } #[cfg(test)] mod tests { use super::*; /// Encode frames the way a client would. fn encode(frames: &[&[u8]]) -> Vec { let mut out = Vec::new(); for f in frames { out.extend_from_slice(&(f.len() as u32).to_be_bytes()); out.extend_from_slice(f); } out } async fn collect(body: Body, max_total: usize) -> Result, DomainError> { let stream = parse_chunk_frames(body, max_total); futures::pin_mut!(stream); let mut out = Vec::new(); while let Some(item) = stream.next().await { out.push(item?); } Ok(out) } #[tokio::test] async fn roundtrips_frames_in_order() { let wire = encode(&[b"first", b"second chunk", &[0xAB; 1000]]); let frames = collect(Body::from(wire), usize::MAX).await.unwrap(); assert_eq!(frames.len(), 3); assert_eq!(&frames[0][..], b"first"); assert_eq!(&frames[1][..], b"second chunk"); assert_eq!(frames[2].len(), 1000); } #[tokio::test] async fn empty_body_yields_no_frames() { let frames = collect(Body::empty(), usize::MAX).await.unwrap(); assert!(frames.is_empty()); } #[tokio::test] async fn rejects_zero_length_frame() { let wire = encode(&[b""]); assert!(collect(Body::from(wire), usize::MAX).await.is_err()); } #[tokio::test] async fn rejects_frame_above_cdc_max() { let mut wire = Vec::new(); wire.extend_from_slice(&((CDC_MAX_CHUNK as u32) + 1).to_be_bytes()); // Header alone is enough — the length is rejected before any data. assert!(collect(Body::from(wire), usize::MAX).await.is_err()); } #[tokio::test] async fn rejects_truncated_frame() { let mut wire = encode(&[b"complete"]); wire.extend_from_slice(&10u32.to_be_bytes()); wire.extend_from_slice(b"only5"); // promises 10, delivers 5 assert!(collect(Body::from(wire), usize::MAX).await.is_err()); } #[tokio::test] async fn rejects_request_above_total_cap() { let wire = encode(&[&[0u8; 600], &[1u8; 600]]); assert!(collect(Body::from(wire), 1000).await.is_err()); } #[tokio::test] async fn accepts_frame_exactly_at_cdc_max() { let big = vec![7u8; CDC_MAX_CHUNK]; let wire = encode(&[&big]); let frames = collect(Body::from(wire), usize::MAX).await.unwrap(); assert_eq!(frames.len(), 1); assert_eq!(frames[0].len(), CDC_MAX_CHUNK); } }