2026-02-22 14:12:53 +01:00
|
|
|
|
//! Chunked Upload Service — TUS-like Protocol for Large File Uploads
|
2026-02-14 01:29:34 +01:00
|
|
|
|
//!
|
2026-02-22 14:12:53 +01:00
|
|
|
|
//! Enables parallel chunk uploads for files >10 MB with:
|
|
|
|
|
|
//! - **Persistent, resumable uploads** — progress survives server restarts.
|
|
|
|
|
|
//! A `session.json` (written once on create) + a `progress.bin` bitmask
|
|
|
|
|
|
//! (updated atomically on each chunk) are stored alongside the chunk files.
|
|
|
|
|
|
//! On boot the service scans `temp_base_dir` and recovers any active sessions.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
//! - Parallel chunk transfers (up to 6 concurrent)
|
2026-03-01 21:47:39 +01:00
|
|
|
|
//! - Automatic reassembly with hash-on-write (BLAKE3)
|
2026-02-22 14:12:53 +01:00
|
|
|
|
//! - Expiration cleanup (24 h)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
//!
|
|
|
|
|
|
//! Protocol:
|
|
|
|
|
|
//! 1. POST /api/uploads → Create upload session, get upload_id
|
|
|
|
|
|
//! 2. PATCH /api/uploads/:id → Upload chunks (parallel OK)
|
|
|
|
|
|
//! 3. HEAD /api/uploads/:id → Check progress
|
|
|
|
|
|
//! 4. POST /api/uploads/:id/complete → Finalize and assemble
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
use chrono::{DateTime, Utc};
|
2026-02-25 23:31:51 +01:00
|
|
|
|
use dashmap::DashMap;
|
2026-02-22 14:12:53 +01:00
|
|
|
|
use serde::{Deserialize, Serialize};
|
2026-03-01 21:47:39 +01:00
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
use std::collections::HashMap;
|
2026-02-22 14:12:53 +01:00
|
|
|
|
use std::path::{Path, PathBuf};
|
2026-02-14 01:29:34 +01:00
|
|
|
|
use std::sync::Arc;
|
2026-02-22 14:12:53 +01:00
|
|
|
|
use std::time::Duration;
|
2026-02-24 15:11:56 +01:00
|
|
|
|
use tokio::fs::{self, File};
|
|
|
|
|
|
use tokio::io::AsyncWriteExt;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
use uuid::Uuid;
|
|
|
|
|
|
|
|
|
|
|
|
use crate::application::ports::chunked_upload_ports::{
|
|
|
|
|
|
ChunkUploadResponseDto, ChunkedUploadPort, CreateUploadResponseDto, UploadStatusResponseDto,
|
|
|
|
|
|
};
|
|
|
|
|
|
use crate::domain::errors::{DomainError, ErrorKind};
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Minimum file size to use chunked upload (10 MB)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub const CHUNKED_UPLOAD_THRESHOLD: usize = 10 * 1024 * 1024;
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Default chunk size (5 MB) — optimised for parallel transfers
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub const DEFAULT_CHUNK_SIZE: usize = 5 * 1024 * 1024;
|
|
|
|
|
|
|
|
|
|
|
|
/// Maximum concurrent chunks per upload
|
|
|
|
|
|
pub const MAX_PARALLEL_CHUNKS: usize = 6;
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Upload session expiration time (24 h)
|
|
|
|
|
|
const SESSION_EXPIRATION: Duration = Duration::from_secs(24 * 60 * 60);
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
/// Prefix every session directory name with this string so the cleanup
|
|
|
|
|
|
/// loop can be safely co-located with unrelated writers (PUT spool
|
|
|
|
|
|
/// tempfiles, the NC chunked subtree, anything else a sysadmin places
|
|
|
|
|
|
/// under the same `OXICLOUD_CHUNK_DIR`). The orphan-cleanup scan
|
|
|
|
|
|
/// filters by this prefix, so non-OxiCloud directories sharing the
|
|
|
|
|
|
/// root are never touched.
|
|
|
|
|
|
const SESSION_DIR_PREFIX: &str = "oxi-chunk-";
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Sentinel file names inside each session directory
|
|
|
|
|
|
const SESSION_META_FILE: &str = "session.json";
|
|
|
|
|
|
const PROGRESS_FILE: &str = "progress.bin";
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
/// Build a session directory name from an upload_id by attaching the
|
|
|
|
|
|
/// well-known prefix. Symmetric with [`strip_session_prefix`].
|
|
|
|
|
|
fn session_dir_name(upload_id: &str) -> String {
|
|
|
|
|
|
format!("{}{}", SESSION_DIR_PREFIX, upload_id)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Extract the upload_id from a session directory name. Returns
|
|
|
|
|
|
/// `None` when the directory wasn't created by this service (no
|
|
|
|
|
|
/// `oxi-chunk-` prefix) — the recovery and cleanup paths use this to
|
|
|
|
|
|
/// skip foreign directories cohabiting under `OXICLOUD_CHUNK_DIR`.
|
|
|
|
|
|
fn strip_session_prefix(dir_name: &str) -> Option<&str> {
|
|
|
|
|
|
dir_name.strip_prefix(SESSION_DIR_PREFIX)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// ─── Serialisable types ──────────────────────────────────────────────────────
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
/// Chunk status
|
2026-02-22 14:12:53 +01:00
|
|
|
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub enum ChunkStatus {
|
|
|
|
|
|
Pending,
|
|
|
|
|
|
Uploading,
|
|
|
|
|
|
Complete,
|
|
|
|
|
|
Failed(String),
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Individual chunk metadata
|
2026-02-22 14:12:53 +01:00
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub struct ChunkInfo {
|
|
|
|
|
|
pub index: usize,
|
|
|
|
|
|
pub offset: u64,
|
|
|
|
|
|
pub size: usize,
|
|
|
|
|
|
pub status: ChunkStatus,
|
|
|
|
|
|
pub checksum: Option<String>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Upload session state — fully serialisable for crash recovery.
|
|
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub struct UploadSession {
|
|
|
|
|
|
pub id: String,
|
2026-03-05 13:15:34 +01:00
|
|
|
|
pub user_id: String,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub filename: String,
|
|
|
|
|
|
pub folder_id: Option<String>,
|
|
|
|
|
|
pub content_type: String,
|
|
|
|
|
|
pub total_size: u64,
|
|
|
|
|
|
pub chunk_size: usize,
|
|
|
|
|
|
pub chunks: Vec<ChunkInfo>,
|
2026-02-22 14:12:53 +01:00
|
|
|
|
pub created_at: DateTime<Utc>,
|
|
|
|
|
|
pub last_activity: DateTime<Utc>,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub temp_dir: PathBuf,
|
|
|
|
|
|
pub bytes_received: u64,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl UploadSession {
|
|
|
|
|
|
/// Calculate number of chunks needed
|
|
|
|
|
|
pub fn calculate_chunk_count(total_size: u64, chunk_size: usize) -> usize {
|
|
|
|
|
|
(total_size as usize).div_ceil(chunk_size).max(1)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Get upload progress (0.0 – 1.0)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub fn progress(&self) -> f64 {
|
|
|
|
|
|
if self.total_size == 0 {
|
|
|
|
|
|
return 1.0;
|
|
|
|
|
|
}
|
|
|
|
|
|
self.bytes_received as f64 / self.total_size as f64
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Check if all chunks are complete
|
|
|
|
|
|
pub fn is_complete(&self) -> bool {
|
|
|
|
|
|
self.chunks
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.all(|c| c.status == ChunkStatus::Complete)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Get pending chunk indices
|
|
|
|
|
|
pub fn pending_chunks(&self) -> Vec<usize> {
|
|
|
|
|
|
self.chunks
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.enumerate()
|
|
|
|
|
|
.filter(|(_, c)| c.status == ChunkStatus::Pending)
|
|
|
|
|
|
.map(|(i, _)| i)
|
|
|
|
|
|
.collect()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Check if session has expired
|
|
|
|
|
|
pub fn is_expired(&self) -> bool {
|
2026-02-22 14:12:53 +01:00
|
|
|
|
let elapsed = Utc::now()
|
|
|
|
|
|
.signed_duration_since(self.last_activity)
|
|
|
|
|
|
.to_std()
|
|
|
|
|
|
.unwrap_or(Duration::ZERO);
|
|
|
|
|
|
elapsed > SESSION_EXPIRATION
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// ── Persistence helpers ──────────────────────────────────────────────
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Build the completed-chunks bitmask (1 bit per chunk).
|
|
|
|
|
|
fn build_progress_bitmask(&self) -> Vec<u8> {
|
2026-02-25 10:28:34 +01:00
|
|
|
|
let len = self.chunks.len().div_ceil(8);
|
2026-02-22 14:12:53 +01:00
|
|
|
|
let mut bitmask = vec![0u8; len];
|
|
|
|
|
|
for chunk in &self.chunks {
|
|
|
|
|
|
if chunk.status == ChunkStatus::Complete {
|
|
|
|
|
|
bitmask[chunk.index / 8] |= 1 << (chunk.index % 8);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
bitmask
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Apply a bitmask read from disk, marking matching chunks as `Complete`
|
|
|
|
|
|
/// and recalculating `bytes_received`.
|
|
|
|
|
|
fn apply_progress_bitmask(&mut self, bitmask: &[u8]) {
|
|
|
|
|
|
self.bytes_received = 0;
|
|
|
|
|
|
for chunk in &mut self.chunks {
|
|
|
|
|
|
let byte_idx = chunk.index / 8;
|
|
|
|
|
|
let bit_idx = chunk.index % 8;
|
|
|
|
|
|
if byte_idx < bitmask.len() && (bitmask[byte_idx] & (1 << bit_idx)) != 0 {
|
|
|
|
|
|
chunk.status = ChunkStatus::Complete;
|
|
|
|
|
|
self.bytes_received += chunk.size as u64;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Persist the full session metadata once (on create).
|
|
|
|
|
|
async fn persist_metadata(&self) -> Result<(), String> {
|
|
|
|
|
|
let path = self.temp_dir.join(SESSION_META_FILE);
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let json =
|
|
|
|
|
|
serde_json::to_vec(self).map_err(|e| format!("Failed to serialise session: {e}"))?;
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// Atomic write: write to .tmp then rename
|
|
|
|
|
|
let tmp = self.temp_dir.join("session.json.tmp");
|
|
|
|
|
|
fs::write(&tmp, &json)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| format!("Failed to write session metadata: {e}"))?;
|
|
|
|
|
|
fs::rename(&tmp, &path)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| format!("Failed to rename session metadata: {e}"))?;
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Persist the lightweight progress bitmask (on each chunk upload).
|
|
|
|
|
|
async fn persist_progress(&self) -> Result<(), String> {
|
|
|
|
|
|
let bitmask = self.build_progress_bitmask();
|
|
|
|
|
|
let path = self.temp_dir.join(PROGRESS_FILE);
|
|
|
|
|
|
// Bitmask is tiny (< 512 B for up to 4 096 chunks).
|
|
|
|
|
|
// A single write() of < 512 B is atomic on POSIX.
|
|
|
|
|
|
fs::write(&path, &bitmask)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| format!("Failed to write progress bitmask: {e}"))?;
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// ─── Service ─────────────────────────────────────────────────────────────────
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// Chunked Upload Service
|
2026-02-25 23:31:51 +01:00
|
|
|
|
///
|
|
|
|
|
|
/// Uses `DashMap` (sharded concurrent map) instead of a global `RwLock<HashMap>`
|
|
|
|
|
|
/// so that operations on independent upload sessions never contend with each
|
|
|
|
|
|
/// other. Disk I/O (temp-dir cleanup) is always performed **outside** any
|
|
|
|
|
|
/// map lock to avoid blocking concurrent uploads.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub struct ChunkedUploadService {
|
2026-02-25 23:31:51 +01:00
|
|
|
|
sessions: Arc<DashMap<String, UploadSession>>,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
temp_base_dir: PathBuf,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl ChunkedUploadService {
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Create the service, recover any persisted sessions, and start the
|
|
|
|
|
|
/// background cleanup task.
|
|
|
|
|
|
pub async fn new(temp_base_dir: PathBuf) -> Self {
|
|
|
|
|
|
// Ensure the base directory exists
|
|
|
|
|
|
let _ = fs::create_dir_all(&temp_base_dir).await;
|
|
|
|
|
|
|
|
|
|
|
|
// Recover sessions that survived a restart
|
|
|
|
|
|
let recovered = Self::recover_sessions(&temp_base_dir).await;
|
|
|
|
|
|
let recovered_count = recovered.len();
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let service = Self {
|
2026-02-25 23:31:51 +01:00
|
|
|
|
sessions: Arc::new(DashMap::from_iter(recovered)),
|
2026-02-14 01:29:34 +01:00
|
|
|
|
temp_base_dir,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
if recovered_count > 0 {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
tracing::info!("♻️ Recovered {recovered_count} chunked-upload session(s) from disk");
|
2026-02-22 14:12:53 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
// Start cleanup task
|
|
|
|
|
|
let sessions_clone = service.sessions.clone();
|
|
|
|
|
|
let temp_dir_clone = service.temp_base_dir.clone();
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
|
Self::cleanup_loop(sessions_clone, temp_dir_clone).await;
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
service
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Lightweight constructor that skips recovery and cleanup.
|
|
|
|
|
|
/// Used only by `AppState::default()` (stub wiring).
|
|
|
|
|
|
pub fn new_stub(temp_base_dir: PathBuf) -> Self {
|
|
|
|
|
|
Self {
|
2026-02-25 23:31:51 +01:00
|
|
|
|
sessions: Arc::new(DashMap::new()),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
temp_base_dir,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// ── Recovery ─────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
/// Scan `temp_base_dir` for directories containing `session.json`,
|
|
|
|
|
|
/// deserialise each session, apply the `progress.bin` bitmask, and
|
|
|
|
|
|
/// verify the chunk files on disk actually exist for completed chunks.
|
|
|
|
|
|
async fn recover_sessions(base: &Path) -> HashMap<String, UploadSession> {
|
|
|
|
|
|
let mut sessions = HashMap::new();
|
|
|
|
|
|
let mut entries = match fs::read_dir(base).await {
|
|
|
|
|
|
Ok(e) => e,
|
|
|
|
|
|
Err(_) => return sessions,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
while let Ok(Some(entry)) = entries.next_entry().await {
|
|
|
|
|
|
let dir = entry.path();
|
|
|
|
|
|
if !dir.is_dir() {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-06-09 10:19:21 +02:00
|
|
|
|
// Only consider directories WE created — anything without the
|
|
|
|
|
|
// `oxi-chunk-` prefix belongs to a sibling writer (NC subtree,
|
|
|
|
|
|
// PUT spool tempfiles, sysadmin-placed dirs) and must be left
|
|
|
|
|
|
// strictly alone. See `SESSION_DIR_PREFIX`.
|
|
|
|
|
|
let dir_name = match dir.file_name().and_then(|n| n.to_str()) {
|
|
|
|
|
|
Some(n) => n,
|
|
|
|
|
|
None => continue,
|
|
|
|
|
|
};
|
|
|
|
|
|
if strip_session_prefix(dir_name).is_none() {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-02-22 14:12:53 +01:00
|
|
|
|
|
|
|
|
|
|
let meta_path = dir.join(SESSION_META_FILE);
|
|
|
|
|
|
let meta_bytes = match fs::read(&meta_path).await {
|
|
|
|
|
|
Ok(b) => b,
|
|
|
|
|
|
Err(_) => continue, // no session.json → orphaned dir, skip
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let mut session: UploadSession = match serde_json::from_slice(&meta_bytes) {
|
|
|
|
|
|
Ok(s) => s,
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!("Skipping corrupt session in {:?}: {e}", dir);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Apply progress bitmask if present
|
|
|
|
|
|
if let Ok(bitmask) = fs::read(dir.join(PROGRESS_FILE)).await {
|
|
|
|
|
|
session.apply_progress_bitmask(&bitmask);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
// No progress file → all chunks still pending (freshly created)
|
|
|
|
|
|
session.bytes_received = 0;
|
|
|
|
|
|
for chunk in &mut session.chunks {
|
|
|
|
|
|
chunk.status = ChunkStatus::Pending;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Verify chunk files on disk — downgrade to Pending if missing
|
|
|
|
|
|
for chunk in &mut session.chunks {
|
|
|
|
|
|
if chunk.status == ChunkStatus::Complete {
|
|
|
|
|
|
let chunk_path = dir.join(format!("chunk_{:06}", chunk.index));
|
|
|
|
|
|
if !chunk_path.exists() {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Chunk {} missing on disk for session {}, marking pending",
|
|
|
|
|
|
chunk.index,
|
|
|
|
|
|
session.id
|
|
|
|
|
|
);
|
|
|
|
|
|
chunk.status = ChunkStatus::Pending;
|
|
|
|
|
|
session.bytes_received =
|
|
|
|
|
|
session.bytes_received.saturating_sub(chunk.size as u64);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Skip expired sessions
|
|
|
|
|
|
if session.is_expired() {
|
|
|
|
|
|
tracing::info!("Skipping expired recovered session: {}", session.id);
|
|
|
|
|
|
let _ = fs::remove_dir_all(&dir).await;
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
tracing::info!(
|
|
|
|
|
|
"♻️ Recovered session {} — {}/{} chunks ({:.0}%)",
|
|
|
|
|
|
session.id,
|
|
|
|
|
|
session
|
|
|
|
|
|
.chunks
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.filter(|c| c.status == ChunkStatus::Complete)
|
|
|
|
|
|
.count(),
|
|
|
|
|
|
session.chunks.len(),
|
|
|
|
|
|
session.progress() * 100.0
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
sessions.insert(session.id.clone(), session);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sessions
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// ── Cleanup ──────────────────────────────────────────────────────────
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// Background task to clean expired sessions
|
2026-02-26 01:02:25 +01:00
|
|
|
|
async fn cleanup_loop(sessions: Arc<DashMap<String, UploadSession>>, temp_base_dir: PathBuf) {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let mut interval = tokio::time::interval(Duration::from_secs(3600)); // Every hour
|
|
|
|
|
|
|
|
|
|
|
|
loop {
|
|
|
|
|
|
interval.tick().await;
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
// Collect expired session ids + temp dirs (lock-free iteration)
|
|
|
|
|
|
let expired: Vec<(String, PathBuf)> = sessions
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.filter(|entry| entry.value().is_expired())
|
|
|
|
|
|
.map(|entry| (entry.key().clone(), entry.value().temp_dir.clone()))
|
|
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
|
|
|
|
// Remove from map (microseconds per entry) then clean disk OUTSIDE lock
|
|
|
|
|
|
for (id, temp_dir) in expired {
|
|
|
|
|
|
sessions.remove(&id);
|
|
|
|
|
|
if let Err(e) = fs::remove_dir_all(&temp_dir).await {
|
|
|
|
|
|
tracing::warn!("Failed to cleanup expired upload {}: {}", id, e);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
tracing::info!("🧹 Cleaned expired upload session: {}", id);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
// Also clean orphaned temp directories (no session.json or very old).
|
|
|
|
|
|
// Filter strictly on the `oxi-chunk-` prefix so we never touch
|
|
|
|
|
|
// sibling directories sharing `OXICLOUD_CHUNK_DIR` (NC subtree
|
|
|
|
|
|
// `nextcloud/`, PUT spool tempfiles which are files anyway,
|
|
|
|
|
|
// operator-placed dirs). Without the prefix filter this loop
|
|
|
|
|
|
// would silently delete anything older than 24 h sitting at the
|
|
|
|
|
|
// root of the chunked-upload dir.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
if let Ok(mut entries) = fs::read_dir(&temp_base_dir).await {
|
|
|
|
|
|
while let Ok(Some(entry)) = entries.next_entry().await {
|
|
|
|
|
|
let path = entry.path();
|
2026-06-09 10:19:21 +02:00
|
|
|
|
if !path.is_dir() {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
let dir_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
|
|
|
|
|
let upload_id = match strip_session_prefix(dir_name) {
|
|
|
|
|
|
Some(id) => id,
|
|
|
|
|
|
None => continue, // not ours — never touch
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
if !sessions.contains_key(upload_id)
|
|
|
|
|
|
&& let Ok(metadata) = fs::metadata(&path).await
|
|
|
|
|
|
&& let Ok(modified) = metadata.modified()
|
|
|
|
|
|
&& modified.elapsed().unwrap_or_default() > SESSION_EXPIRATION
|
|
|
|
|
|
{
|
|
|
|
|
|
let _ = fs::remove_dir_all(&path).await;
|
|
|
|
|
|
tracing::info!("🧹 Cleaned orphaned upload dir: {:?}", path);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// ── Core operations ──────────────────────────────────────────────────
|
|
|
|
|
|
|
2026-03-05 13:15:34 +01:00
|
|
|
|
/// Verify that the given session belongs to the given user.
|
|
|
|
|
|
/// Returns 404 (not 403) to avoid revealing the existence of other users' sessions.
|
2026-03-05 21:28:51 +01:00
|
|
|
|
fn verify_session_owner(&self, upload_id: &str, user_id: &str) -> Result<(), String> {
|
2026-03-05 13:15:34 +01:00
|
|
|
|
let session = self
|
|
|
|
|
|
.sessions
|
|
|
|
|
|
.get(upload_id)
|
|
|
|
|
|
.ok_or_else(|| format!("Upload session not found: {}", upload_id))?;
|
|
|
|
|
|
if session.user_id != user_id {
|
|
|
|
|
|
return Err(format!("Upload session not found: {}", upload_id));
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Create a new upload session (persists `session.json` + empty `progress.bin`)
|
|
|
|
|
|
async fn create_session_inner(
|
2026-02-14 01:29:34 +01:00
|
|
|
|
&self,
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: String,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
filename: String,
|
|
|
|
|
|
folder_id: Option<String>,
|
|
|
|
|
|
content_type: String,
|
|
|
|
|
|
total_size: u64,
|
|
|
|
|
|
chunk_size: Option<usize>,
|
2026-02-22 14:12:53 +01:00
|
|
|
|
) -> Result<CreateUploadResponseDto, String> {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let upload_id = Uuid::new_v4().to_string();
|
|
|
|
|
|
let chunk_size = chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE);
|
|
|
|
|
|
let chunk_count = UploadSession::calculate_chunk_count(total_size, chunk_size);
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
// Create temp directory for chunks. The `oxi-chunk-` prefix
|
|
|
|
|
|
// tags the directory as belonging to this service so the
|
|
|
|
|
|
// shared-`OXICLOUD_CHUNK_DIR` story holds — see
|
|
|
|
|
|
// `SESSION_DIR_PREFIX` for the full rationale.
|
|
|
|
|
|
let temp_dir = self.temp_base_dir.join(session_dir_name(&upload_id));
|
2026-02-14 01:29:34 +01:00
|
|
|
|
fs::create_dir_all(&temp_dir)
|
|
|
|
|
|
.await
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.map_err(|e| format!("Failed to create temp directory: {e}"))?;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// Initialise chunk metadata
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let mut chunks = Vec::with_capacity(chunk_count);
|
|
|
|
|
|
let mut offset: u64 = 0;
|
|
|
|
|
|
|
|
|
|
|
|
for i in 0..chunk_count {
|
|
|
|
|
|
let size = if i == chunk_count - 1 {
|
|
|
|
|
|
(total_size - offset) as usize
|
|
|
|
|
|
} else {
|
|
|
|
|
|
chunk_size
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
chunks.push(ChunkInfo {
|
|
|
|
|
|
index: i,
|
|
|
|
|
|
offset,
|
|
|
|
|
|
size,
|
|
|
|
|
|
status: ChunkStatus::Pending,
|
|
|
|
|
|
checksum: None,
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
offset += size as u64;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
let now = Utc::now();
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let session = UploadSession {
|
|
|
|
|
|
id: upload_id.clone(),
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
filename,
|
|
|
|
|
|
folder_id,
|
|
|
|
|
|
content_type,
|
|
|
|
|
|
total_size,
|
|
|
|
|
|
chunk_size,
|
|
|
|
|
|
chunks,
|
|
|
|
|
|
created_at: now,
|
|
|
|
|
|
last_activity: now,
|
|
|
|
|
|
temp_dir,
|
|
|
|
|
|
bytes_received: 0,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let expires_at = SESSION_EXPIRATION.as_secs();
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// Persist metadata + empty progress to disk BEFORE inserting into RAM
|
|
|
|
|
|
session.persist_metadata().await?;
|
|
|
|
|
|
session.persist_progress().await?;
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
self.sessions.insert(upload_id.clone(), session);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
tracing::info!(
|
|
|
|
|
|
"📤 Created chunked upload session: {} ({} chunks, {} bytes each)",
|
|
|
|
|
|
upload_id,
|
|
|
|
|
|
chunk_count,
|
|
|
|
|
|
chunk_size
|
|
|
|
|
|
);
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
Ok(CreateUploadResponseDto {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
upload_id,
|
|
|
|
|
|
chunk_size,
|
|
|
|
|
|
total_chunks: chunk_count,
|
|
|
|
|
|
expires_at,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-08 23:17:07 +02:00
|
|
|
|
/// Prepare a chunk write — validates session ownership and chunk
|
|
|
|
|
|
/// index, returns the on-disk path the caller should stream the
|
|
|
|
|
|
/// HTTP body to plus the expected byte count for that chunk.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Used by the streaming REST PUT path: the handler calls
|
|
|
|
|
|
/// `prepare_chunk` → streams body to disk via
|
|
|
|
|
|
/// `interfaces::upload_spool::stream_body_to_path` → calls
|
|
|
|
|
|
/// `commit_chunk` to finalise. This lets the body bypass the
|
|
|
|
|
|
/// in-memory `Bytes` allocation entirely (peak heap ~one HTTP
|
|
|
|
|
|
/// frame instead of "chunk size").
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Returns `Err` if the session is unknown, owned by another user,
|
|
|
|
|
|
/// the chunk index is out of range, or the chunk is already complete.
|
|
|
|
|
|
pub async fn prepare_chunk(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
|
|
|
|
|
user_id: Uuid,
|
|
|
|
|
|
chunk_index: usize,
|
|
|
|
|
|
) -> Result<(PathBuf, usize), DomainError> {
|
|
|
|
|
|
self.verify_session_owner(upload_id, &user_id.to_string())
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?;
|
|
|
|
|
|
|
|
|
|
|
|
let session = self.sessions.get(upload_id).ok_or_else(|| {
|
|
|
|
|
|
DomainError::new(
|
|
|
|
|
|
ErrorKind::NotFound,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
format!("Upload session not found: {}", upload_id),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
|
|
|
|
|
|
if chunk_index >= session.chunks.len() {
|
|
|
|
|
|
return Err(DomainError::new(
|
|
|
|
|
|
ErrorKind::InvalidInput,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"Invalid chunk index: {} (max: {})",
|
|
|
|
|
|
chunk_index,
|
|
|
|
|
|
session.chunks.len() - 1
|
|
|
|
|
|
),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
let chunk = &session.chunks[chunk_index];
|
|
|
|
|
|
if chunk.status == ChunkStatus::Complete {
|
|
|
|
|
|
return Err(DomainError::new(
|
|
|
|
|
|
ErrorKind::InvalidInput,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
format!("Chunk {} already uploaded", chunk_index),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Ok((
|
|
|
|
|
|
session.temp_dir.join(format!("chunk_{:06}", chunk_index)),
|
|
|
|
|
|
chunk.size,
|
|
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Finalise a chunk write — verifies the actually-written byte count
|
|
|
|
|
|
/// matches the chunk's declared size, validates an optional
|
|
|
|
|
|
/// algorithm-tagged checksum, and updates session state. The chunk
|
|
|
|
|
|
/// file at `{session.temp_dir}/chunk_{index:06}` must already have
|
|
|
|
|
|
/// been written by the caller (typically via
|
|
|
|
|
|
/// `stream_body_to_path`).
|
|
|
|
|
|
///
|
|
|
|
|
|
/// `actual_size` is the byte count the streaming write reported;
|
|
|
|
|
|
/// `computed_checksum` is the hex digest computed during streaming
|
|
|
|
|
|
/// (or `None` if the client didn't request a checksum). When
|
|
|
|
|
|
/// `expected_checksum` is supplied the two are compared; a
|
|
|
|
|
|
/// mismatch removes the partial file and returns `ValidationError`
|
|
|
|
|
|
/// so a client retry against the same chunk index gets a clean
|
|
|
|
|
|
/// slot. A size mismatch does the same.
|
|
|
|
|
|
pub async fn commit_chunk(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
|
|
|
|
|
user_id: Uuid,
|
|
|
|
|
|
chunk_index: usize,
|
|
|
|
|
|
actual_size: u64,
|
|
|
|
|
|
computed_checksum: Option<String>,
|
|
|
|
|
|
expected_checksum: Option<String>,
|
|
|
|
|
|
) -> Result<ChunkUploadResponseDto, DomainError> {
|
|
|
|
|
|
self.verify_session_owner(upload_id, &user_id.to_string())
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?;
|
|
|
|
|
|
|
|
|
|
|
|
// Re-fetch chunk metadata under fresh lock — guards against the
|
|
|
|
|
|
// (vanishingly unlikely) case of a session expiry / cancellation
|
|
|
|
|
|
// racing with the write.
|
|
|
|
|
|
let (chunk_path, expected_size, persist_path) = {
|
|
|
|
|
|
let session = self.sessions.get(upload_id).ok_or_else(|| {
|
|
|
|
|
|
DomainError::new(
|
|
|
|
|
|
ErrorKind::NotFound,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
"Session disappeared".to_string(),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
if chunk_index >= session.chunks.len() {
|
|
|
|
|
|
return Err(DomainError::new(
|
|
|
|
|
|
ErrorKind::InvalidInput,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
format!("Invalid chunk index: {}", chunk_index),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
(
|
|
|
|
|
|
session.temp_dir.join(format!("chunk_{:06}", chunk_index)),
|
|
|
|
|
|
session.chunks[chunk_index].size,
|
|
|
|
|
|
session.temp_dir.join(PROGRESS_FILE),
|
|
|
|
|
|
)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Size check — the streaming body may have been truncated by
|
|
|
|
|
|
// the client mid-flight or exceeded the chunk's declared
|
|
|
|
|
|
// length. Either way we don't want a partial chunk to count
|
|
|
|
|
|
// as complete; nuke it and ask the client to retry.
|
|
|
|
|
|
if actual_size != expected_size as u64 {
|
|
|
|
|
|
let _ = fs::remove_file(&chunk_path).await;
|
|
|
|
|
|
return Err(DomainError::new(
|
|
|
|
|
|
ErrorKind::InvalidInput,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"Invalid chunk size: expected {} bytes, got {} bytes",
|
|
|
|
|
|
expected_size, actual_size
|
|
|
|
|
|
),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Checksum check — case-insensitive compare so clients that
|
|
|
|
|
|
// send uppercase hex still match.
|
|
|
|
|
|
if let Some(expected) = expected_checksum.as_ref()
|
|
|
|
|
|
&& let Some(actual) = computed_checksum.as_ref()
|
|
|
|
|
|
&& !expected.eq_ignore_ascii_case(actual)
|
|
|
|
|
|
{
|
|
|
|
|
|
let _ = fs::remove_file(&chunk_path).await;
|
|
|
|
|
|
return Err(DomainError::new(
|
|
|
|
|
|
ErrorKind::InvalidInput,
|
|
|
|
|
|
"ChunkedUpload",
|
2026-06-08 23:57:51 +02:00
|
|
|
|
format!("Checksum mismatch: expected {}, got {}", expected, actual),
|
2026-06-08 23:17:07 +02:00
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Update session state — DashMap shard lock held only for the
|
|
|
|
|
|
// RAM updates (~µs). The bitmask write happens AFTER the ref
|
|
|
|
|
|
// is dropped so concurrent uploads to other sessions are never
|
|
|
|
|
|
// blocked. Mirrors the legacy `upload_chunk_inner` semantics.
|
|
|
|
|
|
let (bytes_received, progress, is_complete, persist_bitmask) = {
|
|
|
|
|
|
let mut session = self.sessions.get_mut(upload_id).ok_or_else(|| {
|
|
|
|
|
|
DomainError::new(
|
|
|
|
|
|
ErrorKind::NotFound,
|
|
|
|
|
|
"ChunkedUpload",
|
|
|
|
|
|
"Session disappeared".to_string(),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
session.chunks[chunk_index].status = ChunkStatus::Complete;
|
|
|
|
|
|
session.chunks[chunk_index].checksum = expected_checksum;
|
|
|
|
|
|
session.bytes_received += actual_size;
|
|
|
|
|
|
session.last_activity = Utc::now();
|
|
|
|
|
|
let bitmask = session.build_progress_bitmask();
|
|
|
|
|
|
(
|
|
|
|
|
|
session.bytes_received,
|
|
|
|
|
|
session.progress(),
|
|
|
|
|
|
session.is_complete(),
|
|
|
|
|
|
bitmask,
|
|
|
|
|
|
)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
if let Err(e) = fs::write(&persist_path, &persist_bitmask).await {
|
|
|
|
|
|
tracing::warn!("Failed to persist progress for {upload_id}: {e}");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"📦 Chunk {} committed for {} ({:.1}% complete)",
|
|
|
|
|
|
chunk_index,
|
|
|
|
|
|
upload_id,
|
|
|
|
|
|
progress * 100.0
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
Ok(ChunkUploadResponseDto {
|
|
|
|
|
|
chunk_index,
|
|
|
|
|
|
bytes_received,
|
|
|
|
|
|
progress,
|
|
|
|
|
|
is_complete,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
/// Upload a single chunk (persists `progress.bin` after success)
|
|
|
|
|
|
async fn upload_chunk_inner(
|
2026-02-14 01:29:34 +01:00
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: &str,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
chunk_index: usize,
|
|
|
|
|
|
data: bytes::Bytes,
|
|
|
|
|
|
checksum: Option<String>,
|
2026-02-22 14:12:53 +01:00
|
|
|
|
) -> Result<ChunkUploadResponseDto, String> {
|
2026-03-05 13:15:34 +01:00
|
|
|
|
// Verify session exists AND belongs to the requesting user
|
|
|
|
|
|
self.verify_session_owner(upload_id, user_id)?;
|
|
|
|
|
|
|
|
|
|
|
|
// Validate chunk index is valid
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let (chunk_path, expected_size) = {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let session = self
|
|
|
|
|
|
.sessions
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.get(upload_id)
|
|
|
|
|
|
.ok_or_else(|| format!("Upload session not found: {}", upload_id))?;
|
|
|
|
|
|
|
|
|
|
|
|
if chunk_index >= session.chunks.len() {
|
|
|
|
|
|
return Err(format!(
|
|
|
|
|
|
"Invalid chunk index: {} (max: {})",
|
|
|
|
|
|
chunk_index,
|
|
|
|
|
|
session.chunks.len() - 1
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
let chunk = &session.chunks[chunk_index];
|
|
|
|
|
|
if chunk.status == ChunkStatus::Complete {
|
|
|
|
|
|
return Err(format!("Chunk {} already uploaded", chunk_index));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
(
|
|
|
|
|
|
session.temp_dir.join(format!("chunk_{:06}", chunk_index)),
|
|
|
|
|
|
chunk.size,
|
|
|
|
|
|
)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Validate chunk size
|
|
|
|
|
|
if data.len() != expected_size {
|
|
|
|
|
|
return Err(format!(
|
|
|
|
|
|
"Invalid chunk size: expected {} bytes, got {} bytes",
|
|
|
|
|
|
expected_size,
|
|
|
|
|
|
data.len()
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-23 22:49:34 +01:00
|
|
|
|
// Verify checksum if provided — MD5 is CPU-bound (~1.2 ms per 5 MB),
|
|
|
|
|
|
// so we offload it to the blocking thread-pool to keep the Tokio
|
|
|
|
|
|
// worker free for other connections.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
if let Some(ref expected_checksum) = checksum {
|
2026-02-23 22:49:34 +01:00
|
|
|
|
let data_clone = data.clone(); // Bytes::clone is O(1) — just an Arc increment
|
2026-03-04 23:55:08 +01:00
|
|
|
|
let actual_checksum = tokio::task::spawn_blocking(move || {
|
|
|
|
|
|
use md5::{Digest, Md5};
|
2026-04-14 23:17:39 +02:00
|
|
|
|
let hash = Md5::digest(&data_clone);
|
|
|
|
|
|
hash.iter().map(|b| format!("{b:02x}")).collect::<String>()
|
2026-03-04 23:55:08 +01:00
|
|
|
|
})
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| format!("MD5 checksum task failed: {e}"))?;
|
2026-02-23 22:49:34 +01:00
|
|
|
|
|
|
|
|
|
|
if actual_checksum != *expected_checksum {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
return Err(format!(
|
|
|
|
|
|
"Checksum mismatch: expected {}, got {}",
|
|
|
|
|
|
expected_checksum, actual_checksum
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Write chunk to temp file
|
|
|
|
|
|
let mut file = File::create(&chunk_path)
|
|
|
|
|
|
.await
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.map_err(|e| format!("Failed to create chunk file: {e}"))?;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
file.write_all(&data)
|
|
|
|
|
|
.await
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.map_err(|e| format!("Failed to write chunk: {e}"))?;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
// Update session state — DashMap shard lock held only for RAM updates (~µs).
|
|
|
|
|
|
// Disk I/O (persist_progress) is done AFTER the ref is dropped so
|
|
|
|
|
|
// concurrent uploads to other sessions are never blocked.
|
2026-02-23 22:32:14 +01:00
|
|
|
|
let (bytes_received, progress, is_complete, persist_path, persist_bitmask) = {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let mut session = self
|
|
|
|
|
|
.sessions
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.get_mut(upload_id)
|
|
|
|
|
|
.ok_or_else(|| "Session disappeared".to_string())?;
|
|
|
|
|
|
|
|
|
|
|
|
session.chunks[chunk_index].status = ChunkStatus::Complete;
|
|
|
|
|
|
session.chunks[chunk_index].checksum = checksum;
|
|
|
|
|
|
session.bytes_received += data.len() as u64;
|
2026-02-22 14:12:53 +01:00
|
|
|
|
session.last_activity = Utc::now();
|
|
|
|
|
|
|
2026-02-23 22:32:14 +01:00
|
|
|
|
// Build bitmask while under lock (CPU-only, ~microseconds)
|
|
|
|
|
|
let bitmask = session.build_progress_bitmask();
|
|
|
|
|
|
let path = session.temp_dir.join(PROGRESS_FILE);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
(
|
|
|
|
|
|
session.bytes_received,
|
|
|
|
|
|
session.progress(),
|
|
|
|
|
|
session.is_complete(),
|
2026-02-23 22:32:14 +01:00
|
|
|
|
path,
|
|
|
|
|
|
bitmask,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
)
|
2026-02-25 23:31:51 +01:00
|
|
|
|
}; // DashMap shard ref dropped here — held only for RAM updates (~µs)
|
2026-02-23 22:32:14 +01:00
|
|
|
|
|
|
|
|
|
|
// Persist bitmask to disk OUTSIDE the lock — no longer blocks other uploads
|
|
|
|
|
|
if let Err(e) = fs::write(&persist_path, &persist_bitmask).await {
|
|
|
|
|
|
tracing::warn!("Failed to persist progress for {upload_id}: {e}");
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"📦 Chunk {}/{} uploaded for {} ({:.1}% complete)",
|
|
|
|
|
|
chunk_index + 1,
|
|
|
|
|
|
expected_size,
|
|
|
|
|
|
upload_id,
|
|
|
|
|
|
progress * 100.0
|
|
|
|
|
|
);
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
Ok(ChunkUploadResponseDto {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
chunk_index,
|
|
|
|
|
|
bytes_received,
|
|
|
|
|
|
progress,
|
|
|
|
|
|
is_complete,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Get upload status
|
2026-03-05 21:28:51 +01:00
|
|
|
|
async fn get_status_inner(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
|
|
|
|
|
user_id: &str,
|
|
|
|
|
|
) -> Result<UploadStatusResponseDto, String> {
|
2026-03-05 13:15:34 +01:00
|
|
|
|
self.verify_session_owner(upload_id, user_id)?;
|
|
|
|
|
|
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let session = self
|
|
|
|
|
|
.sessions
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.get(upload_id)
|
|
|
|
|
|
.ok_or_else(|| format!("Upload session not found: {}", upload_id))?;
|
|
|
|
|
|
|
|
|
|
|
|
let completed_chunks = session
|
|
|
|
|
|
.chunks
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.filter(|c| c.status == ChunkStatus::Complete)
|
|
|
|
|
|
.count();
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
Ok(UploadStatusResponseDto {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
upload_id: session.id.clone(),
|
|
|
|
|
|
filename: session.filename.clone(),
|
|
|
|
|
|
total_size: session.total_size,
|
|
|
|
|
|
bytes_received: session.bytes_received,
|
|
|
|
|
|
progress: session.progress(),
|
|
|
|
|
|
total_chunks: session.chunks.len(),
|
|
|
|
|
|
completed_chunks,
|
|
|
|
|
|
pending_chunks: session.pending_chunks(),
|
|
|
|
|
|
is_complete: session.is_complete(),
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-01 21:47:39 +01:00
|
|
|
|
/// Assemble chunks into final file and return the path + pre-computed BLAKE3 hash.
|
2026-02-15 17:53:25 +01:00
|
|
|
|
///
|
2026-03-01 21:47:39 +01:00
|
|
|
|
/// **Hash-on-Write**: BLAKE3 is computed while copying chunks into the
|
2026-02-15 17:53:25 +01:00
|
|
|
|
/// assembled file, eliminating the second sequential read that dedup_service
|
|
|
|
|
|
/// would otherwise need.
|
|
|
|
|
|
///
|
2026-03-01 21:47:39 +01:00
|
|
|
|
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`.
|
2026-02-22 14:12:53 +01:00
|
|
|
|
async fn complete_upload_inner(
|
2026-02-14 01:29:34 +01:00
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: &str,
|
2026-02-15 17:53:25 +01:00
|
|
|
|
) -> Result<(PathBuf, String, Option<String>, String, u64, String), String> {
|
2026-03-05 13:15:34 +01:00
|
|
|
|
// Verify ownership before assembly
|
|
|
|
|
|
self.verify_session_owner(upload_id, user_id)?;
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
// Get session and validate completion.
|
|
|
|
|
|
// Clone the session data and drop the DashMap ref immediately
|
|
|
|
|
|
// so the shard is not held during the expensive assembly step.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let session = {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let entry = self
|
|
|
|
|
|
.sessions
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.get(upload_id)
|
|
|
|
|
|
.ok_or_else(|| format!("Upload session not found: {}", upload_id))?;
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
if !entry.is_complete() {
|
|
|
|
|
|
let pending = entry.pending_chunks();
|
2026-02-14 01:29:34 +01:00
|
|
|
|
return Err(format!(
|
|
|
|
|
|
"Upload not complete. Missing chunks: {:?}",
|
|
|
|
|
|
pending
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
entry.clone()
|
2026-02-14 01:29:34 +01:00
|
|
|
|
};
|
|
|
|
|
|
|
2026-02-24 15:11:56 +01:00
|
|
|
|
// Assemble file with hash-on-write.
|
|
|
|
|
|
//
|
2026-03-01 21:47:39 +01:00
|
|
|
|
// The entire loop is offloaded to spawn_blocking because BLAKE3
|
|
|
|
|
|
// hashing is CPU-bound and would otherwise block a Tokio worker,
|
|
|
|
|
|
// starving all other connections.
|
2026-02-24 15:11:56 +01:00
|
|
|
|
// Synchronous I/O is used inside the blocking thread — it avoids
|
|
|
|
|
|
// the async reactor overhead and is actually faster for this
|
|
|
|
|
|
// sequential workload.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let assembled_path = session.temp_dir.join("assembled");
|
2026-02-24 15:11:56 +01:00
|
|
|
|
let chunks_meta: Vec<(usize, PathBuf)> = session
|
|
|
|
|
|
.chunks
|
|
|
|
|
|
.iter()
|
2026-02-26 01:02:25 +01:00
|
|
|
|
.map(|c| {
|
|
|
|
|
|
(
|
|
|
|
|
|
c.index,
|
|
|
|
|
|
session.temp_dir.join(format!("chunk_{:06}", c.index)),
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
2026-02-24 15:11:56 +01:00
|
|
|
|
.collect();
|
|
|
|
|
|
let total_size = session.total_size;
|
|
|
|
|
|
|
|
|
|
|
|
let hash = tokio::task::spawn_blocking(move || -> Result<String, String> {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
use std::io::{BufWriter as StdBufWriter, Read, Write};
|
2026-02-24 15:11:56 +01:00
|
|
|
|
|
|
|
|
|
|
let raw_output = std::fs::OpenOptions::new()
|
|
|
|
|
|
.create(true)
|
|
|
|
|
|
.write(true)
|
|
|
|
|
|
.truncate(true)
|
|
|
|
|
|
.open(&assembled_path)
|
|
|
|
|
|
.map_err(|e| format!("Failed to create assembled file: {e}"))?;
|
|
|
|
|
|
|
|
|
|
|
|
// Pre-allocate assembled file to reduce fragmentation
|
|
|
|
|
|
let _ = raw_output.set_len(total_size);
|
|
|
|
|
|
|
|
|
|
|
|
// 512 KB I/O buffers — 8× fewer syscalls than 64 KB
|
|
|
|
|
|
let mut output = StdBufWriter::with_capacity(524_288, raw_output);
|
2026-03-01 21:47:39 +01:00
|
|
|
|
let mut hasher = blake3::Hasher::new();
|
2026-02-24 15:11:56 +01:00
|
|
|
|
|
2026-03-03 11:18:40 +01:00
|
|
|
|
// For files >10 MB, use multithreaded BLAKE3 hashing (all cores)
|
|
|
|
|
|
const RAYON_THRESHOLD: u64 = 10 * 1024 * 1024;
|
|
|
|
|
|
let use_rayon = total_size > RAYON_THRESHOLD;
|
|
|
|
|
|
|
2026-02-24 15:11:56 +01:00
|
|
|
|
// Single 512 KB read buffer reused across all chunks (avoids N allocations)
|
|
|
|
|
|
let mut buf = vec![0u8; 524_288];
|
|
|
|
|
|
for (index, chunk_path) in &chunks_meta {
|
|
|
|
|
|
let mut chunk_file = std::fs::File::open(chunk_path)
|
|
|
|
|
|
.map_err(|e| format!("Failed to open chunk {index}: {e}"))?;
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = chunk_file
|
|
|
|
|
|
.read(&mut buf)
|
|
|
|
|
|
.map_err(|e| format!("Failed to read chunk {index}: {e}"))?;
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
2026-03-03 11:18:40 +01:00
|
|
|
|
if use_rayon {
|
|
|
|
|
|
hasher.update_rayon(&buf[..n]);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
hasher.update(&buf[..n]);
|
|
|
|
|
|
}
|
2026-02-24 15:11:56 +01:00
|
|
|
|
output.write_all(&buf[..n]).map_err(|e| {
|
|
|
|
|
|
format!("Failed to write chunk {index} to assembled file: {e}")
|
|
|
|
|
|
})?;
|
2026-02-15 17:53:25 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-24 15:11:56 +01:00
|
|
|
|
output
|
|
|
|
|
|
.flush()
|
|
|
|
|
|
.map_err(|e| format!("Failed to flush assembled file: {e}"))?;
|
2026-06-08 23:57:51 +02:00
|
|
|
|
// ── Durability boundary ────────────────────────────────────
|
|
|
|
|
|
// `flush` drains BufWriter's userspace buffer but leaves the
|
|
|
|
|
|
// bytes in the kernel page cache. Without `sync_all`, a
|
|
|
|
|
|
// power loss between this `complete_upload` returning 2xx
|
|
|
|
|
|
// and the OS writeback timer firing (~5 s default) loses
|
|
|
|
|
|
// the merged blob — and PG's metadata row references a hash
|
|
|
|
|
|
// that no longer exists on disk. Reclaim the BufWriter's
|
|
|
|
|
|
// inner File via `into_inner` so we can `sync_all` it; the
|
|
|
|
|
|
// BufWriter would otherwise drop without flushing on the
|
|
|
|
|
|
// inner handle.
|
|
|
|
|
|
let raw_output = output
|
|
|
|
|
|
.into_inner()
|
|
|
|
|
|
.map_err(|e| format!("into_inner on BufWriter failed: {e}"))?;
|
|
|
|
|
|
raw_output
|
|
|
|
|
|
.sync_all()
|
|
|
|
|
|
.map_err(|e| format!("Failed to fsync assembled file: {e}"))?;
|
2026-02-15 17:53:25 +01:00
|
|
|
|
|
2026-02-24 15:11:56 +01:00
|
|
|
|
// Clean up chunk files (keep assembled) — already on a blocking thread
|
|
|
|
|
|
for (_index, chunk_path) in &chunks_meta {
|
|
|
|
|
|
let _ = std::fs::remove_file(chunk_path);
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-03-01 21:47:39 +01:00
|
|
|
|
Ok(hasher.finalize().to_hex().to_string())
|
2026-02-24 15:11:56 +01:00
|
|
|
|
})
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| format!("Assembly task panicked: {e}"))??;
|
|
|
|
|
|
|
|
|
|
|
|
let assembled_path = session.temp_dir.join("assembled");
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
tracing::info!(
|
|
|
|
|
|
"✅ Assembled chunked upload: {} ({} bytes from {} chunks)",
|
|
|
|
|
|
session.filename,
|
|
|
|
|
|
session.total_size,
|
|
|
|
|
|
session.chunks.len()
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
Ok((
|
|
|
|
|
|
assembled_path,
|
|
|
|
|
|
session.filename.clone(),
|
|
|
|
|
|
session.folder_id.clone(),
|
|
|
|
|
|
session.content_type.clone(),
|
|
|
|
|
|
session.total_size,
|
2026-02-15 17:53:25 +01:00
|
|
|
|
hash,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
/// Finalize upload: remove session from RAM, then clean disk OUTSIDE lock.
|
2026-03-05 13:15:34 +01:00
|
|
|
|
async fn finalize_upload_inner(&self, upload_id: &str, user_id: &str) -> Result<(), String> {
|
|
|
|
|
|
self.verify_session_owner(upload_id, user_id)?;
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
// Remove from map (~µs) — releases shard immediately
|
|
|
|
|
|
let removed = self.sessions.remove(upload_id).map(|(_, s)| s);
|
|
|
|
|
|
|
|
|
|
|
|
// Disk I/O happens with NO lock held
|
2026-02-26 00:57:49 +01:00
|
|
|
|
if let Some(session) = removed
|
|
|
|
|
|
&& let Err(e) = fs::remove_dir_all(&session.temp_dir).await
|
|
|
|
|
|
{
|
|
|
|
|
|
tracing::warn!("Failed to cleanup upload {}: {}", upload_id, e);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
/// Cancel an upload and cleanup — disk I/O outside lock.
|
2026-05-21 19:11:28 +02:00
|
|
|
|
///
|
|
|
|
|
|
/// Returns:
|
|
|
|
|
|
/// - `DomainError::NotFound` if no session matches `upload_id` for `user_id`
|
|
|
|
|
|
/// (covers both "session missing" and "owned by someone else" — same
|
|
|
|
|
|
/// error for anti-enumeration).
|
|
|
|
|
|
/// - `DomainError::InternalError` for unexpected disk I/O failures.
|
|
|
|
|
|
async fn cancel_upload_inner(&self, upload_id: &str, user_id: &str) -> Result<(), DomainError> {
|
|
|
|
|
|
self.verify_session_owner(upload_id, user_id)
|
|
|
|
|
|
.map_err(|_| DomainError::not_found("Upload", upload_id))?;
|
2026-03-05 13:15:34 +01:00
|
|
|
|
|
2026-02-25 23:31:51 +01:00
|
|
|
|
// Remove from map (~µs)
|
|
|
|
|
|
let removed = self.sessions.remove(upload_id).map(|(_, s)| s);
|
|
|
|
|
|
|
|
|
|
|
|
// Disk I/O with NO lock held
|
|
|
|
|
|
if let Some(session) = removed {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
if let Err(e) = fs::remove_dir_all(&session.temp_dir).await {
|
|
|
|
|
|
tracing::warn!("Failed to cleanup cancelled upload {}: {}", upload_id, e);
|
|
|
|
|
|
}
|
|
|
|
|
|
tracing::info!("❌ Cancelled chunked upload: {}", upload_id);
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Check if file size qualifies for chunked upload
|
|
|
|
|
|
pub fn should_use_chunked(size: u64) -> bool {
|
|
|
|
|
|
size as usize >= CHUNKED_UPLOAD_THRESHOLD
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Get active session count (for monitoring)
|
|
|
|
|
|
pub async fn active_sessions(&self) -> usize {
|
2026-02-25 23:31:51 +01:00
|
|
|
|
self.sessions.len()
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// ─── Port implementation ─────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
impl ChunkedUploadPort for ChunkedUploadService {
|
|
|
|
|
|
async fn create_session(
|
|
|
|
|
|
&self,
|
2026-03-07 14:59:32 +01:00
|
|
|
|
user_id: Uuid,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
filename: String,
|
|
|
|
|
|
folder_id: Option<String>,
|
|
|
|
|
|
content_type: String,
|
|
|
|
|
|
total_size: u64,
|
|
|
|
|
|
chunk_size: Option<usize>,
|
|
|
|
|
|
) -> Result<CreateUploadResponseDto, DomainError> {
|
2026-03-05 21:28:51 +01:00
|
|
|
|
self.create_session_inner(
|
2026-03-07 14:59:32 +01:00
|
|
|
|
user_id.to_string(),
|
2026-03-05 21:28:51 +01:00
|
|
|
|
filename,
|
|
|
|
|
|
folder_id,
|
|
|
|
|
|
content_type,
|
|
|
|
|
|
total_size,
|
|
|
|
|
|
chunk_size,
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e))
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn upload_chunk(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
2026-03-07 14:59:32 +01:00
|
|
|
|
user_id: Uuid,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
chunk_index: usize,
|
|
|
|
|
|
data: bytes::Bytes,
|
|
|
|
|
|
checksum: Option<String>,
|
|
|
|
|
|
) -> Result<ChunkUploadResponseDto, DomainError> {
|
2026-03-07 14:59:32 +01:00
|
|
|
|
self.upload_chunk_inner(upload_id, &user_id.to_string(), chunk_index, data, checksum)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.await
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e))
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-05 21:28:51 +01:00
|
|
|
|
async fn get_status(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
2026-03-07 14:59:32 +01:00
|
|
|
|
user_id: Uuid,
|
2026-03-05 21:28:51 +01:00
|
|
|
|
) -> Result<UploadStatusResponseDto, DomainError> {
|
2026-03-07 14:59:32 +01:00
|
|
|
|
self.get_status_inner(upload_id, &user_id.to_string())
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.await
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn complete_upload(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
upload_id: &str,
|
2026-03-07 14:59:32 +01:00
|
|
|
|
user_id: Uuid,
|
2026-02-15 17:53:25 +01:00
|
|
|
|
) -> Result<(PathBuf, String, Option<String>, String, u64, String), DomainError> {
|
2026-03-07 14:59:32 +01:00
|
|
|
|
self.complete_upload_inner(upload_id, &user_id.to_string())
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-07 14:59:32 +01:00
|
|
|
|
async fn finalize_upload(&self, upload_id: &str, user_id: Uuid) -> Result<(), DomainError> {
|
|
|
|
|
|
self.finalize_upload_inner(upload_id, &user_id.to_string())
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-07 14:59:32 +01:00
|
|
|
|
async fn cancel_upload(&self, upload_id: &str, user_id: Uuid) -> Result<(), DomainError> {
|
2026-05-21 19:11:28 +02:00
|
|
|
|
// Inner function now returns DomainError with proper variants
|
|
|
|
|
|
// (NotFound for missing/wrong-owner sessions, InternalError otherwise),
|
|
|
|
|
|
// so no mapping needed here.
|
2026-03-07 14:59:32 +01:00
|
|
|
|
self.cancel_upload_inner(upload_id, &user_id.to_string())
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.await
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn should_use_chunked(&self, size: u64) -> bool {
|
|
|
|
|
|
ChunkedUploadService::should_use_chunked(size)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-22 14:12:53 +01:00
|
|
|
|
// ─── Tests ───────────────────────────────────────────────────────────────────
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
mod tests {
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn test_chunk_count_calculation() {
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
UploadSession::calculate_chunk_count(10 * 1024 * 1024, 5 * 1024 * 1024),
|
|
|
|
|
|
2
|
|
|
|
|
|
);
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
UploadSession::calculate_chunk_count(11 * 1024 * 1024, 5 * 1024 * 1024),
|
|
|
|
|
|
3
|
|
|
|
|
|
);
|
|
|
|
|
|
assert_eq!(UploadSession::calculate_chunk_count(1, 5 * 1024 * 1024), 1);
|
|
|
|
|
|
assert_eq!(UploadSession::calculate_chunk_count(0, 5 * 1024 * 1024), 1);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn test_should_use_chunked() {
|
|
|
|
|
|
assert!(!ChunkedUploadService::should_use_chunked(9 * 1024 * 1024));
|
|
|
|
|
|
assert!(ChunkedUploadService::should_use_chunked(10 * 1024 * 1024));
|
|
|
|
|
|
assert!(ChunkedUploadService::should_use_chunked(100 * 1024 * 1024));
|
|
|
|
|
|
}
|
2026-02-22 14:12:53 +01:00
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn test_bitmask_roundtrip() {
|
|
|
|
|
|
let now = Utc::now();
|
|
|
|
|
|
let mut session = UploadSession {
|
|
|
|
|
|
id: "test-id".into(),
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: "user-1".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
filename: "file.bin".into(),
|
|
|
|
|
|
folder_id: None,
|
|
|
|
|
|
content_type: "application/octet-stream".into(),
|
|
|
|
|
|
total_size: 15 * 1024 * 1024,
|
|
|
|
|
|
chunk_size: 5 * 1024 * 1024,
|
|
|
|
|
|
chunks: (0..3)
|
|
|
|
|
|
.map(|i| ChunkInfo {
|
|
|
|
|
|
index: i,
|
|
|
|
|
|
offset: i as u64 * 5 * 1024 * 1024,
|
|
|
|
|
|
size: 5 * 1024 * 1024,
|
|
|
|
|
|
status: ChunkStatus::Pending,
|
|
|
|
|
|
checksum: None,
|
|
|
|
|
|
})
|
|
|
|
|
|
.collect(),
|
|
|
|
|
|
created_at: now,
|
|
|
|
|
|
last_activity: now,
|
|
|
|
|
|
temp_dir: PathBuf::from("/tmp/test-id"),
|
|
|
|
|
|
bytes_received: 0,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Mark chunks 0 and 2 as complete
|
|
|
|
|
|
session.chunks[0].status = ChunkStatus::Complete;
|
|
|
|
|
|
session.chunks[2].status = ChunkStatus::Complete;
|
|
|
|
|
|
session.bytes_received = 10 * 1024 * 1024;
|
|
|
|
|
|
|
|
|
|
|
|
let bitmask = session.build_progress_bitmask();
|
|
|
|
|
|
assert_eq!(bitmask, vec![0b00000101]); // bits 0 and 2 set
|
|
|
|
|
|
|
|
|
|
|
|
// Reset and re-apply
|
|
|
|
|
|
for chunk in &mut session.chunks {
|
|
|
|
|
|
chunk.status = ChunkStatus::Pending;
|
|
|
|
|
|
}
|
|
|
|
|
|
session.bytes_received = 0;
|
|
|
|
|
|
session.apply_progress_bitmask(&bitmask);
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(session.chunks[0].status, ChunkStatus::Complete);
|
|
|
|
|
|
assert_eq!(session.chunks[1].status, ChunkStatus::Pending);
|
|
|
|
|
|
assert_eq!(session.chunks[2].status, ChunkStatus::Complete);
|
|
|
|
|
|
assert_eq!(session.bytes_received, 10 * 1024 * 1024);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn test_session_serialisation_roundtrip() {
|
|
|
|
|
|
let now = Utc::now();
|
|
|
|
|
|
let session = UploadSession {
|
|
|
|
|
|
id: "abc-123".into(),
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: "user-1".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
filename: "photo.jpg".into(),
|
|
|
|
|
|
folder_id: Some("folder-1".into()),
|
|
|
|
|
|
content_type: "image/jpeg".into(),
|
|
|
|
|
|
total_size: 1024,
|
|
|
|
|
|
chunk_size: 512,
|
|
|
|
|
|
chunks: vec![
|
|
|
|
|
|
ChunkInfo {
|
|
|
|
|
|
index: 0,
|
|
|
|
|
|
offset: 0,
|
|
|
|
|
|
size: 512,
|
|
|
|
|
|
status: ChunkStatus::Complete,
|
|
|
|
|
|
checksum: Some("aabb".into()),
|
|
|
|
|
|
},
|
|
|
|
|
|
ChunkInfo {
|
|
|
|
|
|
index: 1,
|
|
|
|
|
|
offset: 512,
|
|
|
|
|
|
size: 512,
|
|
|
|
|
|
status: ChunkStatus::Pending,
|
|
|
|
|
|
checksum: None,
|
|
|
|
|
|
},
|
|
|
|
|
|
],
|
|
|
|
|
|
created_at: now,
|
|
|
|
|
|
last_activity: now,
|
|
|
|
|
|
temp_dir: PathBuf::from("/tmp/abc-123"),
|
|
|
|
|
|
bytes_received: 512,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let json = serde_json::to_vec(&session).expect("serialise");
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let restored: UploadSession = serde_json::from_slice(&json).expect("deserialise");
|
2026-02-22 14:12:53 +01:00
|
|
|
|
|
|
|
|
|
|
assert_eq!(restored.id, session.id);
|
2026-03-05 13:15:34 +01:00
|
|
|
|
assert_eq!(restored.user_id, session.user_id);
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert_eq!(restored.filename, session.filename);
|
|
|
|
|
|
assert_eq!(restored.folder_id, session.folder_id);
|
|
|
|
|
|
assert_eq!(restored.total_size, session.total_size);
|
|
|
|
|
|
assert_eq!(restored.chunks.len(), 2);
|
|
|
|
|
|
assert_eq!(restored.chunks[0].status, ChunkStatus::Complete);
|
|
|
|
|
|
assert_eq!(restored.chunks[1].status, ChunkStatus::Pending);
|
|
|
|
|
|
assert_eq!(restored.bytes_received, 512);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn test_session_expiry_check() {
|
|
|
|
|
|
let mut session = UploadSession {
|
|
|
|
|
|
id: "exp-test".into(),
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: "user-1".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
filename: "f".into(),
|
|
|
|
|
|
folder_id: None,
|
|
|
|
|
|
content_type: "x".into(),
|
|
|
|
|
|
total_size: 0,
|
|
|
|
|
|
chunk_size: 1,
|
|
|
|
|
|
chunks: vec![],
|
|
|
|
|
|
created_at: Utc::now(),
|
|
|
|
|
|
last_activity: Utc::now(),
|
|
|
|
|
|
temp_dir: PathBuf::from("/tmp"),
|
|
|
|
|
|
bytes_received: 0,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
assert!(!session.is_expired(), "Fresh session must not be expired");
|
|
|
|
|
|
|
|
|
|
|
|
// Simulate old activity
|
|
|
|
|
|
session.last_activity = Utc::now() - chrono::Duration::hours(25);
|
|
|
|
|
|
assert!(session.is_expired(), "25h-old session must be expired");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn test_persist_and_recover_session() {
|
|
|
|
|
|
// Use a unique temp dir for this test
|
|
|
|
|
|
let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4()));
|
|
|
|
|
|
let _ = fs::create_dir_all(&base).await;
|
|
|
|
|
|
|
|
|
|
|
|
let service = ChunkedUploadService::new(base.clone()).await;
|
|
|
|
|
|
|
|
|
|
|
|
// Create a session
|
|
|
|
|
|
let resp = service
|
|
|
|
|
|
.create_session_inner(
|
2026-03-05 13:15:34 +01:00
|
|
|
|
"test-user".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
"bigfile.bin".into(),
|
|
|
|
|
|
Some("folder-x".into()),
|
|
|
|
|
|
"application/octet-stream".into(),
|
|
|
|
|
|
10 * 1024 * 1024,
|
|
|
|
|
|
Some(5 * 1024 * 1024),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("create_session");
|
|
|
|
|
|
|
|
|
|
|
|
let upload_id = resp.upload_id.clone();
|
|
|
|
|
|
|
|
|
|
|
|
// Upload first chunk (5 MB of zeros)
|
|
|
|
|
|
let chunk_data = bytes::Bytes::from(vec![0u8; 5 * 1024 * 1024]);
|
|
|
|
|
|
service
|
2026-03-05 13:15:34 +01:00
|
|
|
|
.upload_chunk_inner(&upload_id, "test-user", 0, chunk_data, None)
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.expect("upload_chunk 0");
|
|
|
|
|
|
|
|
|
|
|
|
// Verify files exist on disk
|
2026-06-09 10:19:21 +02:00
|
|
|
|
let session_dir = base.join(session_dir_name(&upload_id));
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert!(session_dir.join(SESSION_META_FILE).exists());
|
|
|
|
|
|
assert!(session_dir.join(PROGRESS_FILE).exists());
|
|
|
|
|
|
assert!(session_dir.join("chunk_000000").exists());
|
|
|
|
|
|
|
|
|
|
|
|
// Simulate restart: drop service, recover from disk
|
|
|
|
|
|
drop(service);
|
|
|
|
|
|
|
|
|
|
|
|
let recovered = ChunkedUploadService::recover_sessions(&base).await;
|
|
|
|
|
|
assert_eq!(recovered.len(), 1);
|
2026-02-26 01:02:25 +01:00
|
|
|
|
let session = recovered
|
|
|
|
|
|
.get(&upload_id)
|
|
|
|
|
|
.expect("session must be recovered");
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert_eq!(session.filename, "bigfile.bin");
|
|
|
|
|
|
assert_eq!(session.folder_id, Some("folder-x".into()));
|
|
|
|
|
|
assert_eq!(session.chunks[0].status, ChunkStatus::Complete);
|
|
|
|
|
|
assert_eq!(session.chunks[1].status, ChunkStatus::Pending);
|
|
|
|
|
|
assert_eq!(session.bytes_received, 5 * 1024 * 1024);
|
|
|
|
|
|
|
|
|
|
|
|
// Cleanup
|
|
|
|
|
|
let _ = fs::remove_dir_all(&base).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn test_full_upload_lifecycle() {
|
|
|
|
|
|
let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4()));
|
|
|
|
|
|
let service = ChunkedUploadService::new(base.clone()).await;
|
|
|
|
|
|
|
|
|
|
|
|
// 1. Create session (1024 bytes, 512 byte chunks → 2 chunks)
|
|
|
|
|
|
let resp = service
|
|
|
|
|
|
.create_session_inner(
|
2026-03-05 13:15:34 +01:00
|
|
|
|
"test-user".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
"test.txt".into(),
|
|
|
|
|
|
None,
|
|
|
|
|
|
"text/plain".into(),
|
|
|
|
|
|
1024,
|
|
|
|
|
|
Some(512),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("create");
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(resp.total_chunks, 2);
|
|
|
|
|
|
assert_eq!(resp.chunk_size, 512);
|
|
|
|
|
|
let id = resp.upload_id;
|
|
|
|
|
|
|
|
|
|
|
|
// 2. Upload chunks
|
|
|
|
|
|
let chunk0 = bytes::Bytes::from(vec![b'A'; 512]);
|
|
|
|
|
|
let r0 = service
|
2026-03-05 13:15:34 +01:00
|
|
|
|
.upload_chunk_inner(&id, "test-user", 0, chunk0, None)
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.expect("chunk 0");
|
|
|
|
|
|
assert!(!r0.is_complete);
|
|
|
|
|
|
|
|
|
|
|
|
let chunk1 = bytes::Bytes::from(vec![b'B'; 512]);
|
|
|
|
|
|
let r1 = service
|
2026-03-05 13:15:34 +01:00
|
|
|
|
.upload_chunk_inner(&id, "test-user", 1, chunk1, None)
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.expect("chunk 1");
|
|
|
|
|
|
assert!(r1.is_complete);
|
|
|
|
|
|
assert_eq!(r1.bytes_received, 1024);
|
|
|
|
|
|
|
|
|
|
|
|
// 3. Status check
|
2026-03-05 21:28:51 +01:00
|
|
|
|
let status = service
|
|
|
|
|
|
.get_status_inner(&id, "test-user")
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("status");
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert!(status.is_complete);
|
|
|
|
|
|
assert_eq!(status.completed_chunks, 2);
|
|
|
|
|
|
assert!(status.pending_chunks.is_empty());
|
|
|
|
|
|
|
|
|
|
|
|
// 4. Complete (assemble)
|
2026-03-05 21:28:51 +01:00
|
|
|
|
let (path, filename, _folder, _ct, size, hash) = service
|
|
|
|
|
|
.complete_upload_inner(&id, "test-user")
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("complete");
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert_eq!(filename, "test.txt");
|
|
|
|
|
|
assert_eq!(size, 1024);
|
|
|
|
|
|
assert!(!hash.is_empty());
|
|
|
|
|
|
assert!(path.exists());
|
|
|
|
|
|
|
|
|
|
|
|
// 5. Verify assembled content
|
|
|
|
|
|
let content = fs::read(&path).await.expect("read assembled");
|
|
|
|
|
|
assert_eq!(&content[..512], &[b'A'; 512]);
|
|
|
|
|
|
assert_eq!(&content[512..], &[b'B'; 512]);
|
|
|
|
|
|
|
|
|
|
|
|
// 6. Finalize
|
2026-03-05 21:28:51 +01:00
|
|
|
|
service
|
|
|
|
|
|
.finalize_upload_inner(&id, "test-user")
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("finalize");
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert_eq!(service.active_sessions().await, 0);
|
|
|
|
|
|
|
|
|
|
|
|
let _ = fs::remove_dir_all(&base).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn test_cancel_removes_files() {
|
|
|
|
|
|
let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4()));
|
|
|
|
|
|
let service = ChunkedUploadService::new(base.clone()).await;
|
|
|
|
|
|
|
|
|
|
|
|
let resp = service
|
2026-02-26 01:02:25 +01:00
|
|
|
|
.create_session_inner(
|
2026-03-05 13:15:34 +01:00
|
|
|
|
"test-user".into(),
|
2026-02-26 01:02:25 +01:00
|
|
|
|
"x.bin".into(),
|
|
|
|
|
|
None,
|
|
|
|
|
|
"application/octet-stream".into(),
|
|
|
|
|
|
512,
|
|
|
|
|
|
Some(512),
|
|
|
|
|
|
)
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.expect("create");
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
let session_dir = base.join(session_dir_name(&resp.upload_id));
|
2026-02-22 14:12:53 +01:00
|
|
|
|
assert!(session_dir.exists());
|
|
|
|
|
|
|
|
|
|
|
|
service
|
2026-03-05 13:15:34 +01:00
|
|
|
|
.cancel_upload_inner(&resp.upload_id, "test-user")
|
2026-02-22 14:12:53 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.expect("cancel");
|
|
|
|
|
|
|
|
|
|
|
|
assert!(!session_dir.exists(), "temp dir must be removed on cancel");
|
|
|
|
|
|
assert_eq!(service.active_sessions().await, 0);
|
|
|
|
|
|
|
|
|
|
|
|
let _ = fs::remove_dir_all(&base).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn test_recovery_skips_expired_sessions() {
|
|
|
|
|
|
let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4()));
|
|
|
|
|
|
let _ = fs::create_dir_all(&base).await;
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
// Manually create an expired session on disk. The dir name MUST
|
|
|
|
|
|
// carry the `oxi-chunk-` prefix or recovery will (correctly) skip
|
|
|
|
|
|
// it as belonging to another writer co-located in chunk_dir.
|
|
|
|
|
|
let session_dir = base.join(session_dir_name("expired-session"));
|
2026-02-22 14:12:53 +01:00
|
|
|
|
let _ = fs::create_dir_all(&session_dir).await;
|
|
|
|
|
|
|
|
|
|
|
|
let expired_session = UploadSession {
|
|
|
|
|
|
id: "expired-session".into(),
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: "user-1".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
filename: "old.bin".into(),
|
|
|
|
|
|
folder_id: None,
|
|
|
|
|
|
content_type: "application/octet-stream".into(),
|
|
|
|
|
|
total_size: 1024,
|
|
|
|
|
|
chunk_size: 1024,
|
|
|
|
|
|
chunks: vec![ChunkInfo {
|
|
|
|
|
|
index: 0,
|
|
|
|
|
|
offset: 0,
|
|
|
|
|
|
size: 1024,
|
|
|
|
|
|
status: ChunkStatus::Pending,
|
|
|
|
|
|
checksum: None,
|
|
|
|
|
|
}],
|
|
|
|
|
|
created_at: Utc::now() - chrono::Duration::hours(48),
|
|
|
|
|
|
last_activity: Utc::now() - chrono::Duration::hours(48),
|
|
|
|
|
|
temp_dir: session_dir.clone(),
|
|
|
|
|
|
bytes_received: 0,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let json = serde_json::to_vec(&expired_session).unwrap();
|
|
|
|
|
|
fs::write(session_dir.join(SESSION_META_FILE), &json)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
let recovered = ChunkedUploadService::recover_sessions(&base).await;
|
|
|
|
|
|
assert!(
|
|
|
|
|
|
recovered.is_empty(),
|
|
|
|
|
|
"Expired sessions must not be recovered"
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
let _ = fs::remove_dir_all(&base).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn test_recovery_downgrades_missing_chunk_files() {
|
|
|
|
|
|
let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4()));
|
|
|
|
|
|
let _ = fs::create_dir_all(&base).await;
|
|
|
|
|
|
|
2026-06-09 10:19:21 +02:00
|
|
|
|
let session_dir = base.join(session_dir_name("partial-session"));
|
2026-02-22 14:12:53 +01:00
|
|
|
|
let _ = fs::create_dir_all(&session_dir).await;
|
|
|
|
|
|
|
|
|
|
|
|
let session = UploadSession {
|
|
|
|
|
|
id: "partial-session".into(),
|
2026-03-05 13:15:34 +01:00
|
|
|
|
user_id: "user-1".into(),
|
2026-02-22 14:12:53 +01:00
|
|
|
|
filename: "file.bin".into(),
|
|
|
|
|
|
folder_id: None,
|
|
|
|
|
|
content_type: "application/octet-stream".into(),
|
|
|
|
|
|
total_size: 1024,
|
|
|
|
|
|
chunk_size: 512,
|
|
|
|
|
|
chunks: vec![
|
|
|
|
|
|
ChunkInfo {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
index: 0,
|
|
|
|
|
|
offset: 0,
|
|
|
|
|
|
size: 512,
|
|
|
|
|
|
status: ChunkStatus::Pending,
|
|
|
|
|
|
checksum: None,
|
2026-02-22 14:12:53 +01:00
|
|
|
|
},
|
|
|
|
|
|
ChunkInfo {
|
2026-02-26 01:02:25 +01:00
|
|
|
|
index: 1,
|
|
|
|
|
|
offset: 512,
|
|
|
|
|
|
size: 512,
|
|
|
|
|
|
status: ChunkStatus::Pending,
|
|
|
|
|
|
checksum: None,
|
2026-02-22 14:12:53 +01:00
|
|
|
|
},
|
|
|
|
|
|
],
|
|
|
|
|
|
created_at: Utc::now(),
|
|
|
|
|
|
last_activity: Utc::now(),
|
|
|
|
|
|
temp_dir: session_dir.clone(),
|
|
|
|
|
|
bytes_received: 0,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Write metadata
|
|
|
|
|
|
let json = serde_json::to_vec(&session).unwrap();
|
2026-02-26 01:02:25 +01:00
|
|
|
|
fs::write(session_dir.join(SESSION_META_FILE), &json)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
2026-02-22 14:12:53 +01:00
|
|
|
|
|
|
|
|
|
|
// Write progress marking both chunks complete
|
|
|
|
|
|
let bitmask = vec![0b00000011u8]; // bits 0 and 1
|
2026-02-26 01:02:25 +01:00
|
|
|
|
fs::write(session_dir.join(PROGRESS_FILE), &bitmask)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
2026-02-22 14:12:53 +01:00
|
|
|
|
|
|
|
|
|
|
// But only create chunk_000000 on disk — chunk_000001 is "missing"
|
2026-02-26 01:02:25 +01:00
|
|
|
|
fs::write(session_dir.join("chunk_000000"), &[0u8; 512])
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
2026-02-22 14:12:53 +01:00
|
|
|
|
|
|
|
|
|
|
let recovered = ChunkedUploadService::recover_sessions(&base).await;
|
|
|
|
|
|
let s = recovered.get("partial-session").expect("must be recovered");
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(s.chunks[0].status, ChunkStatus::Complete);
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
s.chunks[1].status,
|
|
|
|
|
|
ChunkStatus::Pending,
|
|
|
|
|
|
"Missing chunk file must be downgraded to Pending"
|
|
|
|
|
|
);
|
|
|
|
|
|
assert_eq!(s.bytes_received, 512);
|
|
|
|
|
|
|
|
|
|
|
|
let _ = fs::remove_dir_all(&base).await;
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|