From f9c3adadc53bfdf2b037d8f0bb7fb0108adc0e3f Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Tue, 9 Jun 2026 11:33:02 +0200 Subject: [PATCH] feat(chunked-upload): keep previous pending upload a prefix path as been add to avoid collision if OXICLOUD_UPLOAD_TMPDIR=OXICLOUD_CHUNK_DIR upgrade to this version will convert previous upload chunked into the prefixed version at server start --- .../services/chunked_upload_service.rs | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index dfa50388..39aaa7d8 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -225,6 +225,15 @@ impl ChunkedUploadService { // Ensure the base directory exists let _ = fs::create_dir_all(&temp_base_dir).await; + // One-shot migration of pre-prefix session directories to the + // `oxi-chunk-{uuid}/` layout. Without this, any chunked upload + // in flight at the moment an admin upgrades from a pre-prefix + // build to this one would be orphaned — the recovery scan + // filters strictly on the `oxi-chunk-` prefix and would skip + // legacy `{uuid}/` directories. Idempotent: subsequent boots + // find no legacy dirs left to rename and do nothing. + Self::migrate_pre_prefix_sessions(&temp_base_dir).await; + // Recover sessions that survived a restart let recovered = Self::recover_sessions(&temp_base_dir).await; let recovered_count = recovered.len(); @@ -257,6 +266,84 @@ impl ChunkedUploadService { } } + // ── Upgrade migration ──────────────────────────────────────────────── + + /// One-shot upgrade migration: rename any pre-prefix session + /// directory to the new `oxi-chunk-{uuid}/` layout so the recovery + /// scan picks it up. + /// + /// A pre-prefix session is identified by: a directory under + /// `temp_base_dir` whose name does NOT start with + /// `SESSION_DIR_PREFIX` but whose contents include `session.json`. + /// That signature can only come from a chunked upload created by + /// a pre-prefix OxiCloud build — admins don't normally drop + /// session.json files into the chunk dir. + /// + /// Idempotent: on a fresh boot all dirs are already prefixed, the + /// scan finds nothing to rename, no-op. Safe against concurrent + /// boots: `fs::rename` is atomic, so a racing migration sees the + /// source disappear and proceeds. + async fn migrate_pre_prefix_sessions(base: &Path) { + let mut entries = match fs::read_dir(base).await { + Ok(e) => e, + Err(_) => return, + }; + + let mut migrated_count = 0usize; + + while let Ok(Some(entry)) = entries.next_entry().await { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let dir_name = match path.file_name().and_then(|n| n.to_str()) { + Some(n) => n.to_string(), + None => continue, + }; + if dir_name.starts_with(SESSION_DIR_PREFIX) { + continue; // already migrated + } + // Identify pre-prefix sessions by `session.json` presence. + // Avoids touching the NC subtree (`nextcloud/` — no + // session.json at that level) and any operator-placed + // sibling directories without the marker. + if !fs::try_exists(path.join(SESSION_META_FILE)) + .await + .unwrap_or(false) + { + continue; + } + + let new_path = base.join(session_dir_name(&dir_name)); + match fs::rename(&path, &new_path).await { + Ok(()) => { + migrated_count += 1; + tracing::info!( + old = %path.display(), + new = %new_path.display(), + "Migrated pre-prefix chunked-upload session to new layout" + ); + } + Err(e) => { + tracing::warn!( + error = %e, + old = %path.display(), + "Failed to migrate pre-prefix session — left orphaned on disk; \ + next chunk PATCH from the client will 404 and the client should \ + restart its upload session. Manual rm safe." + ); + } + } + } + + if migrated_count > 0 { + tracing::info!( + count = migrated_count, + "🔧 Upgraded chunked-upload session layout (one-shot migration)" + ); + } + } + // ── Recovery ───────────────────────────────────────────────────────── /// Scan `temp_base_dir` for directories containing `session.json`, @@ -1541,4 +1628,139 @@ mod tests { let _ = fs::remove_dir_all(&base).await; } + + /// Upgrade-path scenario: a pre-prefix session directory (the layout + /// used by builds before the `oxi-chunk-` prefix change) gets renamed + /// in place when the service starts, then recovered normally. Without + /// the migration, the upgrade would orphan every in-flight REST + /// chunked upload because recovery filters strictly on the prefix. + #[tokio::test] + async fn test_migrate_pre_prefix_session_on_boot() { + let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4())); + let _ = fs::create_dir_all(&base).await; + + // Pre-upgrade layout: `{base}/legacy-upload-id/session.json` + // (no `oxi-chunk-` prefix on the directory name). + let legacy_id = "legacy-upload-id"; + let legacy_dir = base.join(legacy_id); + let _ = fs::create_dir_all(&legacy_dir).await; + + let session = UploadSession { + id: legacy_id.into(), + user_id: "user-1".into(), + filename: "in-flight.bin".into(), + folder_id: None, + content_type: "application/octet-stream".into(), + total_size: 1024, + chunk_size: 1024, + chunks: vec![ChunkInfo { + index: 0, + offset: 0, + size: 1024, + status: ChunkStatus::Pending, + checksum: None, + }], + created_at: Utc::now(), + last_activity: Utc::now(), + // `temp_dir` here is the legacy path — after migration the + // session.json's path won't match the new location on disk, + // but recovery doesn't use temp_dir for lookups; chunk + // I/O within commit_chunk computes paths from the current + // session dir, which is the renamed location. + temp_dir: legacy_dir.clone(), + bytes_received: 0, + }; + fs::write( + legacy_dir.join(SESSION_META_FILE), + serde_json::to_vec(&session).unwrap(), + ) + .await + .unwrap(); + + // Boot the service — migration runs as part of `new`. + let svc = ChunkedUploadService::new(base.clone()).await; + + // Legacy path must be gone, prefixed path must exist + carry the + // session.json. + assert!( + !legacy_dir.exists(), + "Legacy un-prefixed dir should have been renamed away" + ); + let new_dir = base.join(session_dir_name(legacy_id)); + assert!( + new_dir.exists(), + "Prefixed dir should exist post-migration at {}", + new_dir.display() + ); + assert!( + new_dir.join(SESSION_META_FILE).exists(), + "session.json must travel with the rename" + ); + + // Recovery picks it up — the session is now in the live map + // keyed by its original upload_id. + assert!( + svc.sessions.contains_key(legacy_id), + "Recovered session must be keyed by its original upload_id" + ); + + let _ = fs::remove_dir_all(&base).await; + } + + /// Idempotency: running migration twice (a second boot after a + /// successful migration) must be a no-op. The already-prefixed + /// directory is left untouched, no spurious double-prefixing. + #[tokio::test] + async fn test_migration_is_idempotent() { + let base = std::env::temp_dir().join(format!("oxicloud_test_{}", Uuid::new_v4())); + let _ = fs::create_dir_all(&base).await; + + let upload_id = "already-prefixed"; + let prefixed_dir = base.join(session_dir_name(upload_id)); + let _ = fs::create_dir_all(&prefixed_dir).await; + + // Minimal session.json — just enough to count as a session for + // the migration's identification heuristic. + let session = UploadSession { + id: upload_id.into(), + user_id: "user-1".into(), + filename: "x.bin".into(), + folder_id: None, + content_type: "application/octet-stream".into(), + total_size: 1, + chunk_size: 1, + chunks: vec![ChunkInfo { + index: 0, + offset: 0, + size: 1, + status: ChunkStatus::Pending, + checksum: None, + }], + created_at: Utc::now(), + last_activity: Utc::now(), + temp_dir: prefixed_dir.clone(), + bytes_received: 0, + }; + fs::write( + prefixed_dir.join(SESSION_META_FILE), + serde_json::to_vec(&session).unwrap(), + ) + .await + .unwrap(); + + // Two migration runs in a row. + ChunkedUploadService::migrate_pre_prefix_sessions(&base).await; + ChunkedUploadService::migrate_pre_prefix_sessions(&base).await; + + // The dir is still there, with the SAME (single) prefix — + // not `oxi-chunk-oxi-chunk-already-prefixed/`. + assert!(prefixed_dir.exists()); + let double_prefixed = base.join(session_dir_name(&session_dir_name(upload_id))); + assert!( + !double_prefixed.exists(), + "Migration must not double-prefix an already-prefixed dir" + ); + + let _ = fs::remove_dir_all(&base).await; + } }