perf: rewrite batch download_zip to stream via temp file instead of RAM

- Replace in-memory Cursor<Vec<u8>> with async_zip + NamedTempFile (O(256KB) RAM)
- Stream ZIP to client via ReaderStream instead of materializing entire archive
- Replace N+1 BFS folder traversal with 2 bulk ltree queries
- Remove dead zip_service field and with_zip_service() from BatchOperationService
- Remove synchronous zip crate dependency (only async_zip remains)
- Eliminates ~2.5GB RAM spike per 2GB batch download
This commit is contained in:
Dionisio
2026-02-25 23:12:19 +01:00
parent 23474b9681
commit 5f883aa0f8
5 changed files with 176 additions and 347 deletions
Generated
-204
View File
@@ -8,17 +8,6 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aes"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures",
]
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "1.1.4" version = "1.1.4"
@@ -70,15 +59,6 @@ version = "1.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea"
[[package]]
name = "arbitrary"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
dependencies = [
"derive_arbitrary",
]
[[package]] [[package]]
name = "argon2" name = "argon2"
version = "0.5.3" version = "0.5.3"
@@ -345,15 +325,6 @@ version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
[[package]]
name = "bzip2"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
dependencies = [
"libbz2-rs-sys",
]
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.55" version = "1.2.55"
@@ -361,8 +332,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29"
dependencies = [ dependencies = [
"find-msvc-tools", "find-msvc-tools",
"jobserver",
"libc",
"shlex", "shlex",
] ]
@@ -403,16 +372,6 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "cipher"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common",
"inout",
]
[[package]] [[package]]
name = "color_quant" name = "color_quant"
version = "1.1.0" version = "1.1.0"
@@ -452,12 +411,6 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
[[package]]
name = "constant_time_eq"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]] [[package]]
name = "core-foundation-sys" name = "core-foundation-sys"
version = "0.8.7" version = "0.8.7"
@@ -589,12 +542,6 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "deflate64"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204"
[[package]] [[package]]
name = "der" name = "der"
version = "0.7.10" version = "0.7.10"
@@ -615,17 +562,6 @@ dependencies = [
"powerfmt", "powerfmt",
] ]
[[package]]
name = "derive_arbitrary"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.10.7" version = "0.10.7"
@@ -831,7 +767,6 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [ dependencies = [
"crc32fast", "crc32fast",
"miniz_oxide", "miniz_oxide",
"zlib-rs",
] ]
[[package]] [[package]]
@@ -1436,15 +1371,6 @@ dependencies = [
"cfb", "cfb",
] ]
[[package]]
name = "inout"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.11.0" version = "2.11.0"
@@ -1467,16 +1393,6 @@ version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]]
name = "jobserver"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
dependencies = [
"getrandom 0.3.4",
"libc",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.85" version = "0.3.85"
@@ -1519,12 +1435,6 @@ dependencies = [
"spin", "spin",
] ]
[[package]]
name = "libbz2-rs-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.180" version = "0.2.180"
@@ -1600,16 +1510,6 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lzma-rust2"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a"
dependencies = [
"crc",
"sha2",
]
[[package]] [[package]]
name = "matchers" name = "matchers"
version = "0.2.0" version = "0.2.0"
@@ -1883,7 +1783,6 @@ dependencies = [
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"uuid", "uuid",
"zip",
] ]
[[package]] [[package]]
@@ -1950,16 +1849,6 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "pbkdf2"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
dependencies = [
"digest",
"hmac",
]
[[package]] [[package]]
name = "pem" name = "pem"
version = "3.0.6" version = "3.0.6"
@@ -2078,12 +1967,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppmd-rust"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.21" version = "0.2.21"
@@ -3884,20 +3767,6 @@ name = "zeroize"
version = "1.8.2" version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
dependencies = [
"zeroize_derive",
]
[[package]]
name = "zeroize_derive"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "zerotrie" name = "zerotrie"
@@ -3932,85 +3801,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "zip"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
dependencies = [
"aes",
"arbitrary",
"bzip2",
"constant_time_eq",
"crc32fast",
"deflate64",
"flate2",
"getrandom 0.3.4",
"hmac",
"indexmap",
"lzma-rust2",
"memchr",
"pbkdf2",
"ppmd-rust",
"sha1",
"time",
"zeroize",
"zopfli",
"zstd",
]
[[package]]
name = "zlib-rs"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c"
[[package]] [[package]]
name = "zmij" name = "zmij"
version = "1.0.19" version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445"
[[package]]
name = "zopfli"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
dependencies = [
"bumpalo",
"crc32fast",
"log",
"simd-adler32",
]
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]
[[package]] [[package]]
name = "zune-core" name = "zune-core"
version = "0.5.1" version = "0.5.1"
-1
View File
@@ -15,7 +15,6 @@ tempfile = "3.25.0"
tower = "0.5.3" tower = "0.5.3"
tower-http = { version = "0.6.8", features = ["fs", "compression-gzip", "compression-br", "trace", "cors", "add-extension", "request-id", "set-header", "limit"] } tower-http = { version = "0.6.8", features = ["fs", "compression-gzip", "compression-br", "trace", "cors", "add-extension", "request-id", "set-header", "limit"] }
flate2 = "1.1.9" flate2 = "1.1.9"
zip = "=6.0.0"
tracing = "0.1.44" tracing = "0.1.44"
tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
chrono = { version = "0.4.43", features = ["serde"] } chrono = { version = "0.4.43", features = ["serde"] }
+143 -134
View File
@@ -1,6 +1,12 @@
use async_zip::base::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use futures::io::AsyncWriteExt as FuturesWriteExt;
use futures::{Future, StreamExt, future::join_all}; use futures::{Future, StreamExt, future::join_all};
use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use tempfile::NamedTempFile;
use thiserror::Error; use thiserror::Error;
use tokio::io::BufWriter;
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
use tracing::info; use tracing::info;
@@ -9,7 +15,6 @@ use crate::application::dtos::folder_dto::{FolderDto, MoveFolderDto};
use crate::application::ports::file_ports::{FileManagementUseCase, FileRetrievalUseCase}; use crate::application::ports::file_ports::{FileManagementUseCase, FileRetrievalUseCase};
use crate::application::ports::inbound::FolderUseCase; use crate::application::ports::inbound::FolderUseCase;
use crate::application::ports::trash_ports::TrashUseCase; use crate::application::ports::trash_ports::TrashUseCase;
use crate::application::ports::zip_ports::ZipPort;
use crate::application::services::folder_service::FolderService; use crate::application::services::folder_service::FolderService;
use crate::common::config::AppConfig; use crate::common::config::AppConfig;
use crate::common::errors::DomainError; use crate::common::errors::DomainError;
@@ -65,7 +70,6 @@ pub struct BatchOperationService {
file_management: Arc<dyn FileManagementUseCase>, file_management: Arc<dyn FileManagementUseCase>,
folder_service: Arc<FolderService>, folder_service: Arc<FolderService>,
trash_service: Option<Arc<dyn TrashUseCase>>, trash_service: Option<Arc<dyn TrashUseCase>>,
zip_service: Option<Arc<dyn ZipPort>>,
config: AppConfig, config: AppConfig,
semaphore: Arc<Semaphore>, semaphore: Arc<Semaphore>,
} }
@@ -86,7 +90,6 @@ impl BatchOperationService {
file_management, file_management,
folder_service, folder_service,
trash_service: None, trash_service: None,
zip_service: None,
config, config,
semaphore: Arc::new(Semaphore::new(max_concurrency)), semaphore: Arc::new(Semaphore::new(max_concurrency)),
} }
@@ -112,12 +115,6 @@ impl BatchOperationService {
self self
} }
/// Set the optional zip service (enables batch download)
pub fn with_zip_service(mut self, zip_service: Arc<dyn ZipPort>) -> Self {
self.zip_service = Some(zip_service);
self
}
/// Copies multiple files in parallel /// Copies multiple files in parallel
pub async fn copy_files( pub async fn copy_files(
&self, &self,
@@ -675,17 +672,16 @@ impl BatchOperationService {
Ok(result) Ok(result)
} }
/// Downloads multiple files/folders as a single ZIP archive /// Downloads multiple files/folders as a single ZIP archive.
///
/// Writes the archive to a temporary file so RAM usage is O(buffer_size)
/// regardless of total archive size. The caller streams the resulting
/// `NamedTempFile` to the client; the OS deletes it on drop.
pub async fn download_zip( pub async fn download_zip(
&self, &self,
file_ids: Vec<String>, file_ids: Vec<String>,
folder_ids: Vec<String>, folder_ids: Vec<String>,
) -> Result<Vec<u8>, BatchOperationError> { ) -> Result<NamedTempFile, BatchOperationError> {
use std::io::{Cursor, Write};
use zip::{ZipWriter, write::SimpleFileOptions};
let zip_service = self.zip_service.as_ref();
info!( info!(
"Starting batch download: {} files, {} folders", "Starting batch download: {} files, {} folders",
file_ids.len(), file_ids.len(),
@@ -693,163 +689,176 @@ impl BatchOperationService {
); );
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
let buf = Cursor::new(Vec::new()); // ── Open temp file + async ZIP writer (all writes go to disk) ────
let mut zip = ZipWriter::new(buf); let temp = NamedTempFile::new()
let options = SimpleFileOptions::default() .map_err(|e| BatchOperationError::Internal(format!("temp file error: {}", e)))?;
.compression_method(zip::CompressionMethod::Deflated) let tokio_file = tokio::fs::File::create(temp.path())
.unix_permissions(0o644); .await
.map_err(|e| BatchOperationError::Internal(format!("temp file open: {}", e)))?;
let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
let mut zip = ZipFileWriter::with_tokio(buf_writer);
// Add individual files at the root of the ZIP // ── Add individual files at the root of the ZIP ──────────────────
for file_id in &file_ids { for file_id in &file_ids {
match self.file_retrieval.get_file(file_id).await { match self.file_retrieval.get_file(file_id).await {
Ok(file_dto) => match self.file_retrieval.get_file_stream(file_id).await { Ok(file_dto) => {
Ok(stream) => { if let Err(e) = self
let mut stream = std::pin::Pin::from(stream); .add_file_entry_streamed(&mut zip, file_id, &file_dto.name)
if let Err(e) = zip.start_file(&file_dto.name, options) { .await
info!("Could not start zip entry for {}: {}", file_dto.name, e); {
continue; info!("Could not add file {} to ZIP: {}", file_dto.name, e);
}
while let Some(chunk) = stream.next().await {
match chunk {
Ok(bytes) => {
if let Err(e) = zip.write_all(&bytes) {
info!(
"Could not write zip chunk for {}: {}",
file_dto.name, e
);
break;
}
}
Err(e) => {
info!("Stream error for {}: {}", file_dto.name, e);
break;
}
}
}
} }
Err(e) => { }
info!("Could not stream file content {}: {}", file_id, e);
}
},
Err(e) => { Err(e) => {
info!("Could not get file metadata {}: {}", file_id, e); info!("Could not get file metadata {}: {}", file_id, e);
} }
} }
} }
// Add folders as sub-trees using the existing ZipPort if available // ── Add folders as sub-trees (bulk subtree queries, not N+1) ─────
// Otherwise fall back to manual folder traversal for folder_id in &folder_ids {
if let Some(zip_svc) = zip_service { match self.folder_service.get_folder(folder_id).await {
// For each folder, create a separate zip and merge its contents Ok(root_folder) => {
// Actually, we need to build the tree ourselves for a single zip if let Err(e) = self
// Use manual approach for consistency within one archive .add_folder_subtree_to_zip(&mut zip, folder_id, &root_folder)
for folder_id in &folder_ids { .await
match self.folder_service.get_folder(folder_id).await { {
Ok(folder) => { info!("Could not add folder {} to ZIP: {}", root_folder.name, e);
self.add_folder_to_zip(&mut zip, folder_id, &folder.name, &options)
.await;
}
Err(e) => {
info!("Could not get folder {}: {}", folder_id, e);
} }
} }
} Err(e) => {
// Suppress unused variable warning info!("Could not get folder {}: {}", folder_id, e);
let _ = zip_svc;
} else {
for folder_id in &folder_ids {
match self.folder_service.get_folder(folder_id).await {
Ok(folder) => {
self.add_folder_to_zip(&mut zip, folder_id, &folder.name, &options)
.await;
}
Err(e) => {
info!("Could not get folder {}: {}", folder_id, e);
}
} }
} }
} }
let mut zip_buf = zip // ── Finalize ─────────────────────────────────────────────────────
.finish() let mut compat_writer = zip.close().await.map_err(|e| {
.map_err(|e| BatchOperationError::Internal(format!("ZIP finalize error: {}", e)))?; BatchOperationError::Internal(format!("ZIP finalize error: {}", e))
})?;
compat_writer.close().await.map_err(|e| {
BatchOperationError::Internal(format!("ZIP flush error: {}", e))
})?;
use std::io::Read; let file_size = temp
let mut bytes = Vec::new(); .as_file()
zip_buf .metadata()
.read_to_end(&mut bytes) .map(|m| m.len())
.map_err(|e| BatchOperationError::Internal(format!("ZIP read error: {}", e)))?; .unwrap_or(0);
info!( info!(
"Batch download ZIP created: {} bytes in {}ms", "Batch download ZIP created: {} bytes in {}ms",
bytes.len(), file_size,
start_time.elapsed().as_millis() start_time.elapsed().as_millis()
); );
Ok(bytes) Ok(temp)
} }
/// Recursively add a folder and its contents to a ZipWriter /// Streams a single file into an async ZIP entry (~64 KB peak RAM per file).
async fn add_folder_to_zip( async fn add_file_entry_streamed(
&self, &self,
zip: &mut zip::ZipWriter<std::io::Cursor<Vec<u8>>>, zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
folder_id: &str, file_id: &str,
path: &str, entry_name: &str,
options: &zip::write::SimpleFileOptions, ) -> Result<(), BatchOperationError> {
) { let entry = ZipEntryBuilder::new(entry_name.to_string().into(), Compression::Deflate);
use std::io::Write; let mut writer = zip
.write_entry_stream(entry)
.await
.map_err(|e| BatchOperationError::Internal(format!("zip entry start: {}", e)))?;
struct PendingFolder { let stream = self
id: String, .file_retrieval
path: String, .get_file_stream(file_id)
.await
.map_err(|e| BatchOperationError::Domain(e))?;
let mut stream = std::pin::Pin::from(stream);
while let Some(chunk) = stream.next().await {
let bytes = chunk.map_err(|e| {
BatchOperationError::Internal(format!("stream read: {}", e))
})?;
writer.write_all(&bytes).await.map_err(|e| {
BatchOperationError::Internal(format!("zip chunk write: {}", e))
})?;
} }
let mut queue = vec![PendingFolder { writer.close().await.map_err(|e| {
id: folder_id.to_string(), BatchOperationError::Internal(format!("zip entry close: {}", e))
path: path.to_string(), })?;
}]; Ok(())
}
let mut visited = std::collections::HashSet::new(); /// Adds an entire folder subtree to the ZIP using 2 bulk SQL queries
/// (ltree `<@`) instead of N+1 per-folder traversal.
async fn add_folder_subtree_to_zip(
&self,
zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
folder_id: &str,
root_folder: &FolderDto,
) -> Result<(), BatchOperationError> {
// Bulk-fetch entire subtree (2 queries total)
let all_folders = self
.folder_service
.list_subtree_folders(folder_id)
.await
.map_err(BatchOperationError::Domain)?;
while let Some(current) = queue.pop() { let all_files = self
if visited.contains(&current.id) { .file_retrieval
continue; .list_files_in_subtree(folder_id)
.await
.map_err(BatchOperationError::Domain)?;
// Group files by folder_id
let mut files_by_folder: HashMap<String, Vec<FileDto>> =
HashMap::with_capacity(all_folders.len());
for file in all_files {
let fid = file.folder_id.clone().unwrap_or_default();
files_by_folder.entry(fid).or_default().push(file);
}
// Build path mapping: folder_id → ZIP-relative path
let root_path = root_folder.path.trim_end_matches('/');
let folder_zip_path = |db_path: &str| -> String {
let db_path = db_path.trim_end_matches('/');
if db_path == root_path {
root_folder.name.clone()
} else {
let suffix = db_path
.strip_prefix(root_path)
.unwrap_or(db_path)
.trim_start_matches('/');
format!("{}/{}", root_folder.name, suffix)
} }
visited.insert(current.id.clone()); };
let dir_path = format!("{}/", current.path); // Write folder + file entries (folders are sorted by path from DB)
let _ = zip.add_directory(&dir_path, *options); for folder in &all_folders {
let zip_dir = format!("{}/", folder_zip_path(&folder.path));
// Add files via streaming (constant ~64 KB memory per file) // Directory entry (Stored, zero-length body)
if let Ok(files) = self.file_retrieval.list_files(Some(&current.id)).await { let dir_entry = ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
if let Err(e) = zip.write_entry_whole(dir_entry, &[]).await {
info!("Could not add folder entry {}: {}", zip_dir, e);
}
// Stream files belonging to this folder
if let Some(files) = files_by_folder.get(&folder.id) {
for file in files { for file in files {
let file_path = format!("{}{}", dir_path, file.name); let file_path = format!("{}{}", zip_dir, file.name);
if let Ok(stream) = self.file_retrieval.get_file_stream(&file.id).await { if let Err(e) = self
let mut stream = std::pin::Pin::from(stream); .add_file_entry_streamed(zip, &file.id, &file_path)
if zip.start_file(&file_path, *options).is_ok() { .await
while let Some(chunk) = stream.next().await { {
match chunk { info!("Could not add file {} to ZIP: {}", file.name, e);
Ok(bytes) => {
let _ = zip.write_all(&bytes);
}
Err(_) => break,
}
}
}
} }
} }
} }
// Enqueue subfolders
if let Ok(subfolders) = self.folder_service.list_folders(Some(&current.id)).await {
for sub in subfolders {
queue.push(PendingFolder {
id: sub.id.clone(),
path: format!("{}/{}", current.path, sub.name),
});
}
}
} }
Ok(())
} }
/// Generic batch operation for any type of async function /// Generic batch operation for any type of async function
+33 -6
View File
@@ -590,7 +590,10 @@ pub async fn move_folders_batch(
Ok((status_code, Json(response)).into_response()) Ok((status_code, Json(response)).into_response())
} }
/// Handler for downloading multiple files and folders as a single ZIP /// Handler for downloading multiple files and folders as a single ZIP.
///
/// The ZIP is written to a temporary file and streamed to the client,
/// so RAM usage is O(buffer_size) regardless of archive size.
pub async fn download_batch( pub async fn download_batch(
State(state): State<BatchHandlerState>, State(state): State<BatchHandlerState>,
Json(request): Json<BatchDownloadRequest>, Json(request): Json<BatchDownloadRequest>,
@@ -602,22 +605,46 @@ pub async fn download_batch(
)); ));
} }
let zip_bytes = state let temp_file = state
.batch_service .batch_service
.download_zip(request.file_ids, request.folder_ids) .download_zip(request.file_ids, request.folder_ids)
.await .await
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
// Read file size for Content-Length before splitting ownership
let file_size = temp_file
.as_file()
.metadata()
.map(|m| m.len())
.map_err(|e| (
StatusCode::INTERNAL_SERVER_ERROR,
format!("Failed to read temp file metadata: {}", e),
))?;
// Split into the already-open fd + auto-delete path
let (std_file, temp_path) = temp_file.into_parts();
let tokio_file = tokio::fs::File::from_std(std_file);
// Stream to client — O(64 KB) RAM regardless of ZIP size
let stream = tokio_util::io::ReaderStream::new(tokio_file);
let body = axum::body::Body::from_stream(stream);
let filename = format!("oxicloud-download-{}.zip", chrono::Utc::now().timestamp()); let filename = format!("oxicloud-download-{}.zip", chrono::Utc::now().timestamp());
Ok(Response::builder() let mut response = Response::builder()
.status(StatusCode::OK) .status(StatusCode::OK)
.header("Content-Type", "application/zip") .header("Content-Type", "application/zip")
.header( .header(
"Content-Disposition", "Content-Disposition",
format!("attachment; filename=\"{}\"", filename), format!("attachment; filename=\"{}\"", filename),
) )
.header("Content-Length", zip_bytes.len().to_string()) .header("Content-Length", file_size)
.body(axum::body::Body::from(zip_bytes)) .body(body)
.unwrap()) .unwrap();
// Keep TempPath alive in response extensions so the file is only
// deleted AFTER the body stream finishes sending.
response.extensions_mut().insert(std::sync::Arc::new(temp_path));
Ok(response)
} }
-2
View File
@@ -94,8 +94,6 @@ pub fn create_api_routes(app_state: &Arc<AppState>) -> Router<Arc<AppState>> {
if let Some(ref ts) = trash_service { if let Some(ref ts) = trash_service {
batch_service_builder = batch_service_builder.with_trash_service(ts.clone()); batch_service_builder = batch_service_builder.with_trash_service(ts.clone());
} }
let zip_service_ref = app_state.core.zip_service.clone();
batch_service_builder = batch_service_builder.with_zip_service(zip_service_ref);
let batch_service = Arc::new(batch_service_builder); let batch_service = Arc::new(batch_service_builder);
// Create state for the batch operations handler // Create state for the batch operations handler