perf: replace sync zip crate with async_zip in ZipService

ZipWriter<std::fs::File> performed every write_all() as a blocking
write(2) syscall on the Tokio worker thread, sequestering it for
10-100ms per file (0.8-12s for a 100-file ZIP).

Replace with async_zip::ZipFileWriter backed by a buffered
tokio::fs::File:
- All I/O (headers, deflate chunks, central directory) is fully async
- 256 KB BufWriter minimises syscall count
- Zero Tokio worker blocking during ZIP creation
- Streaming per-chunk writes keep RAM O(1) regardless of archive size
- Removed dead From<zip::result::ZipError> for DomainError impl
- zip crate retained for batch_operations.rs (separate concern)
This commit is contained in:
Dionisio
2026-02-23 23:11:55 +01:00
parent 958836e96b
commit a162aafd43
3 changed files with 106 additions and 40 deletions
Generated
+51
View File
@@ -99,6 +99,7 @@ checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40"
dependencies = [ dependencies = [
"compression-codecs", "compression-codecs",
"compression-core", "compression-core",
"futures-io",
"pin-project-lite", "pin-project-lite",
"tokio", "tokio",
] ]
@@ -147,6 +148,21 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "async_zip"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
dependencies = [
"async-compression",
"crc32fast",
"futures-lite",
"pin-project",
"thiserror",
"tokio",
"tokio-util",
]
[[package]] [[package]]
name = "atoi" name = "atoi"
version = "2.0.0" version = "2.0.0"
@@ -920,6 +936,19 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-lite"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad"
dependencies = [
"fastrand",
"futures-core",
"futures-io",
"parking",
"pin-project-lite",
]
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.31" version = "0.3.31"
@@ -1793,6 +1822,7 @@ dependencies = [
"argon2", "argon2",
"async-stream", "async-stream",
"async-trait", "async-trait",
"async_zip",
"axum", "axum",
"base64", "base64",
"bytes", "bytes",
@@ -1933,6 +1963,26 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pin-project"
version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.16" version = "0.2.16"
@@ -3060,6 +3110,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures-core", "futures-core",
"futures-io",
"futures-sink", "futures-sink",
"pin-project-lite", "pin-project-lite",
"tokio", "tokio",
+2 -1
View File
@@ -8,7 +8,7 @@ default-run = "oxicloud"
[dependencies] [dependencies]
axum = { version = "0.8.8", features = ["multipart", "http1", "tokio", "macros"] } axum = { version = "0.8.8", features = ["multipart", "http1", "tokio", "macros"] }
tokio = { version = "1.49.0", features = ["full"] } tokio = { version = "1.49.0", features = ["full"] }
tokio-util = { version = "0.7.18", features = ["io", "codec"] } tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] }
tokio-stream = { version = "0.1.18", features = ["fs"] } tokio-stream = { version = "0.1.18", features = ["fs"] }
bytes = "1.11.1" bytes = "1.11.1"
tempfile = "3.25.0" tempfile = "3.25.0"
@@ -49,6 +49,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
base64 = "0.22.1" base64 = "0.22.1"
fs2 = "0.4" fs2 = "0.4"
rayon = "1.10" rayon = "1.10"
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
[features] [features]
default = [] default = []
+53 -39
View File
@@ -7,13 +7,16 @@ use crate::{
common::errors::{DomainError, ErrorKind, Result}, common::errors::{DomainError, ErrorKind, Result},
}; };
use async_trait::async_trait; use async_trait::async_trait;
use async_zip::base::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use futures::io::AsyncWriteExt as FuturesWriteExt;
use futures::StreamExt; use futures::StreamExt;
use std::io::Write;
use std::sync::Arc; use std::sync::Arc;
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
use thiserror::Error; use thiserror::Error;
use tokio::io::BufWriter;
use tokio_util::compat::Compat;
use tracing::*; use tracing::*;
use zip::{ZipWriter, write::SimpleFileOptions};
/// Error related to ZIP file creation /// Error related to ZIP file creation
#[derive(Debug, Error)] #[derive(Debug, Error)]
@@ -22,7 +25,7 @@ pub enum ZipError {
IoError(#[from] std::io::Error), IoError(#[from] std::io::Error),
#[error("ZIP error: {0}")] #[error("ZIP error: {0}")]
ZipError(#[from] zip::result::ZipError), AsyncZipError(#[from] async_zip::error::ZipError),
#[error("Error reading file: {0}")] #[error("Error reading file: {0}")]
FileReadError(String), FileReadError(String),
@@ -34,24 +37,21 @@ pub enum ZipError {
FolderNotFound(String), FolderNotFound(String),
} }
// Implement From<ZipError> for DomainError to allow the use of ?
impl From<ZipError> for DomainError { impl From<ZipError> for DomainError {
fn from(err: ZipError) -> Self { fn from(err: ZipError) -> Self {
DomainError::new(ErrorKind::InternalError, "zip_service", err.to_string()) DomainError::new(ErrorKind::InternalError, "zip_service", err.to_string())
} }
} }
// Implement From<zip::result::ZipError> for DomainError directly /// Type alias for the fully-async ZIP writer backed by a buffered tokio file.
impl From<zip::result::ZipError> for DomainError { type AsyncZipWriter = ZipFileWriter<Compat<BufWriter<tokio::fs::File>>>;
fn from(err: zip::result::ZipError) -> Self {
DomainError::new(ErrorKind::InternalError, "zip_service", err.to_string())
}
}
/// Service for creating ZIP files. /// Service for creating ZIP files.
/// ///
/// Writes the ZIP archive to a temporary file on disk so that only one file's /// Uses `async_zip` for fully-async archive creation. Every write (headers,
/// stream-chunk (~64 KB) is held in memory at a time, regardless of archive size. /// compressed chunk data, central directory) goes through
/// `tokio::io::BufWriter` → `tokio::fs::File`, so **no Tokio worker is ever
/// blocked** by disk I/O or compression.
pub struct ZipService { pub struct ZipService {
file_service: Arc<dyn FileRetrievalUseCase>, file_service: Arc<dyn FileRetrievalUseCase>,
folder_service: Arc<dyn FolderUseCase>, folder_service: Arc<dyn FolderUseCase>,
@@ -70,7 +70,7 @@ impl ZipService {
} }
/// Creates a ZIP file backed by a temporary file, containing the contents /// Creates a ZIP file backed by a temporary file, containing the contents
/// of a folder and all its subfolders. Returns the `NamedTempFile` so the /// of a folder and all its subfolders. Returns the `NamedTempFile` so the
/// caller can stream it and let the OS clean up on drop. /// caller can stream it and let the OS clean up on drop.
pub async fn create_folder_zip( pub async fn create_folder_zip(
&self, &self,
@@ -91,15 +91,15 @@ impl ZipService {
} }
}; };
// Create a temp file to back the ZIP archive (O(1) RAM) // Create a temp file; open a second async handle for writing.
let temp = NamedTempFile::new().map_err(ZipError::IoError)?; let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
let raw_file = temp.reopen().map_err(ZipError::IoError)?; let tokio_file = tokio::fs::File::create(temp.path())
let mut zip = ZipWriter::new(raw_file); .await
.map_err(ZipError::IoError)?;
// Set compression options // 256 KB buffer keeps syscall count low.
let options = SimpleFileOptions::default() let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
.compression_method(zip::CompressionMethod::Deflated) let mut zip = ZipFileWriter::with_tokio(buf_writer);
.unix_permissions(0o755);
// Track processed folders to avoid cycles // Track processed folders to avoid cycles
let mut processed_folders = std::collections::HashSet::new(); let mut processed_folders = std::collections::HashSet::new();
@@ -109,25 +109,24 @@ impl ZipService {
&mut zip, &mut zip,
&folder, &folder,
folder_name, folder_name,
&options,
&mut processed_folders, &mut processed_folders,
) )
.await?; .await?;
// Finalize the ZIP (flushes central directory) // Finalize: writes central directory, then flush buffered data to disk.
zip.finish()?; let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
compat_writer.close().await.map_err(ZipError::IoError)?;
Ok(temp) Ok(temp)
} }
/// Iterative BFS over the folder tree. Writes entries directly to the /// Iterative BFS over the folder tree. Writes entries directly to the
/// file-backed `ZipWriter` so memory stays flat. /// async `ZipFileWriter` so memory stays flat.
async fn process_folder_recursively( async fn process_folder_recursively(
&self, &self,
zip: &mut ZipWriter<std::fs::File>, zip: &mut AsyncZipWriter,
folder: &FolderDto, folder: &FolderDto,
path: &str, path: &str,
options: &SimpleFileOptions,
processed_folders: &mut std::collections::HashSet<String>, processed_folders: &mut std::collections::HashSet<String>,
) -> Result<()> { ) -> Result<()> {
struct PendingFolder { struct PendingFolder {
@@ -148,10 +147,12 @@ impl ZipService {
} }
processed_folders.insert(folder_id.clone()); processed_folders.insert(folder_id.clone());
// Directory entry // Directory entry (Stored, zero-length body)
let folder_path = format!("{}/", current.path); let folder_path = format!("{}/", current.path);
match zip.add_directory(&folder_path, *options) { let dir_entry =
Ok(_) => debug!("Folder added to ZIP: {}", folder_path), ZipEntryBuilder::new(folder_path.clone().into(), Compression::Stored);
match zip.write_entry_whole(dir_entry, &[]).await {
Ok(()) => debug!("Folder added to ZIP: {}", folder_path),
Err(e) => { Err(e) => {
warn!("Could not add folder to ZIP (may already exist): {}", e); warn!("Could not add folder to ZIP (may already exist): {}", e);
} }
@@ -171,7 +172,7 @@ impl ZipService {
}; };
for file in files { for file in files {
self.add_file_to_zip_streamed(zip, &file, &folder_path, options) self.add_file_to_zip_streamed(zip, &file, &folder_path)
.await?; .await?;
} }
@@ -200,29 +201,33 @@ impl ZipService {
Ok(()) Ok(())
} }
/// Streams file content in chunks (~64 KB) into the ZIP entry, keeping /// Streams file content in chunks (~64 KB) into an async ZIP entry,
/// peak memory independent of individual file sizes. /// keeping peak memory independent of individual file sizes.
async fn add_file_to_zip_streamed( async fn add_file_to_zip_streamed(
&self, &self,
zip: &mut ZipWriter<std::fs::File>, zip: &mut AsyncZipWriter,
file: &FileDto, file: &FileDto,
folder_path: &str, folder_path: &str,
options: &SimpleFileOptions,
) -> Result<()> { ) -> Result<()> {
let file_path = format!("{}{}", folder_path, file.name); let file_path = format!("{}{}", folder_path, file.name);
info!("Adding file to ZIP: {}", file_path); info!("Adding file to ZIP: {}", file_path);
let file_id = file.id.to_string(); let file_id = file.id.to_string();
// Start the ZIP entry // Open a streaming entry with Deflate compression
zip.start_file_from_path(std::path::Path::new(&file_path), *options) let entry = ZipEntryBuilder::new(file_path.clone().into(), Compression::Deflate);
.map_err(ZipError::ZipError)?; let mut entry_writer = zip
.write_entry_stream(entry)
.await
.map_err(ZipError::AsyncZipError)?;
// Stream file contents in chunks instead of loading all into RAM // Stream file contents in chunks instead of loading all into RAM
let stream = match self.file_service.get_file_stream(&file_id).await { let stream = match self.file_service.get_file_stream(&file_id).await {
Ok(s) => s, Ok(s) => s,
Err(e) => { Err(e) => {
error!("Error opening file stream {}: {}", file_id, e); error!("Error opening file stream {}: {}", file_id, e);
// Close the partially-opened entry before returning
let _ = entry_writer.close().await;
return Err(ZipError::FileReadError(format!( return Err(ZipError::FileReadError(format!(
"Error streaming file {}: {}", "Error streaming file {}: {}",
file_id, e file_id, e
@@ -236,9 +241,18 @@ impl ZipService {
while let Some(chunk_result) = stream.next().await { while let Some(chunk_result) = stream.next().await {
let bytes = chunk_result.map_err(ZipError::IoError)?; let bytes = chunk_result.map_err(ZipError::IoError)?;
zip.write_all(&bytes).map_err(ZipError::IoError)?; entry_writer
.write_all(&bytes)
.await
.map_err(ZipError::IoError)?;
} }
// Finalize the entry (writes data descriptor with CRC + sizes)
entry_writer
.close()
.await
.map_err(ZipError::AsyncZipError)?;
debug!("File added to ZIP: {}", file_path); debug!("File added to ZIP: {}", file_path);
Ok(()) Ok(())
} }