perf: replace buffered decompress_stream with async-compression streaming
- Add async-compression dependency with tokio+gzip features - Rewrite decompress_stream to use true streaming pipeline: Stream<Bytes> → StreamReader → BufReader(64KB) → GzipDecoder → ReaderStream(64KB) - Memory usage drops from ~4GB (1GB compressed file) to constant ~128KB - Eliminates OOM risk on large compressed file decompression
This commit is contained in:
Generated
+1
@@ -1820,6 +1820,7 @@ version = "0.4.2"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argon2",
|
"argon2",
|
||||||
|
"async-compression",
|
||||||
"async-stream",
|
"async-stream",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"async_zip",
|
"async_zip",
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
|
|||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
fs2 = "0.4"
|
fs2 = "0.4"
|
||||||
rayon = "1.10"
|
rayon = "1.10"
|
||||||
|
async-compression = { version = "0.4", features = ["tokio", "gzip"] }
|
||||||
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ use flate2::write::GzEncoder as GzEncoderWrite;
|
|||||||
use futures::{Stream, StreamExt};
|
use futures::{Stream, StreamExt};
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
|
use tokio_util::io::{ReaderStream, StreamReader};
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
|
|
||||||
use crate::application::ports::compression_ports::{
|
use crate::application::ports::compression_ports::{
|
||||||
@@ -181,10 +182,12 @@ impl CompressionService for GzipCompressionService {
|
|||||||
|
|
||||||
/// Decompresses a byte stream using true streaming — constant memory usage.
|
/// Decompresses a byte stream using true streaming — constant memory usage.
|
||||||
///
|
///
|
||||||
/// Collects compressed chunks, then decompresses in a blocking task.
|
/// Uses `async-compression` to wrap the incoming compressed stream as an
|
||||||
/// For streaming decompression of very large data, a dedicated
|
/// `AsyncBufRead`, then pipes it through a `GzipDecoder` that produces
|
||||||
/// async-compression crate would be better, but this avoids adding
|
/// decompressed bytes on-the-fly.
|
||||||
/// new deps while still being correct.
|
///
|
||||||
|
/// Memory usage: ~128 KB constant (64 KB read buffer + 64 KB output chunks),
|
||||||
|
/// independent of the total file size.
|
||||||
fn decompress_stream<S>(
|
fn decompress_stream<S>(
|
||||||
&self,
|
&self,
|
||||||
compressed_stream: S,
|
compressed_stream: S,
|
||||||
@@ -192,48 +195,11 @@ impl CompressionService for GzipCompressionService {
|
|||||||
where
|
where
|
||||||
S: Stream<Item = io::Result<Bytes>> + Send + 'static + Unpin,
|
S: Stream<Item = io::Result<Bytes>> + Send + 'static + Unpin,
|
||||||
{
|
{
|
||||||
// Decompression is harder to stream without async-compression crate.
|
// Stream<Bytes> → AsyncRead → BufReader → GzipDecoder → Stream<Bytes>
|
||||||
// We keep the collect-then-decompress approach here but decompress in
|
let reader = StreamReader::new(compressed_stream);
|
||||||
// a blocking task to avoid blocking the async runtime.
|
let buf_reader = tokio::io::BufReader::with_capacity(64 * 1024, reader);
|
||||||
// This is acceptable because decompress_stream is rarely used in the
|
let decoder = async_compression::tokio::bufread::GzipDecoder::new(buf_reader);
|
||||||
// hot path (downloads serve raw content, not compressed).
|
ReaderStream::with_capacity(decoder, 64 * 1024)
|
||||||
Box::pin(async_stream::stream! {
|
|
||||||
let mut compressed_data = Vec::new();
|
|
||||||
|
|
||||||
let mut stream = Box::pin(compressed_stream);
|
|
||||||
while let Some(result) = stream.next().await {
|
|
||||||
match result {
|
|
||||||
Ok(bytes) => {
|
|
||||||
compressed_data.extend_from_slice(&bytes);
|
|
||||||
},
|
|
||||||
Err(e) => {
|
|
||||||
yield Err(e);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decompress in a blocking task
|
|
||||||
match tokio::task::spawn_blocking(move || {
|
|
||||||
let mut decoder = GzDecoder::new(&compressed_data[..]);
|
|
||||||
let mut decompressed = Vec::new();
|
|
||||||
decoder.read_to_end(&mut decompressed)?;
|
|
||||||
Ok::<_, io::Error>(decompressed)
|
|
||||||
}).await {
|
|
||||||
Ok(Ok(decompressed)) => {
|
|
||||||
// Yield in 64KB chunks to avoid a single huge allocation in the response
|
|
||||||
for chunk in decompressed.chunks(64 * 1024) {
|
|
||||||
yield Ok(Bytes::copy_from_slice(chunk));
|
|
||||||
}
|
|
||||||
},
|
|
||||||
Ok(Err(e)) => {
|
|
||||||
yield Err(e);
|
|
||||||
},
|
|
||||||
Err(e) => {
|
|
||||||
yield Err(io::Error::other(e.to_string()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Determines whether a file should be compressed based on its MIME type and size
|
/// Determines whether a file should be compressed based on its MIME type and size
|
||||||
|
|||||||
Reference in New Issue
Block a user