perf: replace buffered decompress_stream with async-compression streaming
- Add async-compression dependency with tokio+gzip features - Rewrite decompress_stream to use true streaming pipeline: Stream<Bytes> → StreamReader → BufReader(64KB) → GzipDecoder → ReaderStream(64KB) - Memory usage drops from ~4GB (1GB compressed file) to constant ~128KB - Eliminates OOM risk on large compressed file decompression
This commit is contained in:
@@ -49,6 +49,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
|
||||
base64 = "0.22.1"
|
||||
fs2 = "0.4"
|
||||
rayon = "1.10"
|
||||
async-compression = { version = "0.4", features = ["tokio", "gzip"] }
|
||||
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
||||
|
||||
[features]
|
||||
|
||||
Reference in New Issue
Block a user