perf: replace buffered decompress_stream with async-compression streaming

- Add async-compression dependency with tokio+gzip features
- Rewrite decompress_stream to use true streaming pipeline:
  Stream<Bytes> → StreamReader → BufReader(64KB) → GzipDecoder → ReaderStream(64KB)
- Memory usage drops from ~4GB (1GB compressed file) to constant ~128KB
- Eliminates OOM risk on large compressed file decompression
This commit is contained in:
Dionisio
2026-02-24 11:47:21 +01:00
parent b628a3a166
commit 4fc0ab8831
3 changed files with 14 additions and 46 deletions
+1
View File
@@ -49,6 +49,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
base64 = "0.22.1"
fs2 = "0.4"
rayon = "1.10"
async-compression = { version = "0.4", features = ["tokio", "gzip"] }
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
[features]