perf: thumbnail semaphore, WOPI streaming, store_bytes guard, spawn_blocking SHA-256

- Issue #1: Add Semaphore(4) + 50MP resolution guard to thumbnail_service
  Bounds peak RAM from 4.8GB (50 uploads) to 384MB (4 concurrent decodes)
- Issue #3: Migrate WOPI PutFile from Bytes to streaming temp file + SHA-256
  RAM per WOPI PUT: ~100MB → ~256KB regardless of file size
- Issue #3: Add 10MB guard in dedup store_bytes (defense-in-depth)
- Issue #5: Move chunked upload assembly to spawn_blocking (sync I/O)
  Frees Tokio workers during SHA-256 hashing (~130ms for 500MB)
- Clean up unused tokio imports (OpenOptions, BufWriter)
This commit is contained in:
Dionisio
2026-02-24 16:11:52 +01:00
parent 71c2cb5edb
commit 6aa38d0d24
3 changed files with 154 additions and 11 deletions
@@ -16,6 +16,7 @@ use image::{ImageFormat, imageops::FilterType};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::fs;
use tokio::sync::Semaphore;
use crate::application::ports::thumbnail_ports::{
ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto,
@@ -69,6 +70,14 @@ struct ThumbnailCacheKey {
size: ThumbnailSize,
}
/// Maximum pixel count before rejecting decode (50 megapixels → ~200 MB RGBA).
/// Images above this are silently skipped — protects against single-image OOM.
const MAX_DECODE_PIXELS: u64 = 50_000_000;
/// Default max concurrent thumbnail decode operations.
/// 4 × 96 MB (6000×4000 RGBA) = 384 MB worst-case peak.
const DEFAULT_MAX_CONCURRENT_DECODES: usize = 4;
/// Thumbnail service for generating and caching image thumbnails
pub struct ThumbnailService {
/// Root path for thumbnail storage
@@ -77,6 +86,10 @@ pub struct ThumbnailService {
cache: moka::future::Cache<ThumbnailCacheKey, Bytes>,
/// Configured maximum cache weight (for stats reporting)
max_cache_bytes: u64,
/// Limits how many images are decoded in parallel to bound RAM usage.
/// Without this, 50 simultaneous uploads would decode 50 bitmaps
/// (~96 MB each for 6000×4000) = 4.8 GB peak.
decode_semaphore: Arc<Semaphore>,
}
impl ThumbnailService {
@@ -105,6 +118,7 @@ impl ThumbnailService {
thumbnails_root,
cache,
max_cache_bytes: max_cache_bytes as u64,
decode_semaphore: Arc::new(Semaphore::new(DEFAULT_MAX_CONCURRENT_DECODES)),
}
}
@@ -217,7 +231,11 @@ impl ThumbnailService {
Ok(bytes)
}
/// Generate a thumbnail from an image file
/// Generate a thumbnail from an image file.
///
/// Concurrency is bounded by `decode_semaphore` to prevent OOM when
/// many images are uploaded simultaneously. Resolution is also
/// capped at `MAX_DECODE_PIXELS` to reject pathologically large images.
async fn generate_thumbnail(
&self,
original_path: &Path,
@@ -226,9 +244,25 @@ impl ThumbnailService {
let path = original_path.to_path_buf();
let max_dim = size.max_dimension();
// Acquire semaphore permit — bounds peak RAM from concurrent decodes
let _permit = self.decode_semaphore.acquire().await
.map_err(|_| ThumbnailError::TaskError("Decode semaphore closed".into()))?;
// Run image processing in blocking thread pool
let result = tokio::task::spawn_blocking(move || -> Result<Vec<u8>, ThumbnailError> {
// Load image
// Safety check: read dimensions from headers only (no full decode)
let (w, h) = image::ImageReader::open(&path)
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?
.into_dimensions()
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
if (w as u64) * (h as u64) > MAX_DECODE_PIXELS {
return Err(ThumbnailError::ImageError(format!(
"Image too large for thumbnail: {w}×{h} ({} MP, max {MAX_DECODE_PIXELS})",
w as u64 * h as u64 / 1_000_000
)));
}
// Load image (full decode — now safe within semaphore + resolution guard)
let img = image::open(&path).map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
// Calculate new dimensions preserving aspect ratio
@@ -274,10 +308,31 @@ impl ThumbnailService {
tokio::spawn(async move {
tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id);
// Acquire semaphore permit — bounds peak RAM from concurrent decodes
let _permit = match self.decode_semaphore.acquire().await {
Ok(p) => p,
Err(_) => {
tracing::warn!("Decode semaphore closed, skipping thumbnails for {}", file_id);
return;
}
};
let path = original_path.clone();
// Single spawn_blocking: 1 read + 1 decode + 3 resize + 3 encode
let results = tokio::task::spawn_blocking(move || {
// Safety check: read dimensions from headers only (no full decode)
let (w, h) = image::ImageReader::open(&path)
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?
.into_dimensions()
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
if (w as u64) * (h as u64) > MAX_DECODE_PIXELS {
return Err(ThumbnailError::ImageError(format!(
"Image too large for thumbnail: {w}×{h} ({} MP, max {MAX_DECODE_PIXELS})",
w as u64 * h as u64 / 1_000_000
)));
}
let img = image::open(&path)
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;