2026-02-14 01:29:34 +01:00
|
|
|
|
use bytes::Bytes;
|
2026-03-07 20:37:19 +01:00
|
|
|
|
use image::codecs::jpeg::JpegEncoder;
|
2026-03-09 14:34:07 +01:00
|
|
|
|
use image::imageops::FilterType;
|
2026-04-11 19:13:48 +02:00
|
|
|
|
use rayon::prelude::*;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/**
|
|
|
|
|
|
* Thumbnail Generation Service
|
|
|
|
|
|
*
|
|
|
|
|
|
* Generates and manages image thumbnails for fast gallery previews.
|
|
|
|
|
|
*
|
|
|
|
|
|
* Features:
|
|
|
|
|
|
* - Background thumbnail generation after upload
|
|
|
|
|
|
* - Multiple sizes (icon 150x150, preview 800x600)
|
2026-03-07 20:37:19 +01:00
|
|
|
|
* - JPEG output (lossy q=80) for compact thumbnails
|
2026-02-22 23:28:03 +01:00
|
|
|
|
* - Lock-free moka cache with weight-based eviction
|
2026-02-14 01:29:34 +01:00
|
|
|
|
* - Lazy generation on first request if not pre-generated
|
2026-03-28 18:40:34 +00:00
|
|
|
|
* - Timeout protection for large image processing
|
2026-02-14 01:29:34 +01:00
|
|
|
|
*/
|
|
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
|
|
use std::sync::Arc;
|
2026-03-28 18:40:34 +00:00
|
|
|
|
use std::time::Duration;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
use tokio::fs;
|
2026-02-24 16:11:52 +01:00
|
|
|
|
use tokio::sync::Semaphore;
|
2026-03-28 18:40:34 +00:00
|
|
|
|
use tokio::time::timeout;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
use crate::application::ports::thumbnail_ports::{
|
|
|
|
|
|
ThumbnailPort, ThumbnailSize as PortThumbnailSize, ThumbnailStatsDto,
|
|
|
|
|
|
};
|
|
|
|
|
|
use crate::domain::errors::{DomainError, ErrorKind};
|
2026-05-13 13:27:33 +02:00
|
|
|
|
use crate::infrastructure::services::dedup_service::DedupService;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
/// Thumbnail sizes supported by the system
|
|
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
|
|
|
|
|
pub enum ThumbnailSize {
|
|
|
|
|
|
/// Small icon for file listings (150x150)
|
|
|
|
|
|
Icon,
|
|
|
|
|
|
/// Medium preview for gallery view (400x400)
|
|
|
|
|
|
Preview,
|
|
|
|
|
|
/// Large preview for detail view (800x800)
|
|
|
|
|
|
Large,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl ThumbnailSize {
|
|
|
|
|
|
/// Get the maximum dimension for this size
|
|
|
|
|
|
pub fn max_dimension(&self) -> u32 {
|
|
|
|
|
|
match self {
|
|
|
|
|
|
ThumbnailSize::Icon => 150,
|
|
|
|
|
|
ThumbnailSize::Preview => 400,
|
|
|
|
|
|
ThumbnailSize::Large => 800,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Get the directory name for this size
|
|
|
|
|
|
pub fn dir_name(&self) -> &'static str {
|
|
|
|
|
|
match self {
|
|
|
|
|
|
ThumbnailSize::Icon => "icon",
|
|
|
|
|
|
ThumbnailSize::Preview => "preview",
|
|
|
|
|
|
ThumbnailSize::Large => "large",
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Get all thumbnail sizes
|
|
|
|
|
|
pub fn all() -> &'static [ThumbnailSize] {
|
|
|
|
|
|
&[
|
|
|
|
|
|
ThumbnailSize::Icon,
|
|
|
|
|
|
ThumbnailSize::Preview,
|
|
|
|
|
|
ThumbnailSize::Large,
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Cache key for thumbnails
|
|
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
|
|
|
|
|
struct ThumbnailCacheKey {
|
|
|
|
|
|
file_id: String,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-24 16:11:52 +01:00
|
|
|
|
/// Maximum pixel count before rejecting decode (50 megapixels → ~200 MB RGBA).
|
|
|
|
|
|
/// Images above this are silently skipped — protects against single-image OOM.
|
|
|
|
|
|
const MAX_DECODE_PIXELS: u64 = 50_000_000;
|
|
|
|
|
|
|
2026-06-21 15:32:15 +02:00
|
|
|
|
/// Environment override for the decode-concurrency cap (ops tuning).
|
|
|
|
|
|
const DECODE_CONCURRENCY_ENV: &str = "OXICLOUD_THUMBNAIL_DECODE_CONCURRENCY";
|
|
|
|
|
|
|
2026-04-11 18:14:33 +02:00
|
|
|
|
/// Compute max concurrent thumbnail decode operations at runtime.
|
2026-06-21 15:32:15 +02:00
|
|
|
|
///
|
|
|
|
|
|
/// Uses all available CPUs (min 2). Before shrink-on-load each decode
|
|
|
|
|
|
/// materialised the full-resolution RGBA bitmap (~96 MB for a 12 MP photo), so
|
|
|
|
|
|
/// concurrency was halved to keep peak RAM in check. Decodes are now DCT-shrunk
|
|
|
|
|
|
/// to the thumbnail size (~18–25 MB regardless of source resolution), so the RAM
|
|
|
|
|
|
/// ceiling no longer forces throttling and we can saturate every core. Override
|
|
|
|
|
|
/// with `OXICLOUD_THUMBNAIL_DECODE_CONCURRENCY`. `available_parallelism()`
|
|
|
|
|
|
/// respects cgroup limits (Docker/K8s) and CPU affinity masks.
|
2026-04-11 18:14:33 +02:00
|
|
|
|
fn max_concurrent_decodes() -> usize {
|
2026-06-21 15:32:15 +02:00
|
|
|
|
if let Some(n) = std::env::var(DECODE_CONCURRENCY_ENV)
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.and_then(|v| v.parse::<usize>().ok())
|
|
|
|
|
|
.filter(|&n| n >= 1)
|
|
|
|
|
|
{
|
|
|
|
|
|
return n;
|
|
|
|
|
|
}
|
2026-04-11 18:14:33 +02:00
|
|
|
|
let cpus = std::thread::available_parallelism()
|
|
|
|
|
|
.map(|n| n.get())
|
|
|
|
|
|
.unwrap_or(4);
|
2026-06-21 15:32:15 +02:00
|
|
|
|
cpus.max(2)
|
2026-04-11 18:14:33 +02:00
|
|
|
|
}
|
2026-02-24 16:11:52 +01:00
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// Thumbnail service for generating and caching image thumbnails
|
|
|
|
|
|
pub struct ThumbnailService {
|
|
|
|
|
|
/// Root path for thumbnail storage
|
|
|
|
|
|
thumbnails_root: PathBuf,
|
2026-02-22 23:28:03 +01:00
|
|
|
|
/// Lock-free concurrent cache (moka) with weight-based eviction
|
|
|
|
|
|
cache: moka::future::Cache<ThumbnailCacheKey, Bytes>,
|
|
|
|
|
|
/// Configured maximum cache weight (for stats reporting)
|
|
|
|
|
|
max_cache_bytes: u64,
|
2026-06-21 15:32:15 +02:00
|
|
|
|
/// Limits how many images are decoded in parallel. Post shrink-on-load each
|
|
|
|
|
|
/// decode is bounded (~18–25 MB), so this now exists mainly to avoid CPU
|
|
|
|
|
|
/// oversubscription rather than to cap RAM; defaults to all cores.
|
2026-02-24 16:11:52 +01:00
|
|
|
|
decode_semaphore: Arc<Semaphore>,
|
2026-03-28 18:40:34 +00:00
|
|
|
|
/// Timeout for thumbnail generation operations to prevent hanging on large images.
|
|
|
|
|
|
/// Defaults to 30 seconds.
|
|
|
|
|
|
generation_timeout: Duration,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl ThumbnailService {
|
|
|
|
|
|
/// Create a new thumbnail service
|
|
|
|
|
|
///
|
|
|
|
|
|
/// # Arguments
|
|
|
|
|
|
/// * `storage_root` - Root path of file storage
|
2026-02-22 23:28:03 +01:00
|
|
|
|
/// * `max_cache_entries` - (ignored — moka uses weight-based eviction)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// * `max_cache_bytes` - Maximum total bytes to cache
|
2026-03-28 18:40:34 +00:00
|
|
|
|
/// * `generation_timeout` - Timeout for thumbnail generation operations
|
|
|
|
|
|
pub fn new(
|
|
|
|
|
|
storage_root: &Path,
|
|
|
|
|
|
max_cache_entries: usize,
|
|
|
|
|
|
max_cache_bytes: usize,
|
|
|
|
|
|
generation_timeout: Option<Duration>,
|
|
|
|
|
|
) -> Self {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let thumbnails_root = storage_root.join(".thumbnails");
|
|
|
|
|
|
|
2026-02-22 23:28:03 +01:00
|
|
|
|
// Ignore max_cache_entries — weight-based eviction is more accurate
|
|
|
|
|
|
// for variable-size thumbnails than entry-count limits.
|
|
|
|
|
|
let _ = max_cache_entries;
|
|
|
|
|
|
|
2026-03-07 19:48:35 +01:00
|
|
|
|
// No time_to_live — thumbnails are immutable (content never changes
|
|
|
|
|
|
// for a given file_id). Eviction is purely weight-based: when the
|
|
|
|
|
|
// cache exceeds max_cache_bytes the lightest entries are dropped.
|
|
|
|
|
|
// On eviction the thumbnail is still on disk; the next request
|
|
|
|
|
|
// promotes it back with a single async read (~0.1 ms).
|
2026-02-22 23:28:03 +01:00
|
|
|
|
let cache = moka::future::Cache::builder()
|
|
|
|
|
|
.max_capacity(max_cache_bytes as u64)
|
|
|
|
|
|
.weigher(|_key: &ThumbnailCacheKey, value: &Bytes| -> u32 {
|
|
|
|
|
|
value.len().min(u32::MAX as usize) as u32
|
|
|
|
|
|
})
|
|
|
|
|
|
.build();
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
Self {
|
|
|
|
|
|
thumbnails_root,
|
2026-02-22 23:28:03 +01:00
|
|
|
|
cache,
|
|
|
|
|
|
max_cache_bytes: max_cache_bytes as u64,
|
2026-04-11 18:14:33 +02:00
|
|
|
|
decode_semaphore: Arc::new(Semaphore::new(max_concurrent_decodes())),
|
2026-03-28 18:40:34 +00:00
|
|
|
|
generation_timeout: generation_timeout.unwrap_or(Duration::from_secs(30)),
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Initialize the thumbnail directories
|
|
|
|
|
|
pub async fn initialize(&self) -> std::io::Result<()> {
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let dir = self.thumbnails_root.join(size.dir_name());
|
|
|
|
|
|
fs::create_dir_all(&dir).await?;
|
|
|
|
|
|
}
|
|
|
|
|
|
tracing::info!(
|
|
|
|
|
|
"🖼️ Thumbnail service initialized at {:?}",
|
|
|
|
|
|
self.thumbnails_root
|
|
|
|
|
|
);
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Check if a file is an image that can have thumbnails
|
|
|
|
|
|
pub fn is_supported_image(mime_type: &str) -> bool {
|
|
|
|
|
|
matches!(
|
|
|
|
|
|
mime_type,
|
|
|
|
|
|
"image/jpeg" | "image/jpg" | "image/png" | "image/gif" | "image/webp"
|
|
|
|
|
|
)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
/// Get the path where a thumbnail would be stored (keyed by blob hash for dedup).
|
|
|
|
|
|
fn get_thumbnail_path(&self, blob_hash: &str, size: ThumbnailSize) -> PathBuf {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
self.thumbnails_root
|
|
|
|
|
|
.join(size.dir_name())
|
2026-04-12 00:50:10 +02:00
|
|
|
|
.join(format!("{}.jpg", blob_hash))
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-23 23:22:54 +01:00
|
|
|
|
/// Get a thumbnail, generating it if needed.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
///
|
|
|
|
|
|
/// # Arguments
|
2026-04-12 00:50:10 +02:00
|
|
|
|
/// * `file_id` - ID of the original file (used as moka cache key)
|
|
|
|
|
|
/// * `blob_hash` - Content hash of the file (used as disk key for dedup)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// * `size` - Desired thumbnail size
|
|
|
|
|
|
/// * `original_path` - Path to the original image file
|
|
|
|
|
|
///
|
|
|
|
|
|
/// # Returns
|
2026-03-07 20:37:19 +01:00
|
|
|
|
/// Bytes of the thumbnail image (JPEG format)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub async fn get_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
2026-04-12 00:50:10 +02:00
|
|
|
|
blob_hash: &str,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
original_path: &Path,
|
|
|
|
|
|
) -> Result<Bytes, ThumbnailError> {
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.to_string(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
let thumb_path = self.get_thumbnail_path(blob_hash, size);
|
2026-02-23 23:22:54 +01:00
|
|
|
|
let original_owned = original_path.to_path_buf();
|
|
|
|
|
|
let file_id_owned = file_id.to_string();
|
|
|
|
|
|
|
|
|
|
|
|
// Moka's entry().or_insert_with() guarantees that for the same key
|
|
|
|
|
|
// only ONE init closure runs; concurrent callers await the same
|
|
|
|
|
|
// computation instead of stampeding (thundering-herd protection).
|
|
|
|
|
|
let entry = self
|
|
|
|
|
|
.cache
|
|
|
|
|
|
.entry(cache_key)
|
|
|
|
|
|
.or_insert_with(async {
|
|
|
|
|
|
// 1. Try loading from disk
|
|
|
|
|
|
if let Ok(data) = fs::read(&thumb_path).await {
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"💾 Thumbnail loaded from disk: {} {:?}",
|
|
|
|
|
|
file_id_owned,
|
|
|
|
|
|
size
|
|
|
|
|
|
);
|
|
|
|
|
|
return Bytes::from(data);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 2. Generate thumbnail (CPU-bound, runs in spawn_blocking)
|
2026-02-26 00:47:32 +01:00
|
|
|
|
tracing::info!("🎨 Generating thumbnail: {} {:?}", file_id_owned, size);
|
2026-02-23 23:22:54 +01:00
|
|
|
|
match self.generate_thumbnail(&original_owned, size).await {
|
|
|
|
|
|
Ok(bytes) => {
|
|
|
|
|
|
// Save to disk (best-effort — don't fail the request)
|
|
|
|
|
|
if let Some(parent) = thumb_path.parent() {
|
|
|
|
|
|
let _ = fs::create_dir_all(parent).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = fs::write(&thumb_path, &bytes).await;
|
|
|
|
|
|
bytes
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Thumbnail generation failed for {} {:?}: {e}",
|
|
|
|
|
|
file_id_owned,
|
|
|
|
|
|
size
|
|
|
|
|
|
);
|
|
|
|
|
|
// Return empty sentinel — will be evicted quickly by
|
|
|
|
|
|
// the weigher (weight 0) and retried on next request.
|
|
|
|
|
|
Bytes::new()
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
})
|
|
|
|
|
|
.await;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-23 23:22:54 +01:00
|
|
|
|
let bytes = entry.into_value();
|
|
|
|
|
|
if bytes.is_empty() {
|
|
|
|
|
|
return Err(ThumbnailError::ImageError(
|
|
|
|
|
|
"Thumbnail generation failed".to_string(),
|
|
|
|
|
|
));
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-23 23:22:54 +01:00
|
|
|
|
tracing::debug!("🔥 Thumbnail served: {} {:?}", file_id, size);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
Ok(bytes)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-26 11:55:05 +02:00
|
|
|
|
/// Get a thumbnail from raw image bytes, generating it if needed.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// This is the storage-model-safe entrypoint for CDC/manifest-backed
|
2026-06-10 13:36:29 +00:00
|
|
|
|
/// blobs where no single local source file exists on disk. Prefer
|
|
|
|
|
|
/// [`Self::get_thumbnail_from_blob`] on request paths — it defers the
|
|
|
|
|
|
/// full blob read until a decode permit is held, so a stampede of
|
|
|
|
|
|
/// cache misses cannot stack one source image per request in RAM.
|
2026-04-26 11:55:05 +02:00
|
|
|
|
pub async fn get_thumbnail_from_bytes(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
blob_hash: &str,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
original_data: Bytes,
|
|
|
|
|
|
) -> Result<Bytes, ThumbnailError> {
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.to_string(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(blob_hash, size);
|
|
|
|
|
|
let file_id_owned = file_id.to_string();
|
|
|
|
|
|
|
|
|
|
|
|
let entry = self
|
|
|
|
|
|
.cache
|
|
|
|
|
|
.entry(cache_key)
|
|
|
|
|
|
.or_insert_with(async move {
|
|
|
|
|
|
if let Ok(data) = fs::read(&thumb_path).await {
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"💾 Thumbnail loaded from disk: {} {:?}",
|
|
|
|
|
|
file_id_owned,
|
|
|
|
|
|
size
|
|
|
|
|
|
);
|
|
|
|
|
|
return Bytes::from(data);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-10 13:36:29 +00:00
|
|
|
|
let Ok(_permit) = self.decode_semaphore.acquire().await else {
|
|
|
|
|
|
tracing::warn!("Decode semaphore closed, skipping {}", file_id_owned);
|
|
|
|
|
|
return Bytes::new();
|
|
|
|
|
|
};
|
|
|
|
|
|
self.generate_and_persist(&file_id_owned, &thumb_path, size, original_data)
|
|
|
|
|
|
.await
|
|
|
|
|
|
})
|
|
|
|
|
|
.await;
|
|
|
|
|
|
|
|
|
|
|
|
let bytes = entry.into_value();
|
|
|
|
|
|
if bytes.is_empty() {
|
|
|
|
|
|
return Err(ThumbnailError::ImageError(
|
|
|
|
|
|
"Thumbnail generation failed".to_string(),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
tracing::debug!("🔥 Thumbnail served: {} {:?}", file_id, size);
|
|
|
|
|
|
Ok(bytes)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Get a thumbnail for a content-addressed blob, generating it if needed.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Request-path entrypoint: on a memory+disk cache miss the source blob
|
|
|
|
|
|
/// is read **after** a decode permit is acquired, so peak RAM under a
|
|
|
|
|
|
/// thumbnail stampede is `permits × image size` instead of
|
|
|
|
|
|
/// `in-flight requests × image size`. moka's per-key init additionally
|
|
|
|
|
|
/// collapses concurrent requests for the same thumbnail into one read.
|
|
|
|
|
|
pub async fn get_thumbnail_from_blob(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
blob_hash: &str,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
dedup: Arc<DedupService>,
|
|
|
|
|
|
) -> Result<Bytes, ThumbnailError> {
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.to_string(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(blob_hash, size);
|
|
|
|
|
|
let file_id_owned = file_id.to_string();
|
|
|
|
|
|
let blob_hash_owned = blob_hash.to_string();
|
|
|
|
|
|
|
|
|
|
|
|
let entry = self
|
|
|
|
|
|
.cache
|
|
|
|
|
|
.entry(cache_key)
|
|
|
|
|
|
.or_insert_with(async move {
|
|
|
|
|
|
if let Ok(data) = fs::read(&thumb_path).await {
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"💾 Thumbnail loaded from disk: {} {:?}",
|
|
|
|
|
|
file_id_owned,
|
|
|
|
|
|
size
|
|
|
|
|
|
);
|
|
|
|
|
|
return Bytes::from(data);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
let Ok(_permit) = self.decode_semaphore.acquire().await else {
|
|
|
|
|
|
tracing::warn!("Decode semaphore closed, skipping {}", file_id_owned);
|
|
|
|
|
|
return Bytes::new();
|
|
|
|
|
|
};
|
|
|
|
|
|
let original_data = match dedup.read_blob_bytes(&blob_hash_owned).await {
|
|
|
|
|
|
Ok(bytes) => bytes,
|
2026-04-26 11:55:05 +02:00
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!(
|
2026-06-10 13:36:29 +00:00
|
|
|
|
"Failed to read blob for thumbnail {} {:?}: {e}",
|
2026-04-26 11:55:05 +02:00
|
|
|
|
file_id_owned,
|
|
|
|
|
|
size
|
|
|
|
|
|
);
|
2026-06-10 13:36:29 +00:00
|
|
|
|
return Bytes::new();
|
2026-04-26 11:55:05 +02:00
|
|
|
|
}
|
2026-06-10 13:36:29 +00:00
|
|
|
|
};
|
|
|
|
|
|
self.generate_and_persist(&file_id_owned, &thumb_path, size, original_data)
|
|
|
|
|
|
.await
|
2026-04-26 11:55:05 +02:00
|
|
|
|
})
|
|
|
|
|
|
.await;
|
|
|
|
|
|
|
|
|
|
|
|
let bytes = entry.into_value();
|
|
|
|
|
|
if bytes.is_empty() {
|
|
|
|
|
|
return Err(ThumbnailError::ImageError(
|
|
|
|
|
|
"Thumbnail generation failed".to_string(),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
tracing::debug!("🔥 Thumbnail served: {} {:?}", file_id, size);
|
|
|
|
|
|
Ok(bytes)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-10 13:36:29 +00:00
|
|
|
|
/// Decode `original_data` into one thumbnail size, persist it to its
|
|
|
|
|
|
/// blob-keyed disk path, and return the encoded bytes — empty `Bytes`
|
|
|
|
|
|
/// on failure (moka's zero-weight negative-entry convention).
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Callers must hold a `decode_semaphore` permit.
|
|
|
|
|
|
async fn generate_and_persist(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
thumb_path: &Path,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
original_data: Bytes,
|
|
|
|
|
|
) -> Bytes {
|
|
|
|
|
|
tracing::info!("🎨 Generating thumbnail: {} {:?}", file_id, size);
|
|
|
|
|
|
match Self::generate_thumbnail_from_data(original_data, size, self.generation_timeout).await
|
|
|
|
|
|
{
|
|
|
|
|
|
Ok(bytes) => {
|
|
|
|
|
|
if let Some(parent) = thumb_path.parent() {
|
|
|
|
|
|
let _ = fs::create_dir_all(parent).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = fs::write(&thumb_path, &bytes).await;
|
|
|
|
|
|
bytes
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Thumbnail generation failed for {} {:?}: {e}",
|
|
|
|
|
|
file_id,
|
|
|
|
|
|
size
|
|
|
|
|
|
);
|
|
|
|
|
|
Bytes::new()
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-07 18:55:44 +01:00
|
|
|
|
/// Try to serve a thumbnail from cache only (memory → disk).
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Unlike `get_thumbnail`, this does **not** generate a new thumbnail.
|
|
|
|
|
|
/// Useful for non-image file types (videos) where a client-generated
|
|
|
|
|
|
/// thumbnail may have been uploaded previously.
|
2026-04-12 00:50:10 +02:00
|
|
|
|
///
|
|
|
|
|
|
/// `blob_hash` is used to locate the file on disk (dedup-aware).
|
|
|
|
|
|
/// If `None`, only the in-memory cache is checked (used for video
|
|
|
|
|
|
/// thumbnails where blob_hash is not yet resolved).
|
|
|
|
|
|
pub async fn get_cached_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
blob_hash: Option<&str>,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
) -> Option<Bytes> {
|
2026-03-07 18:55:44 +01:00
|
|
|
|
// 1. Check in-memory cache
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.to_string(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
2026-03-09 14:34:07 +01:00
|
|
|
|
if let Some(bytes) = self.cache.get(&cache_key).await
|
|
|
|
|
|
&& !bytes.is_empty()
|
|
|
|
|
|
{
|
|
|
|
|
|
return Some(bytes);
|
2026-03-07 18:55:44 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:56:58 +02:00
|
|
|
|
// 2. Check disk for external (video-frame) thumbnails stored by file_id.
|
|
|
|
|
|
// These don't require blob_hash since they use ext-{file_id}.jpg paths.
|
|
|
|
|
|
let ext_path = self
|
|
|
|
|
|
.thumbnails_root
|
|
|
|
|
|
.join(size.dir_name())
|
|
|
|
|
|
.join(format!("ext-{}.jpg", file_id));
|
|
|
|
|
|
if let Ok(data) = fs::read(&ext_path).await {
|
|
|
|
|
|
let bytes = Bytes::from(data);
|
|
|
|
|
|
self.cache.insert(cache_key.clone(), bytes.clone()).await;
|
|
|
|
|
|
return Some(bytes);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 3. Check disk for blob-hash thumbnails (needs blob_hash to locate)
|
2026-04-12 00:50:10 +02:00
|
|
|
|
let hash = blob_hash?;
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(hash, size);
|
2026-03-07 18:55:44 +01:00
|
|
|
|
if let Ok(data) = fs::read(&thumb_path).await {
|
|
|
|
|
|
let bytes = Bytes::from(data);
|
|
|
|
|
|
// Populate in-memory cache for next hit
|
|
|
|
|
|
self.cache.insert(cache_key, bytes.clone()).await;
|
|
|
|
|
|
Some(bytes)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Store an externally-generated thumbnail (e.g. client-side video frame).
|
|
|
|
|
|
///
|
2026-03-07 20:37:19 +01:00
|
|
|
|
/// **Fast path**: if the payload is already a correctly-sized JPEG, it is
|
|
|
|
|
|
/// stored as-is — zero decode, zero encode. The browser pre-scales the
|
|
|
|
|
|
/// canvas to 400 px and sends JPEG, so this fast path is hit on every
|
|
|
|
|
|
/// normal video-thumbnail upload.
|
2026-03-07 20:12:12 +01:00
|
|
|
|
///
|
2026-03-07 20:37:19 +01:00
|
|
|
|
/// **Slow path**: decode → optional resize → re-encode to JPEG q=80.
|
|
|
|
|
|
/// Only triggered when a client sends an oversized or non-JPEG image.
|
2026-04-12 00:50:10 +02:00
|
|
|
|
///
|
|
|
|
|
|
/// External thumbnails (video frames) are stored by `file_id` since
|
|
|
|
|
|
/// they are client-generated and not dedup-able.
|
2026-03-07 18:55:44 +01:00
|
|
|
|
pub async fn store_external_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
data: Bytes,
|
|
|
|
|
|
) -> Result<Bytes, ThumbnailError> {
|
|
|
|
|
|
let max_dim = size.max_dimension();
|
|
|
|
|
|
|
2026-03-07 20:12:12 +01:00
|
|
|
|
// Validate + optionally re-encode in blocking thread
|
2026-03-07 20:37:19 +01:00
|
|
|
|
let jpeg_bytes = tokio::task::spawn_blocking(move || -> Result<Vec<u8>, ThumbnailError> {
|
|
|
|
|
|
// ── Fast path: already a correctly-sized JPEG ─────────────
|
|
|
|
|
|
// JPEG files start with SOI marker 0xFF 0xD8.
|
2026-03-09 14:34:07 +01:00
|
|
|
|
if data.len() >= 2
|
|
|
|
|
|
&& data[0] == 0xFF
|
|
|
|
|
|
&& data[1] == 0xD8
|
|
|
|
|
|
&& let Ok(reader) =
|
|
|
|
|
|
image::ImageReader::new(std::io::Cursor::new(&data)).with_guessed_format()
|
|
|
|
|
|
&& let Ok((w, h)) = reader.into_dimensions()
|
|
|
|
|
|
&& w <= max_dim
|
|
|
|
|
|
&& h <= max_dim
|
|
|
|
|
|
{
|
|
|
|
|
|
// Already JPEG at correct size — zero-copy store
|
|
|
|
|
|
return Ok(data.to_vec());
|
2026-03-07 20:12:12 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-03-07 20:37:19 +01:00
|
|
|
|
// ── Slow path: decode, resize, re-encode to JPEG ─────────
|
2026-03-07 18:55:44 +01:00
|
|
|
|
let img = image::load_from_memory(&data)
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(format!("Invalid image data: {e}")))?;
|
|
|
|
|
|
|
|
|
|
|
|
let (w, h) = (img.width(), img.height());
|
|
|
|
|
|
let img = if w > max_dim || h > max_dim {
|
|
|
|
|
|
let filter = FilterType::CatmullRom;
|
|
|
|
|
|
if w > h {
|
|
|
|
|
|
let ratio = max_dim as f32 / w as f32;
|
|
|
|
|
|
img.resize(max_dim, (h as f32 * ratio) as u32, filter)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
let ratio = max_dim as f32 / h as f32;
|
|
|
|
|
|
img.resize((w as f32 * ratio) as u32, max_dim, filter)
|
|
|
|
|
|
}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
img
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-03-07 20:37:19 +01:00
|
|
|
|
let rgb = img.to_rgb8();
|
2026-03-07 18:55:44 +01:00
|
|
|
|
let mut buffer = Vec::new();
|
2026-03-07 20:37:19 +01:00
|
|
|
|
let encoder = JpegEncoder::new_with_quality(&mut buffer, 80);
|
|
|
|
|
|
rgb.write_with_encoder(encoder)
|
2026-03-07 18:55:44 +01:00
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
Ok(buffer)
|
|
|
|
|
|
})
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| ThumbnailError::TaskError(e.to_string()))??;
|
|
|
|
|
|
|
2026-03-07 20:37:19 +01:00
|
|
|
|
let bytes = Bytes::from(jpeg_bytes);
|
2026-03-07 18:55:44 +01:00
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
// External thumbnails are stored by file_id (not dedup-able)
|
|
|
|
|
|
let thumb_path = self
|
|
|
|
|
|
.thumbnails_root
|
|
|
|
|
|
.join(size.dir_name())
|
|
|
|
|
|
.join(format!("ext-{}.jpg", file_id));
|
2026-03-07 18:55:44 +01:00
|
|
|
|
if let Some(parent) = thumb_path.parent() {
|
|
|
|
|
|
let _ = fs::create_dir_all(parent).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
fs::write(&thumb_path, &bytes)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| ThumbnailError::IoError(e.to_string()))?;
|
|
|
|
|
|
|
|
|
|
|
|
// Populate in-memory cache
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.to_string(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
|
|
|
|
|
self.cache.insert(cache_key, bytes.clone()).await;
|
|
|
|
|
|
|
|
|
|
|
|
tracing::info!("✅ Stored external thumbnail: {} {:?}", file_id, size);
|
|
|
|
|
|
Ok(bytes)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-21 15:13:03 +02:00
|
|
|
|
/// Cheap magic-byte check for a JPEG container (SOI + first marker).
|
|
|
|
|
|
fn is_jpeg(data: &[u8]) -> bool {
|
|
|
|
|
|
data.len() >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Decode `data` into a `DynamicImage` sized for a thumbnail whose longest
|
|
|
|
|
|
/// side is `target_long`, with EXIF orientation already applied.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// For JPEG this is **shrink-on-load**: the decoder emits the image at the
|
|
|
|
|
|
/// smallest DCT scale (1/8·1/4·1/2·1/1) whose longest axis is still ≥
|
|
|
|
|
|
/// `target_long`, so a 12 MP photo decodes ~16× fewer pixels for an 800 px
|
|
|
|
|
|
/// thumbnail — and the full-resolution bitmap (the dominant time/RAM cost)
|
|
|
|
|
|
/// is never materialised. PNG/GIF/WebP (no DCT scaling) and unusual JPEG
|
|
|
|
|
|
/// colour spaces (CMYK / 16-bit grey) fall back to a full decode.
|
|
|
|
|
|
fn decode_oriented(
|
2026-04-26 11:55:05 +02:00
|
|
|
|
data: &[u8],
|
2026-06-21 15:13:03 +02:00
|
|
|
|
target_long: u32,
|
|
|
|
|
|
) -> Result<image::DynamicImage, ThumbnailError> {
|
|
|
|
|
|
// JPEG fast path: shrink-on-load. A non-JPEG, or a JPEG colour space we
|
|
|
|
|
|
// don't map (CMYK / 16-bit grey → `None`), falls through to a full decode.
|
|
|
|
|
|
if Self::is_jpeg(data)
|
|
|
|
|
|
&& let Some(img) = Self::decode_jpeg_scaled(data, target_long)?
|
|
|
|
|
|
{
|
|
|
|
|
|
return Ok(Self::apply_exif_orientation(data, img));
|
|
|
|
|
|
}
|
2026-04-26 11:55:05 +02:00
|
|
|
|
|
|
|
|
|
|
let (w, h) = image::ImageReader::new(std::io::Cursor::new(data))
|
|
|
|
|
|
.with_guessed_format()
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?
|
|
|
|
|
|
.into_dimensions()
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
if (w as u64) * (h as u64) > MAX_DECODE_PIXELS {
|
|
|
|
|
|
return Err(ThumbnailError::ImageError(format!(
|
|
|
|
|
|
"Image too large for thumbnail: {w}×{h} ({} MP, max {MAX_DECODE_PIXELS})",
|
|
|
|
|
|
w as u64 * h as u64 / 1_000_000
|
|
|
|
|
|
)));
|
|
|
|
|
|
}
|
|
|
|
|
|
let img =
|
|
|
|
|
|
image::load_from_memory(data).map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
2026-06-21 15:13:03 +02:00
|
|
|
|
Ok(Self::apply_exif_orientation(data, img))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// JPEG shrink-on-load. Returns `Ok(None)` for colour spaces we don't map
|
|
|
|
|
|
/// (CMYK / 16-bit grey), signalling the caller to fall back to a full decode.
|
|
|
|
|
|
fn decode_jpeg_scaled(
|
|
|
|
|
|
data: &[u8],
|
|
|
|
|
|
target_long: u32,
|
|
|
|
|
|
) -> Result<Option<image::DynamicImage>, ThumbnailError> {
|
|
|
|
|
|
let mut decoder = jpeg_decoder::Decoder::new(std::io::Cursor::new(data));
|
|
|
|
|
|
// read_info() first so the dimension guard sees the *original* size.
|
|
|
|
|
|
decoder
|
|
|
|
|
|
.read_info()
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
let info = decoder
|
|
|
|
|
|
.info()
|
|
|
|
|
|
.ok_or_else(|| ThumbnailError::ImageError("missing JPEG metadata".into()))?;
|
|
|
|
|
|
let (orig_w, orig_h) = (info.width as u64, info.height as u64);
|
|
|
|
|
|
if orig_w * orig_h > MAX_DECODE_PIXELS {
|
|
|
|
|
|
return Err(ThumbnailError::ImageError(format!(
|
|
|
|
|
|
"Image too large for thumbnail: {orig_w}×{orig_h} ({} MP, max {MAX_DECODE_PIXELS})",
|
|
|
|
|
|
orig_w * orig_h / 1_000_000
|
|
|
|
|
|
)));
|
|
|
|
|
|
}
|
2026-04-26 11:55:05 +02:00
|
|
|
|
|
2026-06-21 15:13:03 +02:00
|
|
|
|
// Request a `target_long` square box: jpeg-decoder picks the smallest
|
|
|
|
|
|
// scale whose longest axis is still ≥ target_long (its "≥ in at least
|
|
|
|
|
|
// one axis" rule reduces to the long axis since it dominates), so the
|
|
|
|
|
|
// later resample step only ever downscales — never upscales/blurs.
|
|
|
|
|
|
let req = target_long.min(u16::MAX as u32) as u16;
|
|
|
|
|
|
let (sw, sh) = decoder
|
|
|
|
|
|
.scale(req, req)
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
let pixels = decoder
|
|
|
|
|
|
.decode()
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
|
|
|
|
|
|
let (sw, sh) = (sw as u32, sh as u32);
|
|
|
|
|
|
let img = match decoder.info().map(|i| i.pixel_format) {
|
|
|
|
|
|
Some(jpeg_decoder::PixelFormat::RGB24) => {
|
|
|
|
|
|
image::RgbImage::from_raw(sw, sh, pixels).map(image::DynamicImage::ImageRgb8)
|
|
|
|
|
|
}
|
|
|
|
|
|
Some(jpeg_decoder::PixelFormat::L8) => {
|
|
|
|
|
|
image::GrayImage::from_raw(sw, sh, pixels).map(image::DynamicImage::ImageLuma8)
|
|
|
|
|
|
}
|
|
|
|
|
|
_ => None,
|
2026-04-26 11:55:05 +02:00
|
|
|
|
};
|
2026-06-21 15:13:03 +02:00
|
|
|
|
Ok(img)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Read EXIF orientation from the original bytes and rotate/flip the image.
|
|
|
|
|
|
/// (Applied after shrink-on-load, so the rotation works on the small bitmap.)
|
|
|
|
|
|
fn apply_exif_orientation(data: &[u8], img: image::DynamicImage) -> image::DynamicImage {
|
|
|
|
|
|
use crate::infrastructure::services::exif_service::{ExifService, apply_orientation};
|
|
|
|
|
|
let orientation = ExifService::extract(data)
|
|
|
|
|
|
.and_then(|m| m.orientation)
|
|
|
|
|
|
.unwrap_or(1);
|
|
|
|
|
|
apply_orientation(img, orientation)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn render_thumbnail_from_data(
|
|
|
|
|
|
data: &[u8],
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
) -> Result<Vec<u8>, ThumbnailError> {
|
|
|
|
|
|
let max_dim = size.max_dimension();
|
|
|
|
|
|
let img = Self::decode_oriented(data, max_dim)?;
|
2026-04-26 11:55:05 +02:00
|
|
|
|
|
|
|
|
|
|
let (orig_width, orig_height) = (img.width(), img.height());
|
|
|
|
|
|
let (new_width, new_height) = if orig_width > orig_height {
|
|
|
|
|
|
let ratio = max_dim as f32 / orig_width as f32;
|
|
|
|
|
|
(max_dim, (orig_height as f32 * ratio) as u32)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
let ratio = max_dim as f32 / orig_height as f32;
|
|
|
|
|
|
((orig_width as f32 * ratio) as u32, max_dim)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let filter = match size {
|
|
|
|
|
|
ThumbnailSize::Icon => FilterType::Triangle,
|
|
|
|
|
|
ThumbnailSize::Preview => FilterType::CatmullRom,
|
|
|
|
|
|
ThumbnailSize::Large => FilterType::CatmullRom,
|
|
|
|
|
|
};
|
|
|
|
|
|
let thumbnail = img.resize(new_width, new_height, filter);
|
|
|
|
|
|
|
|
|
|
|
|
let rgb = thumbnail.to_rgb8();
|
|
|
|
|
|
let mut buffer = Vec::new();
|
|
|
|
|
|
let encoder = JpegEncoder::new_with_quality(&mut buffer, 80);
|
|
|
|
|
|
rgb.write_with_encoder(encoder)
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
|
|
|
|
|
|
Ok(buffer)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn render_all_thumbnails_from_data(
|
|
|
|
|
|
data: &[u8],
|
|
|
|
|
|
) -> Result<Vec<(ThumbnailSize, Bytes)>, ThumbnailError> {
|
2026-06-21 15:13:03 +02:00
|
|
|
|
// Decode once, shrunk-on-load for the largest size (800 px); all three
|
|
|
|
|
|
// sizes are then resampled from this single shared bitmap. Sizing the
|
|
|
|
|
|
// shared decode to Large keeps quality for every size while paying the
|
|
|
|
|
|
// (now much smaller) decode cost only once.
|
|
|
|
|
|
let img = Self::decode_oriented(data, ThumbnailSize::Large.max_dimension())?;
|
2026-04-26 11:55:05 +02:00
|
|
|
|
let (orig_w, orig_h) = (img.width(), img.height());
|
|
|
|
|
|
|
|
|
|
|
|
ThumbnailSize::all()
|
|
|
|
|
|
.par_iter()
|
|
|
|
|
|
.map(|&size| {
|
|
|
|
|
|
let max_dim = size.max_dimension();
|
|
|
|
|
|
|
|
|
|
|
|
let (new_w, new_h) = if orig_w > orig_h {
|
|
|
|
|
|
let ratio = max_dim as f32 / orig_w as f32;
|
|
|
|
|
|
(max_dim, (orig_h as f32 * ratio) as u32)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
let ratio = max_dim as f32 / orig_h as f32;
|
|
|
|
|
|
((orig_w as f32 * ratio) as u32, max_dim)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let filter = match size {
|
|
|
|
|
|
ThumbnailSize::Icon => FilterType::Triangle,
|
|
|
|
|
|
ThumbnailSize::Preview => FilterType::CatmullRom,
|
|
|
|
|
|
ThumbnailSize::Large => FilterType::CatmullRom,
|
|
|
|
|
|
};
|
|
|
|
|
|
let thumb = img.resize(new_w, new_h, filter);
|
|
|
|
|
|
|
|
|
|
|
|
let rgb = thumb.to_rgb8();
|
|
|
|
|
|
let mut buf = Vec::new();
|
|
|
|
|
|
let encoder = JpegEncoder::new_with_quality(&mut buf, 80);
|
|
|
|
|
|
rgb.write_with_encoder(encoder)
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
|
|
|
|
|
|
Ok((size, Bytes::from(buf)))
|
|
|
|
|
|
})
|
|
|
|
|
|
.collect::<Result<Vec<_>, ThumbnailError>>()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn generate_thumbnail_from_data(
|
|
|
|
|
|
original_data: Bytes,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
timeout_duration: Duration,
|
|
|
|
|
|
) -> Result<Bytes, ThumbnailError> {
|
|
|
|
|
|
let spawn_result = tokio::task::spawn_blocking(move || {
|
|
|
|
|
|
Self::render_thumbnail_from_data(original_data.as_ref(), size)
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
let result = timeout(timeout_duration, spawn_result)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|_| {
|
|
|
|
|
|
ThumbnailError::TaskError(format!(
|
|
|
|
|
|
"Thumbnail generation timed out after {:?}",
|
|
|
|
|
|
timeout_duration
|
|
|
|
|
|
))
|
|
|
|
|
|
})?
|
|
|
|
|
|
.map_err(|e| ThumbnailError::TaskError(e.to_string()))?;
|
|
|
|
|
|
|
|
|
|
|
|
result.map(Bytes::from)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-24 16:11:52 +01:00
|
|
|
|
/// Generate a thumbnail from an image file.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Concurrency is bounded by `decode_semaphore` to prevent OOM when
|
2026-03-18 11:51:39 +08:00
|
|
|
|
/// many images are uploaded simultaneously. Resolution is also
|
2026-02-24 16:11:52 +01:00
|
|
|
|
/// capped at `MAX_DECODE_PIXELS` to reject pathologically large images.
|
2026-03-18 11:51:39 +08:00
|
|
|
|
/// After decoding, the encoded image buffer is explicitly dropped before
|
|
|
|
|
|
/// processing to minimize peak memory usage.
|
2026-03-28 18:40:34 +00:00
|
|
|
|
/// A timeout prevents the operation from hanging indefinitely on large images.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
async fn generate_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
original_path: &Path,
|
|
|
|
|
|
size: ThumbnailSize,
|
|
|
|
|
|
) -> Result<Bytes, ThumbnailError> {
|
|
|
|
|
|
let path = original_path.to_path_buf();
|
2026-03-28 18:40:34 +00:00
|
|
|
|
let timeout_duration = self.generation_timeout;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-02-24 16:11:52 +01:00
|
|
|
|
// Acquire semaphore permit — bounds peak RAM from concurrent decodes
|
2026-02-26 00:47:32 +01:00
|
|
|
|
let _permit = self
|
|
|
|
|
|
.decode_semaphore
|
|
|
|
|
|
.acquire()
|
|
|
|
|
|
.await
|
2026-02-24 16:11:52 +01:00
|
|
|
|
.map_err(|_| ThumbnailError::TaskError("Decode semaphore closed".into()))?;
|
|
|
|
|
|
|
2026-03-28 18:40:34 +00:00
|
|
|
|
// Run image processing in blocking thread pool with timeout
|
|
|
|
|
|
let spawn_result =
|
|
|
|
|
|
tokio::task::spawn_blocking(move || -> Result<Vec<u8>, ThumbnailError> {
|
|
|
|
|
|
let data =
|
|
|
|
|
|
std::fs::read(&path).map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
2026-04-26 11:55:05 +02:00
|
|
|
|
Self::render_thumbnail_from_data(&data, size)
|
2026-03-28 18:40:34 +00:00
|
|
|
|
});
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
2026-03-28 18:40:34 +00:00
|
|
|
|
// Apply timeout to prevent hanging on large images
|
|
|
|
|
|
let result = timeout(timeout_duration, spawn_result)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|_| {
|
|
|
|
|
|
ThumbnailError::TaskError(format!(
|
|
|
|
|
|
"Thumbnail generation timed out after {:?}",
|
|
|
|
|
|
timeout_duration
|
|
|
|
|
|
))
|
|
|
|
|
|
})?
|
|
|
|
|
|
.map_err(|e| ThumbnailError::TaskError(e.to_string()))?;
|
2026-02-14 01:29:34 +01:00
|
|
|
|
|
|
|
|
|
|
result.map(Bytes::from)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-24 12:41:07 +01:00
|
|
|
|
/// Generate all thumbnail sizes for a file in the background.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
///
|
2026-04-12 00:50:10 +02:00
|
|
|
|
/// Thumbnails are stored on disk keyed by `blob_hash`, so duplicate
|
|
|
|
|
|
/// uploads with the same content share a single set of thumbnails.
|
|
|
|
|
|
/// If thumbnails already exist for this `blob_hash`, only the moka
|
|
|
|
|
|
/// cache is populated (zero CPU for image processing).
|
|
|
|
|
|
///
|
2026-02-24 12:41:07 +01:00
|
|
|
|
/// Loads the image **once** and produces all 3 sizes (Icon, Preview,
|
2026-03-18 11:51:39 +08:00
|
|
|
|
/// Large) inside a single `spawn_blocking` call. This avoids 3×
|
2026-02-24 12:41:07 +01:00
|
|
|
|
/// I/O reads and 3× JPEG/PNG decode — reducing CPU time by ~45%
|
|
|
|
|
|
/// and peak RAM from ~540 MB to ~180 MB for concurrent uploads.
|
2026-03-18 11:51:39 +08:00
|
|
|
|
/// The encoded image buffer is explicitly dropped after decoding
|
|
|
|
|
|
/// to further reduce peak memory by the size of the original file.
|
2026-04-12 00:50:10 +02:00
|
|
|
|
pub fn generate_all_sizes_background(
|
|
|
|
|
|
self: Arc<Self>,
|
|
|
|
|
|
file_id: String,
|
|
|
|
|
|
blob_hash: String,
|
|
|
|
|
|
original_path: PathBuf,
|
|
|
|
|
|
) {
|
2026-02-14 01:29:34 +01:00
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
|
tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id);
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
// ── Fast path: blob-hash thumbnails already exist on disk ────
|
|
|
|
|
|
// If another file with the same content was already uploaded,
|
|
|
|
|
|
// the thumbnails exist. Just populate the moka cache for this
|
|
|
|
|
|
// file_id and skip image processing entirely.
|
|
|
|
|
|
let all_exist = {
|
|
|
|
|
|
let mut ok = true;
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(&blob_hash, *size);
|
|
|
|
|
|
if fs::metadata(&thumb_path).await.is_err() {
|
|
|
|
|
|
ok = false;
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
ok
|
|
|
|
|
|
};
|
|
|
|
|
|
if all_exist {
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(&blob_hash, *size);
|
|
|
|
|
|
if let Ok(data) = fs::read(&thumb_path).await {
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.clone(),
|
|
|
|
|
|
size: *size,
|
|
|
|
|
|
};
|
|
|
|
|
|
self.cache.insert(cache_key, Bytes::from(data)).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
tracing::info!(
|
|
|
|
|
|
"🖼️ Thumbnail dedup hit for {} (blob {}): skipped generation",
|
|
|
|
|
|
file_id,
|
|
|
|
|
|
&blob_hash[..12]
|
|
|
|
|
|
);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-24 16:11:52 +01:00
|
|
|
|
// Acquire semaphore permit — bounds peak RAM from concurrent decodes
|
|
|
|
|
|
let _permit = match self.decode_semaphore.acquire().await {
|
|
|
|
|
|
Ok(p) => p,
|
|
|
|
|
|
Err(_) => {
|
2026-02-26 00:47:32 +01:00
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Decode semaphore closed, skipping thumbnails for {}",
|
|
|
|
|
|
file_id
|
|
|
|
|
|
);
|
2026-02-24 16:11:52 +01:00
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-02-24 12:41:07 +01:00
|
|
|
|
let path = original_path.clone();
|
|
|
|
|
|
|
|
|
|
|
|
// Single spawn_blocking: 1 read + 1 decode + 3 resize + 3 encode
|
|
|
|
|
|
let results = tokio::task::spawn_blocking(move || {
|
2026-02-26 00:32:02 +01:00
|
|
|
|
// Single read: load file once into memory
|
2026-02-26 00:47:32 +01:00
|
|
|
|
let data =
|
|
|
|
|
|
std::fs::read(&path).map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
2026-02-26 00:32:02 +01:00
|
|
|
|
|
|
|
|
|
|
// Safety check: read dimensions from in-memory buffer (no 2nd I/O)
|
|
|
|
|
|
let (w, h) = image::ImageReader::new(std::io::Cursor::new(&data))
|
|
|
|
|
|
.with_guessed_format()
|
2026-02-24 16:11:52 +01:00
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?
|
|
|
|
|
|
.into_dimensions()
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
if (w as u64) * (h as u64) > MAX_DECODE_PIXELS {
|
|
|
|
|
|
return Err(ThumbnailError::ImageError(format!(
|
|
|
|
|
|
"Image too large for thumbnail: {w}×{h} ({} MP, max {MAX_DECODE_PIXELS})",
|
|
|
|
|
|
w as u64 * h as u64 / 1_000_000
|
|
|
|
|
|
)));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-26 00:32:02 +01:00
|
|
|
|
// Full decode from the same in-memory buffer (no 2nd disk read)
|
|
|
|
|
|
let img = image::load_from_memory(&data)
|
|
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
2026-02-24 12:41:07 +01:00
|
|
|
|
|
2026-03-05 12:48:47 -05:00
|
|
|
|
// Apply EXIF orientation so thumbnails display correctly
|
|
|
|
|
|
let img = {
|
2026-03-05 14:46:30 -05:00
|
|
|
|
use crate::infrastructure::services::exif_service::{
|
|
|
|
|
|
ExifService, apply_orientation,
|
|
|
|
|
|
};
|
2026-03-05 12:48:47 -05:00
|
|
|
|
let orientation = ExifService::extract(&data)
|
|
|
|
|
|
.and_then(|m| m.orientation)
|
|
|
|
|
|
.unwrap_or(1);
|
2026-03-18 11:51:39 +08:00
|
|
|
|
// Free the encoded image data now that image is decoded and EXIF extracted
|
|
|
|
|
|
drop(data);
|
2026-03-05 12:48:47 -05:00
|
|
|
|
apply_orientation(img, orientation)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-02-24 12:41:07 +01:00
|
|
|
|
let (orig_w, orig_h) = (img.width(), img.height());
|
|
|
|
|
|
|
|
|
|
|
|
ThumbnailSize::all()
|
2026-04-11 19:13:48 +02:00
|
|
|
|
.par_iter()
|
2026-02-24 12:41:07 +01:00
|
|
|
|
.map(|&size| {
|
|
|
|
|
|
let max_dim = size.max_dimension();
|
|
|
|
|
|
|
|
|
|
|
|
let (new_w, new_h) = if orig_w > orig_h {
|
|
|
|
|
|
let ratio = max_dim as f32 / orig_w as f32;
|
|
|
|
|
|
(max_dim, (orig_h as f32 * ratio) as u32)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
} else {
|
2026-02-24 12:41:07 +01:00
|
|
|
|
let ratio = max_dim as f32 / orig_h as f32;
|
|
|
|
|
|
((orig_w as f32 * ratio) as u32, max_dim)
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
let filter = match size {
|
2026-02-26 00:47:32 +01:00
|
|
|
|
ThumbnailSize::Icon => FilterType::Triangle,
|
2026-02-24 12:41:07 +01:00
|
|
|
|
ThumbnailSize::Preview => FilterType::CatmullRom,
|
2026-02-26 00:47:32 +01:00
|
|
|
|
ThumbnailSize::Large => FilterType::CatmullRom,
|
2026-02-24 12:41:07 +01:00
|
|
|
|
};
|
|
|
|
|
|
let thumb = img.resize(new_w, new_h, filter);
|
|
|
|
|
|
|
2026-03-07 20:37:19 +01:00
|
|
|
|
let rgb = thumb.to_rgb8();
|
2026-02-24 12:41:07 +01:00
|
|
|
|
let mut buf = Vec::new();
|
2026-03-07 20:37:19 +01:00
|
|
|
|
let encoder = JpegEncoder::new_with_quality(&mut buf, 80);
|
|
|
|
|
|
rgb.write_with_encoder(encoder)
|
2026-02-24 12:41:07 +01:00
|
|
|
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
|
|
|
|
|
|
|
|
|
|
|
Ok((size, Bytes::from(buf)))
|
|
|
|
|
|
})
|
|
|
|
|
|
.collect::<Result<Vec<_>, ThumbnailError>>()
|
|
|
|
|
|
})
|
|
|
|
|
|
.await;
|
|
|
|
|
|
|
|
|
|
|
|
// Flatten JoinError + inner ThumbnailError
|
|
|
|
|
|
let thumbnails = match results {
|
|
|
|
|
|
Ok(Ok(t)) => t,
|
|
|
|
|
|
Ok(Err(e)) => {
|
2026-02-26 00:47:32 +01:00
|
|
|
|
tracing::warn!("Thumbnail generation failed for {}: {}", file_id, e);
|
2026-02-24 12:41:07 +01:00
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => {
|
2026-02-26 00:47:32 +01:00
|
|
|
|
tracing::warn!("Thumbnail task panicked for {}: {}", file_id, e);
|
2026-02-24 12:41:07 +01:00
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
// Save each size to disk (keyed by blob_hash for dedup)
|
|
|
|
|
|
// AND populate moka (keyed by file_id for fast serving).
|
2026-02-24 12:41:07 +01:00
|
|
|
|
for (size, bytes) in thumbnails {
|
2026-04-12 00:50:10 +02:00
|
|
|
|
let thumb_path = self.get_thumbnail_path(&blob_hash, size);
|
2026-02-24 12:41:07 +01:00
|
|
|
|
if let Some(parent) = thumb_path.parent() {
|
|
|
|
|
|
let _ = fs::create_dir_all(parent).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Err(e) = fs::write(&thumb_path, &bytes).await {
|
|
|
|
|
|
tracing::warn!("Failed to save thumbnail {} {:?}: {}", file_id, size, e);
|
|
|
|
|
|
} else {
|
2026-03-07 19:48:35 +01:00
|
|
|
|
// Populate in-memory cache for instant first-hit serving
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.clone(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
|
|
|
|
|
self.cache.insert(cache_key, bytes).await;
|
2026-02-24 12:41:07 +01:00
|
|
|
|
tracing::debug!("✅ Generated thumbnail: {} {:?}", file_id, size);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
tracing::info!("✅ Background thumbnail generation complete: {}", file_id);
|
|
|
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-10 13:36:29 +00:00
|
|
|
|
/// Generate all thumbnail sizes in the background for a content-addressed
|
|
|
|
|
|
/// blob (CDC/manifest-safe — no physical source file required).
|
2026-04-26 11:55:05 +02:00
|
|
|
|
///
|
2026-06-10 13:36:29 +00:00
|
|
|
|
/// The source blob is read **after** the decode permit is acquired, so N
|
|
|
|
|
|
/// concurrent uploads queue as N small tasks, not N full images in RAM:
|
|
|
|
|
|
/// peak memory is `permits × image size` regardless of upload concurrency.
|
|
|
|
|
|
pub fn generate_all_sizes_background_from_blob(
|
2026-04-26 11:55:05 +02:00
|
|
|
|
self: Arc<Self>,
|
|
|
|
|
|
file_id: String,
|
|
|
|
|
|
blob_hash: String,
|
2026-05-13 11:33:45 +02:00
|
|
|
|
dedup: Arc<DedupService>,
|
2026-04-26 11:55:05 +02:00
|
|
|
|
) {
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
|
tracing::info!("🖼️ Background thumbnail generation starting: {}", file_id);
|
|
|
|
|
|
|
2026-05-13 11:33:45 +02:00
|
|
|
|
// Guard: if the blob was deleted before this task ran, cleanup_if_orphaned
|
|
|
|
|
|
// already fired with no thumbnails on disk — writing them now would leak them.
|
|
|
|
|
|
// Use the DB check (manifest + blobs tables) as the authoritative source.
|
|
|
|
|
|
if !dedup.blob_exists(&blob_hash).await {
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"Blob {}… deleted before thumbnail task ran, skipping",
|
|
|
|
|
|
&blob_hash[..blob_hash.len().min(12)]
|
|
|
|
|
|
);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-26 11:55:05 +02:00
|
|
|
|
let all_exist = {
|
|
|
|
|
|
let mut ok = true;
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(&blob_hash, *size);
|
|
|
|
|
|
if fs::metadata(&thumb_path).await.is_err() {
|
|
|
|
|
|
ok = false;
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
ok
|
|
|
|
|
|
};
|
|
|
|
|
|
if all_exist {
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(&blob_hash, *size);
|
|
|
|
|
|
if let Ok(data) = fs::read(&thumb_path).await {
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.clone(),
|
|
|
|
|
|
size: *size,
|
|
|
|
|
|
};
|
|
|
|
|
|
self.cache.insert(cache_key, Bytes::from(data)).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
tracing::info!(
|
|
|
|
|
|
"🖼️ Thumbnail dedup hit for {} (blob {}): skipped generation",
|
|
|
|
|
|
file_id,
|
|
|
|
|
|
&blob_hash[..12]
|
|
|
|
|
|
);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
let _permit = match self.decode_semaphore.acquire().await {
|
|
|
|
|
|
Ok(p) => p,
|
|
|
|
|
|
Err(_) => {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Decode semaphore closed, skipping thumbnails for {}",
|
|
|
|
|
|
file_id
|
|
|
|
|
|
);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-06-10 13:36:29 +00:00
|
|
|
|
// Read the source only now that a permit bounds how many of
|
|
|
|
|
|
// these full-image buffers can exist at once.
|
|
|
|
|
|
let original_data = match dedup.read_blob_bytes(&blob_hash).await {
|
|
|
|
|
|
Ok(bytes) => bytes,
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Failed to read blob for thumbnail generation {}: {}",
|
|
|
|
|
|
file_id,
|
|
|
|
|
|
e
|
|
|
|
|
|
);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
|
2026-04-26 11:55:05 +02:00
|
|
|
|
let results = tokio::task::spawn_blocking(move || {
|
|
|
|
|
|
Self::render_all_thumbnails_from_data(original_data.as_ref())
|
|
|
|
|
|
})
|
|
|
|
|
|
.await;
|
|
|
|
|
|
|
|
|
|
|
|
let thumbnails = match results {
|
|
|
|
|
|
Ok(Ok(t)) => t,
|
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
|
tracing::warn!("Thumbnail generation failed for {}: {}", file_id, e);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!("Thumbnail task panicked for {}: {}", file_id, e);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
for (size, bytes) in thumbnails {
|
|
|
|
|
|
let thumb_path = self.get_thumbnail_path(&blob_hash, size);
|
|
|
|
|
|
if let Some(parent) = thumb_path.parent() {
|
|
|
|
|
|
let _ = fs::create_dir_all(parent).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Err(e) = fs::write(&thumb_path, &bytes).await {
|
|
|
|
|
|
tracing::warn!("Failed to save thumbnail {} {:?}: {}", file_id, size, e);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.clone(),
|
|
|
|
|
|
size,
|
|
|
|
|
|
};
|
|
|
|
|
|
self.cache.insert(cache_key, bytes).await;
|
|
|
|
|
|
tracing::debug!("✅ Generated thumbnail: {} {:?}", file_id, size);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
tracing::info!("✅ Background thumbnail generation complete: {}", file_id);
|
|
|
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
/// Delete thumbnails for a file.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Only invalidates the in-memory moka cache (keyed by file_id).
|
|
|
|
|
|
/// Disk thumbnails are keyed by blob_hash and may be shared by
|
|
|
|
|
|
/// other files with the same content — they are cleaned up via
|
|
|
|
|
|
/// `delete_blob_thumbnails` when the blob is garbage-collected.
|
|
|
|
|
|
/// Also removes any external (video-frame) thumbnails stored by file_id.
|
2026-02-14 01:29:34 +01:00
|
|
|
|
pub async fn delete_thumbnails(&self, file_id: &str) -> Result<(), ThumbnailError> {
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
2026-04-12 00:50:10 +02:00
|
|
|
|
// Remove from moka cache (lock-free invalidation)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
let cache_key = ThumbnailCacheKey {
|
|
|
|
|
|
file_id: file_id.to_string(),
|
|
|
|
|
|
size: *size,
|
|
|
|
|
|
};
|
2026-02-22 23:28:03 +01:00
|
|
|
|
self.cache.invalidate(&cache_key).await;
|
2026-04-12 00:50:10 +02:00
|
|
|
|
|
|
|
|
|
|
// Remove external (video-frame) thumbnails stored by file_id
|
|
|
|
|
|
let ext_path = self
|
|
|
|
|
|
.thumbnails_root
|
|
|
|
|
|
.join(size.dir_name())
|
|
|
|
|
|
.join(format!("ext-{}.jpg", file_id));
|
|
|
|
|
|
if fs::metadata(&ext_path).await.is_ok() {
|
|
|
|
|
|
let _ = fs::remove_file(&ext_path).await;
|
|
|
|
|
|
}
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
tracing::debug!("🗑️ Invalidated thumbnail cache for: {}", file_id);
|
2026-02-14 01:29:34 +01:00
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
/// Remove orphaned blob-hash thumbnails whose blob no longer exists.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Call during blob garbage collection: pass the hash of the blob
|
|
|
|
|
|
/// being deleted and the corresponding thumbnails are removed from disk.
|
|
|
|
|
|
pub async fn delete_blob_thumbnails(&self, blob_hash: &str) {
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let path = self.get_thumbnail_path(blob_hash, *size);
|
|
|
|
|
|
if fs::metadata(&path).await.is_ok() {
|
|
|
|
|
|
let _ = fs::remove_file(&path).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"🗑️ Deleted blob thumbnails for hash: {}…",
|
|
|
|
|
|
&blob_hash[..blob_hash.len().min(12)]
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// Get cache statistics
|
|
|
|
|
|
pub async fn get_stats(&self) -> ThumbnailStats {
|
|
|
|
|
|
ThumbnailStats {
|
2026-02-22 23:28:03 +01:00
|
|
|
|
cached_thumbnails: self.cache.entry_count() as usize,
|
|
|
|
|
|
cache_size_bytes: self.cache.weighted_size() as usize,
|
|
|
|
|
|
max_cache_bytes: self.max_cache_bytes as usize,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-05-22 13:10:47 +02:00
|
|
|
|
// ─── FileLifecycleHook + BlobLifecycleHook ───────────────────────────────────
|
2026-05-14 00:03:03 +02:00
|
|
|
|
|
2026-05-22 13:10:47 +02:00
|
|
|
|
/// Wires all thumbnail side-effects into the file and blob lifecycle.
|
2026-05-14 00:03:03 +02:00
|
|
|
|
///
|
2026-05-22 13:10:47 +02:00
|
|
|
|
/// Registered once on both [`FileLifecycleService`] and [`BlobLifecycleService`]
|
|
|
|
|
|
/// during DI. Handles thumbnail generation, invalidation, and cleanup.
|
2026-05-14 00:03:03 +02:00
|
|
|
|
pub struct ThumbnailRefreshHook {
|
|
|
|
|
|
thumbnail: Arc<ThumbnailService>,
|
|
|
|
|
|
dedup: Arc<DedupService>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl ThumbnailRefreshHook {
|
|
|
|
|
|
pub fn new(thumbnail: Arc<ThumbnailService>, dedup: Arc<DedupService>) -> Self {
|
|
|
|
|
|
Self { thumbnail, dedup }
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-05-22 13:10:47 +02:00
|
|
|
|
impl crate::application::ports::file_lifecycle::FileLifecycleHook for ThumbnailRefreshHook {
|
|
|
|
|
|
fn on_file_created(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
blob_hash: &str,
|
|
|
|
|
|
content_type: &str,
|
|
|
|
|
|
is_new_blob: bool,
|
|
|
|
|
|
) {
|
|
|
|
|
|
// Blob-hash thumbnail already exists on disk when is_new_blob=false — skip.
|
|
|
|
|
|
if !is_new_blob || !ThumbnailService::is_supported_image(content_type) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
2026-06-10 13:36:29 +00:00
|
|
|
|
self.thumbnail
|
|
|
|
|
|
.clone()
|
|
|
|
|
|
.generate_all_sizes_background_from_blob(
|
|
|
|
|
|
file_id.to_string(),
|
|
|
|
|
|
blob_hash.to_string(),
|
|
|
|
|
|
self.dedup.clone(),
|
|
|
|
|
|
);
|
2026-05-22 13:10:47 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn on_file_copied(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
_file_id: &str,
|
|
|
|
|
|
_blob_hash: &str,
|
|
|
|
|
|
_content_type: &str,
|
|
|
|
|
|
_source_file_id: &str,
|
|
|
|
|
|
) {
|
|
|
|
|
|
// Thumbnails are keyed by blob_hash on disk — the copy shares them automatically.
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn on_file_updated(&self, file_id: &str, blob_hash: &str, content_type: &str) {
|
|
|
|
|
|
if !ThumbnailService::is_supported_image(content_type) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let thumbnail = self.thumbnail.clone();
|
|
|
|
|
|
let file_id = file_id.to_string();
|
|
|
|
|
|
let blob_hash = blob_hash.to_string();
|
|
|
|
|
|
let dedup = self.dedup.clone();
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
|
if let Err(e) = thumbnail.delete_thumbnails(&file_id).await {
|
2026-05-14 00:03:03 +02:00
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
"Failed to invalidate thumbnail cache for {}: {}",
|
|
|
|
|
|
file_id,
|
|
|
|
|
|
e
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
2026-06-10 13:36:29 +00:00
|
|
|
|
thumbnail.generate_all_sizes_background_from_blob(file_id, blob_hash, dedup);
|
2026-05-22 13:10:47 +02:00
|
|
|
|
});
|
2026-05-14 00:03:03 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-05-22 13:10:47 +02:00
|
|
|
|
fn on_file_deleted(&self, file_id: &str) {
|
|
|
|
|
|
let thumbnail = self.thumbnail.clone();
|
|
|
|
|
|
let file_id = file_id.to_string();
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
|
if let Err(e) = thumbnail.delete_thumbnails(&file_id).await {
|
|
|
|
|
|
tracing::warn!("Failed to delete thumbnails for file {}: {}", file_id, e);
|
2026-05-14 00:03:03 +02:00
|
|
|
|
}
|
2026-05-22 13:10:47 +02:00
|
|
|
|
});
|
2026-05-14 00:03:03 +02:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-05-22 13:10:47 +02:00
|
|
|
|
// BlobLifecycleHook is implemented on ThumbnailService (not ThumbnailRefreshHook)
|
|
|
|
|
|
// to avoid a circular Arc: DedupService→BlobLifecycleService→ThumbnailRefreshHook→DedupService.
|
|
|
|
|
|
// ThumbnailService does not hold DedupService so no cycle exists.
|
|
|
|
|
|
|
|
|
|
|
|
// ─── BlobLifecycleHook ───────────────────────────────────────────────────────
|
2026-05-14 00:03:03 +02:00
|
|
|
|
|
2026-05-22 13:10:47 +02:00
|
|
|
|
impl crate::application::ports::blob_lifecycle::BlobLifecycleHook for ThumbnailService {
|
|
|
|
|
|
fn on_blob_created(&self, _blob_hash: &str, _content_type: Option<&str>) {
|
|
|
|
|
|
// Thumbnail generation is driven by file-level events (on_file_created).
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn on_blob_deleted(&self, blob_hash: &str) {
|
|
|
|
|
|
// delete_blob_thumbnails only needs thumbnails_root — capture it to avoid Arc cycle.
|
|
|
|
|
|
let root = self.thumbnails_root.clone();
|
|
|
|
|
|
let blob_hash = blob_hash.to_string();
|
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
|
for size in ThumbnailSize::all() {
|
|
|
|
|
|
let path = root
|
|
|
|
|
|
.join(size.dir_name())
|
|
|
|
|
|
.join(format!("{}.jpg", &blob_hash));
|
|
|
|
|
|
if tokio::fs::metadata(&path).await.is_ok() {
|
|
|
|
|
|
let _ = tokio::fs::remove_file(&path).await;
|
|
|
|
|
|
}
|
2026-05-14 00:03:03 +02:00
|
|
|
|
}
|
2026-05-22 13:10:47 +02:00
|
|
|
|
tracing::debug!(
|
|
|
|
|
|
"🗑️ Deleted blob thumbnails for hash: {}…",
|
|
|
|
|
|
&blob_hash[..blob_hash.len().min(12)]
|
|
|
|
|
|
);
|
|
|
|
|
|
});
|
2026-05-14 00:03:03 +02:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
// ─── Port implementation ─────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
/// Convert port ThumbnailSize to infra ThumbnailSize.
|
|
|
|
|
|
impl From<PortThumbnailSize> for ThumbnailSize {
|
|
|
|
|
|
fn from(size: PortThumbnailSize) -> Self {
|
|
|
|
|
|
match size {
|
|
|
|
|
|
PortThumbnailSize::Icon => ThumbnailSize::Icon,
|
|
|
|
|
|
PortThumbnailSize::Preview => ThumbnailSize::Preview,
|
|
|
|
|
|
PortThumbnailSize::Large => ThumbnailSize::Large,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl ThumbnailPort for ThumbnailService {
|
|
|
|
|
|
fn is_supported_image(&self, mime_type: &str) -> bool {
|
|
|
|
|
|
ThumbnailService::is_supported_image(mime_type)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn get_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
2026-04-12 00:50:10 +02:00
|
|
|
|
blob_hash: &str,
|
2026-02-14 01:29:34 +01:00
|
|
|
|
size: PortThumbnailSize,
|
|
|
|
|
|
original_path: &Path,
|
|
|
|
|
|
) -> Result<Bytes, DomainError> {
|
2026-04-12 00:50:10 +02:00
|
|
|
|
self.get_thumbnail(file_id, blob_hash, size.into(), original_path)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
fn generate_all_sizes_background(
|
|
|
|
|
|
self: Arc<Self>,
|
|
|
|
|
|
file_id: String,
|
|
|
|
|
|
blob_hash: String,
|
|
|
|
|
|
original_path: PathBuf,
|
|
|
|
|
|
) {
|
|
|
|
|
|
ThumbnailService::generate_all_sizes_background(self, file_id, blob_hash, original_path)
|
2026-02-14 01:29:34 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn delete_thumbnails(&self, file_id: &str) -> Result<(), DomainError> {
|
|
|
|
|
|
self.delete_thumbnails(file_id)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-12 00:50:10 +02:00
|
|
|
|
async fn get_cached_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
blob_hash: Option<&str>,
|
|
|
|
|
|
size: PortThumbnailSize,
|
|
|
|
|
|
) -> Option<Bytes> {
|
|
|
|
|
|
self.get_cached_thumbnail(file_id, blob_hash, size.into())
|
|
|
|
|
|
.await
|
2026-03-07 18:55:44 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn store_external_thumbnail(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
file_id: &str,
|
|
|
|
|
|
size: PortThumbnailSize,
|
|
|
|
|
|
data: Bytes,
|
|
|
|
|
|
) -> Result<Bytes, DomainError> {
|
|
|
|
|
|
self.store_external_thumbnail(file_id, size.into(), data)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
async fn get_stats(&self) -> ThumbnailStatsDto {
|
|
|
|
|
|
let stats = self.get_stats().await;
|
|
|
|
|
|
ThumbnailStatsDto {
|
|
|
|
|
|
cached_thumbnails: stats.cached_thumbnails,
|
|
|
|
|
|
cache_size_bytes: stats.cache_size_bytes,
|
|
|
|
|
|
max_cache_bytes: stats.max_cache_bytes,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-21 15:13:03 +02:00
|
|
|
|
/// Benchmark-only public surface (Phase 0 perf harness).
|
|
|
|
|
|
///
|
|
|
|
|
|
/// The real render functions are private (`render_thumbnail_from_data` /
|
|
|
|
|
|
/// `render_all_thumbnails_from_data`). Benches and examples are separate crate
|
|
|
|
|
|
/// targets and can only see `pub` items, so these thin wrappers — gated behind
|
|
|
|
|
|
/// the `bench` feature so they never exist in production builds — expose the
|
|
|
|
|
|
/// exact CPU-bound work (decode → orientation → resize → JPEG encode) for
|
|
|
|
|
|
/// before/after measurement. Errors are flattened to `String` to avoid leaking
|
|
|
|
|
|
/// `ThumbnailError` into the public API.
|
|
|
|
|
|
#[cfg(feature = "bench")]
|
|
|
|
|
|
impl ThumbnailService {
|
|
|
|
|
|
/// Render a single thumbnail size, returning the encoded JPEG bytes.
|
|
|
|
|
|
pub fn bench_render_thumbnail(data: &[u8], size: ThumbnailSize) -> Result<Vec<u8>, String> {
|
|
|
|
|
|
Self::render_thumbnail_from_data(data, size).map_err(|e| e.to_string())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Render all sizes in one decode (the upload-time path), returning each
|
|
|
|
|
|
/// size paired with its encoded byte length (output-size baseline).
|
|
|
|
|
|
pub fn bench_render_all(data: &[u8]) -> Result<Vec<(ThumbnailSize, usize)>, String> {
|
|
|
|
|
|
Self::render_all_thumbnails_from_data(data)
|
|
|
|
|
|
.map(|v| v.into_iter().map(|(s, b)| (s, b.len())).collect())
|
|
|
|
|
|
.map_err(|e| e.to_string())
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-02-14 01:29:34 +01:00
|
|
|
|
/// Thumbnail service errors
|
|
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
|
|
|
|
pub enum ThumbnailError {
|
|
|
|
|
|
#[error("IO error: {0}")]
|
|
|
|
|
|
IoError(String),
|
|
|
|
|
|
|
|
|
|
|
|
#[error("Image processing error: {0}")]
|
|
|
|
|
|
ImageError(String),
|
|
|
|
|
|
|
|
|
|
|
|
#[error("Task error: {0}")]
|
|
|
|
|
|
TaskError(String),
|
|
|
|
|
|
|
|
|
|
|
|
#[error("Unsupported image format")]
|
|
|
|
|
|
UnsupportedFormat,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Statistics about the thumbnail cache
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
|
pub struct ThumbnailStats {
|
|
|
|
|
|
pub cached_thumbnails: usize,
|
|
|
|
|
|
pub cache_size_bytes: usize,
|
|
|
|
|
|
pub max_cache_bytes: usize,
|
|
|
|
|
|
}
|