feat: client-side video thumbnail generation (no ffmpeg)
Backend:
- get_thumbnail: for non-images, serve cached thumb or 204 (not 400)
- store_external_thumbnail: validate + re-encode to WebP + persist
- get_cached_thumbnail: memory → disk lookup without generation
- PUT /api/files/{id}/thumbnail/{size} endpoint for client uploads
- ThumbnailPort trait: add get_cached_thumbnail + store_external_thumbnail
Frontend (photos.js):
- On <img> error for video tiles, use <video> + <canvas> to extract
a frame at 25% duration using browser's native codec
- Show frame immediately via blob URL
- Fire-and-forget PUT to server for permanent caching
- Subsequent visits serve cached WebP instantly (no re-extraction)
Zero server-side dependencies — CPU distributed across clients.
This commit is contained in:
@@ -84,6 +84,27 @@ pub trait ThumbnailPort: Send + Sync + 'static {
|
|||||||
/// Delete all thumbnails for a file.
|
/// Delete all thumbnails for a file.
|
||||||
async fn delete_thumbnails(&self, file_id: &str) -> Result<(), DomainError>;
|
async fn delete_thumbnails(&self, file_id: &str) -> Result<(), DomainError>;
|
||||||
|
|
||||||
|
/// Try to get a cached thumbnail without generating one.
|
||||||
|
///
|
||||||
|
/// Returns `None` if no cached thumbnail exists on disk or in memory.
|
||||||
|
/// Used for non-image file types (videos) where thumbnails are
|
||||||
|
/// generated client-side and uploaded.
|
||||||
|
async fn get_cached_thumbnail(
|
||||||
|
&self,
|
||||||
|
file_id: &str,
|
||||||
|
size: ThumbnailSize,
|
||||||
|
) -> Option<Bytes>;
|
||||||
|
|
||||||
|
/// Store an externally-generated thumbnail (e.g. client-side video frame).
|
||||||
|
///
|
||||||
|
/// Validates the image, re-encodes to WebP, and persists to cache.
|
||||||
|
async fn store_external_thumbnail(
|
||||||
|
&self,
|
||||||
|
file_id: &str,
|
||||||
|
size: ThumbnailSize,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Result<Bytes, DomainError>;
|
||||||
|
|
||||||
/// Get cache statistics.
|
/// Get cache statistics.
|
||||||
async fn get_stats(&self) -> ThumbnailStatsDto;
|
async fn get_stats(&self) -> ThumbnailStatsDto;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -226,6 +226,101 @@ impl ThumbnailService {
|
|||||||
Ok(bytes)
|
Ok(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Try to serve a thumbnail from cache only (memory → disk).
|
||||||
|
///
|
||||||
|
/// Unlike `get_thumbnail`, this does **not** generate a new thumbnail.
|
||||||
|
/// Useful for non-image file types (videos) where a client-generated
|
||||||
|
/// thumbnail may have been uploaded previously.
|
||||||
|
pub async fn get_cached_thumbnail(
|
||||||
|
&self,
|
||||||
|
file_id: &str,
|
||||||
|
size: ThumbnailSize,
|
||||||
|
) -> Option<Bytes> {
|
||||||
|
// 1. Check in-memory cache
|
||||||
|
let cache_key = ThumbnailCacheKey {
|
||||||
|
file_id: file_id.to_string(),
|
||||||
|
size,
|
||||||
|
};
|
||||||
|
if let Some(bytes) = self.cache.get(&cache_key).await {
|
||||||
|
if !bytes.is_empty() {
|
||||||
|
return Some(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Check disk
|
||||||
|
let thumb_path = self.get_thumbnail_path(file_id, size);
|
||||||
|
if let Ok(data) = fs::read(&thumb_path).await {
|
||||||
|
let bytes = Bytes::from(data);
|
||||||
|
// Populate in-memory cache for next hit
|
||||||
|
self.cache.insert(cache_key, bytes.clone()).await;
|
||||||
|
Some(bytes)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Store an externally-generated thumbnail (e.g. client-side video frame).
|
||||||
|
///
|
||||||
|
/// Validates the image data, re-encodes to WebP for cache consistency,
|
||||||
|
/// and persists to both disk and in-memory cache.
|
||||||
|
pub async fn store_external_thumbnail(
|
||||||
|
&self,
|
||||||
|
file_id: &str,
|
||||||
|
size: ThumbnailSize,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Result<Bytes, ThumbnailError> {
|
||||||
|
let max_dim = size.max_dimension();
|
||||||
|
|
||||||
|
// Validate + re-encode in blocking thread (tiny image, ~1 ms)
|
||||||
|
let webp_bytes = tokio::task::spawn_blocking(move || -> Result<Vec<u8>, ThumbnailError> {
|
||||||
|
let img = image::load_from_memory(&data)
|
||||||
|
.map_err(|e| ThumbnailError::ImageError(format!("Invalid image data: {e}")))?;
|
||||||
|
|
||||||
|
// Resize if larger than target size
|
||||||
|
let (w, h) = (img.width(), img.height());
|
||||||
|
let img = if w > max_dim || h > max_dim {
|
||||||
|
let filter = FilterType::CatmullRom;
|
||||||
|
if w > h {
|
||||||
|
let ratio = max_dim as f32 / w as f32;
|
||||||
|
img.resize(max_dim, (h as f32 * ratio) as u32, filter)
|
||||||
|
} else {
|
||||||
|
let ratio = max_dim as f32 / h as f32;
|
||||||
|
img.resize((w as f32 * ratio) as u32, max_dim, filter)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
img
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
img.write_to(&mut std::io::Cursor::new(&mut buffer), ImageFormat::WebP)
|
||||||
|
.map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
|
||||||
|
Ok(buffer)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|e| ThumbnailError::TaskError(e.to_string()))??;
|
||||||
|
|
||||||
|
let bytes = Bytes::from(webp_bytes);
|
||||||
|
|
||||||
|
// Save to disk
|
||||||
|
let thumb_path = self.get_thumbnail_path(file_id, size);
|
||||||
|
if let Some(parent) = thumb_path.parent() {
|
||||||
|
let _ = fs::create_dir_all(parent).await;
|
||||||
|
}
|
||||||
|
fs::write(&thumb_path, &bytes)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ThumbnailError::IoError(e.to_string()))?;
|
||||||
|
|
||||||
|
// Populate in-memory cache
|
||||||
|
let cache_key = ThumbnailCacheKey {
|
||||||
|
file_id: file_id.to_string(),
|
||||||
|
size,
|
||||||
|
};
|
||||||
|
self.cache.insert(cache_key, bytes.clone()).await;
|
||||||
|
|
||||||
|
tracing::info!("✅ Stored external thumbnail: {} {:?}", file_id, size);
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
/// Generate a thumbnail from an image file.
|
/// Generate a thumbnail from an image file.
|
||||||
///
|
///
|
||||||
/// Concurrency is bounded by `decode_semaphore` to prevent OOM when
|
/// Concurrency is bounded by `decode_semaphore` to prevent OOM when
|
||||||
@@ -505,6 +600,25 @@ impl ThumbnailPort for ThumbnailService {
|
|||||||
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_cached_thumbnail(
|
||||||
|
&self,
|
||||||
|
file_id: &str,
|
||||||
|
size: PortThumbnailSize,
|
||||||
|
) -> Option<Bytes> {
|
||||||
|
self.get_cached_thumbnail(file_id, size.into()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_external_thumbnail(
|
||||||
|
&self,
|
||||||
|
file_id: &str,
|
||||||
|
size: PortThumbnailSize,
|
||||||
|
data: Bytes,
|
||||||
|
) -> Result<Bytes, DomainError> {
|
||||||
|
self.store_external_thumbnail(file_id, size.into(), data)
|
||||||
|
.await
|
||||||
|
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
async fn get_stats(&self) -> ThumbnailStatsDto {
|
async fn get_stats(&self) -> ThumbnailStatsDto {
|
||||||
let stats = self.get_stats().await;
|
let stats = self.get_stats().await;
|
||||||
ThumbnailStatsDto {
|
ThumbnailStatsDto {
|
||||||
|
|||||||
@@ -331,13 +331,25 @@ impl FileHandler {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if !thumbnail_service.is_supported_image(&file.mime_type) {
|
if !thumbnail_service.is_supported_image(&file.mime_type) {
|
||||||
return (
|
// For non-images (videos, etc.), serve from cache if available;
|
||||||
StatusCode::BAD_REQUEST,
|
// otherwise return 204 to signal "not yet generated — please
|
||||||
Json(serde_json::json!({
|
// generate client-side and upload via PUT".
|
||||||
"error": "File is not a supported image type"
|
if let Some(data) = thumbnail_service
|
||||||
})),
|
.get_cached_thumbnail(&id, thumb_size.into())
|
||||||
)
|
.await
|
||||||
.into_response();
|
{
|
||||||
|
let etag = format!("\"thumb-{}-{:?}\"", id, thumb_size);
|
||||||
|
return Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header(header::CONTENT_TYPE, "image/webp")
|
||||||
|
.header(header::CONTENT_LENGTH, data.len())
|
||||||
|
.header(header::CACHE_CONTROL, "public, max-age=31536000, immutable")
|
||||||
|
.header(header::ETAG, etag)
|
||||||
|
.body(Body::from(data))
|
||||||
|
.unwrap()
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
return StatusCode::NO_CONTENT.into_response();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve the physical blob path (content-addressable storage)
|
// Resolve the physical blob path (content-addressable storage)
|
||||||
@@ -377,6 +389,73 @@ impl FileHandler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════
|
||||||
|
// UPLOAD THUMBNAIL (client-generated, e.g. video frames)
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// Accept a client-generated thumbnail (e.g. video frame extracted via
|
||||||
|
/// `<video>` + `<canvas>` in the browser) and persist it in the server
|
||||||
|
/// cache. The image is validated, re-encoded to WebP, and stored so
|
||||||
|
/// subsequent `GET …/thumbnail/{size}` requests are served instantly.
|
||||||
|
///
|
||||||
|
/// **Max body: 512 KB** — thumbnails are small.
|
||||||
|
pub async fn upload_thumbnail(
|
||||||
|
State(state): State<GlobalState>,
|
||||||
|
auth_user: AuthUser,
|
||||||
|
Path((id, size)): Path<(String, String)>,
|
||||||
|
body: Bytes,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
use crate::application::ports::thumbnail_ports::ThumbnailSize;
|
||||||
|
|
||||||
|
let thumbnail_service = &state.core.thumbnail_service;
|
||||||
|
|
||||||
|
// Validate size
|
||||||
|
let thumb_size = match size.as_str() {
|
||||||
|
"icon" => ThumbnailSize::Icon,
|
||||||
|
"preview" => ThumbnailSize::Preview,
|
||||||
|
"large" => ThumbnailSize::Large,
|
||||||
|
_ => {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({
|
||||||
|
"error": "Invalid thumbnail size. Use: icon, preview, or large"
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Reject oversized payloads (512 KB)
|
||||||
|
if body.len() > 512 * 1024 {
|
||||||
|
return (
|
||||||
|
StatusCode::PAYLOAD_TOO_LARGE,
|
||||||
|
Json(serde_json::json!({ "error": "Thumbnail exceeds 512 KB limit" })),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate file ownership
|
||||||
|
let file_retrieval_service = &state.applications.file_retrieval_service;
|
||||||
|
if let Err(err) = file_retrieval_service
|
||||||
|
.get_file_owned(&id, auth_user.id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
return AppError::from(err).into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate, re-encode to WebP, and store
|
||||||
|
match thumbnail_service
|
||||||
|
.store_external_thumbnail(&id, thumb_size.into(), body)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(_) => StatusCode::CREATED.into_response(),
|
||||||
|
Err(err) => {
|
||||||
|
AppError::internal_error(format!("Failed to store thumbnail: {}", err))
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════
|
||||||
// DOWNLOAD
|
// DOWNLOAD
|
||||||
// ═══════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ pub fn create_api_routes(app_state: &Arc<AppState>) -> Router<Arc<AppState>> {
|
|||||||
.route("/", get(FileHandler::list_files_query))
|
.route("/", get(FileHandler::list_files_query))
|
||||||
.route("/upload", post(FileHandler::upload_file_with_thumbnails))
|
.route("/upload", post(FileHandler::upload_file_with_thumbnails))
|
||||||
.route("/{id}", get(FileHandler::download_file))
|
.route("/{id}", get(FileHandler::download_file))
|
||||||
.route("/{id}/thumbnail/{size}", get(FileHandler::get_thumbnail))
|
.route("/{id}/thumbnail/{size}", get(FileHandler::get_thumbnail).put(FileHandler::upload_thumbnail))
|
||||||
.route("/{id}/metadata", get(FileHandler::get_file_metadata))
|
.route("/{id}/metadata", get(FileHandler::get_file_metadata))
|
||||||
.layer(DefaultBodyLimit::max(10 * 1024 * 1024 * 1024)) // 10 GB for file uploads
|
.layer(DefaultBodyLimit::max(10 * 1024 * 1024 * 1024)) // 10 GB for file uploads
|
||||||
.with_state(app_state.clone());
|
.with_state(app_state.clone());
|
||||||
|
|||||||
@@ -177,6 +177,90 @@ const photosView = {
|
|||||||
}, { rootMargin: '400px' });
|
}, { rootMargin: '400px' });
|
||||||
this._observer.observe(sentinel);
|
this._observer.observe(sentinel);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set up client-side video thumbnail generation
|
||||||
|
this._setupVideoThumbnails();
|
||||||
|
},
|
||||||
|
|
||||||
|
// ── Client-side video thumbnail generation ──────────────────────
|
||||||
|
// Uses the browser's native video decoder (<video> + <canvas>) to
|
||||||
|
// extract a frame, show it immediately, and upload to the server
|
||||||
|
// for permanent caching. Zero server-side dependencies (no ffmpeg).
|
||||||
|
|
||||||
|
/** Attach error handlers to video tile images; on failure, extract a
|
||||||
|
* frame from the video using the browser's built-in codec. */
|
||||||
|
_setupVideoThumbnails() {
|
||||||
|
const tiles = this._container.querySelectorAll('.photo-tile[data-mime^="video/"]');
|
||||||
|
for (const tile of tiles) {
|
||||||
|
const img = tile.querySelector('img');
|
||||||
|
if (!img) continue;
|
||||||
|
|
||||||
|
// The server returns 204 for videos without a cached thumbnail,
|
||||||
|
// which causes the <img> to fire 'error'. We could also detect
|
||||||
|
// a natural 0×0 image.
|
||||||
|
img.addEventListener('error', () => {
|
||||||
|
this._generateVideoThumbnail(tile, img);
|
||||||
|
}, { once: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
/** Extract a single frame from a video and display it as the tile
|
||||||
|
* thumbnail, then upload the WebP to the server for caching. */
|
||||||
|
_generateVideoThumbnail(tile, img) {
|
||||||
|
const fileId = tile.dataset.id;
|
||||||
|
const video = document.createElement('video');
|
||||||
|
video.crossOrigin = 'anonymous';
|
||||||
|
video.preload = 'metadata';
|
||||||
|
video.muted = true;
|
||||||
|
// Auth is handled via HttpOnly cookie — direct URL works
|
||||||
|
video.src = `/api/files/${fileId}`;
|
||||||
|
|
||||||
|
video.addEventListener('loadeddata', () => {
|
||||||
|
// Seek to 25 % of duration, clamped between 0.5 s and 5 s
|
||||||
|
video.currentTime = Math.min(5, Math.max(0.5, video.duration * 0.25));
|
||||||
|
}, { once: true });
|
||||||
|
|
||||||
|
video.addEventListener('seeked', () => {
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
canvas.width = video.videoWidth;
|
||||||
|
canvas.height = video.videoHeight;
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
ctx.drawImage(video, 0, 0);
|
||||||
|
|
||||||
|
// Try WebP first, fall back to JPEG
|
||||||
|
const mimeType = typeof canvas.toBlob === 'function'
|
||||||
|
? 'image/webp' : 'image/jpeg';
|
||||||
|
|
||||||
|
canvas.toBlob((blob) => {
|
||||||
|
if (!blob) return;
|
||||||
|
|
||||||
|
// Show immediately in the tile
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
img.src = url;
|
||||||
|
|
||||||
|
// Upload to server for permanent caching (fire-and-forget)
|
||||||
|
const token = localStorage.getItem('token')
|
||||||
|
|| sessionStorage.getItem('token');
|
||||||
|
const headers = { 'Content-Type': blob.type };
|
||||||
|
if (token) headers['Authorization'] = `Bearer ${token}`;
|
||||||
|
|
||||||
|
fetch(`/api/files/${fileId}/thumbnail/preview`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers,
|
||||||
|
body: blob,
|
||||||
|
}).catch(() => { /* best-effort */ });
|
||||||
|
|
||||||
|
// Release video resources
|
||||||
|
video.src = '';
|
||||||
|
video.load();
|
||||||
|
}, mimeType, 0.80);
|
||||||
|
}, { once: true });
|
||||||
|
|
||||||
|
// If the video can't be loaded at all, keep the generic play badge
|
||||||
|
video.addEventListener('error', () => {
|
||||||
|
video.src = '';
|
||||||
|
video.load();
|
||||||
|
}, { once: true });
|
||||||
},
|
},
|
||||||
|
|
||||||
/** Render the group mode toolbar */
|
/** Render the group mode toolbar */
|
||||||
|
|||||||
Reference in New Issue
Block a user