From db93b481493d1a72c38df614110f21001acc506f Mon Sep 17 00:00:00 2001
From: Diocrafts <pozo.dionisio@gmail.com>
Date: Sat, 7 Mar 2026 18:55:44 +0100
Subject: [PATCH] feat: client-side video thumbnail generation (no ffmpeg)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Backend:
- get_thumbnail: for non-images, serve cached thumb or 204 (not 400)
- store_external_thumbnail: validate + re-encode to WebP + persist
- get_cached_thumbnail: memory → disk lookup without generation
- PUT /api/files/{id}/thumbnail/{size} endpoint for client uploads
- ThumbnailPort trait: add get_cached_thumbnail + store_external_thumbnail

Frontend (photos.js):
- On <img> error for video tiles, use <video> + <canvas> to extract
  a frame at 25% duration using browser's native codec
- Show frame immediately via blob URL
- Fire-and-forget PUT to server for permanent caching
- Subsequent visits serve cached WebP instantly (no re-extraction)

Zero server-side dependencies — CPU distributed across clients.
---
 src/application/ports/thumbnail_ports.rs      |  21 ++++
 .../services/thumbnail_service.rs             | 114 ++++++++++++++++++
 src/interfaces/api/handlers/file_handler.rs   |  93 ++++++++++++--
 src/interfaces/api/routes.rs                  |   2 +-
 static/js/features/library/photos.js          |  84 +++++++++++++
 5 files changed, 306 insertions(+), 8 deletions(-)

diff --git a/src/application/ports/thumbnail_ports.rs b/src/application/ports/thumbnail_ports.rs
index 6074f625..8dcc04b7 100755
--- a/src/application/ports/thumbnail_ports.rs
+++ b/src/application/ports/thumbnail_ports.rs
@@ -84,6 +84,27 @@ pub trait ThumbnailPort: Send + Sync + 'static {
     /// Delete all thumbnails for a file.
     async fn delete_thumbnails(&self, file_id: &str) -> Result<(), DomainError>;
 
+    /// Try to get a cached thumbnail without generating one.
+    ///
+    /// Returns `None` if no cached thumbnail exists on disk or in memory.
+    /// Used for non-image file types (videos) where thumbnails are
+    /// generated client-side and uploaded.
+    async fn get_cached_thumbnail(
+        &self,
+        file_id: &str,
+        size: ThumbnailSize,
+    ) -> Option<Bytes>;
+
+    /// Store an externally-generated thumbnail (e.g. client-side video frame).
+    ///
+    /// Validates the image, re-encodes to WebP, and persists to cache.
+    async fn store_external_thumbnail(
+        &self,
+        file_id: &str,
+        size: ThumbnailSize,
+        data: Bytes,
+    ) -> Result<Bytes, DomainError>;
+
     /// Get cache statistics.
     async fn get_stats(&self) -> ThumbnailStatsDto;
 }
diff --git a/src/infrastructure/services/thumbnail_service.rs b/src/infrastructure/services/thumbnail_service.rs
index 2ae18fd4..604deb15 100755
--- a/src/infrastructure/services/thumbnail_service.rs
+++ b/src/infrastructure/services/thumbnail_service.rs
@@ -226,6 +226,101 @@ impl ThumbnailService {
         Ok(bytes)
     }
 
+    /// Try to serve a thumbnail from cache only (memory → disk).
+    ///
+    /// Unlike `get_thumbnail`, this does **not** generate a new thumbnail.
+    /// Useful for non-image file types (videos) where a client-generated
+    /// thumbnail may have been uploaded previously.
+    pub async fn get_cached_thumbnail(
+        &self,
+        file_id: &str,
+        size: ThumbnailSize,
+    ) -> Option<Bytes> {
+        // 1. Check in-memory cache
+        let cache_key = ThumbnailCacheKey {
+            file_id: file_id.to_string(),
+            size,
+        };
+        if let Some(bytes) = self.cache.get(&cache_key).await {
+            if !bytes.is_empty() {
+                return Some(bytes);
+            }
+        }
+
+        // 2. Check disk
+        let thumb_path = self.get_thumbnail_path(file_id, size);
+        if let Ok(data) = fs::read(&thumb_path).await {
+            let bytes = Bytes::from(data);
+            // Populate in-memory cache for next hit
+            self.cache.insert(cache_key, bytes.clone()).await;
+            Some(bytes)
+        } else {
+            None
+        }
+    }
+
+    /// Store an externally-generated thumbnail (e.g. client-side video frame).
+    ///
+    /// Validates the image data, re-encodes to WebP for cache consistency,
+    /// and persists to both disk and in-memory cache.
+    pub async fn store_external_thumbnail(
+        &self,
+        file_id: &str,
+        size: ThumbnailSize,
+        data: Bytes,
+    ) -> Result<Bytes, ThumbnailError> {
+        let max_dim = size.max_dimension();
+
+        // Validate + re-encode in blocking thread (tiny image, ~1 ms)
+        let webp_bytes = tokio::task::spawn_blocking(move || -> Result<Vec<u8>, ThumbnailError> {
+            let img = image::load_from_memory(&data)
+                .map_err(|e| ThumbnailError::ImageError(format!("Invalid image data: {e}")))?;
+
+            // Resize if larger than target size
+            let (w, h) = (img.width(), img.height());
+            let img = if w > max_dim || h > max_dim {
+                let filter = FilterType::CatmullRom;
+                if w > h {
+                    let ratio = max_dim as f32 / w as f32;
+                    img.resize(max_dim, (h as f32 * ratio) as u32, filter)
+                } else {
+                    let ratio = max_dim as f32 / h as f32;
+                    img.resize((w as f32 * ratio) as u32, max_dim, filter)
+                }
+            } else {
+                img
+            };
+
+            let mut buffer = Vec::new();
+            img.write_to(&mut std::io::Cursor::new(&mut buffer), ImageFormat::WebP)
+                .map_err(|e| ThumbnailError::ImageError(e.to_string()))?;
+            Ok(buffer)
+        })
+        .await
+        .map_err(|e| ThumbnailError::TaskError(e.to_string()))??;
+
+        let bytes = Bytes::from(webp_bytes);
+
+        // Save to disk
+        let thumb_path = self.get_thumbnail_path(file_id, size);
+        if let Some(parent) = thumb_path.parent() {
+            let _ = fs::create_dir_all(parent).await;
+        }
+        fs::write(&thumb_path, &bytes)
+            .await
+            .map_err(|e| ThumbnailError::IoError(e.to_string()))?;
+
+        // Populate in-memory cache
+        let cache_key = ThumbnailCacheKey {
+            file_id: file_id.to_string(),
+            size,
+        };
+        self.cache.insert(cache_key, bytes.clone()).await;
+
+        tracing::info!("✅ Stored external thumbnail: {} {:?}", file_id, size);
+        Ok(bytes)
+    }
+
     /// Generate a thumbnail from an image file.
     ///
     /// Concurrency is bounded by `decode_semaphore` to prevent OOM when
@@ -505,6 +600,25 @@ impl ThumbnailPort for ThumbnailService {
             .map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
     }
 
+    async fn get_cached_thumbnail(
+        &self,
+        file_id: &str,
+        size: PortThumbnailSize,
+    ) -> Option<Bytes> {
+        self.get_cached_thumbnail(file_id, size.into()).await
+    }
+
+    async fn store_external_thumbnail(
+        &self,
+        file_id: &str,
+        size: PortThumbnailSize,
+        data: Bytes,
+    ) -> Result<Bytes, DomainError> {
+        self.store_external_thumbnail(file_id, size.into(), data)
+            .await
+            .map_err(|e| DomainError::new(ErrorKind::InternalError, "Thumbnail", e.to_string()))
+    }
+
     async fn get_stats(&self) -> ThumbnailStatsDto {
         let stats = self.get_stats().await;
         ThumbnailStatsDto {
diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs
index 68fa15d0..7ddcb488 100755
--- a/src/interfaces/api/handlers/file_handler.rs
+++ b/src/interfaces/api/handlers/file_handler.rs
@@ -331,13 +331,25 @@ impl FileHandler {
         };
 
         if !thumbnail_service.is_supported_image(&file.mime_type) {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "File is not a supported image type"
-                })),
-            )
-                .into_response();
+            // For non-images (videos, etc.), serve from cache if available;
+            // otherwise return 204 to signal "not yet generated — please
+            // generate client-side and upload via PUT".
+            if let Some(data) = thumbnail_service
+                .get_cached_thumbnail(&id, thumb_size.into())
+                .await
+            {
+                let etag = format!("\"thumb-{}-{:?}\"", id, thumb_size);
+                return Response::builder()
+                    .status(StatusCode::OK)
+                    .header(header::CONTENT_TYPE, "image/webp")
+                    .header(header::CONTENT_LENGTH, data.len())
+                    .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable")
+                    .header(header::ETAG, etag)
+                    .body(Body::from(data))
+                    .unwrap()
+                    .into_response();
+            }
+            return StatusCode::NO_CONTENT.into_response();
         }
 
         // Resolve the physical blob path (content-addressable storage)
@@ -377,6 +389,73 @@ impl FileHandler {
         }
     }
 
+    // ═══════════════════════════════════════════════════════════════════════
+    //  UPLOAD THUMBNAIL (client-generated, e.g. video frames)
+    // ═══════════════════════════════════════════════════════════════════════
+
+    /// Accept a client-generated thumbnail (e.g. video frame extracted via
+    /// `<video>` + `<canvas>` in the browser) and persist it in the server
+    /// cache.  The image is validated, re-encoded to WebP, and stored so
+    /// subsequent `GET …/thumbnail/{size}` requests are served instantly.
+    ///
+    /// **Max body: 512 KB** — thumbnails are small.
+    pub async fn upload_thumbnail(
+        State(state): State<GlobalState>,
+        auth_user: AuthUser,
+        Path((id, size)): Path<(String, String)>,
+        body: Bytes,
+    ) -> impl IntoResponse {
+        use crate::application::ports::thumbnail_ports::ThumbnailSize;
+
+        let thumbnail_service = &state.core.thumbnail_service;
+
+        // Validate size
+        let thumb_size = match size.as_str() {
+            "icon" => ThumbnailSize::Icon,
+            "preview" => ThumbnailSize::Preview,
+            "large" => ThumbnailSize::Large,
+            _ => {
+                return (
+                    StatusCode::BAD_REQUEST,
+                    Json(serde_json::json!({
+                        "error": "Invalid thumbnail size. Use: icon, preview, or large"
+                    })),
+                )
+                    .into_response();
+            }
+        };
+
+        // Reject oversized payloads (512 KB)
+        if body.len() > 512 * 1024 {
+            return (
+                StatusCode::PAYLOAD_TOO_LARGE,
+                Json(serde_json::json!({ "error": "Thumbnail exceeds 512 KB limit" })),
+            )
+                .into_response();
+        }
+
+        // Validate file ownership
+        let file_retrieval_service = &state.applications.file_retrieval_service;
+        if let Err(err) = file_retrieval_service
+            .get_file_owned(&id, auth_user.id)
+            .await
+        {
+            return AppError::from(err).into_response();
+        }
+
+        // Validate, re-encode to WebP, and store
+        match thumbnail_service
+            .store_external_thumbnail(&id, thumb_size.into(), body)
+            .await
+        {
+            Ok(_) => StatusCode::CREATED.into_response(),
+            Err(err) => {
+                AppError::internal_error(format!("Failed to store thumbnail: {}", err))
+                    .into_response()
+            }
+        }
+    }
+
     // ═══════════════════════════════════════════════════════════════════════
     //  DOWNLOAD
     // ═══════════════════════════════════════════════════════════════════════
diff --git a/src/interfaces/api/routes.rs b/src/interfaces/api/routes.rs
index 1062df7b..aa10a6dd 100755
--- a/src/interfaces/api/routes.rs
+++ b/src/interfaces/api/routes.rs
@@ -145,7 +145,7 @@ pub fn create_api_routes(app_state: &Arc<AppState>) -> Router<Arc<AppState>> {
         .route("/", get(FileHandler::list_files_query))
         .route("/upload", post(FileHandler::upload_file_with_thumbnails))
         .route("/{id}", get(FileHandler::download_file))
-        .route("/{id}/thumbnail/{size}", get(FileHandler::get_thumbnail))
+        .route("/{id}/thumbnail/{size}", get(FileHandler::get_thumbnail).put(FileHandler::upload_thumbnail))
         .route("/{id}/metadata", get(FileHandler::get_file_metadata))
         .layer(DefaultBodyLimit::max(10 * 1024 * 1024 * 1024)) // 10 GB for file uploads
         .with_state(app_state.clone());
diff --git a/static/js/features/library/photos.js b/static/js/features/library/photos.js
index 6df216e0..1f983968 100755
--- a/static/js/features/library/photos.js
+++ b/static/js/features/library/photos.js
@@ -177,6 +177,90 @@ const photosView = {
             }, { rootMargin: '400px' });
             this._observer.observe(sentinel);
         }
+
+        // Set up client-side video thumbnail generation
+        this._setupVideoThumbnails();
+    },
+
+    // ── Client-side video thumbnail generation ──────────────────────
+    // Uses the browser's native video decoder (<video> + <canvas>) to
+    // extract a frame, show it immediately, and upload to the server
+    // for permanent caching.  Zero server-side dependencies (no ffmpeg).
+
+    /** Attach error handlers to video tile images; on failure, extract a
+     *  frame from the video using the browser's built-in codec. */
+    _setupVideoThumbnails() {
+        const tiles = this._container.querySelectorAll('.photo-tile[data-mime^="video/"]');
+        for (const tile of tiles) {
+            const img = tile.querySelector('img');
+            if (!img) continue;
+
+            // The server returns 204 for videos without a cached thumbnail,
+            // which causes the <img> to fire 'error'.  We could also detect
+            // a natural 0×0 image.
+            img.addEventListener('error', () => {
+                this._generateVideoThumbnail(tile, img);
+            }, { once: true });
+        }
+    },
+
+    /** Extract a single frame from a video and display it as the tile
+     *  thumbnail, then upload the WebP to the server for caching. */
+    _generateVideoThumbnail(tile, img) {
+        const fileId = tile.dataset.id;
+        const video = document.createElement('video');
+        video.crossOrigin = 'anonymous';
+        video.preload = 'metadata';
+        video.muted = true;
+        // Auth is handled via HttpOnly cookie — direct URL works
+        video.src = `/api/files/${fileId}`;
+
+        video.addEventListener('loadeddata', () => {
+            // Seek to 25 % of duration, clamped between 0.5 s and 5 s
+            video.currentTime = Math.min(5, Math.max(0.5, video.duration * 0.25));
+        }, { once: true });
+
+        video.addEventListener('seeked', () => {
+            const canvas = document.createElement('canvas');
+            canvas.width = video.videoWidth;
+            canvas.height = video.videoHeight;
+            const ctx = canvas.getContext('2d');
+            ctx.drawImage(video, 0, 0);
+
+            // Try WebP first, fall back to JPEG
+            const mimeType = typeof canvas.toBlob === 'function'
+                ? 'image/webp' : 'image/jpeg';
+
+            canvas.toBlob((blob) => {
+                if (!blob) return;
+
+                // Show immediately in the tile
+                const url = URL.createObjectURL(blob);
+                img.src = url;
+
+                // Upload to server for permanent caching (fire-and-forget)
+                const token = localStorage.getItem('token')
+                    || sessionStorage.getItem('token');
+                const headers = { 'Content-Type': blob.type };
+                if (token) headers['Authorization'] = `Bearer ${token}`;
+
+                fetch(`/api/files/${fileId}/thumbnail/preview`, {
+                    method: 'PUT',
+                    headers,
+                    body: blob,
+                }).catch(() => { /* best-effort */ });
+
+                // Release video resources
+                video.src = '';
+                video.load();
+            }, mimeType, 0.80);
+        }, { once: true });
+
+        // If the video can't be loaded at all, keep the generic play badge
+        video.addEventListener('error', () => {
+            video.src = '';
+            video.load();
+        }, { once: true });
     },
 
     /** Render the group mode toolbar */