quick fix

TODO-LIST.md

@@ -177,6 +177,43 @@ This document contains the task list for the development of OxiCloud, a minimali
 - [x] Add content-aware compression by file format
 - [ ] Implement dynamic thumbnail resizing based on viewport
 
+### Bandwidth & Transfer Optimization
+- [ ] **Sub-file chunked dedup (Restic/Borg style)**
+  - [ ] Implement Content-Defined Chunking (CDC) with FastCDC/Rabin rolling hash
+  - [ ] Variable-size chunks (target 1-4 MB) instead of whole-file blobs
+  - [ ] Per-chunk BLAKE3 hashing and dedup (saves storage + bandwidth on similar files)
+  - [ ] Chunk-level Zstd compression (better ratio than whole-file)
+  - [ ] Migrate existing whole-file blobs to chunked storage
+- [ ] **Delta sync / rsync-style transfers**
+  - [ ] Implement rolling checksum algorithm for block-level diffing
+  - [ ] Client sends only changed blocks on re-upload (not the full file)
+  - [ ] Server-side block assembly from delta + existing chunks
+  - [ ] Huge savings for large files with small edits (VMs, databases, ISOs)
+- [ ] **Resumable uploads & downloads (RFC 7233 / tus.io)**
+  - [ ] Server tracks partial upload state; client resumes from last byte on failure
+  - [ ] HTTP Range responses for download resume after network drops
+  - [ ] tus.io protocol support for cross-client compatibility
+- [ ] **Client-side optimization before upload**
+  - [ ] Resize images to configurable max dimensions before upload (e.g. 4K cap)
+  - [ ] Re-encode videos to efficient codec (H.265/AV1) client-side before upload
+  - [ ] ⚡ **HIGH IMPACT / QUICK WIN** — Pre-compute BLAKE3 hash client-side (WASM); query server before upload; skip transfer entirely if blob already exists (instant dedup, zero bandwidth)
+- [ ] **Server-side on-demand transcoding**
+  - [ ] Store originals; serve WebP/AVIF for images on request (saves download BW)
+  - [ ] Adaptive video streaming (HLS/DASH) from stored originals
+  - [ ] Lazy generation + cache of transcoded variants
+- [ ] **Smart sync (placeholder/on-demand files)**
+  - [ ] Sync client downloads metadata only; fetch file content on first open
+  - [ ] Pin/unpin files for offline availability
+  - [ ] Automatic eviction of least-recently-used local copies
+- [ ] **Transfer-level compression**
+  - [ ] Zstd streaming compression for HTTP responses (better than gzip for large files)
+  - [ ] Brotli for static assets; Zstd for dynamic/binary content
+  - [ ] Content-aware: skip compression for already-compressed formats (JPEG, ZIP, etc.)
+- [ ] **Batched & multiplexed operations**
+  - [ ] Batch small file uploads into single request (tar-stream or multipart bundle)
+  - [ ] HTTP/2 multiplexing for parallel chunk transfers on single connection
+  - [ ] Server-side ZIP streaming for multi-file download (already partial)
+
 ## Infrastructure and Deployment
 
 - [x] Create Docker configuration

example.env

@@ -157,4 +157,36 @@ OXICLOUD_WOPI_ENABLED=false
 #OXICLOUD_WOPI_TOKEN_TTL_SECS=86400
 
 # WOPI lock expiration in seconds (default: 1800 = 30 minutes)
-#OXICLOUD_WOPI_LOCK_TTL_SECS=1800
+#OXICLOUD_WOPI_LOCK_TTL_SECS=1800
+
+# -----------------------------------------------------------------------------
+# MEMORY ALLOCATOR TUNING (IMPORTANT FOR RAM USAGE)
+# -----------------------------------------------------------------------------
+# OxiCloud uses mimalloc as its global memory allocator for performance.
+# By default, mimalloc RETAINS freed memory in internal free-lists instead of
+# returning it to the operating system. This causes the process RSS to grow
+# over time (e.g., after large file uploads or password hashing) and never
+# shrink back — even though the application has already freed that memory.
+#
+# In containerized / memory-constrained environments (Docker, K8s, VPS with
+# limited RAM), this is critical: without these settings, the container can
+# appear to "leak" hundreds of MiB that are actually just retained by the
+# allocator.
+#
+# These variables are read directly by the mimalloc C library at startup.
+# They are NOT OxiCloud-specific — they are part of mimalloc's official API.
+# Docs: https://microsoft.github.io/mimalloc/environment.html
+
+# MIMALLOC_PURGE_DELAY: Delay (in ms) before freed memory is returned to the OS.
+#   0  = return immediately (RECOMMENDED for Docker / limited RAM)
+#   -1 = never return (maximum performance, highest RAM usage)
+#   10 = mimalloc default (slight delay for reuse optimization)
+# Setting this to 0 can reduce idle RAM by 80-120 MiB in typical deployments.
+MIMALLOC_PURGE_DELAY=0
+
+# MIMALLOC_ALLOW_LARGE_OS_PAGES: Use 2 MiB huge pages for allocations.
+#   0 = disabled (RECOMMENDED for Docker — avoids RSS inflation from THP)
+#   1 = enabled (better TLB performance on bare-metal servers with plenty of RAM)
+# When enabled with Linux Transparent Huge Pages (THP), partially-used 2 MiB
+# pages inflate the reported RSS by up to 20-30 MiB.
+MIMALLOC_ALLOW_LARGE_OS_PAGES=0

src/interfaces/api/handlers/auth_handler.rs

@@ -674,4 +674,4 @@ async fn oidc_exchange(
     );
     cookie_auth::append_csrf_cookie(response.headers_mut(), auth_response.expires_in);
     Ok(response)
-}
+}

src/interfaces/api/handlers/dedup_handler.rs

@@ -158,14 +158,35 @@ impl DedupHandler {
                     .unwrap_or("application/octet-stream")
                     .to_string();
 
-                // Collect all chunks
+                // Collect all chunks — explicit match to detect client disconnection
                 let mut chunks: Vec<Bytes> = Vec::new();
                 let mut total_size: usize = 0;
                 let mut field = field;
 
-                while let Ok(Some(chunk)) = field.chunk().await {
-                    total_size += chunk.len();
-                    chunks.push(chunk);
+                loop {
+                    match field.chunk().await {
+                        Ok(Some(chunk)) => {
+                            total_size += chunk.len();
+                            chunks.push(chunk);
+                        }
+                        Ok(None) => break,
+                        Err(e) => {
+                            tracing::warn!(
+                                "Connection lost during dedup upload (received {} bytes): {}",
+                                total_size,
+                                e
+                            );
+                            return Response::builder()
+                                .status(StatusCode::BAD_REQUEST)
+                                .header(header::CONTENT_TYPE, "application/json")
+                                .body(Body::from(format!(
+                                    r#"{{"error": "Connection lost during upload: {}"}}"#,
+                                    e
+                                )))
+                                .unwrap()
+                                .into_response();
+                        }
+                    }
                 }
 
                 if chunks.is_empty() {

src/interfaces/api/handlers/file_handler.rs

@@ -106,11 +106,7 @@ impl FileHandler {
                 if let Some(ref fid) = folder_id {
                     use crate::application::ports::inbound::FolderUseCase;
                     let folder_service = &state.applications.folder_service;
-                    if folder_service
-                        .get_folder_owned(fid, &auth_user.id)
-                        .await
-                        .is_err()
-                    {
+                    if folder_service.get_folder_owned(fid, &auth_user.id).await.is_err() {
                         tracing::warn!(
                             "⛔ UPLOAD REJECTED (IDOR): user='{}' attempted upload to folder '{}' owned by another user",
                             auth_user.username,
@@ -169,12 +165,26 @@ impl FileHandler {
                     // 512 KB buffer — 8× fewer write syscalls than 64 KB
                     let mut writer = tokio::io::BufWriter::with_capacity(524_288, file);
                     let mut field = field;
-                    while let Ok(Some(chunk)) = field.chunk().await {
-                        total_size += chunk.len() as u64;
-                        hasher.update(&chunk);
-                        tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk)
-                            .await
-                            .map_err(|e| format!("Failed to write chunk: {}", e))?;
+                    // IMPORTANT: use explicit match instead of `while let Ok(Some(..))`.
+                    // The old pattern silently swallowed Err (client disconnect)
+                    // and accepted partially received data as a complete upload.
+                    loop {
+                        match field.chunk().await {
+                            Ok(Some(chunk)) => {
+                                total_size += chunk.len() as u64;
+                                hasher.update(&chunk);
+                                tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk)
+                                    .await
+                                    .map_err(|e| format!("Failed to write chunk: {}", e))?;
+                            }
+                            Ok(None) => break, // End of field — upload complete
+                            Err(e) => {
+                                return Err(format!(
+                                    "Connection lost during upload (received {} bytes): {}",
+                                    total_size, e
+                                ));
+                            }
+                        }
                     }
                     tokio::io::AsyncWriteExt::flush(&mut writer)
                         .await
@@ -326,28 +336,22 @@ impl FileHandler {
                 .into_response();
         }
 
-        // Resolve the actual blob path on disk (not the logical file path).
+        // Resolve the physical blob path (content-addressable storage)
         let blob_hash = match state
             .repositories
             .file_read_repository
             .get_blob_hash(&id)
             .await
         {
-            Ok(h) => h,
-            Err(err) => {
-                return (
-                    StatusCode::NOT_FOUND,
-                    Json(serde_json::json!({
-                        "error": format!("File content not found: {}", err)
-                    })),
-                )
-                    .into_response();
+            Ok(hash) => hash,
+            Err(_) => {
+                return AppError::internal_error("File blob not found").into_response();
             }
         };
-        let blob_path = state.core.dedup_service.blob_path(&blob_hash);
+        let file_path = state.core.dedup_service.blob_path(&blob_hash);
 
         match thumbnail_service
-            .get_thumbnail(&id, thumb_size.into(), &blob_path)
+            .get_thumbnail(&id, thumb_size.into(), &file_path)
             .await
         {
             Ok(data) => {
@@ -362,8 +366,10 @@ impl FileHandler {
                     .unwrap()
                     .into_response()
             }
-            Err(err) => AppError::internal_error(format!("Thumbnail generation failed: {}", err))
-                .into_response(),
+            Err(err) => {
+                AppError::internal_error(format!("Thumbnail generation failed: {}", err))
+                    .into_response()
+            }
         }
     }
 
@@ -536,7 +542,9 @@ impl FileHandler {
                     .unwrap()
                     .into_response(),
             },
-            Err(err) => AppError::from(err).into_response(),
+            Err(err) => {
+                AppError::from(err).into_response()
+            }
         }
     }
 
@@ -586,7 +594,9 @@ impl FileHandler {
                     .insert(header::ETAG, header::HeaderValue::from_str(&etag).unwrap());
                 resp
             }
-            Err(err) => AppError::from(err).into_response(),
+            Err(err) => {
+                AppError::from(err).into_response()
+            }
         }
     }
 
@@ -605,7 +615,7 @@ impl FileHandler {
             Err(response) => return response.into_response(),
         };
 
-        // Generate thumbnails and extract EXIF metadata for supported images in background
+        // Generate thumbnails for supported images in background
         if state
             .core
             .thumbnail_service
@@ -613,47 +623,30 @@ impl FileHandler {
         {
             let file_id = file.id.clone();
             let thumbnail_service = state.core.thumbnail_service.clone();
-            let dedup_service = state.core.dedup_service.clone();
-            let file_read = state.repositories.file_read_repository.clone();
-            let metadata_repo = state.repositories.file_metadata_repository.clone();
 
-            tokio::spawn(async move {
-                // Resolve the actual blob path on disk (not the logical file path,
-                // which doesn't exist when using blob storage).
-                let blob_hash = match file_read.get_blob_hash(&file_id).await {
-                    Ok(h) => h,
-                    Err(e) => {
-                        tracing::warn!("Skipping thumbnails for {}: {}", file_id, e);
-                        return;
-                    }
-                };
-                let file_path = dedup_service.blob_path(&blob_hash);
-
-                // Extract EXIF metadata (reads only header bytes, very fast).
-                // Runs before thumbnail generation so the OS page cache is primed.
-                {
-                    use crate::infrastructure::services::exif_service::ExifService;
-                    match tokio::fs::read(&file_path).await {
-                        Ok(data) => {
-                            if let Some(meta) = ExifService::extract(&data)
-                                && let Err(e) = metadata_repo.upsert(&file_id, &meta).await
-                            {
-                                tracing::warn!("Failed to store EXIF for {}: {}", file_id, e);
-                            }
-                        }
-                        Err(e) => {
-                            tracing::warn!(
-                                "Failed to read file for EXIF extraction {}: {}",
-                                file_id,
-                                e
-                            );
-                        }
-                    }
+            // Resolve physical blob path before spawning
+            match state
+                .repositories
+                .file_read_repository
+                .get_blob_hash(&file_id)
+                .await
+            {
+                Ok(blob_hash) => {
+                    let file_path = state.core.dedup_service.blob_path(&blob_hash);
+                    tokio::spawn(async move {
+                        tracing::info!("🖼️ Generating thumbnails for: {}", file_id);
+                        thumbnail_service
+                            .generate_all_sizes_background(file_id, file_path);
+                    });
                 }
-
-                tracing::info!("🖼️ Generating thumbnails for: {}", file_id);
-                thumbnail_service.generate_all_sizes_background(file_id, file_path);
-            });
+                Err(e) => {
+                    tracing::warn!(
+                        "⚠️ Cannot generate thumbnails for {}: blob hash not found: {}",
+                        file_id,
+                        e
+                    );
+                }
+            }
         }
 
         Self::created_json_response(&file).into_response()
@@ -674,9 +667,10 @@ impl FileHandler {
         // Verify ownership
         let file_read = &state.repositories.file_read_repository;
         if let Err(e) = file_read.verify_file_owner(&file_id, &auth_user.id).await {
+            let msg = e.to_string();
             return (
                 StatusCode::NOT_FOUND,
-                Json(serde_json::json!({ "error": e.to_string() })),
+                Json(serde_json::json!({ "error": msg })),
             )
                 .into_response();
         }
@@ -732,7 +726,7 @@ impl FileHandler {
 
         match result {
             Ok(_) => StatusCode::NO_CONTENT.into_response(),
-            Err(err) => AppError::from(err).into_response(),
+            Err(err) => AppError::from(err).into_response()
         }
     }
 
@@ -764,7 +758,7 @@ impl FileHandler {
         let mgmt = &state.applications.file_management_service;
         match mgmt.rename_file_owned(&id, &auth_user.id, &new_name).await {
             Ok(file_dto) => (StatusCode::OK, Json(file_dto)).into_response(),
-            Err(err) => AppError::from(err).into_response(),
+            Err(err) => AppError::from(err).into_response()
         }
     }
 
@@ -784,7 +778,7 @@ impl FileHandler {
             .await
         {
             Ok(file) => (StatusCode::OK, Json(file)).into_response(),
-            Err(err) => AppError::from(err).into_response(),
+            Err(err) => AppError::from(err).into_response()
         }
     }
 
@@ -803,7 +797,7 @@ impl FileHandler {
         let mgmt = &state.applications.file_management_service;
         match mgmt.move_file_owned(&id, &auth_user.id, folder_id).await {
             Ok(file_dto) => (StatusCode::OK, Json(file_dto)).into_response(),
-            Err(err) => AppError::from(err).into_response(),
+            Err(err) => AppError::from(err).into_response()
         }
     }
 
@@ -862,7 +856,9 @@ impl FileHandler {
             })
             .collect();
 
-        format!("{disposition}; filename=\"{ascii_safe}\"; filename*=UTF-8''{encoded}")
+        format!(
+            "{disposition}; filename=\"{ascii_safe}\"; filename*=UTF-8''{encoded}"
+        )
     }
 
     /// Build a 201 Created JSON response.

static/js/core/notifications.js

@@ -204,21 +204,34 @@ const notifications = (() => {
 
         if (status === 'error') batch.errorCount = (batch.errorCount || 0) + 1;
 
-        // Only update the current-file label (single DOM element)
-        const curEl = $(batchId + '-current');
-        if (curEl && status === 'uploading') {
+        // Update the current-file label AND progress bar during upload
+        if (status === 'uploading') {
             const now = Date.now();
             const fileChanged = batch.lastLabelFile !== fileName;
+            // Throttle DOM updates to avoid reflow storms (every 300ms or on file change)
             const shouldUpdate = fileChanged || now - (batch.lastLabelUpdateTs || 0) >= 300 || pct >= 100;
             if (!shouldUpdate) return;
 
             // Show just the file name being uploaded (truncate long paths)
-            const shortName = fileName.length > 50
-                ? '…' + fileName.slice(-49)
-                : fileName;
-            curEl.textContent = shortName;
+            const curEl = $(batchId + '-current');
+            if (curEl) {
+                const shortName = fileName.length > 50
+                    ? '…' + fileName.slice(-49)
+                    : fileName;
+                curEl.textContent = shortName;
+            }
             batch.lastLabelFile = fileName;
             batch.lastLabelUpdateTs = now;
+
+            // Update progress bar with per-file granularity:
+            // overall% = (completed_files + current_file_fraction) / total_files
+            const overallPct = Math.round(
+                ((batch.completed + (pct / 100)) / batch.totalFiles) * 100
+            );
+            const fillEl  = $(batchId + '-fill');
+            const pctEl   = $(batchId + '-pct');
+            if (fillEl) fillEl.style.width = overallPct + '%';
+            if (pctEl)  pctEl.textContent  = overallPct + '%';
         }
     }
 

static/js/features/files/fileOperations.js

@@ -62,8 +62,12 @@ const fileOps = {
         return new Promise((resolve) => {
             const xhr = new XMLHttpRequest();
             const notif = window.notifications;
-            xhr.timeout = timeoutMs;
-            const hardDeadlineMs = Math.max(timeoutMs * 2, 180000);
+            // Do NOT set xhr.timeout — it is a TOTAL deadline from send() to
+            // response and would kill large uploads even while data is flowing.
+            // Instead we rely on the stall timer (no progress for N seconds)
+            // and a generous hard deadline that scales with file size.
+            xhr.timeout = 0;
+            const hardDeadlineMs = Math.max(timeoutMs * 4, 600000); // min 10 min
             let lastProgressPctSent = -1;
 
             let isSettled = false;
@@ -121,8 +125,8 @@ const fileOps = {
                 resetStallTimer();
                 if (e.lengthComputable) {
                     const pct = Math.round((e.loaded / e.total) * 100);
-                    // Throttle UI updates from very chatty progress events
-                    if (pct === 100 || pct - lastProgressPctSent >= 10) {
+                    // Throttle UI updates: every 2% for smooth progress on large files
+                    if (pct === 100 || pct - lastProgressPctSent >= 2) {
                         lastProgressPctSent = pct;
                         safeUpdateFile(pct, 'uploading');
                     }
@@ -320,7 +324,11 @@ const fileOps = {
                     file: file.name, size: file.size
                 });
 
-                const result = await this._uploadFileXHR(formData, batchId, file.name);
+                // Scale stall timeout with file size:
+                // base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit
+                const sizeGB = file.size / (1024 * 1024 * 1024);
+                const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000);
+                const result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout);
 
                 uploadedCount++;
 

static/sw.js

@@ -1,5 +1,5 @@
 // OxiCloud Service Worker
-const CACHE_NAME = 'oxicloud-cache-v15';
+const CACHE_NAME = 'oxicloud-cache-v16';
 const ASSETS_TO_CACHE = [
   '/',
   '/index.html',