From 616e48b338bc1f2d0a3eadf74d92a68d53c0d10c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 08:39:06 +0000 Subject: [PATCH 01/16] perf(nextcloud): batch oc:fileid resolution to kill PROPFIND N+1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolving the stable numeric oc:fileid for every child in a NextCloud listing issued one `INSERT ... ON CONFLICT DO UPDATE` per entry — a write (row rewrite + WAL + dead tuple) even when the mapping already existed. A Depth:1 PROPFIND of a folder with N children meant N sequential write round-trips on a read-only operation that sync clients repeat constantly. - Repository: replace the single `get_or_create` (DO UPDATE) with `get_or_create_many` — one idempotent bulk `INSERT ... SELECT unnest(...) ON CONFLICT DO NOTHING` (existing rows untouched) plus a single `SELECT ... WHERE object_id = ANY(...)`. Two statements instead of N. - Service: add an Arc-backed moka cache (uuid -> i64; the mapping is immutable, so warm entries never go stale) and batch APIs `get_or_create_file_ids` / `get_or_create_folder_ids` that only query the misses. Warm listings cost zero queries. - Handlers (PROPFIND, REPORT favorites/search, trashbin, OCS unified search): pre-resolve all ids in two batched queries — file and folder run concurrently via `tokio::join!` — and turn the XML/JSON emission into a synchronous map lookup. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx --- .../services/nextcloud_file_id_service.rs | 109 +++++++++++++-- .../pg/nextcloud_object_id_repository.rs | 66 +++++++-- src/interfaces/nextcloud/ocs_handler.rs | 18 ++- src/interfaces/nextcloud/report_handler.rs | 129 ++++++++++-------- src/interfaces/nextcloud/trashbin_handler.rs | 32 +++-- src/interfaces/nextcloud/webdav_handler.rs | 63 ++++++--- 6 files changed, 304 insertions(+), 113 deletions(-) diff --git a/src/application/services/nextcloud_file_id_service.rs b/src/application/services/nextcloud_file_id_service.rs index 30f35614..2190e7f9 100644 --- a/src/application/services/nextcloud_file_id_service.rs +++ b/src/application/services/nextcloud_file_id_service.rs @@ -1,12 +1,25 @@ +use std::collections::HashMap; use std::sync::Arc; +use moka::future::Cache; +use uuid::Uuid; + use crate::common::errors::{DomainError, ErrorKind, Result}; use crate::infrastructure::repositories::pg::NextcloudObjectIdRepository; +/// Capacity of the in-memory UUID→numeric-id cache. The mapping is immutable +/// once created, so a warm entry never goes stale and eviction only costs a +/// re-query; ~100k entries is a few MB. +const ID_CACHE_CAPACITY: u64 = 100_000; + #[derive(Clone)] pub struct NextcloudFileIdService { repo: Option>, instance_id: String, + /// Object-UUID → stable numeric id. `moka` caches are `Arc`-backed, so all + /// clones of the service share one cache and the per-child resolution in a + /// listing costs zero queries once warm. + cache: Cache, } impl NextcloudFileIdService { @@ -14,6 +27,7 @@ impl NextcloudFileIdService { Self { repo: Some(repo), instance_id, + cache: Cache::new(ID_CACHE_CAPACITY), } } @@ -21,29 +35,76 @@ impl NextcloudFileIdService { Self { repo: None, instance_id: "ocnca".to_string(), + cache: Cache::new(ID_CACHE_CAPACITY), } } - pub async fn get_or_create_file_id(&self, file_id: &str) -> Result { - let repo = self.repo.as_ref().ok_or_else(|| { - DomainError::internal_error("NextcloudFileId", "Repository not initialized") - })?; - repo.get_or_create("file", file_id).await + /// Resolve — creating when absent — stable numeric file IDs for many + /// UUIDs at once. Cache hits cost nothing; the misses are resolved with a + /// single backing query. The returned map is keyed by the caller's + /// original id strings; unresolvable inputs are simply absent (mirroring + /// the `.ok()` behaviour the callers relied on). + pub async fn get_or_create_file_ids( + &self, + file_ids: &[String], + ) -> Result> { + self.get_or_create_many("file", file_ids).await } - pub async fn get_or_create_folder_id(&self, folder_id: &str) -> Result { - let repo = self.repo.as_ref().ok_or_else(|| { + /// Folder counterpart of [`Self::get_or_create_file_ids`]. + pub async fn get_or_create_folder_ids( + &self, + folder_ids: &[String], + ) -> Result> { + self.get_or_create_many("folder", folder_ids).await + } + + async fn get_or_create_many( + &self, + object_type: &str, + raw_ids: &[String], + ) -> Result> { + let mut result = HashMap::with_capacity(raw_ids.len()); + // Parsed-UUID → caller's original string; also dedupes the miss list. + let mut pending: HashMap = HashMap::new(); + + for raw in raw_ids { + let Ok(uuid) = Uuid::parse_str(raw) else { + continue; // Unparseable ids never had a mapping — skip silently. + }; + if let Some(id) = self.cache.get(&uuid).await { + result.insert(raw.clone(), id); + } else { + pending.entry(uuid).or_insert_with(|| raw.clone()); + } + } + + if !pending.is_empty() { + let misses: Vec = pending.keys().copied().collect(); + let resolved = self + .repo()? + .get_or_create_many(object_type, &misses) + .await?; + for (uuid, id) in resolved { + self.cache.insert(uuid, id).await; + if let Some(original) = pending.get(&uuid) { + result.insert(original.clone(), id); + } + } + } + + Ok(result) + } + + fn repo(&self) -> Result<&Arc> { + self.repo.as_ref().ok_or_else(|| { DomainError::internal_error("NextcloudFileId", "Repository not initialized") - })?; - repo.get_or_create("folder", folder_id).await + }) } /// Get the OxiCloud file UUID from a Nextcloud numeric ID. pub async fn get_oxicloud_id(&self, nc_file_id: i64) -> Result { - let repo = self.repo.as_ref().ok_or_else(|| { - DomainError::internal_error("NextcloudFileId", "Repository not initialized") - })?; - repo.get_object_id(nc_file_id, "file").await + self.repo()?.get_object_id(nc_file_id, "file").await } pub fn format_oc_id(&self, id: i64) -> String { @@ -59,6 +120,7 @@ impl NextcloudFileIdService { Self { repo: None, instance_id: instance_id.to_string(), + cache: Cache::new(ID_CACHE_CAPACITY), } } @@ -107,4 +169,25 @@ mod tests { let svc = NextcloudFileIdService::new_stub(); assert!(svc.ensure_ready().is_err()); } + + // Empty input resolves to an empty map without ever touching the repo, so + // it succeeds even on the repo-less stub. + #[tokio::test] + async fn test_get_or_create_file_ids_empty_is_noop() { + let svc = NextcloudFileIdService::new_stub(); + let map = svc.get_or_create_file_ids(&[]).await.unwrap(); + assert!(map.is_empty()); + } + + // Unparseable ids never had a mapping, so they are skipped before any repo + // call — the stub (no repo) must not error on them. + #[tokio::test] + async fn test_get_or_create_file_ids_skips_unparseable() { + let svc = NextcloudFileIdService::new_stub(); + let map = svc + .get_or_create_file_ids(&["not-a-uuid".to_string()]) + .await + .unwrap(); + assert!(map.is_empty()); + } } diff --git a/src/infrastructure/repositories/pg/nextcloud_object_id_repository.rs b/src/infrastructure/repositories/pg/nextcloud_object_id_repository.rs index 3ffddcbc..5e83ed10 100644 --- a/src/infrastructure/repositories/pg/nextcloud_object_id_repository.rs +++ b/src/infrastructure/repositories/pg/nextcloud_object_id_repository.rs @@ -1,5 +1,7 @@ use sqlx::{PgPool, Row}; +use std::collections::HashMap; use std::sync::Arc; +use uuid::Uuid; use crate::common::errors::{DomainError, ErrorKind, Result}; @@ -12,29 +14,73 @@ impl NextcloudObjectIdRepository { Self { pool } } - pub async fn get_or_create(&self, object_type: &str, object_id: &str) -> Result { - let row = sqlx::query( + /// Resolve — creating when absent — stable numeric IDs for a batch of + /// object UUIDs sharing one `object_type`. + /// + /// Two statements instead of one per id: an idempotent bulk insert that + /// leaves existing rows untouched (`ON CONFLICT DO NOTHING` — no row + /// rewrite, no WAL churn, no dead tuples, unlike the former `DO UPDATE`), + /// followed by a single read of every requested mapping. The insert + /// auto-commits before the read, so the read observes both our own rows + /// and any created concurrently. Returns a map keyed by object UUID; + /// unresolvable inputs are simply absent. + pub async fn get_or_create_many( + &self, + object_type: &str, + object_ids: &[Uuid], + ) -> Result> { + if object_ids.is_empty() { + return Ok(HashMap::new()); + } + + // 1. Create missing mappings only. `DO NOTHING` skips the write for + // UUIDs that already map, eliminating the per-listing row rewrite. + sqlx::query( r#" INSERT INTO storage.nextcloud_object_ids (object_type, object_id) - VALUES ($1, $2::uuid) - ON CONFLICT (object_type, object_id) - DO UPDATE SET object_id = EXCLUDED.object_id - RETURNING id + SELECT $1, u FROM unnest($2::uuid[]) AS u + ON CONFLICT (object_type, object_id) DO NOTHING "#, ) .bind(object_type) - .bind(object_id) - .fetch_one(&*self.pool) + .bind(object_ids) + .execute(&*self.pool) .await .map_err(|e| { DomainError::new( ErrorKind::DatabaseError, "NextcloudFileId", - format!("Failed to get/create Nextcloud ID: {}", e), + format!("Failed to create Nextcloud IDs: {}", e), ) })?; - Ok(row.get::("id")) + // 2. Read every requested mapping back in a single round-trip. + let rows = sqlx::query( + r#" + SELECT id, object_id + FROM storage.nextcloud_object_ids + WHERE object_type = $1 AND object_id = ANY($2::uuid[]) + "#, + ) + .bind(object_type) + .bind(object_ids) + .fetch_all(&*self.pool) + .await + .map_err(|e| { + DomainError::new( + ErrorKind::DatabaseError, + "NextcloudFileId", + format!("Failed to load Nextcloud IDs: {}", e), + ) + })?; + + let mut map = HashMap::with_capacity(rows.len()); + for row in rows { + let object_id: Uuid = row.get("object_id"); + let id: i64 = row.get("id"); + map.insert(object_id, id); + } + Ok(map) } /// Get the OxiCloud object ID from a Nextcloud numeric ID. diff --git a/src/interfaces/nextcloud/ocs_handler.rs b/src/interfaces/nextcloud/ocs_handler.rs index 5c255107..6c86fb8d 100644 --- a/src/interfaces/nextcloud/ocs_handler.rs +++ b/src/interfaces/nextcloud/ocs_handler.rs @@ -5,6 +5,7 @@ use axum::{ response::{IntoResponse, Response}, }; use serde_json::json; +use std::collections::HashMap; use std::sync::Arc; use crate::application::dtos::search_dto::SearchCriteriaDto; @@ -384,6 +385,17 @@ pub async fn handle_search( let file_id_svc = state.nextcloud.as_ref().map(|n| &n.file_ids); + // Pre-resolve numeric ids for every file result in a single batch query + // (was one INSERT round-trip per result). + let file_uuids: Vec = results.files.iter().map(|f| f.id.clone()).collect(); + let file_id_map: HashMap = match file_id_svc { + Some(svc) => svc + .get_or_create_file_ids(&file_uuids) + .await + .unwrap_or_default(), + None => HashMap::new(), + }; + let mut entries: Vec = Vec::new(); // Map file results @@ -394,11 +406,7 @@ pub async fn handle_search( .unwrap_or(&file.path); let display_path = format!("/{}", display_path); - let numeric_id = if let Some(svc) = file_id_svc { - svc.get_or_create_file_id(&file.id).await.ok() - } else { - None - }; + let numeric_id = file_id_map.get(&file.id).copied(); let thumbnail_url = match numeric_id { Some(nid) => format!("/index.php/core/preview?fileId={}&x=32&y=32", nid), diff --git a/src/interfaces/nextcloud/report_handler.rs b/src/interfaces/nextcloud/report_handler.rs index c59a07f2..d5e81b38 100644 --- a/src/interfaces/nextcloud/report_handler.rs +++ b/src/interfaces/nextcloud/report_handler.rs @@ -25,8 +25,7 @@ use crate::domain::entities::file::File; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::CurrentUser; use crate::interfaces::nextcloud::webdav_handler::{ - format_oc_id, nc_href, resolve_file_id, resolve_folder_id, write_file_response, - write_folder_response, + batch_resolve_ids, format_oc_id, nc_href, write_file_response, write_folder_response, }; /// Handle WebDAV REPORT and SEARCH methods for Nextcloud compatibility. @@ -87,56 +86,71 @@ async fn handle_filter_files( let home_prefix = format!("My Folder - {}/", user.username); + // Pass 1: fetch the favorited DTOs (the per-item fetch is a separate + // concern from the oc:fileid resolution batched below). + let mut files: Vec = Vec::new(); + let mut folders: Vec = Vec::new(); + for fav in &favorites { + match fav.item_type.as_str() { + "file" => { + if let Ok(f) = file_service.get_file(&fav.item_id).await { + files.push(f); + } + } + "folder" => { + if let Ok(f) = folder_service.get_folder(&fav.item_id).await { + folders.push(f); + } + } + _ => {} + } + } + + // Pass 2: resolve every oc:fileid in two batch queries (was one per item). + let file_uuids: Vec = files.iter().map(|f| f.id.clone()).collect(); + let folder_uuids: Vec = folders.iter().map(|f| f.id.clone()).collect(); + let (file_id_map, folder_id_map) = + batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await; + + // Pass 3: write the multistatus XML (pure synchronous map lookups). let mut buf = Vec::new(); { let mut xml = Writer::new(&mut buf); write_multistatus_start(&mut xml)?; - for fav in &favorites { - match fav.item_type.as_str() { - "file" => { - let file = match file_service.get_file(&fav.item_id).await { - Ok(f) => f, - Err(_) => continue, // Deleted or inaccessible -- skip. - }; - let subpath = strip_home_prefix(&file.path, &home_prefix); - let href = nc_href(&user.username, subpath); - let fid = resolve_file_id(file_id_svc, &file.id).await; - let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); - write_file_response( - &mut xml, - &file, - &href, - fid, - oc_id.as_deref(), - &user.username, - &favorite_ids, - ) - .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; - } - "folder" => { - let folder = match folder_service.get_folder(&fav.item_id).await { - Ok(f) => f, - Err(_) => continue, - }; - let subpath = strip_home_prefix(&folder.path, &home_prefix); - let href = format!("{}/", nc_href(&user.username, subpath)); - let fid = resolve_folder_id(file_id_svc, &folder.id).await; - let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); - write_folder_response( - &mut xml, - &folder, - &href, - fid, - oc_id.as_deref(), - &user.username, - &favorite_ids, - ) - .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; - } - _ => continue, - } + for file in &files { + let subpath = strip_home_prefix(&file.path, &home_prefix); + let href = nc_href(&user.username, subpath); + let fid = file_id_map.get(&file.id).copied(); + let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); + write_file_response( + &mut xml, + file, + &href, + fid, + oc_id.as_deref(), + &user.username, + &favorite_ids, + ) + .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; + } + + for folder in &folders { + let subpath = strip_home_prefix(&folder.path, &home_prefix); + let href = format!("{}/", nc_href(&user.username, subpath)); + let fid = folder_id_map.get(&folder.id).copied(); + let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); + write_folder_response( + &mut xml, + folder, + &href, + fid, + oc_id.as_deref(), + &user.username, + &favorite_ids, + ) + .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; } xml.write_event(Event::End(BytesEnd::new("d:multistatus"))) @@ -192,6 +206,15 @@ async fn handle_search( // No favorite checking for search results -- pass an empty set. let favorite_ids: HashSet = HashSet::new(); + // Materialize DTOs, then resolve every oc:fileid in two batch queries + // (was one INSERT round-trip per result). + let files: Vec = results.files.iter().map(file_dto_from_search).collect(); + let folders: Vec = results.folders.iter().map(folder_dto_from_search).collect(); + let file_uuids: Vec = files.iter().map(|f| f.id.clone()).collect(); + let folder_uuids: Vec = folders.iter().map(|f| f.id.clone()).collect(); + let (file_id_map, folder_id_map) = + batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await; + let mut buf = Vec::new(); { let mut xml = Writer::new(&mut buf); @@ -199,15 +222,14 @@ async fn handle_search( write_multistatus_start(&mut xml)?; // Files. - for fr in &results.files { - let file = file_dto_from_search(fr); + for file in &files { let subpath = strip_home_prefix(&file.path, &home_prefix); let href = nc_href(&user.username, subpath); - let fid = resolve_file_id(file_id_svc, &file.id).await; + let fid = file_id_map.get(&file.id).copied(); let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); write_file_response( &mut xml, - &file, + file, &href, fid, oc_id.as_deref(), @@ -218,15 +240,14 @@ async fn handle_search( } // Folders. - for sr in &results.folders { - let folder = folder_dto_from_search(sr); + for folder in &folders { let subpath = strip_home_prefix(&folder.path, &home_prefix); let href = format!("{}/", nc_href(&user.username, subpath)); - let fid = resolve_folder_id(file_id_svc, &folder.id).await; + let fid = folder_id_map.get(&folder.id).copied(); let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); write_folder_response( &mut xml, - &folder, + folder, &href, fid, oc_id.as_deref(), diff --git a/src/interfaces/nextcloud/trashbin_handler.rs b/src/interfaces/nextcloud/trashbin_handler.rs index 7c8cb286..60464ef2 100644 --- a/src/interfaces/nextcloud/trashbin_handler.rs +++ b/src/interfaces/nextcloud/trashbin_handler.rs @@ -14,7 +14,7 @@ use crate::common::di::AppState; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; use crate::interfaces::nextcloud::webdav_handler::{ - format_oc_id, resolve_file_id, resolve_folder_id, write_text_element, + batch_resolve_ids, format_oc_id, write_text_element, }; const HEADER_DAV: HeaderName = HeaderName::from_static("dav"); @@ -198,6 +198,7 @@ fn strip_home_prefix<'a>(original_path: &'a str, username: &str) -> &'a str { use crate::application::dtos::trash_dto::TrashedItemDto; use crate::application::services::nextcloud_file_id_service::NextcloudFileIdService; +use std::collections::HashMap; /// Generate a complete Nextcloud-compatible multistatus XML response for the trashbin. async fn write_trashbin_multistatus( @@ -219,9 +220,25 @@ async fn write_trashbin_multistatus( // Root container entry for the trash collection itself. write_trash_root_response(&mut xml, username)?; + // Pre-resolve every oc:fileid in two batch queries by object type (was one + // INSERT round-trip per item). File and folder UUIDs are disjoint, so the + // two maps merge cleanly into one keyed by original_id. + let mut file_uuids: Vec = Vec::new(); + let mut folder_uuids: Vec = Vec::new(); + for item in items { + if item.item_type == "folder" { + folder_uuids.push(item.original_id.clone()); + } else { + file_uuids.push(item.original_id.clone()); + } + } + let (mut id_map, folder_id_map) = + batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await; + id_map.extend(folder_id_map); + // Individual trashed items. for item in items { - write_trash_item_response(&mut xml, item, username, file_id_svc).await?; + write_trash_item_response(&mut xml, item, username, file_id_svc, &id_map)?; } xml.write_event(Event::End(BytesEnd::new("d:multistatus"))) @@ -267,11 +284,12 @@ fn write_trash_root_response( } /// Write a single trashed item as a `` element. -async fn write_trash_item_response( +fn write_trash_item_response( xml: &mut Writer, item: &TrashedItemDto, username: &str, file_id_svc: Option<&Arc>, + id_map: &HashMap, ) -> Result<(), String> { xml.write_event(Event::Start(BytesStart::new("d:response"))) .map_err(|e| e.to_string())?; @@ -318,12 +336,8 @@ async fn write_trash_item_response( // d:getcontentlength write_text_element(xml, "d:getcontentlength", "0")?; - // oc:fileid and oc:id — resolve numeric ID via file_id service - let file_id = if item.item_type == "folder" { - resolve_folder_id(file_id_svc, &item.original_id).await - } else { - resolve_file_id(file_id_svc, &item.original_id).await - }; + // oc:fileid and oc:id — resolved up front in a batch query. + let file_id = id_map.get(&item.original_id).copied(); if let Some(id) = file_id { write_text_element(xml, "oc:fileid", &id.to_string())?; let oc_id = format_oc_id(id, file_id_svc); diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 5a82baa7..a150ea7d 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -9,7 +9,7 @@ use quick_xml::{ Writer, events::{BytesEnd, BytesStart, BytesText, Event}, }; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use crate::application::adapters::webdav_adapter::{PropFindRequest, WebDavAdapter}; @@ -1001,6 +1001,26 @@ async fn write_nc_multistatus( file_id_svc: Option<&Arc>, favorite_ids: &HashSet, ) -> Result<(), String> { + // When folder is None, files are the target resource itself (single-file + // PROPFIND) and must always be emitted. When folder is Some, files/subfolders + // are children and should only be listed when depth > 0. + let emit_children = folder.is_none() || depth != "0"; + + // Pre-resolve every oc:fileid up front in two batch queries (one per + // object type) instead of one INSERT round-trip per child. The XML writing + // below is then a pure synchronous map lookup. + let mut file_uuids: Vec = Vec::new(); + let mut folder_uuids: Vec = Vec::new(); + if let Some(f) = folder { + folder_uuids.push(f.id.clone()); + } + if emit_children { + file_uuids.extend(files.iter().map(|f| f.id.clone())); + folder_uuids.extend(subfolders.iter().map(|sf| sf.id.clone())); + } + let (file_id_map, folder_id_map) = + batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await; + let mut xml = Writer::new(writer); // Root element with all required namespaces. @@ -1015,7 +1035,7 @@ async fn write_nc_multistatus( // §5.2 + strict NC-client enforcement — see `nc_collection_href`). if let Some(f) = folder { let href = nc_collection_href(username, subpath); - let file_id = resolve_folder_id(file_id_svc, &f.id).await; + let file_id = folder_id_map.get(&f.id).copied(); let oc_id = file_id.map(|id| format_oc_id(id, file_id_svc)); write_folder_response( &mut xml, @@ -1028,11 +1048,6 @@ async fn write_nc_multistatus( )?; } - // When folder is None, files are the target resource itself (single-file - // PROPFIND) and must always be emitted. When folder is Some, files/subfolders - // are children and should only be listed when depth > 0. - let emit_children = folder.is_none() || depth != "0"; - if emit_children { // Files. for file in files { @@ -1045,7 +1060,7 @@ async fn write_nc_multistatus( format!("{}/{}", subpath.trim_end_matches('/'), file.name) }; let href = nc_href(username, &child_sub); - let file_id = resolve_file_id(file_id_svc, &file.id).await; + let file_id = file_id_map.get(&file.id).copied(); let oc_id = file_id.map(|id| format_oc_id(id, file_id_svc)); write_file_response( &mut xml, @@ -1066,7 +1081,7 @@ async fn write_nc_multistatus( format!("{}/{}", subpath.trim_end_matches('/'), sf.name) }; let href = nc_collection_href(username, &child_sub); - let file_id = resolve_folder_id(file_id_svc, &sf.id).await; + let file_id = folder_id_map.get(&sf.id).copied(); let oc_id = file_id.map(|id| format_oc_id(id, file_id_svc)); write_folder_response( &mut xml, @@ -1273,20 +1288,24 @@ pub fn write_text_element( Ok(()) } -pub async fn resolve_file_id( +/// Resolve every `oc:fileid` for a listing in two batch queries (one per +/// object type) instead of one INSERT round-trip per child. Returns +/// `(file_map, folder_map)` keyed by object UUID; entries are absent when the +/// service is disabled or an id can't be resolved, mirroring the previous +/// per-call `Option` behaviour. The two batches run concurrently. +pub async fn batch_resolve_ids( svc: Option<&Arc>, - file_uuid: &str, -) -> Option { - let svc = svc?; - svc.get_or_create_file_id(file_uuid).await.ok() -} - -pub async fn resolve_folder_id( - svc: Option<&Arc>, - folder_uuid: &str, -) -> Option { - let svc = svc?; - svc.get_or_create_folder_id(folder_uuid).await.ok() + file_uuids: &[String], + folder_uuids: &[String], +) -> (HashMap, HashMap) { + let Some(svc) = svc else { + return (HashMap::new(), HashMap::new()); + }; + let (files, folders) = tokio::join!( + svc.get_or_create_file_ids(file_uuids), + svc.get_or_create_folder_ids(folder_uuids), + ); + (files.unwrap_or_default(), folders.unwrap_or_default()) } pub fn format_oc_id(id: i64, svc: Option<&Arc>) -> String { From 214d42d0f6fa6b1defe2313989fe876fa7b026a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 08:39:12 +0000 Subject: [PATCH 02/16] perf(photos): batch trash deletion instead of N sequential DELETEs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting a multi-photo selection fired one `DELETE /api/files/{id}` per photo in series — 300 selected photos meant 300 sequential round-trips with the UI blocked on `confirm()`. Switch to `POST /api/batch/trash` (already used by the file view), chunked to the backend's MAX_BATCH_SIZE of 1000, so a selection of any size collapses to ceil(N/1000) requests. Only items the server reports as successfully trashed are removed from the grid; the selection bar refreshes to reflect any failures. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx --- static/js/features/library/photos.js | 45 ++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/static/js/features/library/photos.js b/static/js/features/library/photos.js index eff922ba..d7f6f6ee 100644 --- a/static/js/features/library/photos.js +++ b/static/js/features/library/photos.js @@ -472,22 +472,43 @@ const photosView = { if (bar_delete) { bar_delete.onclick = async () => { if (!confirm('Delete selected items?')) return; - for (const fid of this.selected) { - try { - await fetch(`/api/files/${fid}`, { - method: 'DELETE', + + // One batch request per chunk instead of one DELETE per photo. + // The photos view is files-only, so every id is a file id. + const ids = [...this.selected]; + const CHUNK_SIZE = 1000; // backend MAX_BATCH_SIZE + const trashed = new Set(); + + try { + for (let i = 0; i < ids.length; i += CHUNK_SIZE) { + const chunk = ids.slice(i, i + CHUNK_SIZE); + const response = await fetch('/api/batch/trash', { + method: 'POST', credentials: 'include', - headers: this._headers() + headers: this._headers(true), + body: JSON.stringify({ file_ids: chunk, folder_ids: [] }) }); - } catch (err) { - console.error('Delete failed:', fid, err); + // 200 = all trashed, 206 = partial; both carry `successful`. + if (!response.ok && response.status !== 206) { + console.error('Batch trash failed:', response.status); + continue; + } + const data = await response.json(); + const ok = Array.isArray(data?.successful) ? data.successful : chunk; + for (const id of ok) trashed.add(id); } + } catch (err) { + console.error('Batch trash error:', err); } - this.items = this.items.filter((f) => !this.selected.has(f.id)); - this.selected.clear(); - this._hideSelectionBar(); - this._renderedCount = 0; - this._renderFull(); + + if (trashed.size > 0) { + this.items = this.items.filter((f) => !trashed.has(f.id)); + for (const id of trashed) this.selected.delete(id); + this._renderedCount = 0; + this._renderFull(); + } + // Refresh (or hide) the bar to reflect any items left selected. + this._updateSelectionBar(); }; } From 3c64354a0d27cd71176b83df7c57754cea97bd2d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 08:44:39 +0000 Subject: [PATCH 03/16] perf(upload): run multi-file selections through the 10-worker pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `uploadFiles` sent files one at a time (await per file), so dropping N files meant N sequential round-trips. Reuse the bounded-concurrency pool already proven in `uploadFolderEntries` (CONCURRENCY = 10): independent files now upload up to 10 at a time — ~10x faster for many small files. Per-file progress (XHR → bell), the legacy progress bar, timeout notifications and the quota short-circuit are preserved. On a quota error the pool stops pulling new files while in-flight uploads finish (matching `uploadFolderEntries`) instead of the old hard `break`. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx --- static/js/features/files/fileOperations.js | 39 +++++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/static/js/features/files/fileOperations.js b/static/js/features/files/fileOperations.js index 8cb6a3a7..de7f5251 100644 --- a/static/js/features/files/fileOperations.js +++ b/static/js/features/files/fileOperations.js @@ -349,7 +349,9 @@ const fileOps = { } // Filter out unreadable entries (typically dropped folders/placeholders) + /** @type {File[]} */ const readableFiles = []; + /** @type {string[]} */ const skippedEntries = []; for (const f of originalFiles) { // eslint-disable-next-line no-await-in-loop @@ -386,13 +388,21 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; + let quotaStop = false; - for (let i = 0; i < totalFiles; i++) { - const file = readableFiles[i]; + const targetFolderId = app.currentPath || app.userHomeFolderId; + + /** + * Upload a single readable file by index. Shared counters are + * mutated here; safe because JS runs the workers cooperatively + * (no true parallelism between awaits). + * @param {number} idx + */ + const uploadOneFile = async (idx) => { + if (quotaStop) return; + const file = readableFiles[idx]; const formData = new FormData(); - - const targetFolderId = app.currentPath || app.userHomeFolderId; if (targetFolderId) formData.append('folder_id', targetFolderId); formData.append('file', file); @@ -436,6 +446,8 @@ const fileOps = { }); } if (result.isQuotaError) { + // Stop pulling new files; in-flight uploads still finish. + quotaStop = true; const msg = result.errorMsg || i18n.t('storage_quota_exceeded'); if (notifications) { notifications.addNotification({ @@ -445,10 +457,27 @@ const fileOps = { text: msg }); } - break; } } + }; + + // Pool-based concurrency: keep up to CONCURRENCY uploads in flight + // instead of one at a time (mirrors uploadFolderEntries). Files are + // independent, so this is ~CONCURRENCY× faster for many small files. + const CONCURRENCY = 10; + let nextIdx = 0; + const runNext = async () => { + while (nextIdx < totalFiles && !quotaStop) { + const idx = nextIdx++; + await uploadOneFile(idx); + } + }; + + const workers = []; + for (let w = 0; w < Math.min(CONCURRENCY, totalFiles); w++) { + workers.push(runNext()); } + await Promise.all(workers); // All done this._finishUploadToast(successCount, totalFiles); From ca698f73d16c936805390ef8348afab8b4ebe5d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 09:01:42 +0000 Subject: [PATCH 04/16] perf(search): render result sets in one batched pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit displaySearchResults inserted up to 100 results one at a time via addItem({ scroll: true, highlight: true }) — each insert ran two DOM scans (duplicate guard + row lookup), a smooth scrollIntoView and a highlight pulse, so every live-search keystroke triggered ~100 competing smooth-scrolls and O(n²) container scans. Add filesView.renderItems(): a single component.render() pass (one DocumentFragment, zero per-item scans/scrolls). addItem keeps the scroll/highlight affordances for its documented purpose — optimistic single-item inserts after upload/create. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx --- static/js/app/filesView.js | 17 ++++++++++++++++- static/js/features/files/search.js | 15 +++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/static/js/app/filesView.js b/static/js/app/filesView.js index 5f7893be..62769d41 100644 --- a/static/js/app/filesView.js +++ b/static/js/app/filesView.js @@ -496,6 +496,21 @@ async function loadFiles(options = { insertHistory: true }) { } } +/** + * Replace the list contents with a whole result set in one batched render + * (single DocumentFragment pass — no per-item DOM scans, smooth-scrolls or + * highlight pulses). Used by search to display results; optimistic + * single-item inserts should keep using `addItem`. + * + * @param {Array} items + */ +function renderItems(items) { + const component = _ensureComponent(); + if (!component) return; + document.getElementById('files-container-error')?.classList.add('hidden'); + component.render(items); +} + /** * Re-evaluate the shared badge for every item currently rendered in the Files list. * Call this after the outgoing grants cache has been refreshed. @@ -504,4 +519,4 @@ function refreshSharedBadges() { _component?.refreshSharedBadges(); } -export { addItem, filesView, loadFiles, refreshSharedBadges }; +export { addItem, filesView, loadFiles, refreshSharedBadges, renderItems }; diff --git a/static/js/features/files/search.js b/static/js/features/files/search.js index 8db9fa5a..73d5bbac 100644 --- a/static/js/features/files/search.js +++ b/static/js/features/files/search.js @@ -7,7 +7,7 @@ * displays the enriched results returned by the server. */ -import { addItem as filesViewAddItem, loadFiles } from '../../app/filesView.js'; +import { loadFiles, renderItems } from '../../app/filesView.js'; import { app } from '../../app/state.js'; import { ui } from '../../app/ui.js'; import { getAuthHeaders } from './fileOperations.js'; @@ -198,15 +198,10 @@ const search = { return; } - // Render folders (server-provided enriched data) - results.folders.forEach((folder) => { - filesViewAddItem(folder); - }); - - // Render files (server-provided enriched data) - results.files.forEach((file) => { - filesViewAddItem(file); - }); + // Render the whole result set (server-provided enriched data) in one + // batched pass — per-item inserts would trigger a DOM scan, a smooth + // scroll and a highlight pulse for each of up to 100 rows. + renderItems([...results.folders, ...results.files]); }, /** From 64c1e4c8e575db2ca435e713d281fbcae4e1562e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 09:01:55 +0000 Subject: [PATCH 05/16] perf(resource-list): fold per-row listeners into the delegated handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _bindItemEvents attached direct click listeners to the favorite star, the shared badge and every custom action button of every row — ~3+ listeners per row on top of the container delegation the component already wires, i.e. ~10-15k listeners with a few thousand rows loaded via load-more, plus the per-row binding cost on every render. The container's delegated click handler already dispatches by closest() checks; move the three button behaviours there (they return before the card-open branch, preserving the old stopPropagation semantics) and drop _bindItemEvents entirely. Row creation is now pure innerHTML — no listener allocation per row. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx --- static/js/components/resourceList.js | 89 +++++++++++----------------- 1 file changed, 33 insertions(+), 56 deletions(-) diff --git a/static/js/components/resourceList.js b/static/js/components/resourceList.js index 0578be27..73b633fc 100644 --- a/static/js/components/resourceList.js +++ b/static/js/components/resourceList.js @@ -643,7 +643,6 @@ export class ResourceListComponent { `; el.querySelector('.resource-icon-slot')?.replaceWith(buildResourceIcon(folder, 'folder')); - this._bindItemEvents(el, folder); return el; } @@ -693,7 +692,6 @@ export class ResourceListComponent { `; el.querySelector('.resource-icon-slot')?.replaceWith(buildResourceIcon(file, 'file')); - this._bindItemEvents(el, file); return el; } @@ -716,57 +714,6 @@ export class ResourceListComponent { .join(''); } - /** - * Attach direct event listeners to interactive elements inside a .file-item. - * This covers buttons that must stop propagation before the delegated listener runs. - * @param {HTMLElement} el - * @param {FileItem|FolderItem} item - */ - _bindItemEvents(el, item) { - const cfg = this._cfg; - - // Favorite-star — direct click, stopPropagation so the card open doesn't fire - if (cfg.showFavorite && cfg.onFavoriteToggle) { - const star = el.querySelector('.favorite-star'); - star?.addEventListener('click', (e) => { - e.stopPropagation(); - e.stopImmediatePropagation(); - e.preventDefault(); - cfg.onFavoriteToggle?.(item); - }); - } - - // Custom inline actions (e.g. restore / delete-permanently on trash) — - // bound directly so they stop propagation before the card-open handler. - if (cfg.customActions?.length) { - el.querySelectorAll('button[data-custom-action]').forEach((btn) => { - btn.addEventListener('click', (e) => { - e.stopPropagation(); - e.stopImmediatePropagation(); - e.preventDefault(); - const idx = Number(/** @type {HTMLElement} */ (btn).dataset.customAction); - const action = cfg.customActions?.[idx]; - if (action) action.onClick(item); - }); - }); - } - - // Shared-badge click → open share modal (or fall back to context menu) - if (cfg.showShareBadge && (cfg.onShareBadgeClick || cfg.onContextMenu)) { - const badge = el.querySelector('.file-badge-shared'); - badge?.addEventListener('click', (e) => { - e.stopPropagation(); - e.stopImmediatePropagation(); - e.preventDefault(); - if (cfg.onShareBadgeClick) { - cfg.onShareBadgeClick(item); - } else { - cfg.onContextMenu?.(item, /** @type {MouseEvent} */ (e)); - } - }); - } - } - /** Wire one delegated listener for all pointer events in this container. */ _initDelegation() { const container = this._container; @@ -791,6 +738,39 @@ export class ResourceListComponent { return; } + // Favorite-star button — never opens the card + if (target.closest('.favorite-star')) { + e.preventDefault(); + const item = this._itemFromCard(card); + if (item) cfg.onFavoriteToggle?.(item); + return; + } + + // Custom inline actions (e.g. restore / delete-permanently on trash) + const customBtn = /** @type {HTMLElement | null} */ (target.closest('button[data-custom-action]')); + if (customBtn) { + e.preventDefault(); + const idx = Number(customBtn.dataset.customAction); + const action = cfg.customActions?.[idx]; + const item = this._itemFromCard(card); + if (action && item) action.onClick(item); + return; + } + + // Shared-badge click → open share modal (or fall back to context menu) + if (target.closest('.file-badge-shared')) { + e.preventDefault(); + const item = this._itemFromCard(card); + if (item) { + if (cfg.onShareBadgeClick) { + cfg.onShareBadgeClick(item); + } else { + cfg.onContextMenu?.(item, /** @type {MouseEvent} */ (e)); + } + } + return; + } + // Checkbox cell → selection (shift extends range) if (cfg.selectable && target.closest('.checkbox-cell')) { if (e.shiftKey) { @@ -801,9 +781,6 @@ export class ResourceListComponent { return; } - // Favorite star is handled by the direct listener in _bindItemEvents - if (target.closest('.favorite-star')) return; - // Modifier-key click → selection toggle if (e.metaKey || e.altKey || e.ctrlKey) { if (cfg.selectable) this._toggleSelection(card); From 8d040314a333b2b0610edfee2ea0b27a7ace8f0e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 09:01:55 +0000 Subject: [PATCH 06/16] perf(icons): scan only inserted subtrees in the MutationObserver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The global childList observer scheduled replaceIconsInElement() with no scope on ANY node insertion — including bare text nodes — so every notification-bell progress tick during an upload and every infinite-scroll batch re-scanned the whole document with an attribute substring selector. Cost grew with total DOM size (5-10k nodes), not with what was inserted. Queue the added element roots per animation frame and scan just those subtrees (an inserted itself is caught via its parent). Text-node churn no longer triggers any scan at all. The icon replacement is idempotent, so overlapping roots are harmless. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx --- static/js/core/icons.js | 44 ++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/static/js/core/icons.js b/static/js/core/icons.js index 66420925..b5c857cd 100644 --- a/static/js/core/icons.js +++ b/static/js/core/icons.js @@ -618,27 +618,53 @@ function replaceIconsInElement(container) { function oxiIconsInit() { let raf = 0; + /** + * Subtree roots added since the last animation frame. Scanning only + * these (instead of the whole document) keeps the cost proportional + * to what was inserted, not to the total DOM size. + * @type {Set} + */ + let pendingRoots = new Set(); + const scan = () => { raf = 0; - replaceIconsInElement(); + const roots = pendingRoots; + pendingRoots = new Set(); + for (const root of roots) { + if (!root.isConnected) continue; // removed (or replaced) meanwhile + if (root.matches('i[class*="fa-"]')) { + // The inserted node IS the icon — scan via its parent so the + // descendant selector pass picks it up. + replaceIconsInElement(root.parentElement || document.body); + } else { + replaceIconsInElement(root); + } + } }; // Initial sweep once the DOM is ready + const fullSweep = () => replaceIconsInElement(); if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', scan); + document.addEventListener('DOMContentLoaded', fullSweep); } else { - scan(); + fullSweep(); } - // Observe future mutations (dynamic renders, modals, etc.) + // Observe future mutations (dynamic renders, modals, etc.). Only element + // insertions can carry icons — text-node churn (progress counters, + // notification text) no longer triggers any scan at all. new MutationObserver((mutations) => { - if (raf) return; - for (let i = 0; i < mutations.length; i++) { - if (mutations[i].addedNodes.length) { - raf = requestAnimationFrame(scan); - return; + for (const mutation of mutations) { + for (let i = 0; i < mutation.addedNodes.length; i++) { + const node = mutation.addedNodes[i]; + if (node.nodeType === Node.ELEMENT_NODE) { + pendingRoots.add(/** @type {Element} */ (node)); + } } } + if (!raf && pendingRoots.size) { + raf = requestAnimationFrame(scan); + } }).observe(document.documentElement, { childList: true, subtree: true }); } From fd80a3de67082fcb6567cbbe989273e72789f8c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 10 Jun 2026 09:02:13 +0000 Subject: [PATCH 07/16] perf(files-ui): stream media/downloads natively, rAF rubber-band, Map selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four related fixes to stop buffering files in page memory and stop hammering layout from the selection paths: - Inline viewer media: video/audio fetched the ENTIRE file into a blob before the first frame (a 2 GB video = 2 GB of tab heap, no progressive playback, no seek). The