cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

perf(nextcloud): batch oc:fileid resolution to kill PROPFIND N+1

Browse Source 
Resolving the stable numeric oc:fileid for every child in a NextCloud
listing issued one `INSERT ... ON CONFLICT DO UPDATE` per entry — a write
(row rewrite + WAL + dead tuple) even when the mapping already existed.
A Depth:1 PROPFIND of a folder with N children meant N sequential write
round-trips on a read-only operation that sync clients repeat constantly.

- Repository: replace the single `get_or_create` (DO UPDATE) with
  `get_or_create_many` — one idempotent bulk `INSERT ... SELECT unnest(...)
  ON CONFLICT DO NOTHING` (existing rows untouched) plus a single
  `SELECT ... WHERE object_id = ANY(...)`. Two statements instead of N.
- Service: add an Arc-backed moka cache (uuid -> i64; the mapping is
  immutable, so warm entries never go stale) and batch APIs
  `get_or_create_file_ids` / `get_or_create_folder_ids` that only query
  the misses. Warm listings cost zero queries.
- Handlers (PROPFIND, REPORT favorites/search, trashbin, OCS unified
  search): pre-resolve all ids in two batched queries — file and folder
  run concurrently via `tokio::join!` — and turn the XML/JSON emission
  into a synchronous map lookup.

https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx
This commit is contained in:
Claude
2026-06-10 08:39:06 +00:00
parent da6dcb3771
commit 616e48b338
6 changed files with 304 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/interfaces/nextcloud/ocs_handler.rs
+13 -5
View File
@@ -5,6 +5,7 @@ use axum::{
response::{IntoResponse, Response},
};
use serde_json::json;
use std::collections::HashMap;
use std::sync::Arc;
use crate::application::dtos::search_dto::SearchCriteriaDto;
@@ -384,6 +385,17 @@ pub async fn handle_search(
let file_id_svc = state.nextcloud.as_ref().map(|n| &n.file_ids);
// Pre-resolve numeric ids for every file result in a single batch query
// (was one INSERT round-trip per result).
let file_uuids: Vec<String> = results.files.iter().map(|f| f.id.clone()).collect();
let file_id_map: HashMap<String, i64> = match file_id_svc {
Some(svc) => svc
.get_or_create_file_ids(&file_uuids)
.await
.unwrap_or_default(),
None => HashMap::new(),
};
let mut entries: Vec<serde_json::Value> = Vec::new();
// Map file results
@@ -394,11 +406,7 @@ pub async fn handle_search(
.unwrap_or(&file.path);
let display_path = format!("/{}", display_path);
let numeric_id = if let Some(svc) = file_id_svc {
svc.get_or_create_file_id(&file.id).await.ok()
} else {
None
};
let numeric_id = file_id_map.get(&file.id).copied();
let thumbnail_url = match numeric_id {
Some(nid) => format!("/index.php/core/preview?fileId={}&x=32&y=32", nid),
src/interfaces/nextcloud/report_handler.rs
+75 -54
View File
@@ -25,8 +25,7 @@ use crate::domain::entities::file::File;
use crate::interfaces::errors::AppError;
use crate::interfaces::middleware::auth::CurrentUser;
use crate::interfaces::nextcloud::webdav_handler::{
format_oc_id, nc_href, resolve_file_id, resolve_folder_id, write_file_response,
write_folder_response,
batch_resolve_ids, format_oc_id, nc_href, write_file_response, write_folder_response,
};
/// Handle WebDAV REPORT and SEARCH methods for Nextcloud compatibility.
@@ -87,56 +86,71 @@ async fn handle_filter_files(
let home_prefix = format!("My Folder - {}/", user.username);
// Pass 1: fetch the favorited DTOs (the per-item fetch is a separate
// concern from the oc:fileid resolution batched below).
let mut files: Vec<FileDto> = Vec::new();
let mut folders: Vec<FolderDto> = Vec::new();
for fav in &favorites {
match fav.item_type.as_str() {
"file" => {
if let Ok(f) = file_service.get_file(&fav.item_id).await {
files.push(f);
}
}
"folder" => {
if let Ok(f) = folder_service.get_folder(&fav.item_id).await {
folders.push(f);
}
}
_ => {}
}
}
// Pass 2: resolve every oc:fileid in two batch queries (was one per item).
let file_uuids: Vec<String> = files.iter().map(|f| f.id.clone()).collect();
let folder_uuids: Vec<String> = folders.iter().map(|f| f.id.clone()).collect();
let (file_id_map, folder_id_map) =
batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await;
// Pass 3: write the multistatus XML (pure synchronous map lookups).
let mut buf = Vec::new();
{
let mut xml = Writer::new(&mut buf);
write_multistatus_start(&mut xml)?;
for fav in &favorites {
match fav.item_type.as_str() {
"file" => {
let file = match file_service.get_file(&fav.item_id).await {
Ok(f) => f,
Err(_) => continue, // Deleted or inaccessible -- skip.
};
let subpath = strip_home_prefix(&file.path, &home_prefix);
let href = nc_href(&user.username, subpath);
let fid = resolve_file_id(file_id_svc, &file.id).await;
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
write_file_response(
&mut xml,
&file,
&href,
fid,
oc_id.as_deref(),
&user.username,
&favorite_ids,
)
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
}
"folder" => {
let folder = match folder_service.get_folder(&fav.item_id).await {
Ok(f) => f,
Err(_) => continue,
};
let subpath = strip_home_prefix(&folder.path, &home_prefix);
let href = format!("{}/", nc_href(&user.username, subpath));
let fid = resolve_folder_id(file_id_svc, &folder.id).await;
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
write_folder_response(
&mut xml,
&folder,
&href,
fid,
oc_id.as_deref(),
&user.username,
&favorite_ids,
)
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
}
_ => continue,
}
for file in &files {
let subpath = strip_home_prefix(&file.path, &home_prefix);
let href = nc_href(&user.username, subpath);
let fid = file_id_map.get(&file.id).copied();
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
write_file_response(
&mut xml,
file,
&href,
fid,
oc_id.as_deref(),
&user.username,
&favorite_ids,
)
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
}
for folder in &folders {
let subpath = strip_home_prefix(&folder.path, &home_prefix);
let href = format!("{}/", nc_href(&user.username, subpath));
let fid = folder_id_map.get(&folder.id).copied();
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
write_folder_response(
&mut xml,
folder,
&href,
fid,
oc_id.as_deref(),
&user.username,
&favorite_ids,
)
.map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?;
}
xml.write_event(Event::End(BytesEnd::new("d:multistatus")))
@@ -192,6 +206,15 @@ async fn handle_search(
// No favorite checking for search results -- pass an empty set.
let favorite_ids: HashSet<String> = HashSet::new();
// Materialize DTOs, then resolve every oc:fileid in two batch queries
// (was one INSERT round-trip per result).
let files: Vec<FileDto> = results.files.iter().map(file_dto_from_search).collect();
let folders: Vec<FolderDto> = results.folders.iter().map(folder_dto_from_search).collect();
let file_uuids: Vec<String> = files.iter().map(|f| f.id.clone()).collect();
let folder_uuids: Vec<String> = folders.iter().map(|f| f.id.clone()).collect();
let (file_id_map, folder_id_map) =
batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await;
let mut buf = Vec::new();
{
let mut xml = Writer::new(&mut buf);
@@ -199,15 +222,14 @@ async fn handle_search(
write_multistatus_start(&mut xml)?;
// Files.
for fr in &results.files {
let file = file_dto_from_search(fr);
for file in &files {
let subpath = strip_home_prefix(&file.path, &home_prefix);
let href = nc_href(&user.username, subpath);
let fid = resolve_file_id(file_id_svc, &file.id).await;
let fid = file_id_map.get(&file.id).copied();
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
write_file_response(
&mut xml,
&file,
file,
&href,
fid,
oc_id.as_deref(),
@@ -218,15 +240,14 @@ async fn handle_search(
}
// Folders.
for sr in &results.folders {
let folder = folder_dto_from_search(sr);
for folder in &folders {
let subpath = strip_home_prefix(&folder.path, &home_prefix);
let href = format!("{}/", nc_href(&user.username, subpath));
let fid = resolve_folder_id(file_id_svc, &folder.id).await;
let fid = folder_id_map.get(&folder.id).copied();
let oc_id = fid.map(|id| format_oc_id(id, file_id_svc));
write_folder_response(
&mut xml,
&folder,
folder,
&href,
fid,
oc_id.as_deref(),
src/interfaces/nextcloud/trashbin_handler.rs
+23 -9
View File
@@ -14,7 +14,7 @@ use crate::common::di::AppState;
use crate::interfaces::errors::AppError;
use crate::interfaces::middleware::auth::{AuthUser, CurrentUser};
use crate::interfaces::nextcloud::webdav_handler::{
format_oc_id, resolve_file_id, resolve_folder_id, write_text_element,
batch_resolve_ids, format_oc_id, write_text_element,
};
const HEADER_DAV: HeaderName = HeaderName::from_static("dav");
@@ -198,6 +198,7 @@ fn strip_home_prefix<'a>(original_path: &'a str, username: &str) -> &'a str {
use crate::application::dtos::trash_dto::TrashedItemDto;
use crate::application::services::nextcloud_file_id_service::NextcloudFileIdService;
use std::collections::HashMap;
/// Generate a complete Nextcloud-compatible multistatus XML response for the trashbin.
async fn write_trashbin_multistatus<W: std::io::Write>(
@@ -219,9 +220,25 @@ async fn write_trashbin_multistatus<W: std::io::Write>(
// Root container entry for the trash collection itself.
write_trash_root_response(&mut xml, username)?;
// Pre-resolve every oc:fileid in two batch queries by object type (was one
// INSERT round-trip per item). File and folder UUIDs are disjoint, so the
// two maps merge cleanly into one keyed by original_id.
let mut file_uuids: Vec<String> = Vec::new();
let mut folder_uuids: Vec<String> = Vec::new();
for item in items {
if item.item_type == "folder" {
folder_uuids.push(item.original_id.clone());
} else {
file_uuids.push(item.original_id.clone());
}
}
let (mut id_map, folder_id_map) =
batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await;
id_map.extend(folder_id_map);
// Individual trashed items.
for item in items {
write_trash_item_response(&mut xml, item, username, file_id_svc).await?;
write_trash_item_response(&mut xml, item, username, file_id_svc, &id_map)?;
}
xml.write_event(Event::End(BytesEnd::new("d:multistatus")))
@@ -267,11 +284,12 @@ fn write_trash_root_response<W: std::io::Write>(
}
/// Write a single trashed item as a `<d:response>` element.
async fn write_trash_item_response<W: std::io::Write>(
fn write_trash_item_response<W: std::io::Write>(
xml: &mut Writer<W>,
item: &TrashedItemDto,
username: &str,
file_id_svc: Option<&Arc<NextcloudFileIdService>>,
id_map: &HashMap<String, i64>,
) -> Result<(), String> {
xml.write_event(Event::Start(BytesStart::new("d:response")))
.map_err(|e| e.to_string())?;
@@ -318,12 +336,8 @@ async fn write_trash_item_response<W: std::io::Write>(
// d:getcontentlength
write_text_element(xml, "d:getcontentlength", "0")?;
// oc:fileid and oc:id — resolve numeric ID via file_id service
let file_id = if item.item_type == "folder" {
resolve_folder_id(file_id_svc, &item.original_id).await
} else {
resolve_file_id(file_id_svc, &item.original_id).await
};
// oc:fileid and oc:id — resolved up front in a batch query.
let file_id = id_map.get(&item.original_id).copied();
if let Some(id) = file_id {
write_text_element(xml, "oc:fileid", &id.to_string())?;
let oc_id = format_oc_id(id, file_id_svc);
src/interfaces/nextcloud/webdav_handler.rs
+41 -22
View File
@@ -9,7 +9,7 @@ use quick_xml::{
Writer,
events::{BytesEnd, BytesStart, BytesText, Event},
};
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use crate::application::adapters::webdav_adapter::{PropFindRequest, WebDavAdapter};
@@ -1001,6 +1001,26 @@ async fn write_nc_multistatus<W: std::io::Write>(
file_id_svc: Option<&Arc<NextcloudFileIdService>>,
favorite_ids: &HashSet<String>,
) -> Result<(), String> {
// When folder is None, files are the target resource itself (single-file
// PROPFIND) and must always be emitted. When folder is Some, files/subfolders
// are children and should only be listed when depth > 0.
let emit_children = folder.is_none() || depth != "0";
// Pre-resolve every oc:fileid up front in two batch queries (one per
// object type) instead of one INSERT round-trip per child. The XML writing
// below is then a pure synchronous map lookup.
let mut file_uuids: Vec<String> = Vec::new();
let mut folder_uuids: Vec<String> = Vec::new();
if let Some(f) = folder {
folder_uuids.push(f.id.clone());
}
if emit_children {
file_uuids.extend(files.iter().map(|f| f.id.clone()));
folder_uuids.extend(subfolders.iter().map(|sf| sf.id.clone()));
}
let (file_id_map, folder_id_map) =
batch_resolve_ids(file_id_svc, &file_uuids, &folder_uuids).await;
let mut xml = Writer::new(writer);
// Root element with all required namespaces.
@@ -1015,7 +1035,7 @@ async fn write_nc_multistatus<W: std::io::Write>(
// §5.2 + strict NC-client enforcement — see `nc_collection_href`).
if let Some(f) = folder {
let href = nc_collection_href(username, subpath);
let file_id = resolve_folder_id(file_id_svc, &f.id).await;
let file_id = folder_id_map.get(&f.id).copied();
let oc_id = file_id.map(|id| format_oc_id(id, file_id_svc));
write_folder_response(
&mut xml,
@@ -1028,11 +1048,6 @@ async fn write_nc_multistatus<W: std::io::Write>(
)?;
}
// When folder is None, files are the target resource itself (single-file
// PROPFIND) and must always be emitted. When folder is Some, files/subfolders
// are children and should only be listed when depth > 0.
let emit_children = folder.is_none() || depth != "0";
if emit_children {
// Files.
for file in files {
@@ -1045,7 +1060,7 @@ async fn write_nc_multistatus<W: std::io::Write>(
format!("{}/{}", subpath.trim_end_matches('/'), file.name)
};
let href = nc_href(username, &child_sub);
let file_id = resolve_file_id(file_id_svc, &file.id).await;
let file_id = file_id_map.get(&file.id).copied();
let oc_id = file_id.map(|id| format_oc_id(id, file_id_svc));
write_file_response(
&mut xml,
@@ -1066,7 +1081,7 @@ async fn write_nc_multistatus<W: std::io::Write>(
format!("{}/{}", subpath.trim_end_matches('/'), sf.name)
};
let href = nc_collection_href(username, &child_sub);
let file_id = resolve_folder_id(file_id_svc, &sf.id).await;
let file_id = folder_id_map.get(&sf.id).copied();
let oc_id = file_id.map(|id| format_oc_id(id, file_id_svc));
write_folder_response(
&mut xml,
@@ -1273,20 +1288,24 @@ pub fn write_text_element<W: std::io::Write>(
Ok(())
}
pub async fn resolve_file_id(
/// Resolve every `oc:fileid` for a listing in two batch queries (one per
/// object type) instead of one INSERT round-trip per child. Returns
/// `(file_map, folder_map)` keyed by object UUID; entries are absent when the
/// service is disabled or an id can't be resolved, mirroring the previous
/// per-call `Option` behaviour. The two batches run concurrently.
pub async fn batch_resolve_ids(
svc: Option<&Arc<NextcloudFileIdService>>,
file_uuid: &str,
) -> Option<i64> {
let svc = svc?;
svc.get_or_create_file_id(file_uuid).await.ok()
}
pub async fn resolve_folder_id(
svc: Option<&Arc<NextcloudFileIdService>>,
folder_uuid: &str,
) -> Option<i64> {
let svc = svc?;
svc.get_or_create_folder_id(folder_uuid).await.ok()
file_uuids: &[String],
folder_uuids: &[String],
) -> (HashMap<String, i64>, HashMap<String, i64>) {
let Some(svc) = svc else {
return (HashMap::new(), HashMap::new());
};
let (files, folders) = tokio::join!(
svc.get_or_create_file_ids(file_uuids),
svc.get_or_create_folder_ids(folder_uuids),
);
(files.unwrap_or_default(), folders.unwrap_or_default())
}
pub fn format_oc_id(id: i64, svc: Option<&Arc<NextcloudFileIdService>>) -> String {