perf(nextcloud): batch oc:fileid resolution to kill PROPFIND N+1
Resolving the stable numeric oc:fileid for every child in a NextCloud listing issued one `INSERT ... ON CONFLICT DO UPDATE` per entry — a write (row rewrite + WAL + dead tuple) even when the mapping already existed. A Depth:1 PROPFIND of a folder with N children meant N sequential write round-trips on a read-only operation that sync clients repeat constantly. - Repository: replace the single `get_or_create` (DO UPDATE) with `get_or_create_many` — one idempotent bulk `INSERT ... SELECT unnest(...) ON CONFLICT DO NOTHING` (existing rows untouched) plus a single `SELECT ... WHERE object_id = ANY(...)`. Two statements instead of N. - Service: add an Arc-backed moka cache (uuid -> i64; the mapping is immutable, so warm entries never go stale) and batch APIs `get_or_create_file_ids` / `get_or_create_folder_ids` that only query the misses. Warm listings cost zero queries. - Handlers (PROPFIND, REPORT favorites/search, trashbin, OCS unified search): pre-resolve all ids in two batched queries — file and folder run concurrently via `tokio::join!` — and turn the XML/JSON emission into a synchronous map lookup. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx
This commit is contained in:
@@ -1,12 +1,25 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use moka::future::Cache;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::common::errors::{DomainError, ErrorKind, Result};
|
||||
use crate::infrastructure::repositories::pg::NextcloudObjectIdRepository;
|
||||
|
||||
/// Capacity of the in-memory UUID→numeric-id cache. The mapping is immutable
|
||||
/// once created, so a warm entry never goes stale and eviction only costs a
|
||||
/// re-query; ~100k entries is a few MB.
|
||||
const ID_CACHE_CAPACITY: u64 = 100_000;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NextcloudFileIdService {
|
||||
repo: Option<Arc<NextcloudObjectIdRepository>>,
|
||||
instance_id: String,
|
||||
/// Object-UUID → stable numeric id. `moka` caches are `Arc`-backed, so all
|
||||
/// clones of the service share one cache and the per-child resolution in a
|
||||
/// listing costs zero queries once warm.
|
||||
cache: Cache<Uuid, i64>,
|
||||
}
|
||||
|
||||
impl NextcloudFileIdService {
|
||||
@@ -14,6 +27,7 @@ impl NextcloudFileIdService {
|
||||
Self {
|
||||
repo: Some(repo),
|
||||
instance_id,
|
||||
cache: Cache::new(ID_CACHE_CAPACITY),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,29 +35,76 @@ impl NextcloudFileIdService {
|
||||
Self {
|
||||
repo: None,
|
||||
instance_id: "ocnca".to_string(),
|
||||
cache: Cache::new(ID_CACHE_CAPACITY),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_or_create_file_id(&self, file_id: &str) -> Result<i64> {
|
||||
let repo = self.repo.as_ref().ok_or_else(|| {
|
||||
DomainError::internal_error("NextcloudFileId", "Repository not initialized")
|
||||
})?;
|
||||
repo.get_or_create("file", file_id).await
|
||||
/// Resolve — creating when absent — stable numeric file IDs for many
|
||||
/// UUIDs at once. Cache hits cost nothing; the misses are resolved with a
|
||||
/// single backing query. The returned map is keyed by the caller's
|
||||
/// original id strings; unresolvable inputs are simply absent (mirroring
|
||||
/// the `.ok()` behaviour the callers relied on).
|
||||
pub async fn get_or_create_file_ids(
|
||||
&self,
|
||||
file_ids: &[String],
|
||||
) -> Result<HashMap<String, i64>> {
|
||||
self.get_or_create_many("file", file_ids).await
|
||||
}
|
||||
|
||||
pub async fn get_or_create_folder_id(&self, folder_id: &str) -> Result<i64> {
|
||||
let repo = self.repo.as_ref().ok_or_else(|| {
|
||||
/// Folder counterpart of [`Self::get_or_create_file_ids`].
|
||||
pub async fn get_or_create_folder_ids(
|
||||
&self,
|
||||
folder_ids: &[String],
|
||||
) -> Result<HashMap<String, i64>> {
|
||||
self.get_or_create_many("folder", folder_ids).await
|
||||
}
|
||||
|
||||
async fn get_or_create_many(
|
||||
&self,
|
||||
object_type: &str,
|
||||
raw_ids: &[String],
|
||||
) -> Result<HashMap<String, i64>> {
|
||||
let mut result = HashMap::with_capacity(raw_ids.len());
|
||||
// Parsed-UUID → caller's original string; also dedupes the miss list.
|
||||
let mut pending: HashMap<Uuid, String> = HashMap::new();
|
||||
|
||||
for raw in raw_ids {
|
||||
let Ok(uuid) = Uuid::parse_str(raw) else {
|
||||
continue; // Unparseable ids never had a mapping — skip silently.
|
||||
};
|
||||
if let Some(id) = self.cache.get(&uuid).await {
|
||||
result.insert(raw.clone(), id);
|
||||
} else {
|
||||
pending.entry(uuid).or_insert_with(|| raw.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !pending.is_empty() {
|
||||
let misses: Vec<Uuid> = pending.keys().copied().collect();
|
||||
let resolved = self
|
||||
.repo()?
|
||||
.get_or_create_many(object_type, &misses)
|
||||
.await?;
|
||||
for (uuid, id) in resolved {
|
||||
self.cache.insert(uuid, id).await;
|
||||
if let Some(original) = pending.get(&uuid) {
|
||||
result.insert(original.clone(), id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn repo(&self) -> Result<&Arc<NextcloudObjectIdRepository>> {
|
||||
self.repo.as_ref().ok_or_else(|| {
|
||||
DomainError::internal_error("NextcloudFileId", "Repository not initialized")
|
||||
})?;
|
||||
repo.get_or_create("folder", folder_id).await
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the OxiCloud file UUID from a Nextcloud numeric ID.
|
||||
pub async fn get_oxicloud_id(&self, nc_file_id: i64) -> Result<String> {
|
||||
let repo = self.repo.as_ref().ok_or_else(|| {
|
||||
DomainError::internal_error("NextcloudFileId", "Repository not initialized")
|
||||
})?;
|
||||
repo.get_object_id(nc_file_id, "file").await
|
||||
self.repo()?.get_object_id(nc_file_id, "file").await
|
||||
}
|
||||
|
||||
pub fn format_oc_id(&self, id: i64) -> String {
|
||||
@@ -59,6 +120,7 @@ impl NextcloudFileIdService {
|
||||
Self {
|
||||
repo: None,
|
||||
instance_id: instance_id.to_string(),
|
||||
cache: Cache::new(ID_CACHE_CAPACITY),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,4 +169,25 @@ mod tests {
|
||||
let svc = NextcloudFileIdService::new_stub();
|
||||
assert!(svc.ensure_ready().is_err());
|
||||
}
|
||||
|
||||
// Empty input resolves to an empty map without ever touching the repo, so
|
||||
// it succeeds even on the repo-less stub.
|
||||
#[tokio::test]
|
||||
async fn test_get_or_create_file_ids_empty_is_noop() {
|
||||
let svc = NextcloudFileIdService::new_stub();
|
||||
let map = svc.get_or_create_file_ids(&[]).await.unwrap();
|
||||
assert!(map.is_empty());
|
||||
}
|
||||
|
||||
// Unparseable ids never had a mapping, so they are skipped before any repo
|
||||
// call — the stub (no repo) must not error on them.
|
||||
#[tokio::test]
|
||||
async fn test_get_or_create_file_ids_skips_unparseable() {
|
||||
let svc = NextcloudFileIdService::new_stub();
|
||||
let map = svc
|
||||
.get_or_create_file_ids(&["not-a-uuid".to_string()])
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(map.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user