perf(nextcloud): batch oc:fileid resolution to kill PROPFIND N+1
Resolving the stable numeric oc:fileid for every child in a NextCloud listing issued one `INSERT ... ON CONFLICT DO UPDATE` per entry — a write (row rewrite + WAL + dead tuple) even when the mapping already existed. A Depth:1 PROPFIND of a folder with N children meant N sequential write round-trips on a read-only operation that sync clients repeat constantly. - Repository: replace the single `get_or_create` (DO UPDATE) with `get_or_create_many` — one idempotent bulk `INSERT ... SELECT unnest(...) ON CONFLICT DO NOTHING` (existing rows untouched) plus a single `SELECT ... WHERE object_id = ANY(...)`. Two statements instead of N. - Service: add an Arc-backed moka cache (uuid -> i64; the mapping is immutable, so warm entries never go stale) and batch APIs `get_or_create_file_ids` / `get_or_create_folder_ids` that only query the misses. Warm listings cost zero queries. - Handlers (PROPFIND, REPORT favorites/search, trashbin, OCS unified search): pre-resolve all ids in two batched queries — file and folder run concurrently via `tokio::join!` — and turn the XML/JSON emission into a synchronous map lookup. https://claude.ai/code/session_01Dp3oWon5GBMVn4j3QXZdgx
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
use sqlx::{PgPool, Row};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::common::errors::{DomainError, ErrorKind, Result};
|
||||
|
||||
@@ -12,29 +14,73 @@ impl NextcloudObjectIdRepository {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
pub async fn get_or_create(&self, object_type: &str, object_id: &str) -> Result<i64> {
|
||||
let row = sqlx::query(
|
||||
/// Resolve — creating when absent — stable numeric IDs for a batch of
|
||||
/// object UUIDs sharing one `object_type`.
|
||||
///
|
||||
/// Two statements instead of one per id: an idempotent bulk insert that
|
||||
/// leaves existing rows untouched (`ON CONFLICT DO NOTHING` — no row
|
||||
/// rewrite, no WAL churn, no dead tuples, unlike the former `DO UPDATE`),
|
||||
/// followed by a single read of every requested mapping. The insert
|
||||
/// auto-commits before the read, so the read observes both our own rows
|
||||
/// and any created concurrently. Returns a map keyed by object UUID;
|
||||
/// unresolvable inputs are simply absent.
|
||||
pub async fn get_or_create_many(
|
||||
&self,
|
||||
object_type: &str,
|
||||
object_ids: &[Uuid],
|
||||
) -> Result<HashMap<Uuid, i64>> {
|
||||
if object_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
// 1. Create missing mappings only. `DO NOTHING` skips the write for
|
||||
// UUIDs that already map, eliminating the per-listing row rewrite.
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO storage.nextcloud_object_ids (object_type, object_id)
|
||||
VALUES ($1, $2::uuid)
|
||||
ON CONFLICT (object_type, object_id)
|
||||
DO UPDATE SET object_id = EXCLUDED.object_id
|
||||
RETURNING id
|
||||
SELECT $1, u FROM unnest($2::uuid[]) AS u
|
||||
ON CONFLICT (object_type, object_id) DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(object_type)
|
||||
.bind(object_id)
|
||||
.fetch_one(&*self.pool)
|
||||
.bind(object_ids)
|
||||
.execute(&*self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::new(
|
||||
ErrorKind::DatabaseError,
|
||||
"NextcloudFileId",
|
||||
format!("Failed to get/create Nextcloud ID: {}", e),
|
||||
format!("Failed to create Nextcloud IDs: {}", e),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(row.get::<i64, _>("id"))
|
||||
// 2. Read every requested mapping back in a single round-trip.
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT id, object_id
|
||||
FROM storage.nextcloud_object_ids
|
||||
WHERE object_type = $1 AND object_id = ANY($2::uuid[])
|
||||
"#,
|
||||
)
|
||||
.bind(object_type)
|
||||
.bind(object_ids)
|
||||
.fetch_all(&*self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::new(
|
||||
ErrorKind::DatabaseError,
|
||||
"NextcloudFileId",
|
||||
format!("Failed to load Nextcloud IDs: {}", e),
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut map = HashMap::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let object_id: Uuid = row.get("object_id");
|
||||
let id: i64 = row.get("id");
|
||||
map.insert(object_id, id);
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
/// Get the OxiCloud object ID from a Nextcloud numeric ID.
|
||||
|
||||
Reference in New Issue
Block a user