Merge pull request #485 from AtalayaLabs/claude/zealous-faraday-58s1at

feat: Photos evolution — Places (map) & People (faces) + gallery polish
This commit is contained in:
Dionisio Pozo
2026-06-19 14:31:14 +02:00
committed by GitHub
50 changed files with 4780 additions and 127 deletions
+26
View File
@@ -0,0 +1,26 @@
//! DTOs for the "Places" (photo map) feature.
use serde::Serialize;
use utoipa::ToSchema;
/// A geographic bounding box in decimal degrees.
#[derive(Debug, Clone, Copy)]
pub struct GeoBounds {
pub west: f64,
pub south: f64,
pub east: f64,
pub north: f64,
}
/// A clustered group of geotagged photos within one aggregation cell.
#[derive(Debug, Clone, Serialize, ToSchema)]
pub struct GeoCluster {
/// Cluster centroid longitude.
pub lng: f64,
/// Cluster centroid latitude.
pub lat: f64,
/// Number of photos in the cluster.
pub count: i64,
/// A representative photo id, for the cluster thumbnail.
pub sample_file_id: String,
}
+2
View File
@@ -10,9 +10,11 @@ pub mod favorites_dto;
pub mod file_dto;
pub mod folder_dto;
pub mod folder_listing_dto;
pub mod geo_dto;
pub mod grant_dto;
pub mod i18n_dto;
pub mod pagination;
pub mod people_dto;
pub mod playlist_dto;
pub mod plugin_dto;
pub mod recent_dto;
+30
View File
@@ -0,0 +1,30 @@
//! DTOs for the People (faces) API.
use serde::Serialize;
use utoipa::ToSchema;
/// A named (or unnamed) identity cluster, with a cover photo for its tile.
#[derive(Debug, Clone, Serialize, ToSchema)]
pub struct PersonDto {
pub id: String,
/// `None` until the user names the person.
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
/// File id of the cover face's photo, for the tile thumbnail.
#[serde(skip_serializing_if = "Option::is_none")]
pub cover_file_id: Option<String>,
pub face_count: i64,
pub is_hidden: bool,
}
/// One face box within a photo (for tagging overlays in the lightbox).
#[derive(Debug, Clone, Serialize, ToSchema)]
pub struct FaceBoxDto {
pub id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub person_id: Option<String>,
pub x: f32,
pub y: f32,
pub w: f32,
pub h: f32,
}
+78
View File
@@ -0,0 +1,78 @@
//! Ports for the People (faces) feature.
use async_trait::async_trait;
use uuid::Uuid;
use crate::common::errors::DomainError;
use crate::domain::entities::face::{DetectedFace, Face, Person};
/// Detects faces in an image and produces an aligned, L2-normalized embedding
/// for each. Takes raw encoded bytes (it decodes internally) so the
/// application layer stays decoupled from any image/ML crate.
///
/// The default implementation ([`NoopFaceAnalyzer`](crate::infrastructure::services::noop_face_analyzer::NoopFaceAnalyzer))
/// is a no-op that reports `is_ready() == false`; a real ONNX-backed
/// implementation is wired in when the operator provides models at runtime.
#[async_trait]
pub trait FaceAnalyzerPort: Send + Sync + 'static {
/// Whether a usable model is loaded. When false, indexing is skipped.
fn is_ready(&self) -> bool;
/// Detect and embed every face in `image_bytes` (an encoded JPEG/PNG/…).
async fn analyze(&self, image_bytes: &[u8]) -> Result<Vec<DetectedFace>, DomainError>;
}
/// Persistence for faces and persons. Every method is user-scoped; the
/// repository enforces `WHERE user_id = …` so callers only ever touch their
/// own biometric data.
#[async_trait]
pub trait FaceRepository: Send + Sync + 'static {
// ── faces ──────────────────────────────────────────────────────
async fn save_faces(&self, faces: &[Face]) -> Result<(), DomainError>;
async fn faces_for_file(&self, file_id: Uuid) -> Result<Vec<Face>, DomainError>;
async fn delete_faces_for_file(&self, file_id: Uuid) -> Result<(), DomainError>;
async fn faces_for_user(&self, user_id: Uuid) -> Result<Vec<Face>, DomainError>;
/// Faces previously computed for any file sharing this content hash —
/// lets indexing reuse results for deduplicated (identical) uploads.
async fn faces_for_blob(
&self,
user_id: Uuid,
blob_hash: &str,
) -> Result<Vec<Face>, DomainError>;
async fn assign_person(
&self,
face_id: Uuid,
person_id: Option<Uuid>,
) -> Result<(), DomainError>;
// ── persons ────────────────────────────────────────────────────
async fn create_person(&self, person: &Person) -> Result<(), DomainError>;
async fn persons_for_user(&self, user_id: Uuid) -> Result<Vec<Person>, DomainError>;
async fn rename_person(
&self,
user_id: Uuid,
person_id: Uuid,
name: Option<String>,
) -> Result<(), DomainError>;
async fn set_person_cover(
&self,
person_id: Uuid,
cover_face_id: Uuid,
) -> Result<(), DomainError>;
async fn set_person_hidden(
&self,
user_id: Uuid,
person_id: Uuid,
hidden: bool,
) -> Result<(), DomainError>;
/// File ids that contain a face assigned to this person (most recent first).
async fn files_for_person(
&self,
user_id: Uuid,
person_id: Uuid,
) -> Result<Vec<Uuid>, DomainError>;
/// Hard-delete every face and person for a user (right to erasure /
/// disabling the feature).
async fn delete_all_for_user(&self, user_id: Uuid) -> Result<(), DomainError>;
}
+1
View File
@@ -10,6 +10,7 @@ pub mod compression_ports;
pub mod content_index_ports;
pub mod dedup_ports;
pub mod email_sender;
pub mod face_ports;
pub mod favorites_ports;
pub mod file_lifecycle;
pub mod file_ports;
+2
View File
@@ -20,6 +20,8 @@ pub mod magic_link_invite_service;
pub mod music_service;
pub mod nextcloud_file_id_service;
pub mod nextcloud_login_flow_service;
pub mod people_service;
pub mod places_service;
pub mod recent_service;
pub mod recipient_notification_service;
pub mod search_service;
+271
View File
@@ -0,0 +1,271 @@
//! People (faces) use cases: identity clustering + the read/mutation methods
//! the HTTP layer calls.
//!
//! Clustering is a full re-cluster over the user's faces: a union-find groups
//! faces whose embeddings are within a cosine threshold (connected
//! components), and groups of at least `min_faces` become a "person". This is
//! O(n²) in the user's face count — fine for moderate libraries; an ANN index
//! (pgvector/VectorChord) is the documented scale-up.
//!
//! Strictly user-scoped (the repository filters by user), so — like
//! `RecentService` / `PlacesService` — no `AuthorizationEngine` check is
//! needed: the `caller_id` parameter is the access scope.
use std::collections::HashMap;
use std::sync::Arc;
use chrono::Utc;
use uuid::Uuid;
use crate::application::dtos::people_dto::{FaceBoxDto, PersonDto};
use crate::application::ports::face_ports::FaceRepository;
use crate::common::errors::DomainError;
use crate::domain::entities::face::Person;
use crate::infrastructure::repositories::pg::FacePgRepository;
/// Cosine similarity of two equal-length vectors. Embeddings are produced
/// L2-normalized, so this is ~a dot product; we normalize anyway for safety.
fn cosine(a: &[f32], b: &[f32]) -> f32 {
if a.len() != b.len() || a.is_empty() {
return 0.0;
}
let (mut dot, mut na, mut nb) = (0.0f32, 0.0f32, 0.0f32);
for (&x, &y) in a.iter().zip(b.iter()) {
dot += x * y;
na += x * x;
nb += y * y;
}
if na == 0.0 || nb == 0.0 {
return 0.0;
}
dot / (na.sqrt() * nb.sqrt())
}
/// Disjoint-set with path-halving + union by rank.
struct UnionFind {
parent: Vec<usize>,
rank: Vec<usize>,
}
impl UnionFind {
fn new(n: usize) -> Self {
Self {
parent: (0..n).collect(),
rank: vec![0; n],
}
}
fn find(&mut self, mut x: usize) -> usize {
while self.parent[x] != x {
self.parent[x] = self.parent[self.parent[x]];
x = self.parent[x];
}
x
}
fn union(&mut self, a: usize, b: usize) {
let (ra, rb) = (self.find(a), self.find(b));
if ra == rb {
return;
}
match self.rank[ra].cmp(&self.rank[rb]) {
std::cmp::Ordering::Less => self.parent[ra] = rb,
std::cmp::Ordering::Greater => self.parent[rb] = ra,
std::cmp::Ordering::Equal => {
self.parent[rb] = ra;
self.rank[ra] += 1;
}
}
}
}
pub struct PeopleService {
repo: Arc<FacePgRepository>,
/// Min cosine similarity to link two faces into the same identity.
cluster_threshold: f32,
/// Min faces in a cluster before it becomes a named-able "person".
min_faces: usize,
}
impl PeopleService {
pub fn new(repo: Arc<FacePgRepository>) -> Self {
Self {
repo,
cluster_threshold: 0.5,
min_faces: 3,
}
}
/// Re-cluster a user's faces. Returns the number of new persons created.
pub async fn recluster(&self, user_id: Uuid) -> Result<usize, DomainError> {
let faces = self.repo.faces_for_user(user_id).await?;
let n = faces.len();
if n == 0 {
return Ok(0);
}
let mut uf = UnionFind::new(n);
for i in 0..n {
for j in (i + 1)..n {
if cosine(&faces[i].embedding, &faces[j].embedding) >= self.cluster_threshold {
uf.union(i, j);
}
}
}
let mut groups: HashMap<usize, Vec<usize>> = HashMap::new();
for i in 0..n {
let root = uf.find(i);
groups.entry(root).or_default().push(i);
}
let mut created = 0usize;
for idxs in groups.into_values() {
if idxs.len() < self.min_faces {
// Too small to be a person — leave/reset these faces unassigned.
for &i in &idxs {
if faces[i].person_id.is_some() {
self.repo.assign_person(faces[i].id, None).await?;
}
}
continue;
}
// Reuse an existing person on this cluster (preserves a user's name)
// or mint a new one.
let existing = idxs.iter().find_map(|&i| faces[i].person_id);
let person_id = match existing {
Some(pid) => pid,
None => {
let pid = Uuid::new_v4();
let person = Person {
id: pid,
user_id,
display_name: None,
cover_face_id: Some(faces[idxs[0]].id),
is_hidden: false,
created_at: Utc::now(),
};
self.repo.create_person(&person).await?;
created += 1;
pid
}
};
for &i in &idxs {
if faces[i].person_id != Some(person_id) {
self.repo
.assign_person(faces[i].id, Some(person_id))
.await?;
}
}
let _ = self
.repo
.set_person_cover(person_id, faces[idxs[0]].id)
.await;
}
Ok(created)
}
/// People (non-empty clusters), most-photographed first.
pub async fn list_people(&self, caller_id: Uuid) -> Result<Vec<PersonDto>, DomainError> {
let persons = self.repo.persons_for_user(caller_id).await?;
let faces = self.repo.faces_for_user(caller_id).await?;
let mut count: HashMap<Uuid, i64> = HashMap::new();
let mut face_file: HashMap<Uuid, Uuid> = HashMap::new();
for f in &faces {
if let Some(pid) = f.person_id {
*count.entry(pid).or_default() += 1;
}
face_file.insert(f.id, f.file_id);
}
let mut out: Vec<PersonDto> = persons
.into_iter()
.filter_map(|p| {
let c = count.get(&p.id).copied().unwrap_or(0);
if c == 0 {
return None; // hide empty clusters (e.g. after a merge)
}
let cover_file_id = p
.cover_face_id
.and_then(|fid| face_file.get(&fid).copied())
.map(|u| u.to_string());
Some(PersonDto {
id: p.id.to_string(),
name: p.display_name,
cover_file_id,
face_count: c,
is_hidden: p.is_hidden,
})
})
.collect();
out.sort_by(|a, b| b.face_count.cmp(&a.face_count));
Ok(out)
}
/// File ids of a person's photos (most recent first).
pub async fn person_photos(
&self,
caller_id: Uuid,
person_id: Uuid,
) -> Result<Vec<String>, DomainError> {
let files = self.repo.files_for_person(caller_id, person_id).await?;
Ok(files.into_iter().map(|u| u.to_string()).collect())
}
/// Face boxes within a photo (for lightbox tagging), caller-scoped.
pub async fn faces_for_file(
&self,
caller_id: Uuid,
file_id: Uuid,
) -> Result<Vec<FaceBoxDto>, DomainError> {
let faces = self.repo.faces_for_file(file_id).await?;
Ok(faces
.into_iter()
.filter(|f| f.user_id == caller_id)
.map(|f| FaceBoxDto {
id: f.id.to_string(),
person_id: f.person_id.map(|u| u.to_string()),
x: f.bbox.x,
y: f.bbox.y,
w: f.bbox.w,
h: f.bbox.h,
})
.collect())
}
pub async fn rename_person(
&self,
caller_id: Uuid,
person_id: Uuid,
name: Option<String>,
) -> Result<(), DomainError> {
self.repo.rename_person(caller_id, person_id, name).await
}
pub async fn set_hidden(
&self,
caller_id: Uuid,
person_id: Uuid,
hidden: bool,
) -> Result<(), DomainError> {
self.repo
.set_person_hidden(caller_id, person_id, hidden)
.await
}
/// Merge `from` into `into` by reassigning all of `from`'s faces. The
/// now-empty `from` person is hidden by `list_people`.
pub async fn merge(&self, caller_id: Uuid, into: Uuid, from: Uuid) -> Result<(), DomainError> {
let faces = self.repo.faces_for_user(caller_id).await?;
for f in faces.into_iter().filter(|f| f.person_id == Some(from)) {
self.repo.assign_person(f.id, Some(into)).await?;
}
Ok(())
}
/// Erase all of the caller's face data (right to erasure / opt-out).
pub async fn delete_all(&self, caller_id: Uuid) -> Result<(), DomainError> {
self.repo.delete_all_for_user(caller_id).await
}
}
@@ -0,0 +1,45 @@
use std::sync::Arc;
use uuid::Uuid;
use crate::application::dtos::geo_dto::{GeoBounds, GeoCluster};
use crate::common::errors::DomainError;
use crate::infrastructure::repositories::pg::FileBlobReadRepository;
/// "Places" use case: the caller's geotagged photos aggregated into map
/// clusters.
///
/// Strictly user-scoped — the repository filters `WHERE fi.user_id = $1`, so,
/// like [`RecentService`](super::recent_service::RecentService) and the photos
/// timeline, it needs no `AuthorizationEngine` check: the `caller_id`
/// parameter *is* the access scope.
pub struct PlacesService {
file_read: Arc<FileBlobReadRepository>,
}
impl PlacesService {
pub fn new(file_read: Arc<FileBlobReadRepository>) -> Self {
Self { file_read }
}
/// Aggregation cell side, in degrees, for a slippy-map zoom level. The
/// world (360°) is split into `2^zoom` tiles; we use ~4 cells per tile so
/// clusters refine as the user zooms in. Clamped to a sane range.
fn cell_for_zoom(zoom: u8) -> f64 {
let z = i32::from(zoom.min(20));
360.0 / (2_f64.powi(z) * 4.0)
}
/// Clustered geotagged photos for `caller_id` within `bounds`.
pub async fn clusters(
&self,
caller_id: Uuid,
bounds: GeoBounds,
zoom: u8,
) -> Result<Vec<GeoCluster>, DomainError> {
let cell = Self::cell_for_zoom(zoom);
self.file_read
.list_geo_clusters(caller_id, bounds, cell)
.await
}
}
+106
View File
@@ -879,6 +879,11 @@ pub struct FeaturesConfig {
pub enable_trash: bool,
pub enable_search: bool,
pub enable_music: bool,
/// Lists the user's geotagged photos on a map (GET /api/photos/geo).
pub enable_places: bool,
/// Face detection + identity clustering for the photo library ("People").
/// Biometric data — OFF by default; opt-in per deployment/user.
pub enable_faces: bool,
/// Expose other OxiCloud users as a read-only "system" address book
/// at GET /api/address-books. Set to false to hide the user directory.
pub expose_system_users: bool,
@@ -893,11 +898,60 @@ impl Default for FeaturesConfig {
enable_trash: true, // Enable trash feature
enable_search: true, // Enable search feature
enable_music: true, // Enable music feature
enable_places: true, // Photo map (GET /api/photos/geo + Places tab)
enable_faces: false, // People/faces (biometric) — opt-in, off by default
expose_system_users: true, // Expose OxiCloud users as address book by default
}
}
}
/// Face-recognition (People) model configuration.
///
/// Only consulted when the `faces-onnx` cargo feature is compiled in *and*
/// [`FeaturesConfig::enable_faces`] is true; otherwise the inert
/// `NoopFaceAnalyzer` is used regardless of these values. The ONNX Runtime
/// dylib and both model files are operator-provided at runtime (never
/// committed) — when any is unset or fails to load, the People pipeline
/// silently falls back to the no-op analyzer and the server still boots.
#[derive(Debug, Clone)]
pub struct FacesConfig {
/// `libonnxruntime.{so,dylib,dll}`. Falls back to the `ORT_DYLIB_PATH`
/// environment variable when unset. Env: `OXICLOUD_FACES_ORT_DYLIB`.
pub ort_dylib: Option<PathBuf>,
/// SCRFD/RetinaFace detector model with 5-point landmarks.
/// Env: `OXICLOUD_FACES_DETECTOR_MODEL`.
pub detector_model: Option<PathBuf>,
/// ArcFace embedder model (112×112 → 512-d).
/// Env: `OXICLOUD_FACES_EMBEDDER_MODEL`.
pub embedder_model: Option<PathBuf>,
/// Detector square input size in pixels (default 640).
/// Env: `OXICLOUD_FACES_DET_SIZE`.
pub det_size: u32,
/// Minimum detector confidence to keep a face (default 0.5).
/// Env: `OXICLOUD_FACES_DET_THRESHOLD`.
pub det_threshold: f32,
/// IoU threshold for non-max suppression (default 0.4).
/// Env: `OXICLOUD_FACES_NMS_THRESHOLD`.
pub nms_threshold: f32,
/// ONNX Runtime intra-op threads (0 = let ORT decide).
/// Env: `OXICLOUD_FACES_INTRA_THREADS`.
pub intra_threads: usize,
}
impl Default for FacesConfig {
fn default() -> Self {
Self {
ort_dylib: None,
detector_model: None,
embedder_model: None,
det_size: 640,
det_threshold: 0.5,
nms_threshold: 0.4,
intra_threads: 0,
}
}
}
/// Content-search configuration (embedded Tantivy index over file names and
/// extracted file content).
///
@@ -1063,6 +1117,8 @@ pub struct AppConfig {
pub content_search: ContentSearchConfig,
/// WASM plugin runtime configuration
pub plugins: PluginConfig,
/// Face-recognition (People) model configuration
pub faces: FacesConfig,
}
/// Server-side i18n knobs.
@@ -1116,6 +1172,7 @@ impl Default for AppConfig {
i18n: I18nConfig::default(),
content_search: ContentSearchConfig::default(),
plugins: PluginConfig::default(),
faces: FacesConfig::default(),
}
}
}
@@ -1378,6 +1435,55 @@ impl AppConfig {
config.features.enable_music = val;
}
if let Ok(enable_places) = env::var("OXICLOUD_ENABLE_PLACES").map(|v| v.parse::<bool>())
&& let Ok(val) = enable_places
{
config.features.enable_places = val;
}
if let Ok(enable_faces) = env::var("OXICLOUD_ENABLE_FACES").map(|v| v.parse::<bool>())
&& let Ok(val) = enable_faces
{
config.features.enable_faces = val;
}
// Faces (People) ONNX runtime + models — operator-provided at runtime.
if let Ok(v) = env::var("OXICLOUD_FACES_ORT_DYLIB").or_else(|_| env::var("ORT_DYLIB_PATH"))
&& !v.is_empty()
{
config.faces.ort_dylib = Some(PathBuf::from(v));
}
if let Ok(v) = env::var("OXICLOUD_FACES_DETECTOR_MODEL")
&& !v.is_empty()
{
config.faces.detector_model = Some(PathBuf::from(v));
}
if let Ok(v) = env::var("OXICLOUD_FACES_EMBEDDER_MODEL")
&& !v.is_empty()
{
config.faces.embedder_model = Some(PathBuf::from(v));
}
if let Ok(v) = env::var("OXICLOUD_FACES_DET_SIZE").map(|v| v.parse::<u32>())
&& let Ok(val) = v
{
config.faces.det_size = val;
}
if let Ok(v) = env::var("OXICLOUD_FACES_DET_THRESHOLD").map(|v| v.parse::<f32>())
&& let Ok(val) = v
{
config.faces.det_threshold = val;
}
if let Ok(v) = env::var("OXICLOUD_FACES_NMS_THRESHOLD").map(|v| v.parse::<f32>())
&& let Ok(val) = v
{
config.faces.nms_threshold = val;
}
if let Ok(v) = env::var("OXICLOUD_FACES_INTRA_THREADS").map(|v| v.parse::<usize>())
&& let Ok(val) = v
{
config.faces.intra_threads = val;
}
// Content search (embedded Tantivy index)
if let Ok(v) = env::var("OXICLOUD_ENABLE_CONTENT_SEARCH").map(|v| v.parse::<bool>())
&& let Ok(val) = v
+112
View File
@@ -17,6 +17,8 @@ use crate::application::services::folder_service::FolderService;
use crate::application::services::i18n_application_service::I18nApplicationService;
use crate::application::services::nextcloud_file_id_service::NextcloudFileIdService;
use crate::application::services::nextcloud_login_flow_service::NextcloudLoginFlowService;
use crate::application::services::people_service::PeopleService;
use crate::application::services::places_service::PlacesService;
use crate::application::services::recent_service::RecentService;
use crate::application::services::search_service::SearchService;
use crate::application::services::share_browse_service::ShareBrowseService;
@@ -359,6 +361,9 @@ impl AppServiceFactory {
fls = fls.with_hook(audio.clone());
}
fls = fls.with_hook(media_metadata_service.clone());
if self.config.features.enable_faces {
fls = fls.with_hook(self.create_face_indexing_service(db_pool));
}
let file_lifecycle = Arc::new(fls);
Ok(CoreServices {
@@ -798,6 +803,95 @@ impl AppServiceFactory {
service
}
/// Creates the Places (photo map) service. Reuses the existing file-read
/// repository — the data is the caller's own geotagged photos.
pub fn create_places_service(
&self,
file_read: &Arc<FileBlobReadRepository>,
) -> Arc<PlacesService> {
let service = Arc::new(PlacesService::new(file_read.clone()));
tracing::info!("Places service initialized");
service
}
/// Creates the face-indexing lifecycle hook (People feature). Picks the
/// real ONNX analyzer when the `faces-onnx` feature is compiled in and the
/// operator has configured the runtime + models; otherwise the inert no-op
/// analyzer (see [`Self::build_face_analyzer`]).
pub fn create_face_indexing_service(
&self,
db_pool: &Arc<PgPool>,
) -> Arc<crate::infrastructure::services::face_indexing_service::FaceIndexingService> {
let blob_root = self.storage_path.join(".blobs");
let analyzer = self.build_face_analyzer();
Arc::new(
crate::infrastructure::services::face_indexing_service::FaceIndexingService::new(
db_pool.clone(),
blob_root,
analyzer,
),
)
}
/// Selects the face analyzer. With the `faces-onnx` feature and a fully
/// configured runtime + models, loads the real ONNX analyzer; any missing
/// piece or load failure degrades gracefully to the no-op analyzer (logged)
/// so startup never fails on biometric configuration.
fn build_face_analyzer(
&self,
) -> Arc<dyn crate::application::ports::face_ports::FaceAnalyzerPort> {
#[cfg(feature = "faces-onnx")]
{
let f = &self.config.faces;
if let (Some(dylib), Some(detector), Some(embedder)) = (
f.ort_dylib.as_ref(),
f.detector_model.as_ref(),
f.embedder_model.as_ref(),
) {
use crate::infrastructure::services::onnx_face_analyzer::{
OnnxFaceAnalyzer, OnnxLoadConfig,
};
let cfg = OnnxLoadConfig {
dylib,
detector,
embedder,
det_size: f.det_size,
det_threshold: f.det_threshold,
nms_threshold: f.nms_threshold,
intra_threads: f.intra_threads,
};
match OnnxFaceAnalyzer::load(&cfg) {
Ok(analyzer) => {
tracing::info!("Face analyzer: ONNX models loaded");
return Arc::new(analyzer);
}
Err(e) => {
tracing::warn!(
"Face analyzer: failed to load ONNX models ({e}); \
falling back to no-op analyzer"
);
}
}
} else {
tracing::info!(
"Face analyzer: faces-onnx compiled but runtime/models not fully \
configured; using no-op analyzer"
);
}
}
Arc::new(crate::infrastructure::services::noop_face_analyzer::NoopFaceAnalyzer)
}
/// Creates the People (faces) read/clustering service.
pub fn create_people_service(&self, db_pool: &Arc<PgPool>) -> Arc<PeopleService> {
let repo = Arc::new(
crate::infrastructure::repositories::pg::FacePgRepository::new(db_pool.clone()),
);
let service = Arc::new(PeopleService::new(repo));
tracing::info!("People service initialized");
service
}
/// Preloads translations for every locale in the registry. Build
/// the registry at startup via `LocaleRegistry::discover` and pass
/// the resulting list here.
@@ -1005,6 +1099,8 @@ impl AppServiceFactory {
// 6. Database-dependent services (PgPool always available in blob model)
let favorites_service: Option<Arc<FavoritesService>>;
let recent_service: Option<Arc<RecentService>>;
let places_service: Option<Arc<PlacesService>>;
let people_service: Option<Arc<PeopleService>>;
let storage_usage_service: Option<Arc<StorageUsageService>>;
let mut auth_services: Option<crate::common::di::AuthServices> = None;
let mut nextcloud_services: Option<NextcloudServices> = None;
@@ -1027,6 +1123,18 @@ impl AppServiceFactory {
recent_service = Some(recent.clone());
apps.recent_service = Some(recent);
places_service = if core.config.features.enable_places {
Some(self.create_places_service(&repos.file_read_repository))
} else {
None
};
people_service = if core.config.features.enable_faces {
Some(self.create_people_service(&pool))
} else {
None
};
storage_usage_service = Some(storage_usage.clone());
self.start_tree_etag_flush_job(&maintenance_pool);
@@ -1253,6 +1361,8 @@ impl AppServiceFactory {
share_browse_service,
favorites_service,
recent_service,
places_service,
people_service,
storage_usage_service,
calendar_service: None,
contact_service: None,
@@ -1699,6 +1809,8 @@ pub struct AppState {
pub share_browse_service: Option<Arc<ShareBrowseService>>,
pub favorites_service: Option<Arc<FavoritesService>>,
pub recent_service: Option<Arc<RecentService>>,
pub places_service: Option<Arc<PlacesService>>,
pub people_service: Option<Arc<PeopleService>>,
pub storage_usage_service: Option<Arc<StorageUsageService>>,
pub calendar_service: Option<Arc<CalendarService>>,
pub contact_service: Option<Arc<ContactStorageAdapter>>,
+72
View File
@@ -0,0 +1,72 @@
//! Domain entities for the People (faces) feature.
use chrono::{DateTime, Utc};
use uuid::Uuid;
/// Length of a face embedding vector (ArcFace-style).
pub const EMBEDDING_DIM: usize = 512;
/// A face bounding box in normalized image coordinates (each component 0..1).
#[derive(Debug, Clone, Copy)]
pub struct BoundingBox {
pub x: f32,
pub y: f32,
pub w: f32,
pub h: f32,
}
impl BoundingBox {
/// `[x, y, w, h]` — the storage representation (Postgres `REAL[]`).
pub fn to_array(self) -> Vec<f32> {
vec![self.x, self.y, self.w, self.h]
}
/// Build from a stored `[x, y, w, h]` array; missing components default to 0.
pub fn from_slice(a: &[f32]) -> Self {
Self {
x: a.first().copied().unwrap_or(0.0),
y: a.get(1).copied().unwrap_or(0.0),
w: a.get(2).copied().unwrap_or(0.0),
h: a.get(3).copied().unwrap_or(0.0),
}
}
}
/// A face produced by the analyzer but not yet persisted: where it is, how
/// confident the detector was, an optional quality score, and a 512-d,
/// L2-normalized embedding.
#[derive(Debug, Clone)]
pub struct DetectedFace {
pub bbox: BoundingBox,
pub det_score: f32,
pub quality: Option<f32>,
pub embedding: Vec<f32>,
}
/// A persisted face detection.
#[derive(Debug, Clone)]
pub struct Face {
pub id: Uuid,
pub file_id: Uuid,
pub user_id: Uuid,
/// Identity cluster this face belongs to, if any.
pub person_id: Option<Uuid>,
pub bbox: BoundingBox,
pub det_score: f32,
pub quality: Option<f32>,
pub embedding: Vec<f32>,
pub blob_hash: Option<String>,
pub created_at: DateTime<Utc>,
}
/// An identity cluster ("person"). `display_name` is `None` until the user
/// names it.
#[derive(Debug, Clone)]
pub struct Person {
pub id: Uuid,
pub user_id: Uuid,
pub display_name: Option<String>,
pub cover_face_id: Option<Uuid>,
pub is_hidden: bool,
pub created_at: DateTime<Utc>,
}
+1
View File
@@ -4,6 +4,7 @@ pub mod calendar_event;
pub mod contact;
pub mod device_code;
pub mod entity_errors;
pub mod face;
pub mod file;
pub mod folder;
pub mod magic_link_token;
@@ -0,0 +1,322 @@
//! PostgreSQL repository for the People (faces) feature.
//!
//! Embeddings are stored as `BYTEA` (512 × little-endian `f32`); there is no
//! pgvector dependency. Similarity search / clustering is done in-app over the
//! decoded vectors (see `PeopleService`).
use std::sync::Arc;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use sqlx::PgPool;
use uuid::Uuid;
use crate::application::ports::face_ports::FaceRepository;
use crate::common::errors::DomainError;
use crate::domain::entities::face::{BoundingBox, Face, Person};
/// Row shape for `faces.faces` selects (avoids `clippy::type_complexity`).
type FaceRow = (
Uuid, // id
Uuid, // file_id
Uuid, // user_id
Option<Uuid>, // person_id
Vec<f32>, // bbox (REAL[])
f32, // det_score
Option<f32>, // quality
Vec<u8>, // embedding (BYTEA)
Option<String>, // blob_hash
DateTime<Utc>, // created_at
);
type PersonRow = (
Uuid, // id
Uuid, // user_id
Option<String>, // display_name
Option<Uuid>, // cover_face_id
bool, // is_hidden
DateTime<Utc>, // created_at
);
fn embedding_to_bytes(e: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity(e.len() * 4);
for v in e {
out.extend_from_slice(&v.to_le_bytes());
}
out
}
fn bytes_to_embedding(b: &[u8]) -> Vec<f32> {
b.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect()
}
fn row_to_face(r: FaceRow) -> Face {
let (
id,
file_id,
user_id,
person_id,
bbox,
det_score,
quality,
embedding,
blob_hash,
created_at,
) = r;
Face {
id,
file_id,
user_id,
person_id,
bbox: BoundingBox::from_slice(&bbox),
det_score,
quality,
embedding: bytes_to_embedding(&embedding),
blob_hash,
created_at,
}
}
fn row_to_person(r: PersonRow) -> Person {
let (id, user_id, display_name, cover_face_id, is_hidden, created_at) = r;
Person {
id,
user_id,
display_name,
cover_face_id,
is_hidden,
created_at,
}
}
fn db_err(ctx: &'static str, e: sqlx::Error) -> DomainError {
DomainError::internal_error("FacePg", format!("{ctx}: {e}"))
}
const FACE_COLS: &str =
"id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash, created_at";
const PERSON_COLS: &str = "id, user_id, display_name, cover_face_id, is_hidden, created_at";
pub struct FacePgRepository {
pool: Arc<PgPool>,
}
impl FacePgRepository {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool }
}
}
#[async_trait]
impl FaceRepository for FacePgRepository {
async fn save_faces(&self, faces: &[Face]) -> Result<(), DomainError> {
if faces.is_empty() {
return Ok(());
}
let mut tx = self.pool.begin().await.map_err(|e| db_err("begin", e))?;
for f in faces {
sqlx::query(
r#"
INSERT INTO faces.faces
(id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
"#,
)
.bind(f.id)
.bind(f.file_id)
.bind(f.user_id)
.bind(f.person_id)
.bind(f.bbox.to_array())
.bind(f.det_score)
.bind(f.quality)
.bind(embedding_to_bytes(&f.embedding))
.bind(f.blob_hash.as_deref())
.execute(&mut *tx)
.await
.map_err(|e| db_err("save_faces", e))?;
}
tx.commit().await.map_err(|e| db_err("commit", e))?;
Ok(())
}
async fn faces_for_file(&self, file_id: Uuid) -> Result<Vec<Face>, DomainError> {
let sql = format!("SELECT {FACE_COLS} FROM faces.faces WHERE file_id = $1");
let rows: Vec<FaceRow> = sqlx::query_as(&sql)
.bind(file_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("faces_for_file", e))?;
Ok(rows.into_iter().map(row_to_face).collect())
}
async fn delete_faces_for_file(&self, file_id: Uuid) -> Result<(), DomainError> {
sqlx::query("DELETE FROM faces.faces WHERE file_id = $1")
.bind(file_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("delete_faces_for_file", e))?;
Ok(())
}
async fn faces_for_user(&self, user_id: Uuid) -> Result<Vec<Face>, DomainError> {
let sql = format!("SELECT {FACE_COLS} FROM faces.faces WHERE user_id = $1");
let rows: Vec<FaceRow> = sqlx::query_as(&sql)
.bind(user_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("faces_for_user", e))?;
Ok(rows.into_iter().map(row_to_face).collect())
}
async fn faces_for_blob(
&self,
user_id: Uuid,
blob_hash: &str,
) -> Result<Vec<Face>, DomainError> {
let sql =
format!("SELECT {FACE_COLS} FROM faces.faces WHERE user_id = $1 AND blob_hash = $2");
let rows: Vec<FaceRow> = sqlx::query_as(&sql)
.bind(user_id)
.bind(blob_hash)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("faces_for_blob", e))?;
Ok(rows.into_iter().map(row_to_face).collect())
}
async fn assign_person(
&self,
face_id: Uuid,
person_id: Option<Uuid>,
) -> Result<(), DomainError> {
sqlx::query("UPDATE faces.faces SET person_id = $2 WHERE id = $1")
.bind(face_id)
.bind(person_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("assign_person", e))?;
Ok(())
}
async fn create_person(&self, person: &Person) -> Result<(), DomainError> {
sqlx::query(
r#"
INSERT INTO faces.persons (id, user_id, display_name, cover_face_id, is_hidden)
VALUES ($1, $2, $3, $4, $5)
"#,
)
.bind(person.id)
.bind(person.user_id)
.bind(person.display_name.as_deref())
.bind(person.cover_face_id)
.bind(person.is_hidden)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("create_person", e))?;
Ok(())
}
async fn persons_for_user(&self, user_id: Uuid) -> Result<Vec<Person>, DomainError> {
let sql = format!(
"SELECT {PERSON_COLS} FROM faces.persons WHERE user_id = $1 ORDER BY created_at"
);
let rows: Vec<PersonRow> = sqlx::query_as(&sql)
.bind(user_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("persons_for_user", e))?;
Ok(rows.into_iter().map(row_to_person).collect())
}
async fn rename_person(
&self,
user_id: Uuid,
person_id: Uuid,
name: Option<String>,
) -> Result<(), DomainError> {
sqlx::query(
"UPDATE faces.persons SET display_name = $3, updated_at = now() WHERE id = $2 AND user_id = $1",
)
.bind(user_id)
.bind(person_id)
.bind(name)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("rename_person", e))?;
Ok(())
}
async fn set_person_cover(
&self,
person_id: Uuid,
cover_face_id: Uuid,
) -> Result<(), DomainError> {
sqlx::query(
"UPDATE faces.persons SET cover_face_id = $2, updated_at = now() WHERE id = $1",
)
.bind(person_id)
.bind(cover_face_id)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("set_person_cover", e))?;
Ok(())
}
async fn set_person_hidden(
&self,
user_id: Uuid,
person_id: Uuid,
hidden: bool,
) -> Result<(), DomainError> {
sqlx::query(
"UPDATE faces.persons SET is_hidden = $3, updated_at = now() WHERE id = $2 AND user_id = $1",
)
.bind(user_id)
.bind(person_id)
.bind(hidden)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("set_person_hidden", e))?;
Ok(())
}
async fn files_for_person(
&self,
user_id: Uuid,
person_id: Uuid,
) -> Result<Vec<Uuid>, DomainError> {
let rows: Vec<(Uuid,)> = sqlx::query_as(
r#"
SELECT file_id
FROM faces.faces
WHERE user_id = $1 AND person_id = $2
GROUP BY file_id
ORDER BY max(created_at) DESC
"#,
)
.bind(user_id)
.bind(person_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("files_for_person", e))?;
Ok(rows.into_iter().map(|(id,)| id).collect())
}
async fn delete_all_for_user(&self, user_id: Uuid) -> Result<(), DomainError> {
let mut tx = self.pool.begin().await.map_err(|e| db_err("begin", e))?;
sqlx::query("DELETE FROM faces.faces WHERE user_id = $1")
.bind(user_id)
.execute(&mut *tx)
.await
.map_err(|e| db_err("delete_all_faces", e))?;
sqlx::query("DELETE FROM faces.persons WHERE user_id = $1")
.bind(user_id)
.execute(&mut *tx)
.await
.map_err(|e| db_err("delete_all_persons", e))?;
tx.commit().await.map_err(|e| db_err("commit", e))?;
Ok(())
}
}
@@ -20,6 +20,8 @@ type MediaFileRow = (
String, // blob_hash
Option<Uuid>, // user_id
i64, // sort_date
Option<i32>, // width
Option<i32>, // height
);
use bytes::Bytes;
@@ -30,6 +32,7 @@ use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use crate::application::dtos::geo_dto::{GeoBounds, GeoCluster};
use crate::application::dtos::search_dto::SearchCriteriaDto;
use crate::application::ports::storage_ports::FileReadPort;
use crate::common::errors::DomainError;
@@ -416,7 +419,7 @@ impl FileBlobReadRepository {
owner_id: Uuid,
before: Option<i64>,
limit: i64,
) -> Result<(Vec<File>, Vec<i64>), DomainError> {
) -> Result<(Vec<File>, Vec<i64>, Vec<(Option<i32>, Option<i32>)>), DomainError> {
let rows: Vec<MediaFileRow> = sqlx::query_as(
r#"
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
@@ -425,9 +428,11 @@ impl FileBlobReadRepository {
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
fi.blob_hash,
fi.user_id,
EXTRACT(EPOCH FROM fi.media_sort_date)::bigint AS sort_date
EXTRACT(EPOCH FROM fi.media_sort_date)::bigint AS sort_date,
fm.width, fm.height
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
LEFT JOIN storage.file_metadata fm ON fm.file_id = fi.id
WHERE fi.user_id = $1
AND NOT fi.is_trashed
AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%')
@@ -446,15 +451,67 @@ impl FileBlobReadRepository {
let mut files = Vec::with_capacity(rows.len());
let mut sort_dates = Vec::with_capacity(rows.len());
let mut dims = Vec::with_capacity(rows.len());
for (id, name, fid, fpath, size, mime, ca, ma, blob_hash, uid, sd) in rows {
for (id, name, fid, fpath, size, mime, ca, ma, blob_hash, uid, sd, w, h) in rows {
files.push(Self::row_to_file(
id, name, fid, fpath, size, mime, ca, ma, blob_hash, uid,
)?);
sort_dates.push(sd);
dims.push((w, h));
}
Ok((files, sort_dates))
Ok((files, sort_dates, dims))
}
/// Aggregate the caller's geotagged photos into grid cells of side `cell`
/// (degrees) within `bounds`. Plain SQL (no PostGIS), scoped to `user_id`.
/// Returns one cluster per non-empty cell with its centroid, photo count
/// and a representative photo id (for the cluster thumbnail).
pub async fn list_geo_clusters(
&self,
user_id: Uuid,
bounds: GeoBounds,
cell: f64,
) -> Result<Vec<GeoCluster>, DomainError> {
let rows: Vec<(i64, f64, f64, String)> = sqlx::query_as(
r#"
SELECT count(*) AS n,
avg(fm.longitude) AS clng,
avg(fm.latitude) AS clat,
min(fm.file_id::text) AS sample_id
FROM storage.file_metadata fm
JOIN storage.files fi ON fi.id = fm.file_id
WHERE fi.user_id = $1
AND NOT fi.is_trashed
AND fm.latitude IS NOT NULL
AND fm.longitude IS NOT NULL
AND fm.longitude BETWEEN $2 AND $3
AND fm.latitude BETWEEN $4 AND $5
GROUP BY round(fm.longitude / $6), round(fm.latitude / $6)
"#,
)
.bind(user_id)
.bind(bounds.west)
.bind(bounds.east)
.bind(bounds.south)
.bind(bounds.north)
.bind(cell)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("FileBlobRead", format!("list_geo_clusters: {e}"))
})?;
Ok(rows
.into_iter()
.map(|(n, clng, clat, sample_id)| GeoCluster {
lng: clng,
lat: clat,
count: n,
sample_file_id: sample_id,
})
.collect())
}
}
@@ -6,6 +6,7 @@ mod contact_group_pg_repository;
mod contact_persistence_dto;
mod contact_pg_repository;
mod device_code_pg_repository;
mod face_pg_repository;
mod favorites_pg_repository;
pub mod file_metadata_repository;
mod magic_link_token_pg_repository;
@@ -33,6 +34,7 @@ pub use contact_group_pg_repository::ContactGroupPgRepository;
pub use contact_persistence_dto::*;
pub use contact_pg_repository::ContactPgRepository;
pub use device_code_pg_repository::DeviceCodePgRepository;
pub use face_pg_repository::FacePgRepository;
pub use favorites_pg_repository::FavoritesPgRepository;
pub use file_blob_read_repository::FileBlobReadRepository;
pub use file_blob_write_repository::FileBlobWriteRepository;
@@ -0,0 +1,473 @@
//! Pure geometry + post-processing for the ONNX face pipeline.
//!
//! Everything here is plain Rust (no `ort`, no `ndarray`) so it compiles in the
//! default build and is exercised by `cargo test` — the error-prone numerical
//! parts (SCRFD anchor decode, NMS, 5-point similarity alignment, the affine
//! warp, normalization) are unit-tested in isolation, while the untestable ONNX
//! session calls live behind the `faces-onnx` feature in `onnx_face_analyzer`.
//!
//! The pipeline mirrors InsightFace's reference implementation:
//! SCRFD detector (distance-to-box anchors over strides 8/16/32) → 5-point
//! similarity transform onto the canonical 112×112 ArcFace template → ArcFace
//! embedder → L2-normalized 512-d vector.
use image::RgbImage;
/// One detected face in **detector-input pixel** coordinates (before scaling
/// back to the original image): an axis-aligned box `[x1, y1, x2, y2]`, the
/// five facial landmarks, and the detector confidence.
#[derive(Debug, Clone, Copy)]
pub struct Detection {
pub bbox: [f32; 4],
pub kps: [[f32; 2]; 5],
pub score: f32,
}
/// A 2×3 affine transform mapping an output/template coordinate to a source
/// coordinate: `src = (a·ox + b·oy + tx, c·ox + d·oy + ty)`. Used to sample the
/// source image when warping an aligned face crop.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Affine {
pub a: f32,
pub b: f32,
pub c: f32,
pub d: f32,
pub tx: f32,
pub ty: f32,
}
/// Canonical ArcFace 5-point template for a 112×112 crop
/// (left eye, right eye, nose, left mouth, right mouth).
pub const ARCFACE_TEMPLATE: [[f32; 2]; 5] = [
[38.2946, 51.6963],
[73.5318, 51.5014],
[56.0252, 71.7366],
[41.5493, 92.3655],
[70.7299, 92.2041],
];
/// Aligned-crop side length expected by the ArcFace embedder.
pub const ALIGN_SIZE: u32 = 112;
/// Letterbox geometry for the detector: the largest scale that fits a
/// `w0 × h0` image into a `det × det` square without distortion, plus the
/// resulting (possibly smaller) dimensions placed at the top-left.
///
/// Returns `(new_w, new_h, scale)` where `scale = min(det/w0, det/h0)` and
/// detector-space coordinates map back to the original by dividing by `scale`.
pub fn letterbox(w0: u32, h0: u32, det: u32) -> (u32, u32, f32) {
if w0 == 0 || h0 == 0 {
return (0, 0, 1.0);
}
let scale = (det as f32 / w0 as f32).min(det as f32 / h0 as f32);
let new_w = ((w0 as f32 * scale).round() as u32).clamp(1, det);
let new_h = ((h0 as f32 * scale).round() as u32).clamp(1, det);
(new_w, new_h, scale)
}
/// `NCHW`, RGB, float input tensor for an ONNX model: `(px − mean) · scale`,
/// channel-major (all R, then all G, then all B). Length is `3 · w · h`.
pub fn chw_normalized(img: &RgbImage, mean: f32, scale: f32) -> Vec<f32> {
let (w, h) = (img.width() as usize, img.height() as usize);
let mut out = vec![0.0f32; 3 * w * h];
let plane = w * h;
for (i, px) in img.pixels().enumerate() {
out[i] = (px[0] as f32 - mean) * scale;
out[plane + i] = (px[1] as f32 - mean) * scale;
out[2 * plane + i] = (px[2] as f32 - mean) * scale;
}
out
}
/// Decode one SCRFD feature-map stride into detections, appending those above
/// `threshold` to `out`. All coordinates are in detector-input pixels.
///
/// `scores` is `[n]`, `bbox` is `[n·4]` (left, top, right, bottom *distances*,
/// already multiplied by `stride`), `kps` (when present) is `[n·10]`
/// (5 × (dx, dy) distances, already multiplied by `stride`), where
/// `n = feat_h · feat_w · num_anchors`. Anchor centers follow InsightFace's
/// row-major `mgrid` order with `num_anchors` consecutive duplicates.
#[allow(clippy::too_many_arguments)]
pub fn decode_stride(
scores: &[f32],
bbox: &[f32],
kps: Option<&[f32]>,
stride: u32,
feat_h: u32,
feat_w: u32,
num_anchors: u32,
threshold: f32,
out: &mut Vec<Detection>,
) {
let stride_f = stride as f32;
let mut idx = 0usize;
for y in 0..feat_h {
for x in 0..feat_w {
let cx = x as f32 * stride_f;
let cy = y as f32 * stride_f;
for _ in 0..num_anchors {
if idx >= scores.len() {
return;
}
let score = scores[idx];
if score >= threshold {
let b = idx * 4;
if b + 3 < bbox.len() {
let det_bbox = [
cx - bbox[b],
cy - bbox[b + 1],
cx + bbox[b + 2],
cy + bbox[b + 3],
];
let mut det_kps = [[0.0f32; 2]; 5];
if let Some(kps) = kps {
let k = idx * 10;
if k + 9 < kps.len() {
for (p, slot) in det_kps.iter_mut().enumerate() {
*slot = [cx + kps[k + p * 2], cy + kps[k + p * 2 + 1]];
}
}
}
out.push(Detection {
bbox: det_bbox,
kps: det_kps,
score,
});
}
}
idx += 1;
}
}
}
}
/// Intersection-over-union of two `[x1, y1, x2, y2]` boxes.
pub fn iou(a: &[f32; 4], b: &[f32; 4]) -> f32 {
let x1 = a[0].max(b[0]);
let y1 = a[1].max(b[1]);
let x2 = a[2].min(b[2]);
let y2 = a[3].min(b[3]);
let iw = (x2 - x1).max(0.0);
let ih = (y2 - y1).max(0.0);
let inter = iw * ih;
let area_a = (a[2] - a[0]).max(0.0) * (a[3] - a[1]).max(0.0);
let area_b = (b[2] - b[0]).max(0.0) * (b[3] - b[1]).max(0.0);
let union = area_a + area_b - inter;
if union <= 0.0 { 0.0 } else { inter / union }
}
/// Greedy non-maximum suppression: keep highest-scoring boxes, drop any whose
/// IoU with an already-kept box exceeds `iou_thresh`. Returns the kept
/// detections, highest score first.
pub fn nms(mut dets: Vec<Detection>, iou_thresh: f32) -> Vec<Detection> {
dets.sort_by(|a, b| b.score.total_cmp(&a.score));
let mut keep: Vec<Detection> = Vec::with_capacity(dets.len());
for d in dets {
if keep.iter().all(|k| iou(&k.bbox, &d.bbox) <= iou_thresh) {
keep.push(d);
}
}
keep
}
/// Least-squares similarity transform (scale + rotation + translation, no
/// shear, no reflection) mapping `src` landmarks onto `dst`, returned as its
/// **inverse** affine (output/template coordinate → source coordinate) ready
/// for backward-warp sampling.
///
/// Solved in closed form via the complex-number formulation: with points as
/// complex numbers, `w = Σ (b'ᵢ · conj(a'ᵢ)) / Σ |a'ᵢ|²` and `t = mean_b −
/// w·mean_a`, which is equivalent to the Umeyama solution InsightFace obtains
/// from `skimage.SimilarityTransform`.
pub fn similarity_transform_inverse(src: &[[f32; 2]; 5], dst: &[[f32; 2]; 5]) -> Affine {
let n = 5.0f32;
let (mut max, mut may, mut mbx, mut mby) = (0.0f32, 0.0f32, 0.0f32, 0.0f32);
for i in 0..5 {
max += src[i][0];
may += src[i][1];
mbx += dst[i][0];
mby += dst[i][1];
}
max /= n;
may /= n;
mbx /= n;
mby /= n;
// num = Σ b'·conj(a') (complex), den = Σ |a'|² (real)
let (mut num_re, mut num_im, mut den) = (0.0f32, 0.0f32, 0.0f32);
for i in 0..5 {
let ax = src[i][0] - max;
let ay = src[i][1] - may;
let bx = dst[i][0] - mbx;
let by = dst[i][1] - mby;
// b' · conj(a') = (bx + i·by)(ax − i·ay)
num_re += bx * ax + by * ay;
num_im += by * ax - bx * ay;
den += ax * ax + ay * ay;
}
let den = if den.abs() < 1e-12 { 1e-12 } else { den };
// w = num/den (forward scale·rotation)
let wr = num_re / den;
let wi = num_im / den;
// t = mean_b − w·mean_a
let tr = mbx - (wr * max - wi * may);
let ti = mby - (wi * max + wr * may);
// Inverse of the similarity: src = Ainv·(out − t), Ainv = [[wr,wi],[−wi,wr]]/|w|²
let det = wr * wr + wi * wi;
let g = if det.abs() < 1e-12 { 0.0 } else { 1.0 / det };
Affine {
a: g * wr,
b: g * wi,
c: -g * wi,
d: g * wr,
tx: -g * (wr * tr + wi * ti),
ty: g * (wi * tr - wr * ti),
}
}
/// Warp `img` into an `ALIGN_SIZE × ALIGN_SIZE` aligned face crop using the
/// inverse affine from [`similarity_transform_inverse`], sampling bilinearly
/// and clamping to the image edge.
pub fn warp_to_aligned(img: &RgbImage, inv: &Affine) -> RgbImage {
let (w, h) = (img.width(), img.height());
let mut out = RgbImage::new(ALIGN_SIZE, ALIGN_SIZE);
for oy in 0..ALIGN_SIZE {
for ox in 0..ALIGN_SIZE {
let sx = inv.a * ox as f32 + inv.b * oy as f32 + inv.tx;
let sy = inv.c * ox as f32 + inv.d * oy as f32 + inv.ty;
let px = bilinear_sample(img, sx, sy, w, h);
out.put_pixel(ox, oy, px);
}
}
out
}
/// Bilinear RGB sample at floating `(x, y)`, clamping out-of-bounds reads to
/// the nearest edge.
fn bilinear_sample(img: &RgbImage, x: f32, y: f32, w: u32, h: u32) -> image::Rgb<u8> {
let x = x.clamp(0.0, (w - 1) as f32);
let y = y.clamp(0.0, (h - 1) as f32);
let x0 = x.floor() as u32;
let y0 = y.floor() as u32;
let x1 = (x0 + 1).min(w - 1);
let y1 = (y0 + 1).min(h - 1);
let dx = x - x0 as f32;
let dy = y - y0 as f32;
let p00 = img.get_pixel(x0, y0);
let p10 = img.get_pixel(x1, y0);
let p01 = img.get_pixel(x0, y1);
let p11 = img.get_pixel(x1, y1);
let mut out = [0u8; 3];
for (ch, slot) in out.iter_mut().enumerate() {
let top = p00[ch] as f32 * (1.0 - dx) + p10[ch] as f32 * dx;
let bot = p01[ch] as f32 * (1.0 - dx) + p11[ch] as f32 * dx;
*slot = (top * (1.0 - dy) + bot * dy).round().clamp(0.0, 255.0) as u8;
}
image::Rgb(out)
}
/// In-place L2 normalization. A zero vector is left unchanged.
pub fn l2_normalize(v: &mut [f32]) {
let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 1e-12 {
for x in v.iter_mut() {
*x /= norm;
}
}
}
/// Variance of the discrete Laplacian over the luminance of an RGB crop — a
/// cheap focus/sharpness proxy (higher = sharper). Used as a face quality
/// score for cover selection and gating.
pub fn laplacian_variance(img: &RgbImage) -> f32 {
let (w, h) = (img.width() as i64, img.height() as i64);
if w < 3 || h < 3 {
return 0.0;
}
let lum = |x: i64, y: i64| -> f32 {
let p = img.get_pixel(x as u32, y as u32);
0.299 * p[0] as f32 + 0.587 * p[1] as f32 + 0.114 * p[2] as f32
};
let mut vals = Vec::with_capacity(((w - 2) * (h - 2)) as usize);
for y in 1..h - 1 {
for x in 1..w - 1 {
let l = 4.0 * lum(x, y) - lum(x - 1, y) - lum(x + 1, y) - lum(x, y - 1) - lum(x, y + 1);
vals.push(l);
}
}
let n = vals.len() as f32;
if n == 0.0 {
return 0.0;
}
let mean = vals.iter().sum::<f32>() / n;
vals.iter().map(|v| (v - mean) * (v - mean)).sum::<f32>() / n
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn letterbox_fits_and_preserves_aspect() {
// Landscape 1000×500 into 640 → width-bound, scale 0.64.
let (nw, nh, s) = letterbox(1000, 500, 640);
assert_eq!(nw, 640);
assert_eq!(nh, 320);
assert!((s - 0.64).abs() < 1e-6);
// Square fills exactly.
let (nw, nh, s) = letterbox(800, 800, 640);
assert_eq!((nw, nh), (640, 640));
assert!((s - 0.8).abs() < 1e-6);
}
#[test]
fn letterbox_degenerate_is_safe() {
assert_eq!(letterbox(0, 10, 640), (0, 0, 1.0));
}
#[test]
fn chw_layout_and_normalization() {
let mut img = RgbImage::new(2, 1);
img.put_pixel(0, 0, image::Rgb([127, 0, 255]));
img.put_pixel(1, 0, image::Rgb([128, 255, 0]));
let t = chw_normalized(&img, 127.5, 1.0 / 128.0);
// Length = 3 channels × 2 px.
assert_eq!(t.len(), 6);
// R plane first, then G, then B (NCHW).
assert!((t[0] - (127.0 - 127.5) / 128.0).abs() < 1e-6);
assert!((t[1] - (128.0 - 127.5) / 128.0).abs() < 1e-6);
assert!((t[2] - (0.0 - 127.5) / 128.0).abs() < 1e-6); // G of px0
assert!((t[4] - (255.0 - 127.5) / 128.0).abs() < 1e-6); // B of px0
}
#[test]
fn distance_decode_recovers_box_and_kps() {
// 1×2 grid, stride 8, 1 anchor → cell centers (0,0) then (8,0).
let scores = [0.9f32, 0.9];
// distances left/top/right/bottom (already × stride), identical per cell.
let bbox = [2.0, 1.0, 3.0, 4.0, 2.0, 1.0, 3.0, 4.0];
let kps: Vec<f32> = vec![
1.0, 1.0, 2.0, 2.0, 0.0, 0.0, -1.0, 1.0, 1.0, -1.0, // cell 0
1.0, 1.0, 2.0, 2.0, 0.0, 0.0, -1.0, 1.0, 1.0, -1.0, // cell 1
];
let mut out = Vec::new();
decode_stride(&scores, &bbox, Some(&kps), 8, 1, 2, 1, 0.5, &mut out);
assert_eq!(out.len(), 2);
// Cell 0, center (0,0): box = center ± distances, kps = center + offset.
assert_eq!(out[0].bbox, [-2.0, -1.0, 3.0, 4.0]);
assert_eq!(out[0].kps[0], [1.0, 1.0]);
assert_eq!(out[0].kps[1], [2.0, 2.0]);
// Cell 1, center (8,0): anchor center advanced by one stride in x.
assert_eq!(out[1].bbox, [8.0 - 2.0, -1.0, 8.0 + 3.0, 4.0]);
assert_eq!(out[1].kps[0], [9.0, 1.0]);
}
#[test]
fn decode_thresholds_out_low_scores() {
let scores = [0.2f32, 0.8];
let bbox = [0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0];
let mut out = Vec::new();
// 1×2 grid, 1 anchor → two cells.
decode_stride(&scores, &bbox, None, 8, 1, 2, 1, 0.5, &mut out);
assert_eq!(out.len(), 1);
assert!((out[0].score - 0.8).abs() < 1e-6);
}
#[test]
fn iou_and_nms() {
let a = [0.0, 0.0, 10.0, 10.0];
let b = [0.0, 0.0, 10.0, 10.0];
assert!((iou(&a, &b) - 1.0).abs() < 1e-6);
let c = [100.0, 100.0, 110.0, 110.0];
assert_eq!(iou(&a, &c), 0.0);
let dets = vec![
Detection {
bbox: a,
kps: [[0.0; 2]; 5],
score: 0.9,
},
Detection {
bbox: b,
kps: [[0.0; 2]; 5],
score: 0.8,
}, // dup of a
Detection {
bbox: c,
kps: [[0.0; 2]; 5],
score: 0.7,
}, // separate
];
let kept = nms(dets, 0.4);
assert_eq!(kept.len(), 2);
assert!((kept[0].score - 0.9).abs() < 1e-6);
}
#[test]
fn similarity_identity() {
let inv = similarity_transform_inverse(&ARCFACE_TEMPLATE, &ARCFACE_TEMPLATE);
assert!((inv.a - 1.0).abs() < 1e-4);
assert!(inv.b.abs() < 1e-4);
assert!(inv.c.abs() < 1e-4);
assert!((inv.d - 1.0).abs() < 1e-4);
assert!(inv.tx.abs() < 1e-3);
assert!(inv.ty.abs() < 1e-3);
}
#[test]
fn similarity_pure_translation() {
// src = dst shifted by (+10, +5); inverse must map out→src by the same shift.
let mut src = ARCFACE_TEMPLATE;
for p in &mut src {
p[0] += 10.0;
p[1] += 5.0;
}
let inv = similarity_transform_inverse(&src, &ARCFACE_TEMPLATE);
assert!((inv.a - 1.0).abs() < 1e-4);
assert!(inv.b.abs() < 1e-4);
assert!((inv.tx - 10.0).abs() < 1e-3);
assert!((inv.ty - 5.0).abs() < 1e-3);
}
#[test]
fn warp_identity_preserves_template_region() {
// A 112×112 gradient warped by identity returns (close to) itself.
let mut img = RgbImage::new(ALIGN_SIZE, ALIGN_SIZE);
for y in 0..ALIGN_SIZE {
for x in 0..ALIGN_SIZE {
img.put_pixel(x, y, image::Rgb([x as u8, y as u8, 128]));
}
}
let inv = similarity_transform_inverse(&ARCFACE_TEMPLATE, &ARCFACE_TEMPLATE);
let out = warp_to_aligned(&img, &inv);
let a = out.get_pixel(40, 60);
assert!((a[0] as i32 - 40).abs() <= 1);
assert!((a[1] as i32 - 60).abs() <= 1);
}
#[test]
fn l2_normalize_unit_length() {
let mut v = vec![3.0f32, 4.0];
l2_normalize(&mut v);
assert!((v[0] - 0.6).abs() < 1e-6);
assert!((v[1] - 0.8).abs() < 1e-6);
let mut z = vec![0.0f32, 0.0];
l2_normalize(&mut z); // unchanged, no NaN
assert_eq!(z, vec![0.0, 0.0]);
}
#[test]
fn laplacian_variance_sharp_vs_flat() {
let flat = RgbImage::from_pixel(8, 8, image::Rgb([100, 100, 100]));
assert!(laplacian_variance(&flat) < 1e-3);
let mut checker = RgbImage::new(8, 8);
for y in 0..8 {
for x in 0..8 {
let v = if (x + y) % 2 == 0 { 0 } else { 255 };
checker.put_pixel(x, y, image::Rgb([v, v, v]));
}
}
assert!(laplacian_variance(&checker) > 1000.0);
}
}
@@ -0,0 +1,191 @@
//! Face indexing as a `FileLifecycleHook`.
//!
//! On image upload it detects + embeds faces (off the request path, in a
//! background task) and stores them. It mirrors `MediaMetadataService`: reads
//! the blob from the local `.blobs` tree, is dedup-aware (identical uploads
//! clone an existing file's faces instead of re-running inference), and is
//! completely inert when no model is configured (`FaceAnalyzerPort::is_ready()
//! == false`) — so the feature compiles and runs with the default no-op
//! analyzer until the operator wires a real ONNX model.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use chrono::Utc;
use sqlx::PgPool;
use uuid::Uuid;
use crate::application::ports::face_ports::{FaceAnalyzerPort, FaceRepository};
use crate::application::ports::file_lifecycle::FileLifecycleHook;
use crate::common::errors::DomainError;
use crate::domain::entities::face::Face;
use crate::infrastructure::repositories::pg::FacePgRepository;
/// Minimum detector confidence for a face to be stored.
const MIN_DET_SCORE: f32 = 0.6;
fn is_image(content_type: &str) -> bool {
content_type.starts_with("image/")
}
pub struct FaceIndexingService {
pool: Arc<PgPool>,
repo: Arc<FacePgRepository>,
analyzer: Arc<dyn FaceAnalyzerPort>,
blob_root: PathBuf,
}
impl FaceIndexingService {
pub fn new(pool: Arc<PgPool>, blob_root: PathBuf, analyzer: Arc<dyn FaceAnalyzerPort>) -> Self {
let repo = Arc::new(FacePgRepository::new(pool.clone()));
Self {
pool,
repo,
analyzer,
blob_root,
}
}
/// Local path of a blob: `.blobs/{prefix}/{hash}.blob`.
fn blob_path(&self, hash: &str) -> PathBuf {
let prefix = if hash.len() >= 2 { &hash[0..2] } else { hash };
self.blob_root.join(prefix).join(format!("{hash}.blob"))
}
/// Spawn a background indexing task. `reuse_dedup` clones faces from an
/// existing file with the same blob hash instead of re-running inference;
/// `delete_first` clears prior faces (used on overwrite).
fn spawn_index(&self, file_id: Uuid, blob_hash: String, reuse_dedup: bool, delete_first: bool) {
let pool = self.pool.clone();
let repo = self.repo.clone();
let analyzer = self.analyzer.clone();
let blob_path = self.blob_path(&blob_hash);
tokio::spawn(async move {
if delete_first {
let _ = repo.delete_faces_for_file(file_id).await;
}
if let Err(e) = index_file(
&pool,
&repo,
analyzer.as_ref(),
file_id,
&blob_path,
&blob_hash,
reuse_dedup,
)
.await
{
tracing::warn!(target: "oxicloud::faces", "face indexing failed for {file_id}: {e}");
}
});
}
}
impl FileLifecycleHook for FaceIndexingService {
fn on_file_created(
&self,
file_id: &str,
blob_hash: &str,
content_type: &str,
is_new_blob: bool,
) {
if !is_image(content_type) || !self.analyzer.is_ready() {
return;
}
if let Ok(fid) = file_id.parse::<Uuid>() {
// Dedup hit (blob already existed) → clone an existing file's faces.
self.spawn_index(fid, blob_hash.to_string(), !is_new_blob, false);
}
}
fn on_file_copied(
&self,
file_id: &str,
blob_hash: &str,
content_type: &str,
_source_file_id: &str,
) {
if !is_image(content_type) || !self.analyzer.is_ready() {
return;
}
if let Ok(fid) = file_id.parse::<Uuid>() {
self.spawn_index(fid, blob_hash.to_string(), true, false);
}
}
fn on_file_updated(&self, file_id: &str, blob_hash: &str, content_type: &str) {
if !is_image(content_type) || !self.analyzer.is_ready() {
return;
}
if let Ok(fid) = file_id.parse::<Uuid>() {
self.spawn_index(fid, blob_hash.to_string(), false, true);
}
}
fn on_file_deleted(&self, _file_id: &str) {
// faces.faces.file_id has ON DELETE CASCADE — the DB cleans up.
}
}
async fn lookup_user(pool: &PgPool, file_id: Uuid) -> Result<Uuid, DomainError> {
let row: (Uuid,) = sqlx::query_as("SELECT user_id FROM storage.files WHERE id = $1")
.bind(file_id)
.fetch_one(pool)
.await
.map_err(|e| DomainError::internal_error("Faces", format!("lookup user: {e}")))?;
Ok(row.0)
}
async fn index_file(
pool: &PgPool,
repo: &FacePgRepository,
analyzer: &dyn FaceAnalyzerPort,
file_id: Uuid,
blob_path: &Path,
blob_hash: &str,
reuse_dedup: bool,
) -> Result<(), DomainError> {
let user_id = lookup_user(pool, file_id).await?;
// Dedup-aware fast path: reuse faces already computed for an identical blob.
if reuse_dedup {
let peers = repo.faces_for_blob(user_id, blob_hash).await?;
let cloned: Vec<Face> = peers
.into_iter()
.filter(|f| f.file_id != file_id)
.map(|f| Face {
id: Uuid::new_v4(),
file_id,
..f
})
.collect();
if !cloned.is_empty() {
repo.save_faces(&cloned).await?;
return Ok(());
}
// No peer found — fall through and analyze.
}
let bytes = tokio::fs::read(blob_path)
.await
.map_err(|e| DomainError::internal_error("Faces", format!("read blob: {e}")))?;
let detected = analyzer.analyze(&bytes).await?;
let faces: Vec<Face> = detected
.into_iter()
.filter(|d| d.det_score >= MIN_DET_SCORE)
.map(|d| Face {
id: Uuid::new_v4(),
file_id,
user_id,
person_id: None,
bbox: d.bbox,
det_score: d.det_score,
quality: d.quality,
embedding: d.embedding,
blob_hash: Some(blob_hash.to_string()),
created_at: Utc::now(),
})
.collect();
repo.save_faces(&faces).await
}
+5
View File
@@ -6,6 +6,8 @@ pub mod compression_service;
pub mod dedup_service;
pub mod encrypted_blob_backend;
pub mod exif_service;
pub mod face_geometry;
pub mod face_indexing_service;
pub mod file_content_cache;
pub mod file_system_i18n_service;
pub mod image_transcode_service;
@@ -17,7 +19,10 @@ pub mod migration_blob_backend;
pub mod migration_job;
pub mod mock_email_sender;
pub mod nextcloud_chunked_upload_service;
pub mod noop_face_analyzer;
pub mod oidc_service;
#[cfg(feature = "faces-onnx")]
pub mod onnx_face_analyzer;
pub mod password_hasher;
pub mod path_resolver_service;
pub mod path_service;
@@ -0,0 +1,26 @@
//! Default no-op face analyzer.
//!
//! Used when no ML model is configured: it reports `is_ready() == false` and
//! returns no faces, so the whole People pipeline compiles and runs inert
//! until a real ONNX-backed analyzer (provided by the operator) replaces it.
use async_trait::async_trait;
use crate::application::ports::face_ports::FaceAnalyzerPort;
use crate::common::errors::DomainError;
use crate::domain::entities::face::DetectedFace;
/// Analyzer that never detects anything.
#[derive(Debug, Default, Clone, Copy)]
pub struct NoopFaceAnalyzer;
#[async_trait]
impl FaceAnalyzerPort for NoopFaceAnalyzer {
fn is_ready(&self) -> bool {
false
}
async fn analyze(&self, _image_bytes: &[u8]) -> Result<Vec<DetectedFace>, DomainError> {
Ok(Vec::new())
}
}
@@ -0,0 +1,340 @@
//! ONNX-backed face analyzer (SCRFD detector + ArcFace embedder).
//!
//! Compiled only with the `faces-onnx` cargo feature. Mirrors the
//! immich/InsightFace pipeline: detect faces + 5-point landmarks (SCRFD),
//! similarity-align each face to the canonical 112×112 template, then embed
//! (ArcFace) into an L2-normalized 512-d vector. All inference runs on a
//! blocking thread (`spawn_blocking`) so it never stalls a Tokio worker, and
//! each ONNX session is serialized behind a `Mutex` (ORT's `run` needs `&mut`).
//!
//! The heavy numerical post-processing lives in [`super::face_geometry`] (plain
//! Rust, unit-tested); this module only wires it to ONNX Runtime.
//!
//! **Models are operator-provided at runtime, never committed.** `load` returns
//! an error (→ caller falls back to the no-op analyzer) if the ONNX Runtime
//! dylib or either model file is missing or incompatible — the server still
//! boots. The dylib is loaded via [`ort::init_from`] (a fallible path) rather
//! than ORT's lazy loader, which would `panic` on a missing library (fatal
//! under `panic = "abort"`).
use std::path::Path;
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use image::RgbImage;
use ort::session::Session;
use ort::value::Tensor;
use super::face_geometry as geom;
use crate::application::ports::face_ports::FaceAnalyzerPort;
use crate::common::errors::DomainError;
use crate::domain::entities::face::{BoundingBox, DetectedFace, EMBEDDING_DIM};
/// SCRFD pyramid strides for the 3- and 5-level model variants.
const STRIDES_3: [u32; 3] = [8, 16, 32];
const STRIDES_5: [u32; 5] = [8, 16, 32, 64, 128];
/// Discard faces smaller than this (original-image pixels) — embeddings of tiny
/// faces are unreliable.
const MIN_FACE_PX: f32 = 24.0;
/// Hard cap on faces processed per image (bounds work on crowd shots).
const MAX_FACES: usize = 64;
/// Output layout of an InsightFace SCRFD model, inferred from its output count.
#[derive(Clone, Copy)]
struct ScrfdLayout {
/// Feature-map count per output kind (3 for strides 8/16/32, 5 with 64/128).
fmc: usize,
num_anchors: u32,
use_kps: bool,
}
impl ScrfdLayout {
fn from_num_outputs(n: usize) -> Option<Self> {
match n {
6 => Some(Self {
fmc: 3,
num_anchors: 2,
use_kps: false,
}),
9 => Some(Self {
fmc: 3,
num_anchors: 2,
use_kps: true,
}),
10 => Some(Self {
fmc: 5,
num_anchors: 1,
use_kps: false,
}),
15 => Some(Self {
fmc: 5,
num_anchors: 1,
use_kps: true,
}),
_ => None,
}
}
fn strides(&self) -> &'static [u32] {
if self.fmc == 3 {
&STRIDES_3
} else {
&STRIDES_5
}
}
}
/// Where to find the runtime + models, plus detector knobs. Borrowed paths;
/// nothing is retained after [`OnnxFaceAnalyzer::load`].
pub struct OnnxLoadConfig<'a> {
/// Path to `libonnxruntime.{so,dylib,dll}`.
pub dylib: &'a Path,
/// SCRFD detector `.onnx`.
pub detector: &'a Path,
/// ArcFace embedder `.onnx`.
pub embedder: &'a Path,
pub det_size: u32,
pub det_threshold: f32,
pub nms_threshold: f32,
/// ORT intra-op threads (0 = let ONNX Runtime decide).
pub intra_threads: usize,
}
struct Inner {
detector: Mutex<Session>,
embedder: Mutex<Session>,
layout: ScrfdLayout,
det_size: u32,
det_threshold: f32,
nms_threshold: f32,
}
/// Real face analyzer. Cheap to clone (`Arc` inside).
#[derive(Clone)]
pub struct OnnxFaceAnalyzer {
inner: Arc<Inner>,
}
fn dom(e: impl std::fmt::Display) -> DomainError {
DomainError::internal_error("Faces", e.to_string())
}
fn build_session(path: &Path, intra_threads: usize) -> Result<Session, DomainError> {
let mut builder = Session::builder().map_err(dom)?;
if intra_threads > 0 {
builder = builder.with_intra_threads(intra_threads).map_err(dom)?;
}
builder.commit_from_file(path).map_err(dom)
}
impl OnnxFaceAnalyzer {
/// Load the ONNX Runtime dylib and both models. Returns an error (caller
/// falls back to the no-op analyzer) on any missing/incompatible artifact.
pub fn load(cfg: &OnnxLoadConfig<'_>) -> Result<Self, DomainError> {
// Fallible dylib load — populates ORT's global handle so later calls
// never hit the panicking lazy loader.
ort::init_from(cfg.dylib)
.map_err(|e| dom(format!("ONNX Runtime dylib: {e}")))?
.commit();
let detector = build_session(cfg.detector, cfg.intra_threads)?;
let embedder = build_session(cfg.embedder, cfg.intra_threads)?;
let n_out = detector.outputs().len();
let layout = ScrfdLayout::from_num_outputs(n_out).ok_or_else(|| {
dom(format!(
"detector has {n_out} outputs; expected an SCRFD model (6/9/10/15)"
))
})?;
if !layout.use_kps {
tracing::warn!(
target: "oxicloud::faces",
"SCRFD model has no landmark outputs; face alignment will be approximate"
);
}
tracing::info!(
target: "oxicloud::faces",
"ONNX face analyzer ready (detector {} outputs, embedder loaded, det_size={})",
n_out, cfg.det_size
);
Ok(Self {
inner: Arc::new(Inner {
detector: Mutex::new(detector),
embedder: Mutex::new(embedder),
layout,
det_size: cfg.det_size,
det_threshold: cfg.det_threshold,
nms_threshold: cfg.nms_threshold,
}),
})
}
}
impl Inner {
/// Full synchronous pipeline for one encoded image.
fn analyze_blocking(&self, image_bytes: &[u8]) -> Result<Vec<DetectedFace>, DomainError> {
let orig = image::load_from_memory(image_bytes)
.map_err(|e| dom(format!("decode image: {e}")))?
.to_rgb8();
let (w0, h0) = (orig.width(), orig.height());
if w0 == 0 || h0 == 0 {
return Ok(Vec::new());
}
let dets = self.detect(&orig)?;
let mut faces = Vec::new();
for det in dets.into_iter().take(MAX_FACES) {
let fw = det.bbox[2] - det.bbox[0];
let fh = det.bbox[3] - det.bbox[1];
if fw < MIN_FACE_PX || fh < MIN_FACE_PX {
continue;
}
let Some(embedding) = self.embed(&orig, &det)? else {
continue;
};
let aligned_quality = {
let inv = geom::similarity_transform_inverse(&det.kps, &geom::ARCFACE_TEMPLATE);
let aligned = geom::warp_to_aligned(&orig, &inv);
geom::laplacian_variance(&aligned)
};
let x = (det.bbox[0] / w0 as f32).clamp(0.0, 1.0);
let y = (det.bbox[1] / h0 as f32).clamp(0.0, 1.0);
let bw = (fw / w0 as f32).clamp(0.0, 1.0);
let bh = (fh / h0 as f32).clamp(0.0, 1.0);
faces.push(DetectedFace {
bbox: BoundingBox { x, y, w: bw, h: bh },
det_score: det.score,
quality: Some(aligned_quality),
embedding,
});
}
Ok(faces)
}
/// Run SCRFD and return detections in **original-image pixels**.
fn detect(&self, orig: &RgbImage) -> Result<Vec<geom::Detection>, DomainError> {
let det = self.det_size;
let (nw, nh, scale) = geom::letterbox(orig.width(), orig.height(), det);
let resized = image::imageops::resize(orig, nw, nh, image::imageops::FilterType::Triangle);
let mut canvas = RgbImage::new(det, det);
image::imageops::overlay(&mut canvas, &resized, 0, 0);
let input = geom::chw_normalized(&canvas, 127.5, 1.0 / 128.0);
let tensor =
Tensor::from_array(([1_i64, 3, det as i64, det as i64], input)).map_err(dom)?;
let layout = self.layout;
let total = layout.fmc * if layout.use_kps { 3 } else { 2 };
let raw: Vec<Vec<f32>> = {
let mut sess = self
.detector
.lock()
.map_err(|_| dom("detector mutex poisoned"))?;
let outputs = sess.run(ort::inputs![tensor]).map_err(dom)?;
(0..total)
.map(|i| {
outputs[i]
.try_extract_tensor::<f32>()
.map(|(_, data)| data.to_vec())
.map_err(dom)
})
.collect::<Result<_, _>>()?
};
let mut dets = Vec::new();
for (si, &stride) in layout.strides().iter().enumerate() {
let scores = &raw[si];
let bbox: Vec<f32> = raw[layout.fmc + si]
.iter()
.map(|v| v * stride as f32)
.collect();
let kps: Option<Vec<f32>> = if layout.use_kps {
Some(
raw[2 * layout.fmc + si]
.iter()
.map(|v| v * stride as f32)
.collect(),
)
} else {
None
};
let feat = det / stride;
geom::decode_stride(
scores,
&bbox,
kps.as_deref(),
stride,
feat,
feat,
layout.num_anchors,
self.det_threshold,
&mut dets,
);
}
// Scale detector-space coordinates back to the original image.
let inv_scale = if scale.abs() < 1e-9 { 1.0 } else { 1.0 / scale };
for d in &mut dets {
for v in &mut d.bbox {
*v *= inv_scale;
}
for k in &mut d.kps {
k[0] *= inv_scale;
k[1] *= inv_scale;
}
}
Ok(geom::nms(dets, self.nms_threshold))
}
/// Align one detection and run the ArcFace embedder. Returns `None` if the
/// embedder produces an unexpected output length.
fn embed(
&self,
orig: &RgbImage,
det: &geom::Detection,
) -> Result<Option<Vec<f32>>, DomainError> {
let inv = geom::similarity_transform_inverse(&det.kps, &geom::ARCFACE_TEMPLATE);
let aligned = geom::warp_to_aligned(orig, &inv);
let input = geom::chw_normalized(&aligned, 127.5, 1.0 / 127.5);
let size = geom::ALIGN_SIZE as i64;
let tensor = Tensor::from_array(([1_i64, 3, size, size], input)).map_err(dom)?;
let mut embedding: Vec<f32> = {
let mut sess = self
.embedder
.lock()
.map_err(|_| dom("embedder mutex poisoned"))?;
let outputs = sess.run(ort::inputs![tensor]).map_err(dom)?;
let (_, data) = outputs[0].try_extract_tensor::<f32>().map_err(dom)?;
data.to_vec()
};
if embedding.len() != EMBEDDING_DIM {
tracing::warn!(
target: "oxicloud::faces",
"embedder returned {} dims, expected {EMBEDDING_DIM}; skipping face",
embedding.len()
);
return Ok(None);
}
geom::l2_normalize(&mut embedding);
Ok(Some(embedding))
}
}
#[async_trait]
impl FaceAnalyzerPort for OnnxFaceAnalyzer {
fn is_ready(&self) -> bool {
true
}
async fn analyze(&self, image_bytes: &[u8]) -> Result<Vec<DetectedFace>, DomainError> {
let inner = self.inner.clone();
let bytes = image_bytes.to_vec();
tokio::task::spawn_blocking(move || inner.analyze_blocking(&bytes))
.await
.map_err(|e| dom(format!("inference task join: {e}")))?
}
}
+1
View File
@@ -16,6 +16,7 @@ pub mod grant_handler;
pub mod i18n_handler;
pub mod magic_link_handler;
pub mod music_handler;
pub mod people_handler;
pub mod photos_handler;
pub mod recent_handler;
pub mod search_handler;
@@ -0,0 +1,177 @@
//! HTTP handlers for the People (faces) feature.
//!
//! Every route is mounted only when `OXICLOUD_ENABLE_FACES` is on (the service
//! is present in `AppState`); each handler is also defensive. All work is
//! strictly caller-scoped by `PeopleService` (the repository filters by user).
use std::sync::Arc;
use axum::{
Json,
extract::{Path, State},
http::StatusCode,
response::{IntoResponse, Response},
};
use serde::Deserialize;
use uuid::Uuid;
use crate::common::di::AppState;
use crate::interfaces::errors::AppError;
use crate::interfaces::middleware::auth::AuthUser;
fn disabled() -> Response {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({ "error": "People feature is disabled" })),
)
.into_response()
}
fn bad_id() -> Response {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": "invalid id" })),
)
.into_response()
}
/// GET /api/people — identity clusters for the caller.
pub async fn list_people(State(state): State<Arc<AppState>>, auth_user: AuthUser) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
match svc.list_people(auth_user.id).await {
Ok(people) => Json(people).into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
/// GET /api/people/{id}/photos — file ids of a person's photos.
pub async fn person_photos(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Path(id): Path<String>,
) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
let Ok(person_id) = Uuid::parse_str(&id) else {
return bad_id();
};
match svc.person_photos(auth_user.id, person_id).await {
Ok(files) => Json(files).into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
#[derive(Deserialize)]
pub struct RenameBody {
pub name: Option<String>,
}
/// PATCH /api/people/{id} — name (or clear the name of) a person.
pub async fn rename_person(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Path(id): Path<String>,
Json(body): Json<RenameBody>,
) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
let Ok(person_id) = Uuid::parse_str(&id) else {
return bad_id();
};
match svc.rename_person(auth_user.id, person_id, body.name).await {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
#[derive(Deserialize)]
pub struct HideBody {
pub hidden: bool,
}
/// POST /api/people/{id}/hide — hide/unhide a person from the grid.
pub async fn hide_person(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Path(id): Path<String>,
Json(body): Json<HideBody>,
) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
let Ok(person_id) = Uuid::parse_str(&id) else {
return bad_id();
};
match svc.set_hidden(auth_user.id, person_id, body.hidden).await {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
#[derive(Deserialize)]
pub struct MergeBody {
pub into: String,
pub from: String,
}
/// POST /api/people/merge — merge `from` into `into`.
pub async fn merge_people(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Json(body): Json<MergeBody>,
) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
let (Ok(into), Ok(from)) = (Uuid::parse_str(&body.into), Uuid::parse_str(&body.from)) else {
return bad_id();
};
match svc.merge(auth_user.id, into, from).await {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
/// POST /api/people/recluster — re-run identity clustering for the caller.
pub async fn recluster(State(state): State<Arc<AppState>>, auth_user: AuthUser) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
match svc.recluster(auth_user.id).await {
Ok(n) => Json(serde_json::json!({ "persons_created": n })).into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
/// DELETE /api/people/data — erase all of the caller's face data.
pub async fn delete_all(State(state): State<Arc<AppState>>, auth_user: AuthUser) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
match svc.delete_all(auth_user.id).await {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
/// GET /api/people/faces/{file_id} — face boxes within a photo (lightbox tags).
pub async fn faces_for_file(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Path(file_id): Path<String>,
) -> Response {
let Some(svc) = state.people_service.as_ref() else {
return disabled();
};
let Ok(fid) = Uuid::parse_str(&file_id) else {
return bad_id();
};
match svc.faces_for_file(auth_user.id, fid).await {
Ok(boxes) => Json(boxes).into_response(),
Err(e) => AppError::from(e).into_response(),
}
}
+99 -6
View File
@@ -4,11 +4,12 @@ use axum::{
http::StatusCode,
response::IntoResponse,
};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tracing::{error, info};
use crate::application::dtos::file_dto::FileDto;
use crate::application::dtos::geo_dto::GeoBounds;
use crate::common::di::AppState;
use crate::interfaces::middleware::auth::AuthUser;
@@ -21,6 +22,20 @@ pub struct PhotosQueryParams {
pub limit: Option<i64>,
}
/// Photos-timeline item: a `FileDto` plus the image's original pixel
/// dimensions (from EXIF/metadata), flattened into the same JSON shape so
/// the gallery can lay tiles out at their true aspect ratio without a
/// second per-file metadata round-trip.
#[derive(Serialize)]
struct PhotoDto {
#[serde(flatten)]
file: FileDto,
#[serde(skip_serializing_if = "Option::is_none")]
width: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
height: Option<u32>,
}
/// Lists all image/video files for the authenticated user, sorted by
/// capture date (EXIF DateTimeOriginal) falling back to upload date.
///
@@ -55,17 +70,22 @@ pub async fn list_photos(
.list_media_files(user_id, params.before, limit)
.await
{
Ok((files, sort_dates)) => {
Ok((files, sort_dates, dims)) => {
info!("Photos: returned {} media files for user", files.len());
// Convert to DTOs with sort_date populated
let dtos: Vec<FileDto> = files
// Convert to DTOs with sort_date + pixel dimensions populated.
let dtos: Vec<PhotoDto> = files
.into_iter()
.zip(sort_dates.iter())
.map(|(file, &sd)| {
.zip(dims.iter())
.map(|((file, &sd), &(w, h))| {
let mut dto = FileDto::from(file);
dto.sort_date = Some(sd as u64);
dto
PhotoDto {
file: dto,
width: w.map(|v| v.max(0) as u32),
height: h.map(|v| v.max(0) as u32),
}
})
.collect();
@@ -91,3 +111,76 @@ pub async fn list_photos(
}
}
}
/// Query parameters for the photos map (clustered) endpoint.
#[derive(Deserialize)]
pub struct GeoQueryParams {
/// Bounding box as `west,south,east,north` (decimal degrees).
pub bbox: String,
/// Slippy-map zoom level (0–20); controls cluster granularity.
pub zoom: Option<u8>,
}
/// Lists the caller's geotagged photos aggregated into map clusters within a
/// bounding box. Gated on `OXICLOUD_ENABLE_PLACES` (the route is only mounted
/// when the Places service is present).
#[utoipa::path(
get,
path = "/api/photos/geo",
params(
("bbox" = String, Query, description = "Bounding box 'west,south,east,north' (decimal degrees)"),
("zoom" = Option<u8>, Query, description = "Map zoom level (0-20), controls cluster size")
),
responses(
(status = 200, description = "Geotagged photos aggregated into map clusters"),
(status = 400, description = "Invalid bounding box"),
(status = 401, description = "Unauthorized")
),
security(("bearerAuth" = [])),
tag = "photos"
)]
pub async fn list_photos_geo(
State(state): State<Arc<AppState>>,
auth_user: AuthUser,
Query(params): Query<GeoQueryParams>,
) -> impl IntoResponse {
let Some(places) = state.places_service.as_ref() else {
return (
StatusCode::NOT_FOUND,
Json(serde_json::json!({ "error": "Places feature is disabled" })),
)
.into_response();
};
let coords: Vec<f64> = params
.bbox
.split(',')
.filter_map(|s| s.trim().parse::<f64>().ok())
.collect();
if coords.len() != 4 {
return (
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": "bbox must be 'west,south,east,north'" })),
)
.into_response();
}
let bounds = GeoBounds {
west: coords[0],
south: coords[1],
east: coords[2],
north: coords[3],
};
let zoom = params.zoom.unwrap_or(3);
match places.clusters(auth_user.id, bounds, zoom).await {
Ok(clusters) => Json(clusters).into_response(),
Err(err) => {
error!("Error listing photo geo clusters: {}", err);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("{}", err) })),
)
.into_response()
}
}
}
+1
View File
@@ -164,6 +164,7 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload;
handlers::recent_handler::clear_recent_items,
// Photos handler (free function)
handlers::photos_handler::list_photos,
handlers::photos_handler::list_photos_geo,
// Batch handlers (free functions)
handlers::batch_handler::move_files_batch,
handlers::batch_handler::copy_files_batch,
+24 -4
View File
@@ -6,7 +6,7 @@ use axum::{
extract::{DefaultBodyLimit, State},
http::StatusCode,
response::{IntoResponse, Json as AxumJson, Response},
routing::{any, delete, get, post, put},
routing::{any, delete, get, patch, post, put},
};
use serde_json::json;
use std::sync::Arc;
@@ -431,13 +431,33 @@ pub fn create_api_routes(app_state: &Arc<AppState>) -> Router<Arc<AppState>> {
{
use crate::interfaces::api::handlers::photos_handler;
let photos_router = Router::new()
.route("/", get(photos_handler::list_photos))
.with_state(app_state.clone());
let mut photos_router = Router::new().route("/", get(photos_handler::list_photos));
if app_state.places_service.is_some() {
photos_router = photos_router.route("/geo", get(photos_handler::list_photos_geo));
}
let photos_router = photos_router.with_state(app_state.clone());
router = router.nest("/photos", photos_router);
}
// People (faces) routes — mounted only when OXICLOUD_ENABLE_FACES is on.
if app_state.people_service.is_some() {
use crate::interfaces::api::handlers::people_handler;
let people_router = Router::new()
.route("/", get(people_handler::list_people))
.route("/merge", post(people_handler::merge_people))
.route("/recluster", post(people_handler::recluster))
.route("/data", delete(people_handler::delete_all))
.route("/faces/{file_id}", get(people_handler::faces_for_file))
.route("/{id}", patch(people_handler::rename_person))
.route("/{id}/photos", get(people_handler::person_photos))
.route("/{id}/hide", post(people_handler::hide_person))
.with_state(app_state.clone());
router = router.nest("/people", people_router);
}
// Re-enable trash routes to make the trash view work
if let Some(_trash_service_ref) = trash_service.clone() {
tracing::info!("Setting up trash routes for trash view");