diff --git a/Dockerfile b/Dockerfile index c9820a28..185f9929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ COPY --from=cacher /usr/local/cargo/registry /usr/local/cargo/registry COPY Cargo.toml Cargo.lock ./ COPY src src COPY static static +COPY db db # Build with all optimizations (DATABASE_URL only needed at compile-time for sqlx) ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud" RUN DATABASE_URL="${DATABASE_URL}" cargo build --release diff --git a/doc/DATABASE-MIGRATIONS.md b/doc/DATABASE-MIGRATIONS.md deleted file mode 100644 index d6566043..00000000 --- a/doc/DATABASE-MIGRATIONS.md +++ /dev/null @@ -1,184 +0,0 @@ -# Sistema de Migraciones de Base de Datos - -Este documento describe el sistema de migraciones de base de datos implementado en OxiCloud para gestionar cambios de esquema de forma controlada y segura. - -## Descripción General - -OxiCloud utiliza un sistema de migraciones basado en archivos SQL versionados para garantizar que los cambios en la estructura de la base de datos sean: - -- Versionados y rastreables -- Aplicados de forma consistente en todos los entornos -- Reproducibles y comprobables -- Independientes del código de la aplicación - -## Estructura de Directorios - -``` -OxiCloud/ -├── migrations/ # Directorio principal de migraciones -│ ├── 20250408000000_initial_schema.sql # Migración 1: Esquema inicial -│ ├── 20250408000001_default_users.sql # Migración 2: Usuarios por defecto -│ └── ... # Futuras migraciones -├── src/ - ├── bin/ - │ └── migrate.rs # Herramienta CLI para ejecutar migraciones -``` - -## Convenciones de Nomenclatura - -Las migraciones siguen el formato: `YYYYMMDDHHMMSS_descripción_breve.sql`, donde: - -- `YYYYMMDDHHMMSS`: Timestamp que garantiza el orden correcto (año, mes, día, hora, minuto, segundo) -- `descripción_breve`: Descripción concisa del propósito de la migración -- `.sql`: Extensión de archivo SQL - -## Ejecución de Migraciones - -Las migraciones se ejecutan mediante una herramienta CLI dedicada: - -```bash -cargo run --bin migrate --features migrations -``` - -Este comando: -1. Conecta con la base de datos configurada en el entorno -2. Busca migraciones en el directorio `/migrations/` -3. Compara las migraciones aplicadas con las disponibles -4. Ejecuta secuencialmente las migraciones pendientes -5. Registra las migraciones aplicadas en una tabla de control - -## Creación de Nuevas Migraciones - -Para crear una nueva migración: - -1. Crea un nuevo archivo en el directorio `migrations/` siguiendo la convención de nomenclatura -2. Define los cambios SQL en el archivo -3. Asegúrate de que los cambios sean compatibles con la versión actual del esquema -4. Ejecuta las migraciones con el comando correspondiente - -Ejemplo de estructura para una nueva migración: - -```sql --- Migración: Añadir tabla de etiquetas --- Descripción: Crea la tabla para almacenar etiquetas de archivos y sus relaciones - --- Crear tabla de etiquetas -CREATE TABLE IF NOT EXISTS auth.tags ( - id SERIAL PRIMARY KEY, - user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, - name TEXT NOT NULL, - color TEXT NOT NULL DEFAULT '#3498db', - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE(user_id, name) -); - --- Crear índices -CREATE INDEX IF NOT EXISTS idx_tags_user_id ON auth.tags(user_id); - --- Tabla de relación entre archivos y etiquetas -CREATE TABLE IF NOT EXISTS auth.file_tags ( - id SERIAL PRIMARY KEY, - tag_id INTEGER NOT NULL REFERENCES auth.tags(id) ON DELETE CASCADE, - file_id TEXT NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE(tag_id, file_id) -); - --- Comentarios de documentación -COMMENT ON TABLE auth.tags IS 'Almacena etiquetas definidas por usuarios'; -COMMENT ON TABLE auth.file_tags IS 'Relación muchos-a-muchos entre archivos y etiquetas'; -``` - -## Guía de Buenas Prácticas - -1. **Migraciones Incrementales**: Cada migración debe representar un cambio atómico y coherente. - -2. **Migraciones Idempotentes**: Cuando sea posible, usa comandos que pueden ejecutarse múltiples veces sin errores (ej. `CREATE TABLE IF NOT EXISTS`). - -3. **Migraciones Forward-Only**: Diseña las migraciones para avanzar, no para revertir. Si necesitas deshacer un cambio, crea una nueva migración. - -4. **Compatibilidad Hacia Adelante**: Las migraciones deben ser compatibles con el código existente y el que se va a desplegar. - -5. **Prueba Antes de Desplegar**: Prueba las migraciones en un entorno similar al de producción antes de aplicarlas. - -6. **Documentación**: Documenta el propósito y los cambios clave de cada migración con comentarios dentro del archivo SQL. - -## Solución de Problemas - -### Verificación del Estado de las Migraciones - -Para verificar qué migraciones se han aplicado, OxiCloud incluye detección en tiempo de inicio: - -```rust -// Desde src/common/db.rs -let migration_check = sqlx::query("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'auth' AND tablename = 'users')") - .fetch_one(&pool) - .await; - -match migration_check { - Ok(row) => { - let tables_exist: bool = row.get(0); - if !tables_exist { - tracing::warn!("Las tablas de la base de datos no existen. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations"); - } - }, - Err(_) => { - tracing::warn!("No se pudo verificar el estado de las migraciones. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations"); - } -} -``` - -### Problemas Comunes - -1. **Error de conexión a la base de datos**: Verifica la URL de conexión en la variable de entorno `DATABASE_URL`. - -2. **Conflictos de migración**: Si una migración falla, revisa los mensajes de error para identificar conflictos con el esquema existente. - -3. **Permisos insuficientes**: Asegúrate de que el usuario de la base de datos tenga permisos suficientes para crear esquemas, tablas e índices. - -4. **Error "Admin already exists"**: Si al intentar registrar un usuario admin recibes el error "El usuario 'admin' ya existe", sigue estos pasos: - - a. Conéctate al contenedor de PostgreSQL: - ```bash - # Encuentra el contenedor - docker ps - # Ejemplo: oxicloud-postgres-1 - docker exec -it oxicloud-postgres-1 bash - ``` - - b. Conéctate a la base de datos: - ```bash - psql -U postgres -d oxicloud - ``` - - c. Establece el esquema y borra el usuario admin existente: - ```sql - SET search_path TO auth; - DELETE FROM auth.users WHERE username = 'admin'; - ``` - - d. Verifica la eliminación: - ```sql - SELECT username, email, role FROM auth.users; - ``` - - e. Sal de PostgreSQL: - ```sql - \q - exit - ``` - - f. Ahora puedes registrar un nuevo usuario admin a través de la interfaz de OxiCloud. - - Alternativamente, utiliza el script proporcionado: - ```bash - cat scripts/reset_admin.sql | docker exec -i oxicloud-postgres-1 psql -U postgres -d oxicloud - ``` - -## Beneficios del Enfoque Basado en Migraciones - -- **Separación de Responsabilidades**: Las migraciones están separadas del código de la aplicación. -- **Automatización**: Facilita la automatización de despliegues y CI/CD. -- **Historial de Cambios**: Proporciona un historial claro de cómo ha evolucionado el esquema. -- **Colaboración**: Permite que múltiples desarrolladores contribuyan cambios al esquema de forma ordenada. -- **Entornos Múltiples**: Garantiza que todos los entornos (desarrollo, pruebas, producción) tengan estructuras de base de datos idénticas. \ No newline at end of file diff --git a/doc/DATABASE-TRANSACTIONS.md b/doc/DATABASE-TRANSACTIONS.md deleted file mode 100644 index 1d2e3e81..00000000 --- a/doc/DATABASE-TRANSACTIONS.md +++ /dev/null @@ -1,171 +0,0 @@ -# Base de Datos y Transacciones en OxiCloud - -## Introducción a Transacciones Explícitas en la Base de Datos - -Este documento describe la implementación de transacciones explícitas en OxiCloud para garantizar la integridad de los datos en operaciones de base de datos PostgreSQL. - -## ¿Qué son las Transacciones? - -Una transacción es una secuencia de operaciones de base de datos tratadas como una única unidad lógica. Las transacciones siguen las propiedades ACID: - -- **Atomicidad**: Una transacción es "todo o nada". Si cualquier parte falla, toda la transacción falla. -- **Consistencia**: La base de datos pasa de un estado válido a otro estado válido. -- **Aislamiento**: Las transacciones simultáneas se comportan como si fueran secuenciales. -- **Durabilidad**: Una vez confirmada, la transacción permanece confirmada incluso en caso de fallo del sistema. - -## Implementación en OxiCloud - -OxiCloud ahora utiliza un enfoque consistente para las transacciones de base de datos mediante la función `with_transaction`, que: - -1. Comienza una transacción -2. Ejecuta operaciones -3. Confirma automáticamente si todo fue exitoso -4. Revierte (rollback) automáticamente en caso de error - -### Utilidad de Transacciones - -En `src/infrastructure/repositories/pg/transaction_utils.rs` hemos implementado: - -```rust -/// Helper function to execute database operations in a transaction -pub async fn with_transaction( - pool: &Arc, - operation_name: &str, - operation: F, -) -> Result -where - F: for<'c> FnOnce(&'c mut Transaction<'_, Postgres>) -> futures::future::BoxFuture<'c, Result>, - E: From + std::fmt::Display -{ ... } -``` - -Esta función: -- Recibe un pool de conexiones y un closure con operaciones -- Maneja begin/commit/rollback automáticamente -- Proporciona logging detallado del ciclo de vida de la transacción - -### Ejemplo de Uso en Repositorios - -```rust -// Creación de un usuario con transacción explícita -async fn create_user(&self, user: User) -> UserRepositoryResult { - with_transaction( - &self.pool, - "create_user", - |tx| { - Box::pin(async move { - // Operación principal - insertar usuario - sqlx::query("INSERT INTO auth.users ...") - .bind(...) - .execute(&mut **tx) - .await?; - - // Operaciones adicionales dentro de la misma transacción - // ... - - Ok(user_clone) - }) - } - ).await -} -``` - -## Casos de Uso Implementados - -### En UserPgRepository - -1. **Creación de Usuario** - - Garantiza que todas las operaciones de inserción son atómicas - - Permite agregar operaciones relacionadas (como configuración de permisos) - -2. **Actualización de Usuario** - - Asegura que las modificaciones se apliquen completamente o no se apliquen en absoluto - - Soporta operaciones combinadas como actualización de información de perfil y preferencias - -### En SessionPgRepository - -1. **Creación de Sesión** - - Inserta la sesión y actualiza el timestamp de último acceso del usuario en una única transacción - - Garantiza consistencia entre sesiones y datos de usuario - -2. **Revocación de Sesiones** - - Asegura que la revocación de una sesión o de todas las sesiones de un usuario sea atómica - - Permite registrar eventos de seguridad dentro de la misma transacción - -## Niveles de Aislamiento - -OxiCloud admite diferentes niveles de aislamiento de transacciones mediante `with_transaction_isolation`: - -```rust -// Ejemplo de uso con nivel de aislamiento específico -with_transaction_isolation( - &pool, - "operacion_critica", - sqlx::postgres::PgIsolationLevel::Serializable, - |tx| { ... } -).await -``` - -Los niveles de aislamiento disponibles son: - -1. **Read Committed** (predeterminado) - - Garantiza que los datos leídos están confirmados - - No previene lecturas no repetibles o fantasma - -2. **Repeatable Read** - - Garantiza que las lecturas sean consistentes durante toda la transacción - - Previene lecturas no repetibles pero no lecturas fantasma - -3. **Serializable** - - Nivel más alto de aislamiento - - Garantiza que las transacciones se comporten como si se ejecutaran en serie - - Puede causar errores de serialización que requieren reintento - -## Mejores Prácticas - -1. **Duración de Transacciones** - - Mantén las transacciones lo más cortas posible - - Evita operaciones de larga duración dentro de transacciones - -2. **Manejo de Errores** - - Los errores dentro de una transacción provocan rollback automático - - Utiliza logging adecuado para diagnosticar fallos - -3. **Límites de Transacción** - - Define claramente dónde comienzan y terminan las transacciones - - Agrupa operaciones relacionadas en una sola transacción - -4. **Aislamiento Apropiado** - - Usa el nivel de aislamiento más bajo adecuado para tu caso de uso - - Considera serializable para operaciones críticas con posibilidad de conflicto - -## Ventajas de Transacciones Explícitas - -1. **Integridad de Datos Mejorada** - - Garantía ACID para operaciones complejas - - Prevención de estados inconsistentes - -2. **Mejor Manejo de Errores** - - Rollback automático ante fallos - - Comportamiento predecible en caso de error - -3. **Concurrencia Segura** - - Manejo adecuado de operaciones simultáneas - - Prevención de condiciones de carrera - -4. **Rendimiento** - - Reducción de trips a la base de datos - - Operaciones en lote para mejor eficiencia - -## Consideraciones de Rendimiento - -- Las transacciones añaden cierta sobrecarga -- El rendimiento puede verse afectado por: - - Duración de la transacción - - Nivel de aislamiento - - Número de registros afectados - - Contención por bloqueos - -## Conclusión - -La implementación de transacciones explícitas en OxiCloud mejora significativamente la robustez del sistema y garantiza la integridad de los datos en escenarios complejos. El enfoque modular y la API de transacciones simplificada permiten extender fácilmente estos beneficios a nuevas funcionalidades. \ No newline at end of file diff --git a/doc/DAV-IMPLEMENTATION-PLAN.md b/doc/DAV-IMPLEMENTATION-PLAN.md deleted file mode 100644 index 56d6eca9..00000000 --- a/doc/DAV-IMPLEMENTATION-PLAN.md +++ /dev/null @@ -1,286 +0,0 @@ -# Plan de Implementación DAV para OxiCloud - -Este documento presenta un plan de implementación estructurado para añadir soporte WebDAV, CalDAV y CardDAV a OxiCloud. - -## Resumen Ejecutivo - -La implementación de los protocolos DAV (WebDAV, CalDAV y CardDAV) permitirá a OxiCloud interoperar con una amplia gama de clientes y dispositivos, aumentando significativamente su versatilidad y utilidad. Este plan propone un enfoque por fases que prioriza primero WebDAV (para acceso a archivos), seguido de CalDAV (para calendarios) y finalmente CardDAV (para contactos). - -## Fases de Implementación - -### Fase 1: Infraestructura DAV Común (Estimado: 2-3 semanas) - -**Objetivos:** -- Establecer la infraestructura básica compartida por todos los protocolos DAV -- Implementar el manejo de solicitudes XML y respuestas -- Crear adaptadores para las operaciones básicas DAV - -**Tareas:** -1. **Semana 1: Diseño y Arquitectura** - - Diseñar la arquitectura de los componentes DAV - - Definir interfaces para adaptadores DAV - - Seleccionar bibliotecas para procesamiento XML y RFC4918 - -2. **Semana 2: Implementación Base** - - Implementar manejadores de serialización/deserialización XML - - Desarrollar middleware para procesamiento de solicitudes DAV - - Crear estructuras comunes (propiedades, espacios de nombres) - - Implementar validación de solicitudes DAV - -3. **Semana 3: Framework de Pruebas** - - Configurar entorno de pruebas para protocolos DAV - - Implementar clientes de prueba automatizados - - Crear casos de prueba para operaciones DAV básicas - -**Entregables:** -- Framework de procesamiento XML para solicitudes/respuestas DAV -- Adaptadores base para las entidades existentes -- Suite de pruebas para operaciones DAV - -### Fase 2: WebDAV (Estimado: 3-4 semanas) - -**Objetivos:** -- Implementar el protocolo WebDAV completo (RFC4918) -- Permitir acceso a archivos y carpetas vía WebDAV -- Asegurar compatibilidad con clientes WebDAV comunes - -**Tareas:** -1. **Semana 1: Operaciones Básicas** - - Implementar métodos PROPFIND y PROPPATCH - - Desarrollar endpoint OPTIONS (descubrimiento de capacidades) - - Implementar operaciones GET, HEAD, PUT (lectura/escritura) - -2. **Semana 2: Operaciones Avanzadas** - - Implementar MKCOL (creación de directorios) - - Desarrollar DELETE para recursos WebDAV - - Implementar COPY y MOVE para archivos y directorios - -3. **Semana 3: Bloqueo y Características Extendidas** - - Implementar LOCK y UNLOCK para recursos - - Añadir soporte para propiedades personalizadas - - Desarrollar características de WebDAV extendidas (si es necesario) - -4. **Semana 4: Pruebas y Optimización** - - Realizar pruebas con clientes reales (Windows, macOS, Linux) - - Optimizar rendimiento para transferencias grandes - - Documentar APIs y comportamiento WebDAV - -**Entregables:** -- Implementación completa de WebDAV (RFC4918) -- Documentación de uso de WebDAV con OxiCloud -- Compatibilidad con los clientes WebDAV más comunes - -### Fase 3: CalDAV (Estimado: 4-5 semanas) - -**Objetivos:** -- Implementar el protocolo CalDAV (RFC4791) -- Crear entidades y repositorios para calendarios y eventos -- Soportar operaciones de calendario con clientes comunes - -**Tareas:** -1. **Semana 1: Modelo de Datos** - - Implementar entidades Calendar y CalendarEvent - - Desarrollar repositorios para almacenamiento de datos - - Crear DTOs y adaptadores CalDAV - -2. **Semana 2: Endpoints Básicos** - - Implementar PROPFIND para detección de calendarios - - Desarrollar MKCALENDAR para creación de calendarios - - Implementar GET/PUT para eventos individuales - -3. **Semana 3: Consultas Avanzadas** - - Implementar REPORT para consultas de calendario - - Desarrollar soporte para búsqueda por rango de fechas - - Añadir manejo de recurrencias (reglas RRULE) - -4. **Semana 4: Interoperabilidad** - - Implementar sincronización eficiente (collection-sync) - - Añadir soporte para zonas horarias - - Desarrollar manejo de alarmas y notificaciones - -5. **Semana 5: Pruebas y Refinamiento** - - Probar con clientes CalDAV populares - - Optimizar rendimiento para calendarios grandes - - Documentar APIs y comportamiento CalDAV - -**Entregables:** -- Implementación completa de CalDAV (RFC4791) -- Soporte para creación y gestión de calendarios -- Compatibilidad con clientes CalDAV populares -- Documentación de uso de CalDAV con OxiCloud - -### Fase 4: CardDAV (Estimado: 3-4 semanas) - -**Objetivos:** -- Implementar el protocolo CardDAV (RFC6352) -- Crear entidades y repositorios para libretas de direcciones y contactos -- Soportar operaciones de contactos con clientes comunes - -**Tareas:** -1. **Semana 1: Modelo de Datos** - - Implementar entidades AddressBook y Contact - - Desarrollar repositorios para almacenamiento de datos - - Crear DTOs y adaptadores CardDAV - -2. **Semana 2: Endpoints Básicos** - - Implementar PROPFIND para detección de libretas - - Desarrollar MKCOL para creación de libretas de direcciones - - Implementar GET/PUT para contactos individuales - -3. **Semana 3: Consultas y Búsqueda** - - Implementar REPORT para consultas de contactos - - Desarrollar búsqueda de contactos por criterios - - Añadir soporte para grupos de contactos - -4. **Semana 4: Pruebas y Refinamiento** - - Probar con clientes CardDAV populares - - Optimizar rendimiento para libretas grandes - - Documentar APIs y comportamiento CardDAV - -**Entregables:** -- Implementación completa de CardDAV (RFC6352) -- Soporte para creación y gestión de libretas de direcciones -- Compatibilidad con clientes CardDAV populares -- Documentación de uso de CardDAV con OxiCloud - -### Fase 5: Integración y Lanzamiento (Estimado: 2-3 semanas) - -**Objetivos:** -- Integrar todos los protocolos DAV en una solución cohesiva -- Asegurar compatibilidad cruzada entre protocolos -- Preparar la documentación y materiales para lanzamiento - -**Tareas:** -1. **Semana 1: Integración** - - Consolidar código compartido entre protocolos - - Asegurar coherencia de comportamiento - - Refinar manejo de errores y recuperación - -2. **Semana 2: Pruebas de Sistema** - - Realizar pruebas de integración end-to-end - - Validar rendimiento bajo carga - - Verificar seguridad y permisos - -3. **Semana 3: Documentación y Lanzamiento** - - Finalizar guías de usuario para clientes DAV - - Crear documentación para desarrolladores - - Preparar materiales de lanzamiento - -**Entregables:** -- Solución DAV completa e integrada -- Documentación comprensiva para usuarios y desarrolladores -- Paquete de lanzamiento listo para despliegue - -## Requisitos de Infraestructura - -### Dependencias de Bibliotecas - -```toml -# Añadir a Cargo.toml -[dependencies] -# Procesamiento XML -quick-xml = "0.30.0" -xml-rs = "0.8.14" - -# Soporte para iCalendar -icalendar = "0.15.0" - -# Soporte para vCard -vcard = "0.2.0" - -# Utilidades para DAV -http-multipart = "0.3.0" -``` - -### Esquema de Base de Datos - -Las nuevas tablas para CalDAV y CardDAV deben ser creadas como parte de la fase correspondiente. Ver el esquema completo en el documento principal de implementación. - -## Estrategia de Pruebas - -### Pruebas Unitarias - -- Pruebas de serialización/deserialización XML -- Pruebas de validación de entradas -- Pruebas de lógica de negocio para cada operación DAV - -### Pruebas de Integración - -- Pruebas end-to-end con clientes simulados -- Pruebas de flujos completos (creación, actualización, eliminación) -- Pruebas de concurrencia y manejo de conflictos - -### Pruebas de Compatibilidad - -- Matriz de pruebas con clientes reales (al menos 3 por protocolo) -- Pruebas en diferentes sistemas operativos -- Verificación de conformidad con RFCs - -## Consideraciones de Rendimiento - -1. **Optimización de Consultas** - - Implementar paginación para conjuntos grandes de resultados - - Optimizar consultas SQL para calendarios y contactos - - Utilizar índices adecuados para búsqueda rápida - -2. **Caché** - - Implementar caché de propiedades para respuestas PROPFIND - - Usar ETags para validación de caché - - Aplicar caché de consultas para reportes frecuentes - -3. **Procesamiento Eficiente** - - Procesamiento XML eficiente para solicitudes grandes - - Streaming de datos para archivos grandes - - Procesamiento asíncrono para operaciones costosas - -## Riesgos y Mitigación - -| Riesgo | Impacto | Probabilidad | Estrategia de Mitigación | -|--------|---------|--------------|--------------------------| -| Problemas de compatibilidad con clientes | Alto | Medio | Pruebas tempranas con variedad de clientes, seguir estrictamente las especificaciones | -| Rendimiento insuficiente | Medio | Bajo | Pruebas de carga desde el inicio, diseño para escalabilidad | -| Complejidad excesiva | Medio | Medio | Enfoque modular, abstracciones claras, revisiones de código frecuentes | -| Problemas de seguridad | Alto | Bajo | Revisiones de seguridad, validación estricta de entradas, pruebas de penetración | -| Retrasos en el cronograma | Medio | Medio | Planificación conservadora, hitos claros, enfoque iterativo | - -## Criterios de Éxito - -1. **Compatibilidad** - - Todos los protocolos cumplen con sus respectivos RFCs - - Compatibilidad verificada con al menos 3 clientes principales por protocolo - - Funciona en todos los sistemas operativos principales - -2. **Rendimiento** - - Tiempo de respuesta para operaciones típicas < 500ms - - Soporta calendarios con >1000 eventos sin degradación significativa - - Soporta libretas con >1000 contactos sin degradación significativa - -3. **Usabilidad** - - Proceso de configuración de cliente sencillo y documentado - - Mensajes de error claros y específicos - - Documentación completa para usuarios y desarrolladores - -## Recursos Necesarios - -1. **Equipo de Desarrollo** - - 1-2 desarrolladores de backend (Rust) - - 1 desarrollador de frontend (para integración UI si es necesario) - - 1 tester - -2. **Infraestructura** - - Entorno de pruebas con múltiples sistemas operativos - - Clientes DAV variados para pruebas - - Servidor de CI/CD para pruebas automatizadas - -3. **Habilidades** - - Experiencia con protocolos HTTP avanzados - - Conocimiento de procesamiento XML - - Familiaridad con los estándares WebDAV, CalDAV y CardDAV - -## Próximos Pasos - -1. Asignar recursos al proyecto -2. Establecer repositorio de código y estructura inicial -3. Iniciar la Fase 1 (Infraestructura DAV Común) -4. Configurar entorno de CI/CD para pruebas -5. Revisar y refinar el plan según sea necesario durante la implementación \ No newline at end of file diff --git a/doc/FILE-SYSTEM-SAFETY.md b/doc/FILE-SYSTEM-SAFETY.md deleted file mode 100644 index 2f4abaa5..00000000 --- a/doc/FILE-SYSTEM-SAFETY.md +++ /dev/null @@ -1,168 +0,0 @@ -# File System Safety in OxiCloud - -This document describes the implementation of file system safety mechanisms in OxiCloud to ensure data integrity and durability during file operations. - -## Introduction - -Data integrity is critical in a file storage system like OxiCloud. When files are written to disk, it's important to ensure that: - -1. Writes are atomic - they either complete fully or not at all -2. Data is properly synchronized to persistent storage -3. Directory entries are properly updated and persisted -4. The system can recover from unexpected crashes or power failures - -OxiCloud implements several mechanisms to achieve these goals. - -## The Problem: Buffered I/O and Data Loss - -Standard file system operations in many programming languages and operating systems use buffered I/O by default: - -```rust -// This operation may not immediately persist to disk -fs::write(path, content) -``` - -When an application writes data, the operating system typically: - -1. Accepts the write into memory buffers -2. Acknowledges completion to the application -3. Schedules the actual disk write for later - -This creates a window where a system crash or power failure can result in data loss, as the data may exist only in memory buffers that haven't been flushed to disk. - -## OxiCloud's Solution - -OxiCloud implements a comprehensive approach to file system safety through the `FileSystemUtils` service, which provides: - -### 1. Atomic Write Pattern - -Files are written using a safe atomic pattern: - -```rust -/// Writes data to a file with fsync to ensure durability -/// Uses a safe atomic write pattern: write to temp file, fsync, rename -pub async fn atomic_write>(path: P, contents: &[u8]) -> Result<(), IoError> -``` - -This implements a write-then-rename pattern: -1. Write to a temporary file in the same directory -2. Call `fsync` to ensure data is on disk -3. Atomically rename the temp file to the target file -4. Sync the parent directory to ensure the rename is persisted - -### 2. Directory Synchronization - -Directory operations are also synchronized: - -```rust -/// Creates directories with fsync -pub async fn create_dir_with_sync>(path: P) -> Result<(), IoError> -``` - -This ensures that: -1. Directories are properly created -2. Directory entries are persisted to disk -3. Parent directories are also synchronized - -### 3. Rename and Delete Operations - -Renames and delete operations follow the same pattern: - -```rust -/// Renames a file or directory with proper syncing -pub async fn rename_with_sync, Q: AsRef>(from: P, to: Q) -> Result<(), IoError> - -/// Removes a file with directory syncing -pub async fn remove_file_with_sync>(path: P) -> Result<(), IoError> -``` - -These operations ensure that: -1. The operation itself is completed -2. The parent directory entry is updated and synchronized - -## Implementation Details - -### Implementing fsync on Files - -```rust -// Write file content -file.write_all(contents).await?; - -// Ensure data is synced to disk -file.flush().await?; -file.sync_all().await?; -``` - -The `sync_all()` call is critical as it instructs the operating system to flush data and metadata to the physical storage device. - -### Implementing fsync on Directories - -```rust -// Sync a directory to ensure its contents (entries) are durable -async fn sync_directory>(path: P) -> Result<(), IoError> { - let dir_file = OpenOptions::new().read(true).open(path).await?; - dir_file.sync_all().await -} -``` - -This is essential after operations that modify directory entries, such as creating, renaming, or deleting files. - -## Usage in the Codebase - -The `FileSystemUtils` service is integrated throughout OxiCloud's file operations: - -### In File Write Repository - -```rust -// Write the file to disk using atomic write with fsync -tokio::time::timeout( - self.config.timeouts.file_write_timeout(), - FileSystemUtils::atomic_write(&abs_path, &content) -).await -``` - -### In File Move Operations - -```rust -// Move the file physically with fsync -time::timeout( - self.config.timeouts.file_timeout(), - FileSystemUtils::rename_with_sync(&old_abs_path, &new_abs_path) -).await -``` - -### For Directory Creation - -```rust -// Ensure the parent directory exists with proper syncing -self.ensure_parent_directory(&abs_path).await?; - -// Implementation uses FileSystemUtils -async fn ensure_parent_directory(&self, abs_path: &PathBuf) -> FileRepositoryResult<()> { - if let Some(parent) = abs_path.parent() { - time::timeout( - self.config.timeouts.dir_timeout(), - FileSystemUtils::create_dir_with_sync(parent) - ).await - } -} -``` - -## Benefits - -By implementing these safety measures, OxiCloud provides: - -1. **Data Durability**: Critical data is properly synchronized to persistent storage -2. **Crash Resilience**: The system can recover from unexpected failures without data loss -3. **Consistency**: File operations maintain a consistent file system state -4. **Atomic Operations**: File writes appear as all-or-nothing operations - -## Performance Considerations - -These safety measures do have some performance impact, as synchronizing to disk is more expensive than buffered writes. However, OxiCloud: - -1. Applies these measures only to critical operations -2. Uses timeouts to prevent operations from blocking indefinitely -3. Implements parallel processing for large files - -The safety-performance tradeoff favors safety for critical data while still maintaining good performance for most operations. \ No newline at end of file diff --git a/doc/LTO-OPTIMIZATIONS.md b/doc/LTO-OPTIMIZATIONS.md deleted file mode 100644 index ee67f6cf..00000000 --- a/doc/LTO-OPTIMIZATIONS.md +++ /dev/null @@ -1,78 +0,0 @@ -# Link Time Optimization (LTO) in OxiCloud - -## Overview - -OxiCloud uses Link Time Optimization (LTO) to significantly improve runtime performance. LTO is a technique that allows the compiler to perform optimizations across module boundaries during the linking phase, which can lead to better inlining, dead code elimination, and overall more efficient binaries. - -## Implemented Optimizations - -This project uses the following optimization settings: - -### Release Profile -```toml -[profile.release] -lto = "fat" # Full cross-module optimization -codegen-units = 1 # Maximum optimization but slower compile time -opt-level = 3 # Maximum optimization level -panic = "abort" # Smaller binary size by removing panic unwinding -strip = true # Removes debug symbols for smaller binary -``` - -### Development Profile -```toml -[profile.dev] -opt-level = 1 # Light optimization for faster build time -debug = true # Keep debug information for development -``` - -### Benchmark Profile -```toml -[profile.bench] -lto = "fat" # Full optimization for benchmarks -codegen-units = 1 # Maximum optimization -opt-level = 3 # Maximum optimization level -``` - -## Performance Improvements - -The optimizations typically result in: - -1. **Smaller binary size**: Removing unused code and metadata -2. **Faster execution**: Better inlining and code optimizations -3. **Reduced memory usage**: More efficient code layout and execution - -## LTO Options Explained - -- **fat**: Also known as "full" LTO, performs optimizations across all crate boundaries. Maximum optimization but longest compile time. -- **thin**: A faster version of LTO that trades some optimization for compile speed. Good for development. -- **off**: No cross-module optimization. - -## Build Time Impact - -While LTO provides runtime performance benefits, it increases compilation time. For OxiCloud, we chose: - -- Development builds: Minimal LTO (`opt-level = 1`) for faster iteration -- Release builds: Full LTO for maximum end-user performance -- Benchmark builds: Full LTO to measure actual optimized performance - -## Measuring the Impact - -To measure the impact of these optimizations, run our benchmarks: - -```bash -# Run benchmarks with all optimizations -cargo bench - -# Compare with non-optimized build (remove for comparison only) -RUSTFLAGS="-C lto=off" cargo bench -``` - -## When to Adjust Settings - -Consider adjusting these settings if: - -1. You need faster compile times during development -2. You're experiencing unexpected runtime behavior -3. You want to experiment with optimization/binary size tradeoffs - -For most users, the default settings should provide a good balance of performance and usability. \ No newline at end of file diff --git a/doc/OIDC-CONFIG-EXAMPLES.md b/doc/OIDC-CONFIG-EXAMPLES.md deleted file mode 100644 index ff6cc9a3..00000000 --- a/doc/OIDC-CONFIG-EXAMPLES.md +++ /dev/null @@ -1,218 +0,0 @@ -# Ejemplos de Configuración de OIDC para OxiCloud - -Esta guía proporciona ejemplos de configuración para integrar OxiCloud con diferentes proveedores OIDC (OpenID Connect). - -## Índice - -1. [Configuración General de OIDC](#configuración-general-de-oidc) -2. [Authentik](#authentik) -3. [Authelia](#authelia) -4. [KeyCloak](#keycloak) -5. [Resolución de Problemas](#resolución-de-problemas) - -## Configuración General de OIDC - -Para habilitar la integración OIDC en OxiCloud, necesitará establecer las siguientes variables de entorno: - -```bash -# Habilitar OIDC -OXICLOUD_ENABLE_OIDC=true - -# Configuración para cada proveedor OIDC (puede configurar múltiples proveedores) -OXICLOUD_OIDC_PROVIDER__NAME="Nombre Visible" -OXICLOUD_OIDC_PROVIDER__CLIENT_ID="su-client-id" -OXICLOUD_OIDC_PROVIDER__CLIENT_SECRET="su-client-secret" -OXICLOUD_OIDC_PROVIDER__DISCOVERY_URL="https://proveedor.example.com/.well-known/openid-configuration" -OXICLOUD_OIDC_PROVIDER__REDIRECT_URI="https://su-oxicloud.example.com/oidc/callback/" -OXICLOUD_OIDC_PROVIDER__SCOPES="openid profile email" -OXICLOUD_OIDC_PROVIDER__USER_ID_ATTRIBUTE="sub" -OXICLOUD_OIDC_PROVIDER__DEFAULT_ROLE="user" -OXICLOUD_OIDC_PROVIDER__AUTO_CREATE_USERS="true" -``` - -## Authentik - -[Authentik](https://goauthentik.io/) es una plataforma de identidad de código abierto que proporciona autenticación, autorización y gestión de usuarios. - -### 1. Configurar una aplicación en Authentik - -1. Inicia sesión en tu panel de administración de Authentik -2. Ve a "Applications" → "Create" -3. Introduce un nombre para tu aplicación (ej. "OxiCloud") -4. Selecciona "OAuth2/OpenID Provider" como tipo de proveedor -5. En la configuración de OAuth2: - - **Redirect URI/Callback URL**: `https://su-oxicloud.example.com/oidc/callback/authentik` - - **Client Type**: Confidential - - **Client ID**: Se generará automáticamente (anótalo) - - **Client Secret**: Se generará automáticamente (anótalo) - - **Scopes**: openid, email, profile -6. En la configuración de UI: - - **Launch URL**: `https://su-oxicloud.example.com/` - - **Icon**: Opcional, puedes subir un icono para OxiCloud - -### 2. Configurar OxiCloud para Authentik - -```yaml -# docker-compose.yml -version: '3' -services: - oxicloud: - image: oxicloud:latest - environment: - # Configuración general - OXICLOUD_ENABLE_OIDC: "true" - - # Configuración de Authentik - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_NAME: "Authentik" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_CLIENT_ID: "tu-client-id-de-authentik" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_CLIENT_SECRET: "tu-client-secret-de-authentik" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_DISCOVERY_URL: "https://authentik.example.com/application/o/oxicloud/.well-known/openid-configuration" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_REDIRECT_URI: "https://oxicloud.example.com/oidc/callback/authentik" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_SCOPES: "openid profile email" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_USER_ID_ATTRIBUTE: "sub" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_DEFAULT_ROLE: "user" - OXICLOUD_OIDC_PROVIDER_AUTHENTIK_AUTO_CREATE_USERS: "true" - ports: - - "8085:8085" - volumes: - - ./storage:/app/storage -``` - -## Authelia - -[Authelia](https://www.authelia.com/) es una solución de autenticación multi-factor de código abierto. - -### 1. Configurar Authelia para OxiCloud - -Edita tu configuración de Authelia (`configuration.yml`): - -```yaml -identity_providers: - oidc: - hmac_secret: tu-secreto-seguro # Cambia esto por un valor aleatorio seguro - issuer_private_key: /config/private.pem # Ruta a tu clave privada - cors: - endpoints: ['authorization', 'token', 'revocation', 'introspection'] - allowed_origins: - - https://oxicloud.example.com - clients: - - id: oxicloud - description: OxiCloud - secret: tu-client-secret-seguro # Cambia esto - public: false - authorization_policy: two_factor - redirect_uris: - - https://oxicloud.example.com/oidc/callback/authelia - scopes: ['openid', 'profile', 'email', 'groups'] - userinfo_signing_algorithm: none -``` - -### 2. Configurar OxiCloud para Authelia - -```yaml -# docker-compose.yml -version: '3' -services: - oxicloud: - image: oxicloud:latest - environment: - # Configuración general - OXICLOUD_ENABLE_OIDC: "true" - - # Configuración de Authelia - OXICLOUD_OIDC_PROVIDER_AUTHELIA_NAME: "Authelia" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_CLIENT_ID: "oxicloud" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_CLIENT_SECRET: "tu-client-secret-seguro" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_DISCOVERY_URL: "https://authelia.example.com/.well-known/openid-configuration" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_REDIRECT_URI: "https://oxicloud.example.com/oidc/callback/authelia" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_SCOPES: "openid profile email groups" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_USER_ID_ATTRIBUTE: "sub" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_DEFAULT_ROLE: "user" - OXICLOUD_OIDC_PROVIDER_AUTHELIA_AUTO_CREATE_USERS: "true" - ports: - - "8085:8085" - volumes: - - ./storage:/app/storage -``` - -## KeyCloak - -[KeyCloak](https://www.keycloak.org/) es una solución de gestión de identidad y acceso de código abierto. - -### 1. Configurar un cliente en KeyCloak - -1. Inicia sesión en la consola de administración de KeyCloak -2. Selecciona tu Reino (Realm) -3. Ve a "Clients" → "Create" -4. Completa el formulario: - - **Client ID**: `oxicloud` - - **Client Protocol**: `openid-connect` - - **Root URL**: `https://oxicloud.example.com` -5. En la configuración del cliente: - - **Access Type**: `confidential` - - **Valid Redirect URIs**: `https://oxicloud.example.com/oidc/callback/keycloak` - - **Web Origins**: `https://oxicloud.example.com` (o `+` para permitir todos los orígenes) -6. Guarda la configuración -7. Ve a la pestaña "Credentials" y copia el "Secret" generado - -### 2. Configurar OxiCloud para KeyCloak - -```yaml -# docker-compose.yml -version: '3' -services: - oxicloud: - image: oxicloud:latest - environment: - # Configuración general - OXICLOUD_ENABLE_OIDC: "true" - - # Configuración de KeyCloak - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_NAME: "KeyCloak" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_CLIENT_ID: "oxicloud" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_CLIENT_SECRET: "tu-client-secret-de-keycloak" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_DISCOVERY_URL: "https://keycloak.example.com/realms/tu-realm/.well-known/openid-configuration" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_REDIRECT_URI: "https://oxicloud.example.com/oidc/callback/keycloak" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_SCOPES: "openid profile email" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_USER_ID_ATTRIBUTE: "sub" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_DEFAULT_ROLE: "user" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_AUTO_CREATE_USERS: "true" - ports: - - "8085:8085" - volumes: - - ./storage:/app/storage -``` - -## Resolución de Problemas - -### Error: "Failed to discover OIDC provider" - -Este error ocurre cuando OxiCloud no puede acceder al punto de descubrimiento del proveedor OIDC. - -**Soluciones:** -1. Verifica que la URL de descubrimiento sea correcta -2. Asegúrate de que OxiCloud pueda acceder a la URL (verifique firewalls, DNS, etc.) -3. Si tu proveedor utiliza un certificado autofirmado, asegúrate de configurar la confianza adecuada - -### Error: "Invalid redirect URI" - -Tu proveedor OIDC rechaza la URI de redirección. - -**Soluciones:** -1. Asegúrate de que la URI de redirección configurada en OxiCloud coincida exactamente con la registrada en tu proveedor OIDC -2. Verifica que no haya diferencias en protocolo (http vs https), puerto o ruta - -### Error: "User does not exist and auto-creation is disabled" - -**Soluciones:** -1. Habilita la creación automática de usuarios: `OXICLOUD_OIDC_PROVIDER__AUTO_CREATE_USERS="true"` -2. O crea manualmente el usuario en OxiCloud antes de intentar iniciar sesión con OIDC - -### Error: "Could not extract user ID from claim" - -OxiCloud no puede encontrar el atributo de ID de usuario especificado en los claims del token. - -**Soluciones:** -1. Verifica que el atributo configurado (`USER_ID_ATTRIBUTE`) exista en los claims del token -2. Prueba con un atributo diferente, como "sub", "email" o "preferred_username" -3. Configura tu proveedor OIDC para incluir el atributo necesario en los tokens \ No newline at end of file diff --git a/doc/OIDC-INTEGRATION.md b/doc/OIDC-INTEGRATION.md deleted file mode 100644 index bc5d8e34..00000000 --- a/doc/OIDC-INTEGRATION.md +++ /dev/null @@ -1,702 +0,0 @@ -# OIDC Integration for OxiCloud - -This document outlines the implementation plan for adding OpenID Connect (OIDC) support to OxiCloud, enabling Single Sign-On (SSO) with identity providers like Authentik, Authelia, KeyCloak, and others. - -## Overview - -OpenID Connect (OIDC) is an identity layer built on top of the OAuth 2.0 protocol. It allows clients to verify the identity of end-users based on the authentication performed by an authorization server, as well as to obtain basic profile information about the end-user. - -Implementing OIDC in OxiCloud will: -1. Allow users to authenticate using their existing identity provider (IdP) credentials -2. Reduce the need for separate username/password management in OxiCloud -3. Enhance security by leveraging modern authentication best practices -4. Provide a seamless experience for users already using SSO in their environment - -## Implementation Plan - -### 1. Add OIDC Configuration Options - -Extend the `AuthConfig` struct in `src/common/config.rs`: - -```rust -pub struct AuthConfig { - pub jwt_secret: String, - pub access_token_expiry_secs: i64, - pub refresh_token_expiry_secs: i64, - pub hash_memory_cost: u32, - pub hash_time_cost: u32, - - // New OIDC configuration - pub enable_oidc: bool, - pub oidc_providers: Vec, -} - -pub struct OidcProviderConfig { - pub name: String, // Display name (e.g., "Authentik", "KeyCloak") - pub client_id: String, // OIDC client ID - pub client_secret: String, // OIDC client secret - pub discovery_url: String, // OIDC discovery URL (.well-known/openid-configuration) - pub redirect_uri: String, // Redirect URI after authentication - pub scopes: Vec, // Scopes to request - pub user_id_attribute: String, // Which claim to use as user ID - pub default_role: String, // Default role for new users - pub auto_create_users: bool, // Create users on first login -} - -impl Default for AuthConfig { - fn default() -> Self { - Self { - // Existing defaults... - - // OIDC defaults - enable_oidc: false, - oidc_providers: Vec::new(), - } - } -} -``` - -Update the environment variable handling in `AppConfig::from_env()` to include OIDC configurations. - -### 2. Create OIDC Service Implementation - -Add a new file `src/domain/services/oidc_service.rs`: - -```rust -use openid::{Client, Discovered, DiscoveredClient, Options, Token, StandardClaims}; -use std::sync::Arc; -use reqwest::Client as HttpClient; -use async_trait::async_trait; -use uuid::Uuid; - -use crate::common::config::OidcProviderConfig; -use crate::domain::entities::user::{User, UserRole}; -use crate::domain::repositories::user_repository::UserRepository; -use crate::common::errors::{DomainError, ErrorKind}; - -pub struct OidcService { - providers: Vec, - user_repository: Arc, -} - -struct OidcProvider { - config: OidcProviderConfig, - client: DiscoveredClient, -} - -impl OidcService { - pub async fn new( - configs: Vec, - user_repository: Arc, - ) -> Result { - let http_client = HttpClient::new(); - let mut providers = Vec::new(); - - for config in configs { - let client = openid::Client::discover( - http_client.clone(), - &config.client_id, - &config.client_secret, - &config.redirect_uri, - &config.discovery_url, - ) - .await - .map_err(|e| DomainError::new( - ErrorKind::InternalError, - "OIDC", - format!("Failed to discover OIDC provider {}: {}", config.name, e) - ))?; - - providers.push(OidcProvider { - config: config.clone(), - client, - }); - } - - Ok(Self { - providers, - user_repository, - }) - } - - pub fn get_provider(&self, provider_name: &str) -> Option<&OidcProvider> { - self.providers.iter().find(|p| p.config.name == provider_name) - } - - pub fn get_providers_info(&self) -> Vec { - self.providers.iter().map(|p| OidcProviderInfo { - name: p.config.name.clone(), - display_name: p.config.name.clone(), - }).collect() - } - - pub fn generate_authorization_url(&self, provider_name: &str, state: &str) -> Result { - let provider = self.get_provider(provider_name).ok_or_else(|| DomainError::new( - ErrorKind::NotFound, - "OIDC", - format!("Provider {} not found", provider_name) - ))?; - - let mut options = Options::default(); - options.scope = Some(provider.config.scopes.join(" ")); - - let auth_url = provider.client.auth_url(&options, Some(state)); - Ok(auth_url.to_string()) - } - - pub async fn process_callback( - &self, - provider_name: &str, - code: &str, - state: &str - ) -> Result<(User, Token), DomainError> { - let provider = self.get_provider(provider_name).ok_or_else(|| DomainError::new( - ErrorKind::NotFound, - "OIDC", - format!("Provider {} not found", provider_name) - ))?; - - // Exchange code for token - let token = provider.client.request_token(code).await.map_err(|e| DomainError::new( - ErrorKind::AccessDenied, - "OIDC", - format!("Failed to exchange code for token: {}", e) - ))?; - - // Extract user information from claims - let claims = token.id_token.payload().clone(); - - // Get user ID from configured attribute - let user_id_attr = &provider.config.user_id_attribute; - let external_user_id = match user_id_attr.as_str() { - "sub" => claims.sub.clone(), - "email" => claims.email.clone().unwrap_or_default(), - // Add other standard claims as needed - _ => claims.additional_claims.get(user_id_attr) - .and_then(|v| v.as_str().map(|s| s.to_string())) - .unwrap_or_default(), - }; - - if external_user_id.is_empty() { - return Err(DomainError::new( - ErrorKind::InvalidInput, - "OIDC", - format!("Could not extract user ID from claim '{}'", user_id_attr) - )); - } - - // Check if user exists with this external ID - let mapped_user_id = format!("{}:{}", provider_name, external_user_id); - - let user = match self.user_repository.get_user_by_external_id(&mapped_user_id).await { - Ok(existing_user) => existing_user, - Err(_) => { - // User doesn't exist, create if allowed - if !provider.config.auto_create_users { - return Err(DomainError::new( - ErrorKind::AccessDenied, - "OIDC", - "User does not exist and auto-creation is disabled" - )); - } - - // Get user information from claims - let email = claims.email.clone().unwrap_or_else(|| - format!("{}@oidc.oxicloud.local", Uuid::new_v4()) - ); - - let username = claims.preferred_username.clone() - .or_else(|| claims.email.clone()) - .unwrap_or_else(|| format!("user_{}", Uuid::new_v4())); - - // Create the user - let role = match provider.config.default_role.as_str() { - "admin" => UserRole::Admin, - _ => UserRole::User, - }; - - // Default quota - let quota = 1024 * 1024 * 1024; // 1GB - - let mut new_user = User::new( - username, - email, - Uuid::new_v4().to_string(), // Random password, not used for OIDC - role, - quota, - )?; - - // Set external ID - new_user.set_external_id(Some(mapped_user_id)); - - // Save user - self.user_repository.create_user(new_user).await? - } - }; - - Ok((user, token)) - } -} - -#[derive(Clone, Debug, serde::Serialize)] -pub struct OidcProviderInfo { - pub name: String, - pub display_name: String, -} -``` - -### 3. Update User Entity - -Modify `src/domain/entities/user.rs` to support external IDs for OIDC users: - -```rust -#[derive(Debug, Clone)] -pub struct User { - // Existing fields... - external_id: Option, // For OIDC users: "provider:external_id" -} - -impl User { - // Existing methods... - - pub fn external_id(&self) -> Option<&str> { - self.external_id.as_deref() - } - - pub fn set_external_id(&mut self, external_id: Option) { - self.external_id = external_id; - } - - pub fn is_oidc_user(&self) -> bool { - self.external_id.is_some() - } -} -``` - -### 4. Update Database Schema - -Add a new column to the users table in `db/schema.sql`: - -```sql -ALTER TABLE auth.users ADD COLUMN IF NOT EXISTS external_id VARCHAR(255) UNIQUE; -``` - -### 5. Update the Auth Application Service - -Modify `src/application/services/auth_application_service.rs` to add OIDC methods: - -```rust -use crate::domain::services::oidc_service::{OidcService, OidcProviderInfo}; -use crate::application::dtos::user_dto::{OidcAuthUrlDto, OidcCallbackDto, OidcProviderDto}; - -impl AuthApplicationService { - // Add OIDC service - pub fn with_oidc_service(mut self, oidc_service: Arc) -> Self { - self.oidc_service = Some(oidc_service); - self - } - - // Get available OIDC providers - pub fn get_oidc_providers(&self) -> Result, DomainError> { - let oidc_service = self.oidc_service.as_ref() - .ok_or_else(|| DomainError::new( - ErrorKind::UnsupportedOperation, - "Auth", - "OIDC is not configured" - ))?; - - let providers = oidc_service.get_providers_info(); - Ok(providers.into_iter().map(OidcProviderDto::from).collect()) - } - - // Generate authorization URL - pub fn generate_oidc_auth_url(&self, dto: OidcAuthUrlDto) -> Result { - let oidc_service = self.oidc_service.as_ref() - .ok_or_else(|| DomainError::new( - ErrorKind::UnsupportedOperation, - "Auth", - "OIDC is not configured" - ))?; - - oidc_service.generate_authorization_url(&dto.provider, &dto.state) - } - - // Process OIDC callback - pub async fn process_oidc_callback(&self, dto: OidcCallbackDto) -> Result { - let oidc_service = self.oidc_service.as_ref() - .ok_or_else(|| DomainError::new( - ErrorKind::UnsupportedOperation, - "Auth", - "OIDC is not configured" - ))?; - - let (user, _token) = oidc_service.process_callback( - &dto.provider, - &dto.code, - &dto.state - ).await?; - - // Generate access token - let access_token = self.auth_service.generate_access_token(&user) - .map_err(DomainError::from)?; - - // Generate refresh token - let refresh_token = self.auth_service.generate_refresh_token(); - - // Create session - let session = Session::new( - user.id().to_string(), - refresh_token.clone(), - None, - None, - self.auth_service.refresh_token_expiry_days(), - ); - - self.session_storage.create_session(session).await?; - - // Return auth response - Ok(AuthResponseDto { - user: UserDto::from(user), - access_token, - refresh_token, - token_type: "Bearer".to_string(), - expires_in: self.auth_service.refresh_token_expiry_secs(), - }) - } -} -``` - -### 6. Add Auth Handler Routes for OIDC - -Update `src/interfaces/api/handlers/auth_handler.rs`: - -```rust -pub fn auth_routes() -> Router> { - Router::new() - .route("/register", post(register)) - .route("/login", post(login)) - .route("/refresh", post(refresh_token)) - .route("/me", get(get_current_user)) - .route("/change-password", put(change_password)) - .route("/logout", post(logout)) - // Add OIDC routes - .route("/oidc/providers", get(get_oidc_providers)) - .route("/oidc/auth", post(generate_oidc_auth_url)) - .route("/oidc/callback", post(process_oidc_callback)) -} - -// Get available OIDC providers -async fn get_oidc_providers( - State(state): State>, -) -> Result { - let auth_service = state.auth_service.as_ref() - .ok_or_else(|| AppError::internal_error("Servicio de autenticación no configurado"))?; - - match auth_service.auth_application_service.get_oidc_providers() { - Ok(providers) => Ok((StatusCode::OK, Json(providers))), - Err(err) => Err(err.into()), - } -} - -// Generate OIDC authorization URL -async fn generate_oidc_auth_url( - State(state): State>, - Json(dto): Json, -) -> Result { - let auth_service = state.auth_service.as_ref() - .ok_or_else(|| AppError::internal_error("Servicio de autenticación no configurado"))?; - - match auth_service.auth_application_service.generate_oidc_auth_url(dto) { - Ok(url) => Ok((StatusCode::OK, Json(json!({ "url": url })))), - Err(err) => Err(err.into()), - } -} - -// Process OIDC callback -async fn process_oidc_callback( - State(state): State>, - Json(dto): Json, -) -> Result { - let auth_service = state.auth_service.as_ref() - .ok_or_else(|| AppError::internal_error("Servicio de autenticación no configurado"))?; - - match auth_service.auth_application_service.process_oidc_callback(dto).await { - Ok(auth_response) => Ok((StatusCode::OK, Json(auth_response))), - Err(err) => Err(err.into()), - } -} -``` - -### 7. Update DTOs for OIDC - -Create new DTOs in `src/application/dtos/user_dto.rs`: - -```rust -use crate::domain::services::oidc_service::OidcProviderInfo; - -#[derive(Debug, Clone, Serialize)] -pub struct OidcProviderDto { - pub name: String, - pub display_name: String, -} - -impl From for OidcProviderDto { - fn from(info: OidcProviderInfo) -> Self { - Self { - name: info.name, - display_name: info.display_name, - } - } -} - -#[derive(Debug, Clone, Deserialize)] -pub struct OidcAuthUrlDto { - pub provider: String, - pub state: String, - pub redirect_uri: Option, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct OidcCallbackDto { - pub provider: String, - pub code: String, - pub state: String, -} -``` - -### 8. Add Frontend Integration - -Create a new JavaScript file `static/js/oidcAuth.js`: - -```javascript -// OIDC Authentication Module -const oidcAuth = { - // Get available OIDC providers - async getProviders() { - try { - const response = await fetch('/api/auth/oidc/providers'); - if (!response.ok) { - throw new Error(`Failed to get OIDC providers: ${response.statusText}`); - } - return await response.json(); - } catch (error) { - console.error('Error fetching OIDC providers:', error); - return []; - } - }, - - // Generate random state for CSRF protection - generateState() { - const array = new Uint8Array(16); - window.crypto.getRandomValues(array); - return Array.from(array, byte => byte.toString(16).padStart(2, '0')).join(''); - }, - - // Start OIDC authentication flow - async startAuth(providerName) { - try { - // Generate and store state - const state = this.generateState(); - localStorage.setItem('oidc_state', state); - - // Get authorization URL - const response = await fetch('/api/auth/oidc/auth', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - provider: providerName, - state: state, - }), - }); - - if (!response.ok) { - throw new Error(`Failed to get auth URL: ${response.statusText}`); - } - - const data = await response.json(); - - // Redirect to authorization URL - window.location.href = data.url; - } catch (error) { - console.error('Error starting OIDC auth:', error); - alert('Failed to start authentication. Please try again.'); - } - }, - - // Handle OIDC callback - async handleCallback() { - // Parse URL parameters - const urlParams = new URLSearchParams(window.location.search); - const code = urlParams.get('code'); - const state = urlParams.get('state'); - const error = urlParams.get('error'); - - // Check for errors - if (error) { - console.error('OIDC authentication error:', error); - alert(`Authentication failed: ${error}`); - window.location.href = '/login.html'; - return; - } - - // Verify code and state - if (!code || !state) { - console.error('Missing code or state in callback'); - alert('Authentication failed: Invalid response'); - window.location.href = '/login.html'; - return; - } - - // Verify state matches - const savedState = localStorage.getItem('oidc_state'); - if (state !== savedState) { - console.error('State mismatch - potential CSRF attack'); - alert('Authentication failed: Invalid state'); - window.location.href = '/login.html'; - return; - } - - // Clear stored state - localStorage.removeItem('oidc_state'); - - try { - // Extract provider from URL path or from saved data - const pathParts = window.location.pathname.split('/'); - const provider = localStorage.getItem('oidc_provider') || - (pathParts.length > 2 ? pathParts[2] : 'default'); - - // Process callback - const response = await fetch('/api/auth/oidc/callback', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - provider: provider, - code: code, - state: state, - }), - }); - - if (!response.ok) { - throw new Error(`Failed to process callback: ${response.statusText}`); - } - - const authData = await response.json(); - - // Store auth data and redirect to dashboard - localStorage.setItem('auth_token', authData.access_token); - localStorage.setItem('refresh_token', authData.refresh_token); - localStorage.setItem('user', JSON.stringify(authData.user)); - - window.location.href = '/index.html'; - } catch (error) { - console.error('Error handling OIDC callback:', error); - alert('Failed to complete authentication. Please try again.'); - window.location.href = '/login.html'; - } - } -}; - -// Check if current page is callback page -if (window.location.pathname.includes('/oidc/callback')) { - document.addEventListener('DOMContentLoaded', () => { - oidcAuth.handleCallback(); - }); -} -``` - -### 9. Update Login Page - -Add OIDC login buttons to `static/login.html`: - -```html - - - - -``` - -## Configuration Example - -Here's how to configure OxiCloud to use OIDC with KeyCloak: - -```yaml -# docker-compose.yml -version: '3' -services: - oxicloud: - image: oxicloud:latest - environment: - OXICLOUD_ENABLE_OIDC: "true" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_NAME: "KeyCloak" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_CLIENT_ID: "oxicloud" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_CLIENT_SECRET: "your-client-secret" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_DISCOVERY_URL: "https://keycloak.example.com/realms/your-realm/.well-known/openid-configuration" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_REDIRECT_URI: "https://oxicloud.example.com/oidc/callback/keycloak" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_SCOPES: "openid profile email" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_USER_ID_ATTRIBUTE: "sub" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_DEFAULT_ROLE: "user" - OXICLOUD_OIDC_PROVIDER_KEYCLOAK_AUTO_CREATE_USERS: "true" - ports: - - "8085:8085" - volumes: - - ./storage:/app/storage -``` - -## Additional Considerations - -1. **Security**: OIDC connections should always use HTTPS. Ensure proper TLS configuration. - -2. **User mapping**: Consider how user attributes from OIDC map to your application (roles, groups, etc.). - -3. **Multiple providers**: The design supports multiple OIDC providers simultaneously. - -4. **Session management**: Implement proper session handling for OIDC users. - -5. **Access control**: Review how OIDC integration affects your application's permission model. - -6. **Testing**: Create separate test IdP configurations for development and testing. \ No newline at end of file diff --git a/doc/POSTGRESQL-BEST-PRACTICES.md b/doc/POSTGRESQL-BEST-PRACTICES.md deleted file mode 100644 index 2008fe0a..00000000 --- a/doc/POSTGRESQL-BEST-PRACTICES.md +++ /dev/null @@ -1,250 +0,0 @@ -# Mejores Prácticas para PostgreSQL en OxiCloud - -Este documento describe las mejores prácticas para el uso de PostgreSQL en OxiCloud, siguiendo recomendaciones oficiales y la guía ["Don't Do This"](https://wiki.postgresql.org/wiki/Don%27t_Do_This) de PostgreSQL. - -## Diseño de Esquema - -### Tipos de Datos - -#### Uso de TEXT en lugar de VARCHAR(n) - -OxiCloud utiliza el tipo `TEXT` en lugar de `VARCHAR(n)` con límites arbitrarios para campos de texto: - -```sql --- Recomendado ✅ -username TEXT NOT NULL UNIQUE - --- Evitar ❌ -username VARCHAR(32) NOT NULL UNIQUE -``` - -**Razones:** -- `TEXT` y `VARCHAR` tienen el mismo rendimiento y ocupan el mismo espacio. -- `VARCHAR(n)` impone un límite arbitrario que puede causar errores inesperados. -- PostgreSQL optimiza internamente ambos tipos de manera idéntica. - -#### Uso de TIMESTAMPTZ para Fechas y Horas - -OxiCloud utiliza `TIMESTAMP WITH TIME ZONE` (o `TIMESTAMPTZ`) para todos los campos de fecha/hora: - -```sql --- Recomendado ✅ -created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP - --- Evitar ❌ -created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -``` - -**Razones:** -- `TIMESTAMPTZ` almacena un punto en el tiempo unívoco. -- Gestiona correctamente las zonas horarias y cambios de horario de verano. -- Evita problemas de ambigüedad al trabajar con diferentes husos horarios. - -#### Evitar CHAR(n) - -OxiCloud no utiliza el tipo `CHAR(n)` en ningún caso: - -```sql --- Recomendado ✅ -country_code TEXT NOT NULL CHECK (length(country_code) = 2) - --- Evitar ❌ -country_code CHAR(2) NOT NULL -``` - -**Razones:** -- `CHAR(n)` rellena con espacios hasta la longitud declarada. -- Este comportamiento puede causar problemas sutiles en comparaciones. -- Para valores de longitud fija, es mejor usar `TEXT` con una restricción CHECK. - -#### Usar SERIAL con Precaución - -OxiCloud utiliza `SERIAL` solo en casos específicos, prefiriendo `IDENTITY` cuando es posible: - -```sql --- Recomendado para PostgreSQL 10+ ✅ -id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY - --- Alternativa aceptable para compatibilidad ✅ -id SERIAL PRIMARY KEY -``` - -**Razones:** -- `SERIAL` tiene comportamientos extraños con gestión de dependencias y permisos. -- Las columnas `IDENTITY` (PostgreSQL 10+) ofrecen mejor integración con el sistema. - -### Índices y Restricciones - -#### Nombrado Consistente de Índices - -OxiCloud sigue una convención de nomenclatura para índices: - -```sql --- Índice en una columna -CREATE INDEX IF NOT EXISTS idx_table_column ON schema.table(column); - --- Índice en múltiples columnas -CREATE INDEX IF NOT EXISTS idx_table_col1_col2 ON schema.table(col1, col2); -``` - -#### Uso de Restricciones Explícitas - -OxiCloud define restricciones explícitas en lugar de depender de convenciones implícitas: - -```sql --- Restricción de unicidad -UNIQUE(user_id, item_id, item_type) - --- Restricción de comprobación -CHECK (storage_quota_bytes >= 0) -``` - -## Consultas SQL - -### Evitar NOT IN con Subconsultas - -OxiCloud evita el uso de `NOT IN` con subconsultas: - -```sql --- Recomendado ✅ -SELECT * FROM files -WHERE NOT EXISTS (SELECT 1 FROM deleted_files WHERE deleted_files.id = files.id); - --- Evitar ❌ -SELECT * FROM files -WHERE id NOT IN (SELECT id FROM deleted_files); -``` - -**Razones:** -- `NOT IN` se comporta de manera inesperada con valores NULL. -- `NOT EXISTS` es más eficiente y predecible. - -### Usar BETWEEN con Precaución - -OxiCloud evita `BETWEEN` para rangos de fechas, prefiriendo comparaciones explícitas: - -```sql --- Recomendado ✅ -WHERE timestamp_col >= '2025-01-01' AND timestamp_col < '2025-01-02' - --- Evitar ❌ -WHERE timestamp_col BETWEEN '2025-01-01' AND '2025-01-02' -``` - -**Razones:** -- `BETWEEN` incluye ambos extremos, lo que puede ser problemático para rangos de tiempo. -- Usar `>=` y `<` es más claro para expresar rangos de tiempo. - -## Transacciones - -### Uso Explícito de Transacciones - -OxiCloud implementa transacciones explícitas para operaciones que deben ser atómicas: - -```rust -// Ejemplo de transacción explícita -let mut tx = pool.begin().await?; - -// Operaciones dentro de la transacción -sqlx::query("INSERT INTO users (id, username) VALUES ($1, $2)") - .bind(id) - .bind(username) - .execute(&mut *tx) - .await?; - -sqlx::query("INSERT INTO profiles (user_id, display_name) VALUES ($1, $2)") - .bind(id) - .bind(display_name) - .execute(&mut *tx) - .await?; - -// Confirmar la transacción -tx.commit().await?; -``` - -### Manejo de Errores en Transacciones - -Las transacciones incluyen manejo adecuado de errores con rollback automático: - -```rust -let result = sqlx::Transaction::begin(&pool).await.and_then(|mut tx| async move { - // Operaciones dentro de la transacción - let result1 = operation1(&mut tx).await?; - let result2 = operation2(&mut tx).await?; - - // Confirmar la transacción si todo fue exitoso - tx.commit().await?; - - Ok((result1, result2)) -}).await; - -// Si ocurre un error, la transacción se revierte automáticamente -if let Err(e) = &result { - log::error!("Error en la transacción: {}", e); -} -``` - -## Migraciones y Gestión de Esquema - -### Separación del Esquema del Código - -OxiCloud separa la definición del esquema del código de la aplicación: - -``` -OxiCloud/ -├── migrations/ # Archivos SQL de migración -├── src/ - ├── bin/migrate.rs # Herramienta de migración - ├── common/db.rs # Solo conecta a la BD, no crea esquema -``` - -### Uso de Migraciones Versionadas - -Las migraciones siguen un formato versionado y se aplican secuencialmente: - -``` -20250408000000_initial_schema.sql -20250408000001_default_users.sql -``` - -## Seguridad - -### Uso de Consultas Parametrizadas - -OxiCloud utiliza consultas parametrizadas para todas las operaciones SQL: - -```rust -// Recomendado ✅ -sqlx::query("SELECT * FROM users WHERE username = $1") - .bind(username) - .fetch_one(&pool) - .await?; - -// Evitar ❌ -sqlx::query(&format!("SELECT * FROM users WHERE username = '{}'", username)) - .fetch_one(&pool) - .await?; -``` - -**Razones:** -- Previene ataques de inyección SQL. -- Permite la reutilización de planes de consulta. -- Mejora el rendimiento general. - -### Configuración de Autenticación Segura - -OxiCloud evita el uso de autenticación `trust` para conexiones TCP/IP: - -``` -# pg_hba.conf recomendado ✅ -hostssl all all 0.0.0.0/0 scram-sha-256 - -# Evitar ❌ -host all all 0.0.0.0/0 trust -``` - -## Recursos Adicionales - -- [Wiki PostgreSQL - Don't Do This](https://wiki.postgresql.org/wiki/Don%27t_Do_This) -- [Documentación oficial de PostgreSQL](https://www.postgresql.org/docs/) -- [Guía de migraciones de OxiCloud](DATABASE-MIGRATIONS.md) \ No newline at end of file diff --git a/doc/SHARE-INTEGRATION.md b/doc/SHARE-INTEGRATION.md deleted file mode 100644 index 5a066cee..00000000 --- a/doc/SHARE-INTEGRATION.md +++ /dev/null @@ -1,452 +0,0 @@ -# Documentación Técnica: Sistema de Compartición en OxiCloud - -## Resumen Ejecutivo - -La funcionalidad de compartición de archivos y carpetas en OxiCloud permite a los usuarios generar enlaces de acceso para compartir sus recursos con otros usuarios, incluso aquellos sin cuenta en el sistema. La implementación sigue los principios de Arquitectura Hexagonal, manteniendo una clara separación entre dominio, aplicación e infraestructura. - -## Arquitectura y Componentes - -### 1. Entidades de Dominio - -**Share (src/domain/entities/share.rs)** - -La entidad principal que representa un recurso compartido: - -```rust -pub struct Share { - pub id: String, // Identificador único del enlace - pub item_id: String, // ID del archivo o carpeta compartido - pub item_type: ShareItemType, // Tipo (File o Folder) - pub token: String, // Token único para acceso público - pub password_hash: Option, // Hash de contraseña opcional - pub expires_at: Option, // Timestamp de expiración opcional - pub permissions: SharePermissions, // Permisos otorgados - pub created_at: u64, // Timestamp de creación - pub created_by: String, // ID del usuario creador - pub access_count: u64, // Contador de accesos -} - -pub enum ShareItemType { - File, - Folder -} - -pub struct SharePermissions { - pub read: bool, // Permiso de lectura - pub write: bool, // Permiso de escritura - pub reshare: bool, // Permiso para volver a compartir -} -``` - -La entidad implementa métodos para: -- Validar la expiración del enlace -- Verificar contraseñas -- Incrementar el contador de accesos -- Modificar propiedades (permisos, contraseña, expiración) - -### 2. Interfaces del Repositorio - -**ShareRepository (src/domain/repositories/share_repository.rs)** - -Define las operaciones de persistencia para los enlaces compartidos: - -```rust -#[async_trait] -pub trait ShareRepository: Send + Sync + 'static { - async fn save(&self, share: &Share) -> Result; - async fn find_by_id(&self, id: &str) -> Result; - async fn find_by_token(&self, token: &str) -> Result; - async fn find_by_item(&self, item_id: &str, item_type: &ShareItemType) -> Result, ShareRepositoryError>; - async fn update(&self, share: &Share) -> Result; - async fn delete(&self, id: &str) -> Result<(), ShareRepositoryError>; - async fn find_by_user(&self, user_id: &str, offset: usize, limit: usize) -> Result<(Vec, usize), ShareRepositoryError>; -} -``` - -### 3. Puertos de Aplicación - -**ShareUseCase y ShareStoragePort (src/application/ports/share_ports.rs)** - -Define las interfaces para la capa de aplicación: - -```rust -#[async_trait] -pub trait ShareUseCase: Send + Sync + 'static { - // Crear un nuevo enlace compartido - async fn create_shared_link(&self, user_id: &str, dto: CreateShareDto) -> Result; - - // Obtener un enlace compartido por ID - async fn get_shared_link(&self, id: &str) -> Result; - - // Obtener un enlace compartido por token - async fn get_shared_link_by_token(&self, token: &str) -> Result; - - // Obtener todos los enlaces compartidos para un elemento - async fn get_shared_links_for_item(&self, item_id: &str, item_type: &ShareItemType) -> Result, DomainError>; - - // Actualizar un enlace compartido - async fn update_shared_link(&self, id: &str, dto: UpdateShareDto) -> Result; - - // Eliminar un enlace compartido - async fn delete_shared_link(&self, id: &str) -> Result<(), DomainError>; - - // Obtener enlaces compartidos de un usuario con paginación - async fn get_user_shared_links(&self, user_id: &str, page: usize, per_page: usize) -> Result, DomainError>; - - // Verificar la contraseña de un enlace protegido - async fn verify_shared_link_password(&self, token: &str, password: &str) -> Result; - - // Registrar un acceso a un enlace compartido - async fn register_shared_link_access(&self, token: &str) -> Result<(), DomainError>; -} - -#[async_trait] -pub trait ShareStoragePort: Send + Sync + 'static { - // Métodos para interactuar con el almacenamiento - async fn save_share(&self, share: &Share) -> Result; - async fn find_share_by_id(&self, id: &str) -> Result; - // ... otros métodos -} -``` - -### 4. Objetos de Transferencia de Datos (DTOs) - -**DTOs (src/application/dtos/share_dto.rs)** - -```rust -// DTO para la creación de enlaces compartidos -pub struct CreateShareDto { - pub item_id: String, - pub item_type: String, - pub password: Option, - pub expires_at: Option, - pub permissions: Option, -} - -// DTO para actualizar enlaces compartidos -pub struct UpdateShareDto { - pub password: Option, - pub expires_at: Option, - pub permissions: Option, -} - -// DTO de permisos -pub struct SharePermissionsDto { - pub read: bool, - pub write: bool, - pub reshare: bool, -} - -// DTO para respuestas -pub struct ShareDto { - pub id: String, - pub item_id: String, - pub item_type: String, - pub token: String, - pub url: String, - pub password_protected: bool, - pub expires_at: Option, - pub permissions: SharePermissionsDto, - pub created_at: u64, - pub created_by: String, - pub access_count: u64, -} -``` - -### 5. Servicios de Aplicación - -**ShareService (src/application/services/share_service.rs)** - -Implementa la lógica de negocio para la compartición de archivos: - -```rust -pub struct ShareService { - config: Arc, - share_repository: Arc, - file_repository: Arc, - folder_repository: Arc, -} -``` - -El servicio implementa: -- Validación de elementos compartidos -- Gestión de permisos -- Generación de enlaces y tokens únicos -- Protección con contraseña -- Control de expiración -- Seguimiento de accesos - -### 6. Implementación de Infraestructura - -**ShareFsRepository (src/infrastructure/repositories/share_fs_repository.rs)** - -Implementa la persistencia de enlaces compartidos usando el sistema de archivos: - -```rust -pub struct ShareFsRepository { - config: Arc, -} - -// Almacena los enlaces en un archivo JSON -struct ShareRecord { - id: String, - item_id: String, - item_type: String, - token: String, - password_hash: Option, - expires_at: Option, - permissions_read: bool, - permissions_write: bool, - permissions_reshare: bool, - created_at: u64, - created_by: String, - access_count: u64, -} -``` - -La implementación: -- Guarda los enlaces compartidos en un archivo JSON -- Gestiona consultas y actualizaciones -- Proporciona búsqueda por ID, token o usuario -- Implementa paginación - -### 7. Controladores API y Rutas - -**Manejadores (src/interfaces/api/handlers/share_handler.rs)** - -```rust -// Crear un nuevo enlace compartido -pub async fn create_shared_link( - State(share_use_case): State>, - Json(dto): Json, -) -> impl IntoResponse { - // Implementación... -} - -// Obtener un enlace compartido -pub async fn get_shared_link( - State(share_use_case): State>, - Path(id): Path, -) -> impl IntoResponse { - // Implementación... -} - -// Obtener enlaces compartidos de un usuario -pub async fn get_user_shares( - State(share_use_case): State>, - Query(query): Query, -) -> impl IntoResponse { - // Implementación... -} - -// Actualizar un enlace compartido -pub async fn update_shared_link( - State(share_use_case): State>, - Path(id): Path, - Json(dto): Json, -) -> impl IntoResponse { - // Implementación... -} - -// Eliminar un enlace compartido -pub async fn delete_shared_link( - State(share_use_case): State>, - Path(id): Path, -) -> impl IntoResponse { - // Implementación... -} - -// Acceder a un elemento compartido a través de su token -pub async fn access_shared_item( - State(share_use_case): State>, - Path(token): Path, -) -> impl IntoResponse { - // Implementación... -} - -// Verificar la contraseña de un elemento compartido protegido -pub async fn verify_shared_item_password( - State(share_use_case): State>, - Path(token): Path, - Json(req): Json, -) -> impl IntoResponse { - // Implementación... -} -``` - -**Rutas (src/interfaces/api/routes.rs)** - -```rust -// Rutas privadas para la gestión de enlaces compartidos -let share_router = Router::new() - .route("/", post(share_handler::create_shared_link)) - .route("/", get(share_handler::get_user_shares)) - .route("/{id}", get(share_handler::get_shared_link)) - .route("/{id}", put(share_handler::update_shared_link)) - .route("/{id}", delete(share_handler::delete_shared_link)); - -// Rutas públicas para acceder a los enlaces compartidos -let public_share_router = Router::new() - .route("/{token}", get(share_handler::access_shared_item)) - .route("/{token}/verify", post(share_handler::verify_shared_item_password)); - -// Configuración en el router principal -router - .nest("/shares", share_router) // API privada: /api/shares/... - .nest("/s", public_share_router); // API pública: /api/s/... -``` - -### 8. Integración en el Sistema - -La funcionalidad de compartición está integrada con: - -1. **Configuración del sistema**: Se puede habilitar/deshabilitar mediante la configuración: -```rust -pub struct FeaturesConfig { - // ... - pub enable_file_sharing: bool, - // ... -} -``` - -2. **Inyección de dependencias**: El servicio se instancia en main.rs y se inyecta en las rutas: -```rust -// Inicializar el repositorio y servicio de compartición -let share_service: Option> = if config.features.enable_file_sharing { - let share_repository = Arc::new(ShareFsRepository::new(Arc::new(config.clone()))); - let share_service = Arc::new(ShareService::new( - Arc::new(config.clone()), - share_repository, - file_repository.clone(), - folder_repository.clone() - )); - Some(share_service) -} else { - None -}; - -// Agregar a los servicios de aplicación -let application_services = ApplicationServices { - // ... - share_service: share_service.clone(), -}; - -// Configurar las rutas -let api_routes = create_api_routes( - folder_service, - file_service, - Some(i18n_service), - trash_service, - search_service, - share_service -); -``` - -## Flujos de Trabajo - -### 1. Creación de un Enlace Compartido - -1. El usuario selecciona un archivo o carpeta para compartir -2. El frontend envía una petición POST a `/api/shares/` con los detalles (contraseña opcional, expiración, permisos) -3. `ShareService.create_shared_link()` valida los datos y verifica que el elemento existe -4. Se genera un token único y una URL de acceso -5. El enlace se guarda en el repositorio -6. Se devuelve la URL y detalles del enlace compartido - -### 2. Acceso a un Recurso Compartido - -1. El usuario recibe y accede a un enlace compartido (ej: `http://oxicloud.example/api/s/{token}`) -2. El backend verifica: - - Que el token es válido - - Que el enlace no ha expirado - - Si está protegido por contraseña -3. Si requiere contraseña, se solicita al usuario -4. El contador de accesos se incrementa -5. Se devuelven los metadatos del recurso compartido para mostrar en la interfaz -6. El usuario puede acceder al contenido según los permisos otorgados - -## Seguridad - -### Protección por Contraseña - -- Las contraseñas se almacenan como hashes en lugar de texto plano -- El sistema utiliza un hash simple por ahora, pero está diseñado para implementar algoritmos más seguros como bcrypt - -### Control de Expiración - -- Los enlaces pueden configurarse para expirar automáticamente -- El sistema verifica la expiración antes de permitir accesos - -### Control de Permisos - -- El sistema implementa un modelo de permisos granular (lectura, escritura, recompartir) -- Cada operación valida los permisos antes de permitir la acción - -## Manejo de Errores - -El sistema implementa manejo de errores consistente: - -```rust -pub enum ShareServiceError { - #[error("Share not found: {0}")] - NotFound(String), - - #[error("Item not found: {0}")] - ItemNotFound(String), - - #[error("Access denied: {0}")] - AccessDenied(String), - - #[error("Invalid password: {0}")] - InvalidPassword(String), - - #[error("Share expired")] - Expired, - - #[error("Repository error: {0}")] - Repository(String), - - #[error("Invalid item type: {0}")] - InvalidItemType(String), - - #[error("Validation error: {0}")] - Validation(String), -} -``` - -Estos errores se mapean a códigos HTTP apropiados en los controladores: -- `NotFound` → HTTP 404 Not Found -- `PasswordRequired` → HTTP 401 Unauthorized + metadata -- `Expired` → HTTP 410 Gone -- `AccessDenied` → HTTP 403 Forbidden -- `ValidationError` → HTTP 400 Bad Request - -## Extensibilidad y Futuras Mejoras - -La arquitectura está diseñada para permitir futuras mejoras: - -1. **Notificaciones**: Integración con un sistema de notificaciones para alertar a los usuarios cuando se accede a sus recursos compartidos. - -2. **Registro de Actividad**: Implementación de un registro detallado de actividades para auditar quién accedió a qué recursos y cuándo. - -3. **Límites de Uso**: Establecer límites de uso (número máximo de accesos, ancho de banda) para enlaces compartidos. - -4. **Estadísticas Avanzadas**: Proporcionar métricas detalladas sobre el uso de recursos compartidos. - -5. **Persistencia Alternativa**: La arquitectura permite implementar fácilmente alternativas de almacenamiento (base de datos, servicios en la nube) manteniendo la misma interfaz. - -## Estado Actual - -La funcionalidad de compartición está completamente implementada en el backend y lista para integrarse con el frontend. La característica está habilitada por defecto en la configuración actual. - -## Consideraciones Técnicas - -- **Rendimiento**: El sistema utiliza un enfoque de almacenamiento basado en archivos JSON, lo que es adecuado para un volumen moderado de enlaces compartidos. Para una carga mayor, se recomienda migrar a una base de datos. - -- **Escalabilidad**: El diseño permite escalar horizontalmente la funcionalidad implementando repositorios distribuidos o basados en la nube. - -- **Mantenimiento**: La clara separación de responsabilidades facilita el mantenimiento y las pruebas de la funcionalidad. - -## Conclusión - -La implementación del sistema de compartición en OxiCloud sigue los principios de la Arquitectura Hexagonal, permitiendo una clara separación entre el dominio, la aplicación y la infraestructura. Esto facilita la evolución del sistema y la adaptación a requisitos cambiantes. La funcionalidad proporciona todas las características básicas esperadas de un sistema de compartición moderno, incluyendo protección por contraseña, expiración y permisos granulares. \ No newline at end of file diff --git a/doc/TRASH-FEATURE-SUMMARY.md b/doc/TRASH-FEATURE-SUMMARY.md deleted file mode 100644 index d1b51c44..00000000 --- a/doc/TRASH-FEATURE-SUMMARY.md +++ /dev/null @@ -1,89 +0,0 @@ -# Trash Feature Implementation Summary - -This document summarizes the implementation of the trash/recycle bin feature in OxiCloud. - -## Architecture Overview - -The trash feature is implemented following the hexagonal architecture (clean architecture) principles of OxiCloud: - -1. **Domain Layer** (`/src/domain/`): - - Entities: `TrashedItem` representing files and folders in the trash bin - - Repository interfaces: `TrashRepository` defining operations for trash management - -2. **Application Layer** (`/src/application/`): - - DTOs: `TrashedItemDto` for data transfer between layers - - Ports: `TrashUseCase` defining the operations available to clients - - Services: `TrashService` implementing the trash use cases - -3. **Infrastructure Layer** (`/src/infrastructure/`): - - Repositories: `TrashFsRepository` for filesystem-based trash storage - - Extensions to existing repositories: `FileRepositoryTrash` and `FolderRepositoryTrash` - - Services: `TrashCleanupService` for automatic cleanup of expired trash items - -4. **Interface Layer** (`/src/interfaces/`): - - API handlers: `trash_handler.rs` providing HTTP endpoints for trash operations - - Routes: Updated `routes.rs` to include trash-related endpoints - -## Key Features - -1. **Soft Deletion**: Moving files and folders to trash instead of immediate permanent deletion -2. **Per-User Trash**: Each user has their own isolated trash bin -3. **Retention Policy**: Items are automatically deleted after a configurable time period -4. **Restoration**: Items can be restored to their original location -5. **Permanent Deletion**: Items can be permanently deleted before the retention period expires -6. **Empty Trash**: All items in the trash can be permanently deleted at once - -## API Endpoints - -The trash feature exposes the following REST API endpoints: - -- `GET /api/trash`: List all items in the user's trash bin -- `DELETE /api/files/trash/:file_id`: Move a file to trash -- `DELETE /api/folders/trash/:folder_id`: Move a folder to trash -- `POST /api/trash/:trash_id/restore`: Restore an item from trash to its original location -- `DELETE /api/trash/:trash_id`: Permanently delete an item from trash -- `DELETE /api/trash/empty`: Empty the entire trash bin - -## Testing - -The trash feature includes comprehensive testing: - -1. **Unit Tests**: Testing the `TrashService` application service - - Test moving files and folders to trash - - Test restoring items from trash - - Test permanent deletion - - Test empty trash operation - -2. **Integration Tests**: Python script to test the API endpoints - - End-to-end testing of all trash operations - - Verification of proper behavior for moving, listing, restoring, and deleting - -3. **Shell Script**: For manual testing and demonstration - - Individual tests for each operation - - Visual feedback of successful operations - -## Configuration - -The trash feature can be configured via environment variables: - -- `TRASH_ENABLED`: Enable/disable the trash feature (default: true) -- `TRASH_RETENTION_DAYS`: Number of days to keep items in trash before automatic deletion (default: 30) - -## Implementation Details - -1. **Physical File Storage**: When items are moved to trash, they are physically moved to a `.trash` directory -2. **Metadata Storage**: Information about trashed items is stored in a separate database table or file -3. **User Isolation**: Trash items are isolated by user ID to prevent access to other users' trash -4. **Automatic Cleanup**: A background job runs periodically to clean up expired trash items -5. **Transaction Safety**: Operations are designed to be atomic and safe, with proper error handling - -## Future Enhancements - -Potential improvements for the trash feature: - -1. **Trash Quotas**: Limit the amount of storage a user can use for trash -2. **Batch Operations**: Add support for trashing, restoring, or deleting multiple items at once -3. **Storage Optimization**: Implement deduplication for trashed items to save storage space -4. **Version Control**: Keep track of file versions when moving to trash -5. **Scheduled Cleanup**: Allow users to configure custom retention periods -6. **Trash Monitoring**: Add metrics and alerts for trash usage and cleanup operations \ No newline at end of file diff --git a/doc/admin-settings.md b/doc/admin-settings.md new file mode 100644 index 00000000..8c8b7e9d --- /dev/null +++ b/doc/admin-settings.md @@ -0,0 +1,160 @@ +# 17 - Admin Settings + +Admin panel API for managing server settings, OIDC configuration, user management, and dashboard stats. All endpoints under `/api/admin` require a valid JWT with `role = "admin"`. + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Domain Port | `SettingsRepository` trait | `src/domain/repositories/settings_repository.rs` | +| Application Service | `AdminSettingsService` | `src/application/services/admin_settings_service.rs` | +| Application DTOs | Settings and user management DTOs | `src/application/dtos/settings_dto.rs` | +| Infrastructure | `SettingsPgRepository` | `src/infrastructure/repositories/pg/settings_pg_repository.rs` | +| Interfaces | `admin_handler` functions | `src/interfaces/api/handlers/admin_handler.rs` | + +## REST API + +All routes under `/api/admin`, require admin JWT. + +### Settings + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/admin/settings/oidc` | `get_oidc_settings` | Get current OIDC configuration | +| `PUT` | `/api/admin/settings/oidc` | `save_oidc_settings` | Update OIDC configuration | +| `POST` | `/api/admin/settings/oidc/test` | `test_oidc_connection` | Test OIDC provider connectivity | +| `GET` | `/api/admin/settings/general` | `get_general_settings` | Get general server settings | + +### Dashboard + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/admin/dashboard` | `get_dashboard_stats` | Server dashboard statistics | + +### User Management + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/admin/users` | `list_users` | List all users (paginated) | +| `GET` | `/api/admin/users/{id}` | `get_user` | Get user details | +| `DELETE` | `/api/admin/users/{id}` | `delete_user` | Delete a user | +| `PUT` | `/api/admin/users/{id}/role` | `update_user_role` | Change user role | +| `PUT` | `/api/admin/users/{id}/active` | `update_user_active` | Activate/deactivate user | +| `PUT` | `/api/admin/users/{id}/quota` | `update_user_quota` | Set storage quota | + +### Safety Guards + +- **Self-deletion blocked** -- admins cannot delete their own account +- **Self-role-change blocked** -- admins cannot change their own role +- **Self-deactivation blocked** -- admins cannot deactivate themselves + +## OIDC Settings Management + +### Get Settings Response + +```json +{ + "enabled": true, + "issuer_url": "https://keycloak.example.com/realms/main", + "client_id": "oxicloud", + "client_secret_set": true, + "scopes": "openid profile email", + "auto_provision": true, + "admin_groups": "oxicloud-admins", + "disable_password_login": false, + "provider_name": "KeyCloak", + "callback_url": "https://oxicloud.example.com/api/auth/oidc/callback", + "env_overrides": ["issuer_url", "client_id"] +} +``` + +The **env_overrides** field lists which settings are overridden by environment variables. Env vars always take priority over DB settings. + +### Save Settings Request + +```json +{ + "enabled": true, + "issuer_url": "https://keycloak.example.com/realms/main", + "client_id": "oxicloud", + "client_secret": "new-secret", + "scopes": "openid profile email", + "auto_provision": true, + "admin_groups": "oxicloud-admins", + "disable_password_login": false, + "provider_name": "KeyCloak" +} +``` + +After saving, the service hot-reloads OIDC via **auth_app_service.reload_oidc()** or **disable_oidc()**. + +### Test OIDC Connection + +```json +// Request +{ "issuer_url": "https://keycloak.example.com/realms/main" } + +// Response +{ + "success": true, + "message": "Successfully connected to OIDC provider", + "issuer": "https://keycloak.example.com/realms/main", + "authorization_endpoint": "https://keycloak.example.com/realms/main/protocol/openid-connect/auth", + "token_endpoint": "https://keycloak.example.com/realms/main/protocol/openid-connect/token", + "userinfo_endpoint": "https://keycloak.example.com/realms/main/protocol/openid-connect/userinfo", + "provider_name_suggestion": "KeyCloak" +} +``` + +## Dashboard Statistics + +```json +{ + "server_version": "0.3.1", + "auth_enabled": true, + "oidc_configured": true, + "quotas_enabled": false, + "total_users": 42, + "active_users": 38, + "admin_users": 2, + "total_quota_bytes": 107374182400, + "total_used_bytes": 53687091200, + "storage_usage_percent": 50.0, + "users_over_80_percent": 5, + "users_over_quota": 1 +} +``` + +## User Management DTOs + +```rust +pub struct UpdateUserRoleDto { pub role: String } // "user" | "admin" +pub struct UpdateUserActiveDto { pub active: bool } +pub struct UpdateUserQuotaDto { pub quota_bytes: i64 } +pub struct ListUsersQueryDto { pub limit: Option, pub offset: Option } +``` + +## Config Priority + +Settings resolve in this order (highest first): + +1. **Environment variables** (`OXICLOUD_OIDC_*`) +2. **Database settings** (`auth.admin_settings` table) +3. **Defaults** + +## Database Schema + +```sql +CREATE TABLE IF NOT EXISTS auth.admin_settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + category TEXT NOT NULL, + is_secret BOOLEAN DEFAULT FALSE, + updated_by VARCHAR(36), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); +``` + +## Frontend + +The admin panel is served from `static/admin.html`. diff --git a/doc/batch-operations.md b/doc/batch-operations.md new file mode 100644 index 00000000..c9df3e11 --- /dev/null +++ b/doc/batch-operations.md @@ -0,0 +1,140 @@ +# 09 - Batch Operations + +Batch endpoints let you perform bulk file and folder operations (move, copy, delete, get, create) in a single request. Operations run concurrently behind a configurable semaphore. Each response includes detailed per-item success/failure info. + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Application Service | **BatchOperationService** | `src/application/services/batch_operations.rs` | +| Interfaces | `batch_handler` functions | `src/interfaces/api/handlers/batch_handler.rs` | + +## REST API + +All routes live under `/api/batch` and require authentication. + +### File Operations + +| Method | Path | Handler | Description | +|---|---|---|---| +| `POST` | `/api/batch/files/move` | `move_files_batch` | Move files to target folder | +| `POST` | `/api/batch/files/copy` | `copy_files_batch` | Copy files to target folder | +| `POST` | `/api/batch/files/delete` | `delete_files_batch` | Delete multiple files | +| `POST` | `/api/batch/files/get` | `get_files_batch` | Get metadata for multiple files | + +### Folder Operations + +| Method | Path | Handler | Description | +|---|---|---|---| +| `POST` | `/api/batch/folders/delete` | `delete_folders_batch` | Delete multiple folders | +| `POST` | `/api/batch/folders/create` | `create_folders_batch` | Create multiple folders | +| `POST` | `/api/batch/folders/get` | `get_folders_batch` | Get metadata for multiple folders | + +## Request DTOs + +### File Operations + +```json +{ + "file_ids": ["id-1", "id-2", "id-3"], + "target_folder_id": "folder-abc" // required for move/copy +} +``` + +### Folder Delete + +```json +{ + "folder_ids": ["folder-1", "folder-2"], + "recursive": true, + "target_folder_id": null +} +``` + +### Folder Create + +```json +{ + "folders": [ + { "name": "Documents", "parent_id": null }, + { "name": "Photos", "parent_id": "folder-abc" } + ] +} +``` + +## Response Format + +Every batch endpoint returns the same structure: + +```json +{ + "successful": [ ... ], + "failed": [ + { "id": "bad-id", "error": "File not found" } + ], + "stats": { + "total": 5, + "successful": 4, + "failed": 1, + "execution_time_ms": 245 + } +} +``` + +### Status Codes + +| Code | Meaning | +|---|---| +| `200 OK` / `201 Created` | All operations succeeded | +| `206 Partial Content` | Some succeeded, some failed | +| `400 Bad Request` | All operations failed | + +## Concurrency + +Operations run concurrently via `tokio::sync::Semaphore`, capped by **max_concurrent_files** (default: 10). + +```rust +pub struct BatchOperationService { + file_retrieval: Arc, + file_management: Arc, + folder_service: Arc, + config: AppConfig, + semaphore: Arc, +} +``` + +## Error Handling + +```rust +pub enum BatchOperationError { + Domain(DomainError), // individual operation error + Cancelled(String), // operation was cancelled + ConcurrencyLimit(String), // semaphore exhausted + PartialFailure(String, usize, usize), // message, success_count, fail_count + Internal(String), +} +``` + +Individual failures do not abort the batch. They get collected in the `failed` array and the response returns `206 Partial Content`. + +## Example + +```bash +# Move 3 files to a folder +curl -X POST -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"file_ids":["id-1","id-2","id-3"],"target_folder_id":"folder-abc"}' \ + "https://oxicloud.example.com/api/batch/files/move" + +# Delete folders recursively +curl -X POST -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"folder_ids":["old-1","old-2"],"recursive":true}' \ + "https://oxicloud.example.com/api/batch/folders/delete" + +# Create multiple folders +curl -X POST -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"folders":[{"name":"Docs","parent_id":null},{"name":"Photos","parent_id":"root-id"}]}' \ + "https://oxicloud.example.com/api/batch/folders/create" +``` diff --git a/doc/caching-architecture.md b/doc/caching-architecture.md new file mode 100644 index 00000000..ddf3a288 --- /dev/null +++ b/doc/caching-architecture.md @@ -0,0 +1,370 @@ +# 04 - Caching Architecture + +OxiCloud uses a multi-layer caching system spanning HTTP-level caching down to kernel-level memory mapping. Covers both uploads and downloads. + +## Cache Layers Summary + +``` +┌─────────────────────────────────────────────────────┐ +│ Layer 0: HTTP Cache Middleware (ETag + 304) │ All endpoints +├─────────────────────────────────────────────────────┤ +│ Layer 1: File Content Cache (LRU, <10MB files) │ Downloads +├─────────────────────────────────────────────────────┤ +│ Layer 2: MMAP (memmap2, 10-100MB files) │ Downloads +├─────────────────────────────────────────────────────┤ +│ Layer 3: Streaming (FramedRead, ≥100MB files) │ Downloads +├─────────────────────────────────────────────────────┤ +│ Layer 4: File Metadata Cache (adaptive TTL) │ All file ops +├─────────────────────────────────────────────────────┤ +│ Layer 5: Write-Behind Cache (<256KB uploads) │ Uploads +├─────────────────────────────────────────────────────┤ +│ Layer 6: Buffer Pool (reusable I/O buffers) │ Compression +└─────────────────────────────────────────────────────┘ +``` + +--- + +## Layer 0: HTTP Cache Middleware + +**File**: `src/interfaces/middleware/cache.rs` + +Generic HTTP caching layer applied to API endpoints. + +| Parameter | Value | +|---|---| +| Max entries | 1,000 | +| Default max-age | 60 seconds | +| Eviction | LRU (oldest 10% when full) | +| Cleanup | Background task every 5 minutes | + +Features: +- ETag-based conditional requests (`If-None-Match` → `304 Not Modified`) +- `Cache-Control` header injection +- Implements Tower `Layer` + `Service` traits for Axum integration +- Per-request key: method + URI + +--- + +## Layer 1: File Content Cache (Download Tier 1) + +**File**: `src/infrastructure/services/file_content_cache.rs` + +In-memory LRU cache for small files, served directly from RAM. + +| Parameter | Value | +|---|---| +| Max file size | 10 MB per file | +| Max total cache size | 512 MB | +| Max entries | 10,000 | +| Structure | `lru::LruCache` | +| Latency | ~0.1ms | + +**CacheEntry**: `{ data: Bytes, etag: String, content_type: String, size: usize }` + +Methods: +- `should_cache(size)` -- checks if file fits in cache +- `get(file_id)` → `Option<(Bytes, String, String)>` -- returns (data, etag, content_type) +- `put(file_id, content, etag, content_type)` -- inserts with LRU eviction +- `invalidate(file_id)`, `clear()` +- `stats()` → `CacheStats { current_size_bytes, max_size_bytes, hits, misses, hit_rate_percent }` + +Port: implements **ContentCachePort** trait. + +--- + +## Layer 2: MMAP (Download Tier 2) + +**File**: `src/infrastructure/repositories/file_fs_read_repository.rs` + +Memory-mapped I/O for medium files using `memmap2`. + +| Parameter | Value | +|---|---| +| File range | 10 MB - 100 MB | +| Implementation | `memmap2::Mmap` via `spawn_blocking` | +| Latency | ~1-5ms | + +Current implementation copies mmap'd data to `Bytes` (`Bytes::copy_from_slice(&mmap[..])`). Not true zero-copy, but still benefits from kernel page cache. + +--- + +## Layer 3: Streaming (Download Tier 3) + +**File**: `src/infrastructure/repositories/file_fs_read_repository.rs` + +Chunked streaming for large files using tokio-util codecs. + +| Parameter | Value | +|---|---| +| File range | ≥100 MB | +| Chunk size | 1 MB (configurable via **ResourceConfig.chunk_size_bytes**) | +| Implementation | `FramedRead` + `BytesCodec` | +| RAM usage | Near zero (one chunk at a time) | + +--- + +## Layer 4: File Metadata Cache + +**File**: `src/infrastructure/services/file_metadata_cache.rs` + +Caches filesystem metadata (existence, size, MIME type, timestamps) to avoid repeated `stat()` calls. + +| Parameter | Value | +|---|---| +| Default file TTL | 60 seconds | +| Default directory TTL | 120 seconds | +| Max entries | 10,000 | +| Adaptive TTL multiplier | 5x for popular entries (≥10 accesses) | +| LRU eviction | Frees 10% capacity when full | +| Cleanup | Background task runs periodically | + +**CachedMetadata:** +```rust +pub struct FileMetadata { + pub path: PathBuf, + pub exists: bool, + pub entry_type: CacheEntryType, // File | Directory | Unknown + pub size: Option, + pub mime_type: Option, + pub created_at: Option, + pub modified_at: Option, + pub last_access: Instant, + pub expires_at: Instant, + pub access_count: usize, +} +``` + +**Adaptive TTL**: entries accessed ≥10 times get 5x the configured TTL, keeping frequently accessed file metadata in cache longer. + +Port: implements **MetadataCachePort** trait. + +--- + +## Layer 5: Write-Behind Cache + +**File**: `src/infrastructure/services/write_behind_cache.rs` + +Buffers small uploads in RAM and confirms immediately. Flushes to disk asynchronously. + +| Parameter | Value | +|---|---| +| Max file size | 1 MB per file | +| Max total cache | 100 MB | +| Max pending duration | 30 seconds | +| Flush interval | 100 ms | +| Write strategy | Atomic (temp file + rename) | + +Architecture: +- `put_pending(file_id, content, target_path)` stores bytes in `HashMap` +- Background `flush_worker` processes **FlushCommands** via `mpsc` channel +- Periodic checker force-flushes entries older than 30 seconds +- `get_pending(file_id)` serves reads while data is still in RAM (before flush) + +Port: implements **WriteBehindCachePort** trait. + +Statistics: +```rust +pub struct WriteBehindStatsDto { + pub pending_count: usize, + pub pending_bytes: usize, + pub total_writes: u64, + pub total_bytes_written: u64, + pub cache_hits: u64, + pub avg_flush_time_us: u64, +} +``` + +--- + +## Layer 6: Buffer Pool + +**File**: `src/infrastructure/services/buffer_pool.rs` + +Reusable byte buffer pool to reduce allocation pressure during compression operations. + +| Parameter | Value | +|---|---| +| Buffer size | 64 KB | +| Max buffers | 100 | +| Buffer TTL | 60 seconds | +| Concurrency control | `tokio::sync::Semaphore` | + +Features: +- `get_buffer()` -- borrows a buffer (blocks if pool exhausted) +- **BorrowedBuffer** auto-returns to pool on `Drop` via `tokio::spawn` +- Expired buffers are cleaned periodically via `start_cleaner()` +- Tracks stats: gets, hits, misses, returns, evictions, waits + +--- + +## Configuration + +All cache-related config in `src/common/config.rs`: + +```rust +pub struct CacheConfig { + pub file_ttl_ms: u64, // default: 60,000 (1 min) + pub directory_ttl_ms: u64, // default: 120,000 (2 min) + pub max_entries: usize, // default: 10,000 +} + +pub struct ResourceConfig { + pub large_file_threshold_mb: u64, // 100 MB (mmap→streaming boundary) + pub chunk_size_bytes: usize, // 1 MB (streaming chunk size) + pub max_in_memory_file_size_mb: u64, // 50 MB +} +``` + +## Download Flow + +``` +Request → ETag check (304?) → Range request (206?) + → file size < 10MB? → Tier 1: LRU cache (RAM) + → file size < 100MB? → Tier 2: MMAP (kernel page cache) + → file size ≥ 100MB → Tier 3: Streaming (chunked) +``` + +--- + +## Range Requests (HTTP 206 Partial Content) + +**Files**: `src/interfaces/api/handlers/file_handler.rs`, `src/infrastructure/repositories/file_fs_read_repository.rs` + +**Crate**: `http-range-header = "0.4"` for parsing. + +### Request Processing Flow + +``` +Range header present? + ├─ parse_range_header(range_str) + │ ├─ Parse OK → ranges.validate(file_size) + │ │ ├─ Valid → take first range → get_file_range_stream(start, end+1) + │ │ │ ├─ Stream OK → 206 Partial Content + │ │ │ └─ Stream Err → fall through to normal download (200) + │ │ └─ Invalid → 416 Range Not Satisfiable + │ └─ Parse Err → fall through to normal download (200) + └─ No Range header → normal 3-tier download +``` + +### Response Headers (206) + +| Header | Value | +|---|---| +| `Content-Type` | File MIME type | +| `Content-Range` | `bytes {start}-{end}/{total_size}` | +| `Content-Length` | Range length (end - start + 1) | +| `Accept-Ranges` | `bytes` | +| `ETag` | `"{file_id}-{modified_at}"` | +| `Cache-Control` | `private, max-age=3600, must-revalidate` | + +### 416 Range Not Satisfiable + +Returned when `ranges.validate(file_size)` fails: +``` +HTTP/1.1 416 Range Not Satisfiable +Content-Range: bytes */12345 +``` + +### File Seek Implementation + +`get_file_range_stream()` at the repository level: + +```rust +async fn get_file_range_stream( + &self, id: &str, start: u64, end: Option, +) -> Result + Send>, DomainError> +``` + +1. Opens the file with `TokioFile::open()` +2. Seeks to `start` via `fh.seek(SeekFrom::Start(start))` +3. Limits read to `range_length` via `fh.take(range_length)` +4. Wraps in `FramedRead` + `BytesCodec` + +Adaptive chunk size: + +| Range size | Chunk size | +|---|---| +| ≤ 1 MB | 8 KB | +| > 1 MB | 1 MB (from **ResourceConfig.chunk_size_bytes**) | + +### Tier Interaction + +Range requests **bypass all download tiers** (LRU, MMAP, write-behind). They always use direct file seek + streaming. On stream creation error, the handler falls through to the normal `get_file_optimized()` 3-tier path. + +### Limitations + +- **Multipart ranges not supported**: only the first range in a multi-range request is served. Additional ranges are ignored. +- **`If-Range` not handled**: no conditional range support. +- **`If-Modified-Since` not handled**: only `If-None-Match` (ETag) is checked. + +--- + +## Upload Flow + +``` +Request → file size < 256KB? → Write-behind cache (instant 201, async flush) + → file size < 1MB? → Buffered write (sync) + → file size ≥ 1MB → Streaming write (chunk-by-chunk to temp + rename) +``` + +### Upload Strategy Selection + +**File**: `src/application/services/file_upload_service.rs` + +```rust +pub enum UploadStrategy { + WriteBehind, // < 256 KB — instant response, async disk write + Buffered, // 256 KB – 1 MB — sync write to final path + Streaming, // ≥ 1 MB — chunk-by-chunk write to temp file + rename +} +``` + +| Constant | Value | +|---|---| +| `WRITE_BEHIND_THRESHOLD` | 256 KB | +| `STREAMING_UPLOAD_THRESHOLD` | 1 MB | + +### Handler-Level Buffering + +Both upload handlers (`upload_file` and `upload_file_with_cache`) buffer the **entire multipart body in RAM** as `Vec` before calling the service layer: + +```rust +let mut chunks: Vec = Vec::new(); +while let Some(chunk) = field.chunk().await { + chunks.push(chunk); +} +// All bytes are now in RAM +upload_service.smart_upload(..., chunks, total_size).await +``` + +The "streaming" in `UploadStrategy::Streaming` refers to the **service→repository** path, not the HTTP-body→disk path. By the time `save_file_from_stream()` is called, data is already in memory. + +### Streaming Path (≥ 1 MB): Service → Repository + +`smart_upload()` converts the in-memory `Vec` into a `futures::stream::iter()` and passes it to `save_file_from_stream()`: + +```rust +let chunk_stream = stream::iter(chunks.into_iter().map(|c| Ok(c))); +self.file_write.save_file_from_stream(name, folder_id, content_type, chunk_stream).await +``` + +**`save_file_from_stream()` implementation** (`file_fs_write_repository.rs`): + +1. Resolves target path + generates unique name if collision +2. Creates temp file: `{target_path}.tmp.upload` +3. Iterates stream, writing each chunk with `fh.write_all(&chunk)` +4. Calls `fh.flush()` + `fh.sync_all()` for durability +5. Atomic rename: `fs::rename(temp_path, final_path)` +6. Post-write: ID mapping, cache invalidation, metadata update + +### Buffered Path (256 KB - 1 MB) + +Uses `save_file()` -- writes all bytes directly to the **final path** (no temp file). For larger content, writes in chunks of **ResourceConfig.chunk_size_bytes** (1 MB). + +### Write-Behind Path (< 256 KB) + +See **Layer 5** above. Instant `201`, background flush within 30 seconds. + +### Dedup Pre-Check + +Runs for **all upload strategies** before writing. Re-combines all chunks into a single `Vec` for hash computation, which means data is temporarily duplicated in RAM during dedup processing. diff --git a/doc/caldav-technical-spec.md b/doc/caldav-technical-spec.md new file mode 100644 index 00000000..9590f20a --- /dev/null +++ b/doc/caldav-technical-spec.md @@ -0,0 +1,239 @@ +# 26 - CalDAV Technical Spec + +CalDAV (RFC 4791) provides calendar synchronization. Clients like Thunderbird, Apple Calendar, GNOME Calendar, and DAVx5 (Android) connect to manage calendars and events via standard CalDAV. + +## Protocol Compliance + +- **DAV compliance**: `1, 2, calendar-access` +- **RFC 4791**: Calendar Access (CalDAV) +- **iCalendar**: RFC 5545 (VEVENT parsing and generation) + +## Endpoint Structure + +All CalDAV endpoints are mounted at the top level (not under `/api`): + +``` +ANY /caldav → handle_caldav_methods_root +ANY /caldav/ → handle_caldav_methods_root +ANY /caldav/{*path} → handle_caldav_methods +``` + +### Path Hierarchy + +| Path | Resource | Description | +|---|---|---| +| `/caldav/` | Calendar home | User's calendar collection | +| `/caldav/{calendar_id}/` | Calendar | Individual calendar | +| `/caldav/{calendar_id}/{ical_uid}.ics` | Event | Individual calendar event | + +### Supported HTTP Methods + +| Method | Description | +|---|---| +| `OPTIONS` | Returns DAV capabilities and allowed methods | +| `PROPFIND` | List calendars, calendar properties, events | +| `REPORT` | CalendarQuery, CalendarMultiget, SyncCollection | +| `MKCALENDAR` | Create a new calendar | +| `PUT` | Create/update events (iCalendar format) | +| `GET` | Retrieve individual event as iCalendar | +| `DELETE` | Delete calendars or events | +| `PROPPATCH` | Update calendar properties | + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Domain Entity | **Calendar**, **CalendarEvent** | `src/domain/entities/calendar.rs`, `calendar_event.rs` | +| Domain Repository | **CalendarRepository**, **CalendarEventRepository** | `src/domain/repositories/calendar_repository.rs`, `calendar_event_repository.rs` | +| Application Port | **CalendarUseCase**, **CalendarStoragePort** | `src/application/ports/calendar_ports.rs` | +| Application Service | **CalendarService** | `src/application/services/calendar_service.rs` | +| Application Adapter | **CalDavAdapter** (XML parsing/generation) | `src/application/adapters/caldav_adapter.rs` | +| Infrastructure | **CalendarPgRepository**, **CalendarEventPgRepository** | `src/infrastructure/repositories/pg/` | +| Interfaces | **CalDavHandler** | `src/interfaces/api/handlers/caldav_handler.rs` | + +## Domain Entities + +### Calendar + +```rust +pub struct Calendar { + id: Uuid, + name: String, + owner_id: String, + description: Option, + color: Option, // #RRGGBB format + created_at: DateTime, + updated_at: DateTime, + custom_properties: HashMap, +} +``` + +Validation: **name** non-empty, **owner_id** non-empty, **color** must be `#RRGGBB` hex format. + +### CalendarEvent + +```rust +pub struct CalendarEvent { + id: Uuid, + calendar_id: Uuid, + summary: String, + description: Option, + location: Option, + start_time: DateTime, + end_time: DateTime, + all_day: bool, + rrule: Option, // iCal RRULE format (e.g., "FREQ=WEEKLY") + ical_uid: String, // unique iCal identifier + ical_data: String, // full VEVENT block + created_at: DateTime, + updated_at: DateTime, +} +``` + +Validation: **summary** non-empty, `end_time >= start_time`, **rrule** starts with `FREQ=`, **ical_data** contains `BEGIN:VEVENT`/`END:VEVENT`. + +The constructor `from_ical(calendar_id, ical_data)` parses iCalendar data to extract summary, dates, location, etc. + +## REPORT Types + +The CalDAV **REPORT** method supports three report types: + +### CalendarQuery + +Filters events by time range. Used for initial sync and view rendering. + +```xml + + + + + + + + + + + + + +``` + +### CalendarMultiget + +Fetches specific events by href. Used for selective sync. + +```xml + + + + + + /caldav/cal-1/event-1.ics + /caldav/cal-1/event-2.ics + +``` + +### SyncCollection + +Incremental sync using sync tokens. Used for ongoing synchronization. + +```xml + + sync-token-value + + + + +``` + +## XML Namespaces + +| Prefix | Namespace | +|---|---| +| `D:` | `DAV:` | +| `C:` | `urn:ietf:params:xml:ns:caldav` | +| `CS:` | `http://calendarserver.org/ns/` | + +## CalDAV Adapter + +**CalDavAdapter** in `src/application/adapters/caldav_adapter.rs` handles all XML parsing and generation: + +- `parse_report(reader)` -> **CalDavReportType** +- `parse_mkcalendar(reader)` -> `(name, description, color)` +- `generate_calendars_propfind_response(...)` -- multi-calendar PROPFIND +- `generate_calendar_collection_propfind(...)` -- single calendar with events +- `generate_calendar_events_response(...)` -- REPORT response + +## Calendar Sharing + +Calendars support sharing with access levels: + +| Level | Permissions | +|---|---| +| `read` | View calendar and events | +| `write` | Create, modify, delete events | +| `owner` | Full control including sharing and deletion | + +## Database Schema + +```sql +CREATE SCHEMA IF NOT EXISTS caldav; + +-- Calendars +CREATE TABLE caldav.calendars ( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + owner_id VARCHAR(36) REFERENCES auth.users(id) ON DELETE CASCADE, + description TEXT, + color VARCHAR(9), + is_public BOOLEAN DEFAULT FALSE, + ctag VARCHAR(64) DEFAULT '0', + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); + +-- Calendar Events +CREATE TABLE caldav.calendar_events ( + id UUID PRIMARY KEY, + calendar_id UUID REFERENCES caldav.calendars(id) ON DELETE CASCADE, + summary TEXT NOT NULL, + description TEXT, + location TEXT, + start_time TIMESTAMPTZ NOT NULL, + end_time TIMESTAMPTZ NOT NULL, + all_day BOOLEAN DEFAULT FALSE, + rrule TEXT, + ical_uid VARCHAR(255) NOT NULL, + ical_data TEXT, + etag VARCHAR(64), + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); + +-- Calendar Shares +CREATE TABLE caldav.calendar_shares ( + id SERIAL PRIMARY KEY, + calendar_id UUID REFERENCES caldav.calendars(id) ON DELETE CASCADE, + user_id VARCHAR(36) REFERENCES auth.users(id) ON DELETE CASCADE, + access_level VARCHAR(10) DEFAULT 'read', + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + UNIQUE(calendar_id, user_id) +); + +-- Calendar Properties +CREATE TABLE caldav.calendar_properties ( + id SERIAL PRIMARY KEY, + calendar_id UUID REFERENCES caldav.calendars(id) ON DELETE CASCADE, + property_name TEXT NOT NULL, + property_value TEXT NOT NULL, + UNIQUE(calendar_id, property_name) +); +``` + +Indexes: `idx_calendars_owner_id`, `idx_calendar_events_calendar_id`, `idx_calendar_events_ical_uid`, `idx_calendar_events_time_range (start_time, end_time)`. + +## Client Configuration + +See `dav-client-setup.md` for client-specific configuration instructions. + +CalDAV URL: `https://oxicloud.example.com/caldav/` diff --git a/doc/CARDDAV-IMPLEMENTATION-PLAN.md b/doc/carddav-implementation-plan.md similarity index 53% rename from doc/CARDDAV-IMPLEMENTATION-PLAN.md rename to doc/carddav-implementation-plan.md index 4175af8f..28693064 100644 --- a/doc/CARDDAV-IMPLEMENTATION-PLAN.md +++ b/doc/carddav-implementation-plan.md @@ -1,125 +1,80 @@ -# CardDAV Implementation Plan +# 28 - CardDAV Implementation Plan -## Introduction - -This document outlines the plan for implementing CardDAV support in OxiCloud. CardDAV is an open protocol for synchronizing address books/contacts between different applications and devices. +CardDAV is an open protocol for syncing address books and contacts across applications and devices. This plan breaks the implementation into five phases over five weeks. ## Implementation Roadmap -The implementation will follow these steps: - ### Phase 1: Core Infrastructure (Week 1) -#### Database Schema -- Create new migration for CardDAV tables: - - `address_books` - For storing address book collections - - `contacts` - For storing contact information - - `address_book_shares` - For sharing address books between users - - `contact_groups` - For organizing contacts into groups - - `group_memberships` - For associating contacts with groups +**Database Schema** -- new migration for CardDAV tables: +- **address_books** -- stores address book collections +- **contacts** -- stores contact data +- **address_book_shares** -- sharing between users +- **contact_groups** -- organizing contacts into groups +- **group_memberships** -- associating contacts with groups -#### Domain Layer -- Define entity models: - - `Contact` - Core contact entity - - `AddressBook` - Collection entity - - `ContactGroup` - For grouping contacts -- Create repository interfaces: - - `ContactRepository` - For contact CRUD operations - - `AddressBookRepository` - For address book management - - `ContactGroupRepository` - For group management +**Domain Layer** -- entity models: +- **Contact** -- core contact entity +- **AddressBook** -- collection entity +- **ContactGroup** -- grouping contacts -#### Testing -- Unit tests for entity models -- Repository interface contract tests +Repository interfaces: +- **ContactRepository** -- contact CRUD +- **AddressBookRepository** -- address book management +- **ContactGroupRepository** -- group management + +**Testing** -- unit tests for entity models and repository interface contract tests. ### Phase 2: Infrastructure Layer (Week 2) -#### Repository Implementations -- Implement PostgreSQL repositories: - - `ContactPgRepository` - - `AddressBookPgRepository` - - `ContactGroupPgRepository` -- Implement vCard parsing and generation utilities -- Create data migration tools (if needed) +**Repository Implementations** -- PostgreSQL: +- **ContactPgRepository** +- **AddressBookPgRepository** +- **ContactGroupPgRepository** -#### Integration -- Update dependency injection system to include new repositories -- Connect with existing auth system +Also implement vCard parsing/generation utilities and any data migration tools needed. -#### Testing -- Repository implementation tests -- vCard parsing/generation tests -- Integration tests with database +**Integration** -- update DI system to include new repositories and connect with existing auth system. + +**Testing** -- repository implementation tests, vCard parsing/generation tests, integration tests with the database. ### Phase 3: Application Layer (Week 3) -#### Services -- Implement business logic services: - - `ContactService` - Contact management - - `AddressBookService` - Address book management - - `ContactGroupService` - Group management +**Services** -- business logic: +- **ContactService** -- contact management +- **AddressBookService** -- address book management +- **ContactGroupService** -- group management -#### DTOs and Ports -- Create DTOs for contact operations -- Define service interface ports -- Implement request/response mapping +**DTOs and Ports** -- create DTOs for contact operations, define service interface ports, implement request/response mapping. -#### CardDAV Adapter -- Create adapter for CardDAV protocol translation -- Implement vCard conversion logic -- Create XML parsing and generation utilities +**CardDAV Adapter** -- protocol translation adapter with vCard conversion logic and XML parsing/generation utilities. -#### Testing -- Service unit tests -- Integration tests for adapter +**Testing** -- service unit tests and integration tests for the adapter. ### Phase 4: Interface Layer (Week 4) -#### REST API -- Create REST endpoints for address book operations -- Implement contact management endpoints -- Add contact group endpoints -- Document API with OpenAPI +**REST API** -- endpoints for address book operations, contact management, contact groups. Document with OpenAPI. -#### CardDAV Protocol Endpoints -- Implement WebDAV method handlers: - - PROPFIND - For discovery and property retrieval - - REPORT - For querying contacts - - MKCOL - For creating address books - - GET/PUT/DELETE - For contact operations -- Add CardDAV-specific XML handling +**CardDAV Protocol Endpoints** -- WebDAV method handlers: +- PROPFIND -- discovery and property retrieval +- REPORT -- querying contacts +- MKCOL -- creating address books +- GET/PUT/DELETE -- contact operations +- CardDAV-specific XML handling -#### Integration -- Connect all layers -- Perform end-to-end testing -- Test with various CardDAV clients +**Integration** -- connect all layers, end-to-end testing, test with various CardDAV clients. -#### Testing -- API endpoint tests -- CardDAV protocol compliance tests -- Client compatibility tests +**Testing** -- API endpoint tests, CardDAV protocol compliance tests, client compatibility tests. ### Phase 5: Refinement and Optimization (Week 5) -#### Performance Optimization -- Add caching for frequently accessed resources -- Optimize database queries -- Implement efficient synchronization mechanisms +**Performance** -- caching for frequently accessed resources, query optimization, efficient sync mechanisms. -#### Security Hardening -- Review authentication and authorization -- Validate input and output -- Add rate limiting +**Security** -- review auth, validate I/O, add rate limiting. -#### Final Testing -- Stress testing with large address books -- Security testing -- User acceptance testing +**Final Testing** -- stress testing with large address books, security testing, user acceptance testing. -#### Documentation -- Update API documentation -- Create user guides -- Document client setup procedures +**Documentation** -- update API docs, user guides, client setup procedures. ## Technical Specifications @@ -227,36 +182,34 @@ CREATE TABLE IF NOT EXISTS carddav.group_memberships ( ## Resources Required - Developer time: 1 full-time developer for 5 weeks -- Testing resources: Multiple CardDAV clients (Apple Contacts, Thunderbird, Android) -- Server resources: Test environment with PostgreSQL +- Testing resources: multiple CardDAV clients (Apple Contacts, Thunderbird, Android) +- Server resources: test environment with PostgreSQL ## Success Criteria -The implementation will be considered successful when: - 1. Users can create, update, and delete address books 2. Contacts can be managed within address books 3. Address books can be shared between users -4. Standard CardDAV clients can synchronize with the server -5. Performance is acceptable with large address books (1000+ contacts) -6. Security measures are properly implemented +4. Standard CardDAV clients can sync with the server +5. Acceptable performance with large address books (1000+ contacts) +6. Security measures properly implemented ## Client Setup Guides -After implementation, we will create setup guides for: +After implementation, setup guides will be created for: - Apple Contacts (macOS/iOS) - Thunderbird/Evolution -- Android (using DAVx⁵) +- Android (using DAVx5) - Other common CardDAV clients +See `dav-client-setup.md` for general DAV client configuration. + ## Future Enhancements -After the initial implementation, we may consider: - -1. Advanced contact search capabilities +1. Advanced contact search 2. Contact merging for duplicate detection -3. Bulk import/export options +3. Bulk import/export 4. Contact photo management -5. Extended fields for specialized contact information -6. Integration with other systems (e.g., LDAP directories) \ No newline at end of file +5. Extended fields for specialized contact info +6. Integration with external systems (e.g., LDAP directories) diff --git a/doc/CARDDAV-TECHNICAL-SPEC.md b/doc/carddav-technical-spec.md similarity index 72% rename from doc/CARDDAV-TECHNICAL-SPEC.md rename to doc/carddav-technical-spec.md index 53f01c8d..ee2fa6f4 100644 --- a/doc/CARDDAV-TECHNICAL-SPEC.md +++ b/doc/carddav-technical-spec.md @@ -1,14 +1,10 @@ -# CardDAV Integration Technical Specification +# 27 - CardDAV Technical Spec -## Overview - -This document outlines the technical specification for implementing CardDAV support in OxiCloud, allowing users to synchronize their contacts across various devices and applications. - -CardDAV (Card Distributed Authoring and Versioning) is an address book client/server protocol designed to allow users to access and share contact data on a server. It's an extension of WebDAV (RFC 4918) and is defined in RFC 6352. +CardDAV (RFC 6352) enables contact synchronization across devices and applications. It extends WebDAV (RFC 4918) to manage address books and vCard-formatted contacts. ## Architecture -The CardDAV implementation will follow the established hexagonal architecture pattern used throughout OxiCloud: +The CardDAV implementation follows the hexagonal architecture pattern: ``` ┌───────────────────┐ ┌────────────────────┐ ┌────────────────────┐ @@ -32,17 +28,17 @@ The CardDAV implementation will follow the established hexagonal architecture pa ### Components 1. **Domain Layer** - - `Contact` entity - Represents a contact with properties like name, email, phone, etc. - - `AddressBook` entity - Represents a collection of contacts + - **Contact** entity -- name, email, phone, etc. + - **AddressBook** entity -- a collection of contacts - Repository interfaces for contact management 2. **Application Layer** - - `ContactService` - Business logic for managing contacts - - `CardDAVAdapter` - Converts between CardDAV protocol requests/responses and domain objects + - **ContactService** -- business logic for managing contacts + - **CardDAVAdapter** -- converts between CardDAV protocol requests/responses and domain objects 3. **Infrastructure Layer** - - `ContactPgRepository` - PostgreSQL implementation of contact repositories - - `AddressBookPgRepository` - PostgreSQL implementation of address book repositories + - **ContactPgRepository** -- PostgreSQL implementation of contact repositories + - **AddressBookPgRepository** -- PostgreSQL implementation of address book repositories 4. **Interface Layer** - REST API endpoints for contact management @@ -50,8 +46,6 @@ The CardDAV implementation will follow the established hexagonal architecture pa ## Database Schema -The following database schema will be used to store contact information: - ```sql -- Address books table CREATE TABLE IF NOT EXISTS carddav.address_books ( @@ -120,8 +114,6 @@ CREATE TABLE IF NOT EXISTS carddav.group_memberships ( ### REST API -The following REST endpoints will be implemented for managing contacts: - | Method | Endpoint | Description | |--------|----------|-------------| | GET | `/api/address-books` | List all address books | @@ -139,12 +131,10 @@ The following REST endpoints will be implemented for managing contacts: ### CardDAV Protocol Endpoints -The following CardDAV protocol endpoints will be implemented: - | Method | Endpoint | Description | |--------|----------|-------------| | PROPFIND | `/carddav/` | List all address books | -| PROPFIND | `/carddav/:addressBookId/` | Get address book information | +| PROPFIND | `/carddav/:addressBookId/` | Get address book info | | REPORT | `/carddav/:addressBookId/` | Query contacts in an address book | | GET | `/carddav/:addressBookId/:contactId.vcf` | Get a specific contact (vCard) | | PUT | `/carddav/:addressBookId/:contactId.vcf` | Create or update a contact | @@ -225,7 +215,7 @@ pub struct AddressBook { ## Repositories -### Contact Repository Interface +### ContactRepository Interface ```rust #[async_trait] @@ -241,7 +231,7 @@ pub trait ContactRepository: Send + Sync + 'static { } ``` -### AddressBook Repository Interface +### AddressBookRepository Interface ```rust #[async_trait] @@ -259,51 +249,47 @@ pub trait AddressBookRepository: Send + Sync + 'static { } ``` -## CardDAV Protocol Implementation +## CardDAV Protocol Features -The CardDAV implementation will support the following features: - -1. **Address Book Discovery** - Allow clients to discover available address books -2. **Address Book Collection** - Manage contacts within address books -3. **vCard Support** - Store and retrieve contacts in vCard format (3.0 and 4.0) -4. **Query Support** - Filter contacts by properties -5. **Multiget Support** - Retrieve multiple contacts in a single request -6. **Sync-Collection** - Efficient synchronization of changes +1. **Address Book Discovery** -- clients discover available address books +2. **Address Book Collection** -- manage contacts within address books +3. **vCard Support** -- store and retrieve contacts in vCard format (3.0 and 4.0) +4. **Query Support** -- filter contacts by properties +5. **Multiget Support** -- retrieve multiple contacts in a single request +6. **Sync-Collection** -- efficient incremental synchronization ### CardDAV Adapter -The CardDAV adapter will handle: +The **CardDAVAdapter** handles: 1. Parsing CardDAV XML requests -2. Converting between vCard and Contact entities +2. Converting between vCard and **Contact** entities 3. Generating CardDAV XML responses -4. Supporting PROPFIND, REPORT, and other WebDAV methods -5. Implementing the proper WebDAV properties for CardDAV +4. Supporting **PROPFIND**, **REPORT**, and other WebDAV methods +5. Implementing proper WebDAV properties for CardDAV ## Integration Points -The CardDAV implementation will integrate with: - -1. **Authentication System** - Reuse existing auth mechanisms -2. **WebDAV Infrastructure** - Extend the existing WebDAV implementation -3. **Database Layer** - Store contacts in PostgreSQL -4. **User Management** - Connect contacts with user accounts +1. **Authentication** -- reuses existing auth mechanisms +2. **WebDAV Infrastructure** -- extends the existing WebDAV implementation +3. **Database Layer** -- stores contacts in PostgreSQL +4. **User Management** -- connects contacts with user accounts ## Client Compatibility -The implementation should be compatible with the following clients: +Target clients: - Apple Contacts - Google Contacts - Thunderbird - Outlook -- Android DAVx⁵ +- Android DAVx5 - iOS native contacts app - Evolution -## Implementation Phases +See `dav-client-setup.md` for detailed connection instructions. -The implementation will be divided into the following phases: +## Implementation Phases ### Phase 1: Core Infrastructure - Database schema creation @@ -322,7 +308,7 @@ The implementation will be divided into the following phases: - Contact group endpoints ### Phase 4: CardDAV Protocol -- CardDAV adapter implementation +- **CardDAVAdapter** implementation - WebDAV method handlers - XML parsing and generation - Protocol compliance testing @@ -332,32 +318,26 @@ The implementation will be divided into the following phases: - Performance optimization - Edge case handling -## Security Considerations +## Security -The CardDAV implementation must address the following security concerns: +1. **Authentication** -- proper authentication for all operations +2. **Authorization** -- verify permissions for each address book operation +3. **Data Validation** -- validate vCard input to prevent injection attacks +4. **Resource Limits** -- limits to prevent abuse +5. **Error Handling** -- appropriate error responses without leaking sensitive data -1. **Authentication** - Ensure proper authentication for all operations -2. **Authorization** - Verify permissions for each address book operation -3. **Data Validation** - Validate vCard input to prevent injection attacks -4. **Resource Limits** - Implement limits to prevent abuse -5. **Error Handling** - Provide appropriate error responses without revealing sensitive information +## Performance -## Performance Considerations - -To ensure good performance: - -1. **Indexing** - Proper database indexes for contact queries -2. **Caching** - Cache frequently accessed address books and contacts -3. **Pagination** - Support pagination for large address books -4. **Incremental Sync** - Efficient synchronization with client devices -5. **ETags** - Use ETags to prevent unnecessary data transfers +1. **Indexing** -- proper database indexes for contact queries +2. **Caching** -- cache frequently accessed address books and contacts +3. **Pagination** -- support pagination for large address books +4. **Incremental Sync** -- efficient sync with client devices +5. **ETags** -- prevent unnecessary data transfers ## Testing Strategy -The CardDAV implementation will be tested using: - -1. **Unit Tests** - Test individual components in isolation -2. **Integration Tests** - Test the interaction between components -3. **Protocol Compliance Tests** - Verify adherence to the CardDAV specification -4. **Client Compatibility Tests** - Test with various CardDAV clients -5. **Performance Tests** - Measure performance with large address books \ No newline at end of file +1. **Unit Tests** -- test individual components in isolation +2. **Integration Tests** -- test component interactions +3. **Protocol Compliance Tests** -- verify adherence to the CardDAV spec (RFC 6352) +4. **Client Compatibility Tests** -- test with various CardDAV clients +5. **Performance Tests** -- measure performance with large address books diff --git a/doc/chunked-uploads.md b/doc/chunked-uploads.md new file mode 100644 index 00000000..453bcfa1 --- /dev/null +++ b/doc/chunked-uploads.md @@ -0,0 +1,175 @@ +# 07 - Chunked Uploads + +OxiCloud implements a TUS-like chunked upload protocol for large files (≥10 MB). Files are split into chunks (default 5 MB) that can be uploaded in parallel (up to 6 concurrent), with progress tracking, optional MD5 checksums, and automatic session expiration. + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **ChunkedUploadPort** trait + DTOs | `src/application/ports/chunked_upload_ports.rs` | +| Infrastructure | **ChunkedUploadService** | `src/infrastructure/services/chunked_upload_service.rs` | +| Interfaces | **ChunkedUploadHandler** | `src/interfaces/api/handlers/chunked_upload_handler.rs` | + +## Constants + +| Constant | Value | Description | +|---|---|---| +| `CHUNKED_UPLOAD_THRESHOLD` | 10 MB | Files above this should use chunked upload | +| `DEFAULT_CHUNK_SIZE` | 5 MB | Default chunk size (minimum 1 MB) | +| `MAX_PARALLEL_CHUNKS` | 6 | Maximum concurrent chunk uploads | +| `SESSION_EXPIRATION` | 24 hours | Sessions expire after this duration | + +## REST API + +All routes under `/api/uploads`, authentication required. + +| Method | Path | Handler | Description | +|---|---|---|---| +| `POST` | `/api/uploads` | `create_upload` | Create upload session | +| `PATCH` | `/api/uploads/{upload_id}` | `upload_chunk` | Upload a single chunk | +| `HEAD` | `/api/uploads/{upload_id}` | `get_upload_status` | Query upload progress | +| `POST` | `/api/uploads/{upload_id}/complete` | `complete_upload` | Assemble chunks → create file | +| `DELETE` | `/api/uploads/{upload_id}` | `cancel_upload` | Cancel and cleanup | + +## Protocol Flow + +``` +1. POST /api/uploads + Body: { "filename": "video.mp4", "total_size": 104857600, "content_type": "video/mp4" } + Response: { "upload_id": "abc-123", "chunk_size": 5242880, "total_chunks": 20, "expires_at": 1707868800 } + +2. PATCH /api/uploads/abc-123?chunk_index=0 ──┐ + PATCH /api/uploads/abc-123?chunk_index=1 ──┼── Up to 6 in parallel + PATCH /api/uploads/abc-123?chunk_index=2 ──┘ + Body: raw chunk bytes + Response: { "chunk_index": 0, "bytes_received": 52428800, "progress": 50.0, "is_complete": false } + +3. HEAD /api/uploads/abc-123 + Response headers: Upload-Offset, Upload-Length, Upload-Progress, Upload-Chunks-Total, Upload-Chunks-Complete + +4. POST /api/uploads/abc-123/complete + Response: { "file_id": "def-456", "filename": "video.mp4", "size": 104857600, "path": "/videos" } + Status: 201 Created +``` + +## Request/Response DTOs + +### Create Upload Request + +```rust +pub struct CreateUploadRequest { + pub filename: String, + pub folder_id: Option, + pub content_type: Option, // default: "application/octet-stream" + pub total_size: u64, + pub chunk_size: Option, // default: 5 MB, minimum: 1 MB +} +``` + +### Chunk Upload Query Parameters + +```rust +pub struct ChunkUploadParams { + pub chunk_index: usize, + pub checksum: Option, // MD5 hash (also via Content-MD5 header) +} +``` + +### Upload Status Response + +```rust +pub struct UploadStatusResponseDto { + pub upload_id: String, + pub filename: String, + pub total_size: u64, + pub bytes_received: u64, + pub progress: f64, // 0.0 - 100.0 + pub total_chunks: usize, + pub completed_chunks: usize, + pub pending_chunks: Vec, + pub is_complete: bool, +} +``` + +### Complete Upload Response + +```rust +pub struct CompleteUploadResponse { + pub file_id: String, + pub filename: String, + pub size: u64, + pub path: String, +} +``` + +## Custom Response Headers + +### On upload_chunk (PATCH) +- `Upload-Offset`: total bytes received so far +- `Upload-Progress`: percentage complete (0-100) +- `Upload-Complete: true` (only when all chunks are uploaded) + +### On get_upload_status (HEAD) +- `Upload-Offset`: bytes received +- `Upload-Length`: total expected size +- `Upload-Progress`: percentage +- `Upload-Chunks-Total`: total chunk count +- `Upload-Chunks-Complete`: completed chunk count + +## Internal Storage + +``` +/ + / + chunk_000000 ← individual chunk files + chunk_000001 + chunk_000002 + ... + assembled ← final assembled file (after complete) +``` + +## Completion Flow + +1. `complete_upload()` assembles all chunks in order into a single `assembled` file +2. Reads the assembled file content +3. Delegates to **FileUploadService::upload_file()** to create the permanent file record +4. Calls `finalize_upload()` to clean up the session and temp directory +5. Returns `201 Created` with file metadata + +## Session Management + +- **In-memory sessions**: `HashMap` protected by `RwLock` +- **Chunk status tracking**: each chunk has a status: `Pending` → `Uploading` → `Complete` (or `Failed`) +- **Expiration**: sessions expire after 24 hours of inactivity +- **Cleanup**: background task runs hourly to remove expired sessions and orphaned temp directories + +## Client Usage Example + +```bash +# 1. Create upload session +RESPONSE=$(curl -s -X POST -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"filename":"large-file.zip","total_size":52428800}' \ + "https://oxicloud.example.com/api/uploads") + +UPLOAD_ID=$(echo $RESPONSE | jq -r '.upload_id') +CHUNK_SIZE=$(echo $RESPONSE | jq -r '.chunk_size') +TOTAL_CHUNKS=$(echo $RESPONSE | jq -r '.total_chunks') + +# 2. Upload chunks in parallel +for i in $(seq 0 $((TOTAL_CHUNKS - 1))); do + dd if=large-file.zip bs=$CHUNK_SIZE skip=$i count=1 2>/dev/null | \ + curl -s -X PATCH -H "Authorization: Bearer $TOKEN" \ + --data-binary @- \ + "https://oxicloud.example.com/api/uploads/$UPLOAD_ID?chunk_index=$i" & +done +wait + +# 3. Complete upload +curl -X POST -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/uploads/$UPLOAD_ID/complete" + +# 4. Check progress (optional) +curl -I -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/uploads/$UPLOAD_ID" +``` diff --git a/doc/database-migrations.md b/doc/database-migrations.md new file mode 100644 index 00000000..de7adfc6 --- /dev/null +++ b/doc/database-migrations.md @@ -0,0 +1,177 @@ +# 18 - Database Migrations + +OxiCloud uses versioned SQL files to manage database schema changes. The migration system ensures changes are versioned, trackable, consistently applied across environments, reproducible, and independent of application code. + +## Directory Structure + +``` +OxiCloud/ +├── db/ +│ └── schema.sql # Main database schema +├── src/ + ├── bin/ + │ └── migrate.rs # CLI tool for running migrations + ├── common/ + │ └── db.rs # Database connection with schema verification +``` + +> The schema is currently applied from `db/schema.sql` at application startup (when it detects the `auth` tables don't exist). The `migrations/` directory doesn't exist yet, but `src/bin/migrate.rs` is ready to use sqlx migrations once the `migrations` feature is enabled. + +## Naming Conventions + +Migration files follow this format: `YYYYMMDDHHMMSS_brief_description.sql` + +- `YYYYMMDDHHMMSS` -- timestamp that guarantees correct ordering (year, month, day, hour, minute, second) +- `brief_description` -- short description of the migration purpose +- `.sql` -- SQL file extension + +## Running Migrations + +Migrations run via a dedicated CLI tool: + +```bash +cargo run --bin migrate --features migrations +``` + +This command: +1. Connects to the database configured in the environment +2. Looks for migrations in the `/migrations/` directory +3. Compares applied migrations against available ones +4. Sequentially executes pending migrations +5. Records applied migrations in a control table + +## Creating New Migrations + +To create a new migration: + +1. Create a new file in `migrations/` following the naming convention +2. Define the SQL changes in the file +3. Make sure the changes are compatible with the current schema version +4. Run the migrations + +Example migration structure: + +```sql +-- Migración: Añadir tabla de etiquetas +-- Descripción: Crea la tabla para almacenar etiquetas de archivos y sus relaciones + +-- Crear tabla de etiquetas +CREATE TABLE IF NOT EXISTS auth.tags ( + id SERIAL PRIMARY KEY, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + name TEXT NOT NULL, + color TEXT NOT NULL DEFAULT '#3498db', + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, name) +); + +-- Crear índices +CREATE INDEX IF NOT EXISTS idx_tags_user_id ON auth.tags(user_id); + +-- Tabla de relación entre archivos y etiquetas +CREATE TABLE IF NOT EXISTS auth.file_tags ( + id SERIAL PRIMARY KEY, + tag_id INTEGER NOT NULL REFERENCES auth.tags(id) ON DELETE CASCADE, + file_id TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(tag_id, file_id) +); + +-- Comentarios de documentación +COMMENT ON TABLE auth.tags IS 'Almacena etiquetas definidas por usuarios'; +COMMENT ON TABLE auth.file_tags IS 'Relación muchos-a-muchos entre archivos y etiquetas'; +``` + +## Best Practices + +1. **Incremental migrations** -- each migration should represent one atomic, coherent change. + +2. **Idempotent migrations** -- use commands that can run multiple times without errors (e.g., `CREATE TABLE IF NOT EXISTS`). + +3. **Forward-only migrations** -- design migrations to move forward, not roll back. If you need to undo a change, create a new migration. + +4. **Forward compatibility** -- migrations must be compatible with both the existing code and the code about to be deployed. + +5. **Test before deploying** -- test migrations in a production-like environment before applying them. + +6. **Documentation** -- document the purpose and key changes of each migration with comments inside the SQL file. + +## Troubleshooting + +### Checking Migration State + +OxiCloud includes startup-time detection to verify which migrations have been applied: + +```rust +// Desde src/common/db.rs +let migration_check = sqlx::query("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'auth' AND tablename = 'users')") + .fetch_one(&pool) + .await; + +match migration_check { + Ok(row) => { + let tables_exist: bool = row.get(0); + if !tables_exist { + tracing::warn!("Las tablas de la base de datos no existen. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations"); + } + }, + Err(_) => { + tracing::warn!("No se pudo verificar el estado de las migraciones. Por favor, ejecuta las migraciones con: cargo run --bin migrate --features migrations"); + } +} +``` + +### Common Issues + +1. **Database connection error** -- verify the connection URL in the **DATABASE_URL** environment variable. + +2. **Migration conflicts** -- if a migration fails, check the error messages for conflicts with the existing schema. + +3. **Insufficient permissions** -- make sure the database user has permissions to create schemas, tables, and indexes. + +4. **"Admin already exists" error** -- if you get this error when trying to register an admin user, follow these steps: + + a. Connect to the PostgreSQL container: + ```bash + # Find the container + docker ps + # Example: oxicloud-postgres-1 + docker exec -it oxicloud-postgres-1 bash + ``` + + b. Connect to the database: + ```bash + psql -U postgres -d oxicloud + ``` + + c. Set the schema and delete the existing admin user: + ```sql + SET search_path TO auth; + DELETE FROM auth.users WHERE username = 'admin'; + ``` + + d. Verify the deletion: + ```sql + SELECT username, email, role FROM auth.users; + ``` + + e. Exit PostgreSQL: + ```sql + \q + exit + ``` + + f. You can now register a new admin user through the OxiCloud interface. + + Alternatively, use the provided script: + ```bash + cat scripts/reset_admin.sql | docker exec -i oxicloud-postgres-1 psql -U postgres -d oxicloud + ``` + +## Benefits of Migration-Based Approach + +- **Separation of concerns** -- migrations live separately from application code. +- **Automation** -- simplifies deployment automation and CI/CD. +- **Change history** -- provides a clear history of schema evolution. +- **Collaboration** -- lets multiple developers contribute schema changes in an orderly way. +- **Multiple environments** -- guarantees identical database structures across dev, test, and production. diff --git a/doc/database-transactions.md b/doc/database-transactions.md new file mode 100644 index 00000000..402e3bc3 --- /dev/null +++ b/doc/database-transactions.md @@ -0,0 +1,119 @@ +# 19 - Database Transactions + +OxiCloud uses explicit transactions on PostgreSQL to guarantee data integrity. All transactional operations follow ACID properties: + +- **Atomicity** -- all-or-nothing. If any part fails, the entire transaction fails. +- **Consistency** -- the database moves from one valid state to another. +- **Isolation** -- concurrent transactions behave as if sequential. +- **Durability** -- once committed, the transaction survives system failures. + +## Implementation + +The **with_transaction** helper wraps the standard pattern: begin, execute, commit on success, rollback on error. + +### Transaction Utility + +Located in `src/infrastructure/repositories/pg/transaction_utils.rs`: + +```rust +/// Helper function to execute database operations in a transaction +pub async fn with_transaction( + pool: &Arc, + operation_name: &str, + operation: F, +) -> Result +where + F: for<'c> FnOnce(&'c mut Transaction<'_, Postgres>) -> futures::future::BoxFuture<'c, Result>, + E: From + std::fmt::Display +{ ... } +``` + +This function takes a connection pool and a closure with operations, handles begin/commit/rollback automatically, and provides detailed logging of the transaction lifecycle. + +### Repository Usage Example + +```rust +// Creación de un usuario con transacción explícita +async fn create_user(&self, user: User) -> UserRepositoryResult { + with_transaction( + &self.pool, + "create_user", + |tx| { + Box::pin(async move { + // Operación principal - insertar usuario + sqlx::query("INSERT INTO auth.users ...") + .bind(...) + .execute(&mut **tx) + .await?; + + // Operaciones adicionales dentro de la misma transacción + // ... + + Ok(user_clone) + }) + } + ).await +} +``` + +## Use Cases + +### UserPgRepository + +1. **User creation** -- guarantees all insert operations are atomic. Allows related operations (like permission setup) to be bundled. + +2. **User update** -- ensures modifications apply fully or not at all. Supports combined operations like profile info and preference updates. + +### SessionPgRepository + +1. **Session creation** -- inserts the session and updates the user's last-access timestamp in a single transaction. Keeps sessions and user data consistent. + +2. **Session revocation** -- ensures revoking one or all sessions for a user is atomic. Allows logging security events within the same transaction. + +## Isolation Levels + +OxiCloud supports different transaction isolation levels via **with_transaction_isolation**: + +```rust +// Ejemplo de uso con nivel de aislamiento específico +with_transaction_isolation( + &pool, + "operacion_critica", + sqlx::postgres::PgIsolationLevel::Serializable, + |tx| { ... } +).await +``` + +Available isolation levels: + +1. **Read Committed** (default) -- guarantees reads see only committed data. Does not prevent non-repeatable or phantom reads. + +2. **Repeatable Read** -- guarantees consistent reads throughout the transaction. Prevents non-repeatable reads but not phantom reads. + +3. **Serializable** -- highest isolation level. Transactions behave as if executed serially. Can cause serialization errors that require retry. + +## Best Practices + +1. **Transaction duration** -- keep transactions as short as possible. Avoid long-running operations inside them. + +2. **Error handling** -- errors inside a transaction trigger automatic rollback. Use proper logging to diagnose failures. + +3. **Transaction boundaries** -- define clearly where transactions begin and end. Group related operations into a single transaction. + +4. **Appropriate isolation** -- use the lowest isolation level that fits the use case. Consider serializable for critical operations with conflict potential. + +## Benefits + +- **Data integrity** -- ACID guarantees for complex operations, prevents inconsistent states. +- **Error handling** -- automatic rollback on failure, predictable behavior. +- **Safe concurrency** -- proper handling of simultaneous operations, prevents race conditions. +- **Performance** -- fewer round-trips to the database, batch operations for better efficiency. + +## Performance Considerations + +Transactions add some overhead. Performance can be affected by: + +- Transaction duration +- Isolation level +- Number of affected records +- Lock contention diff --git a/doc/DAV-CLIENT-SETUP.md b/doc/dav-client-setup.md similarity index 54% rename from doc/DAV-CLIENT-SETUP.md rename to doc/dav-client-setup.md index e9a63a2e..0318604d 100644 --- a/doc/DAV-CLIENT-SETUP.md +++ b/doc/dav-client-setup.md @@ -1,24 +1,20 @@ -# DAV Client Setup Guide for OxiCloud +# 25 - DAV Client Setup -This document provides detailed instructions for connecting clients to OxiCloud using WebDAV, CalDAV, and CardDAV protocols, enabling seamless integration with your operating system's file browser, calendar, and contacts applications. +Step-by-step instructions for connecting native OS clients via WebDAV, CalDAV, and CardDAV. Covers Windows, macOS, Linux, iOS, and Android. ## Table of Contents - [WebDAV Setup](#webdav-setup) (File Access) -- [CalDAV Setup](#caldav-setup) (Calendar Synchronization) -- [CardDAV Setup](#carddav-setup) (Contact Synchronization) +- [CalDAV Setup](#caldav-setup) (Calendar Sync) +- [CardDAV Setup](#carddav-setup) (Contact Sync) - [Troubleshooting](#troubleshooting) --- ## WebDAV Setup -WebDAV (Web Distributed Authoring and Versioning) is an extension of the HTTP protocol that allows users to collaboratively edit and manage files on remote web servers. - ### Connection Information -Use the following details to connect to OxiCloud via WebDAV: - - **Server URL**: `https://[your-oxicloud-server]/webdav/` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password @@ -32,13 +28,13 @@ Use the following details to connect to OxiCloud via WebDAV: 3. Click "Next" 4. Select "Choose a custom network location" and click "Next" 5. Enter the WebDAV URL: `https://[your-oxicloud-server]/webdav/` -6. When prompted, enter your OxiCloud username and password +6. When prompted, enter your username and password 7. Give the connection a name (e.g., "OxiCloud") and click "Next" 8. Click "Finish" -Your OxiCloud files will now appear as a network drive in File Explorer. +Files now appear as a network drive in File Explorer. -#### Alternative Method: Map Network Drive +#### Alternative: Map Network Drive 1. Open File Explorer 2. Right-click on "This PC" and select "Map network drive" @@ -46,19 +42,19 @@ Your OxiCloud files will now appear as a network drive in File Explorer. 4. Enter the WebDAV URL: `https://[your-oxicloud-server]/webdav/` 5. Check "Connect using different credentials" 6. Click "Finish" -7. Enter your OxiCloud username and password +7. Enter your username and password -**Troubleshooting Windows Connections:** +**Windows Troubleshooting:** -If you experience issues connecting on Windows: +If connections fail on Windows: 1. Open Registry Editor (regedit.exe) 2. Navigate to `HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters` -3. Modify `BasicAuthLevel` to value `2` -4. Restart the WebClient service or restart your computer +3. Modify **BasicAuthLevel** to value `2` +4. Restart the WebClient service or reboot -You may also need to increase the file size limit: -1. In the same registry location, modify `FileSizeLimitInBytes` to a higher value (e.g., `4294967295` for 4GB) +To increase the file size limit: +1. In the same registry location, modify **FileSizeLimitInBytes** to a higher value (e.g., `4294967295` for 4GB) 2. Restart the WebClient service ### macOS @@ -66,39 +62,39 @@ You may also need to increase the file size limit: #### Finder 1. Open Finder -2. From the menu bar, click "Go" > "Connect to Server" (or press ⌘K) +2. From the menu bar, click "Go" > "Connect to Server" (or press Cmd+K) 3. Enter the WebDAV URL: `https://[your-oxicloud-server]/webdav/` 4. Click "Connect" -5. Enter your OxiCloud username and password +5. Enter your username and password 6. Click "Connect" -Your OxiCloud files will now appear as a mounted drive in Finder. +Files appear as a mounted drive in Finder. ### Linux -#### GNOME (Nautilus File Manager) +#### GNOME (Nautilus) 1. Open Files (Nautilus) 2. Click the "+" button in the sidebar or press Ctrl+L -3. Enter the WebDAV URL: `davs://[your-oxicloud-server]/webdav/` -4. Enter your credentials when prompted +3. Enter: `davs://[your-oxicloud-server]/webdav/` +4. Enter credentials when prompted 5. Click "Connect" -#### KDE (Dolphin File Manager) +#### KDE (Dolphin) 1. Open Dolphin 2. In the address bar, enter: `webdavs://[your-oxicloud-server]/webdav/` -3. Enter your credentials when prompted +3. Enter credentials when prompted 4. Click "Connect" #### Command Line (davfs2) -1. Install davfs2: `sudo apt-get install davfs2` (Debian/Ubuntu) or equivalent for your distribution +1. Install davfs2: `sudo apt-get install davfs2` (Debian/Ubuntu) or equivalent 2. Create a mount point: `sudo mkdir /mnt/oxicloud` 3. Edit `/etc/davfs2/secrets` and add: `/mnt/oxicloud [username] [password]` -4. Mount the WebDAV share: `sudo mount -t davfs https://[your-oxicloud-server]/webdav/ /mnt/oxicloud` +4. Mount: `sudo mount -t davfs https://[your-oxicloud-server]/webdav/ /mnt/oxicloud` -To automatically mount at boot, add to `/etc/fstab`: +To auto-mount at boot, add to `/etc/fstab`: ``` https://[your-oxicloud-server]/webdav/ /mnt/oxicloud davfs user,rw,auto 0 0 ``` @@ -107,15 +103,13 @@ https://[your-oxicloud-server]/webdav/ /mnt/oxicloud davfs user,rw,auto 0 0 ## CalDAV Setup -CalDAV is an extension of WebDAV specifically designed for calendar access, allowing you to synchronize calendars between different devices and applications. - ### Apple Calendar (macOS/iOS) #### macOS: 1. Open the Calendar app 2. Go to **Calendar** > **Add Account** > **Other CalDAV Account** -3. Enter the following information: +3. Enter: - **Account Type**: Advanced - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password @@ -127,11 +121,11 @@ CalDAV is an extension of WebDAV specifically designed for calendar access, allo 1. Go to **Settings** > **Calendar** > **Accounts** > **Add Account** > **Other** 2. Tap **Add CalDAV Account** -3. Enter the following information: +3. Enter: - **Server**: `https://[your-oxicloud-server]/caldav` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password - - **Description**: OxiCloud Calendar (or any name you prefer) + - **Description**: OxiCloud Calendar (or any name) 4. Tap **Next** 5. Turn on **Calendars** and tap **Save** @@ -145,14 +139,14 @@ CalDAV is an extension of WebDAV specifically designed for calendar access, allo 6. Click **Next** 7. Enter a name for the calendar and choose a color 8. Click **Next** and then **Finish** -9. When prompted, enter your OxiCloud username and password +9. When prompted, enter your username and password -### Android (DAVx⁵) +### Android (DAVx5) -1. Install [DAVx⁵](https://play.google.com/store/apps/details?id=at.bitfire.davdroid) from Google Play Store -2. Open DAVx⁵ and tap the **+** button +1. Install [DAVx5](https://play.google.com/store/apps/details?id=at.bitfire.davdroid) from Google Play Store +2. Open DAVx5 and tap the **+** button 3. Select **Login with URL and username** -4. Enter the following information: +4. Enter: - **Base URL**: `https://[your-oxicloud-server]/caldav` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password @@ -166,8 +160,8 @@ CalDAV is an extension of WebDAV specifically designed for calendar access, allo 2. Open Outlook and navigate to the **CalDAV Synchronizer** tab 3. Click **Synchronization Profiles** 4. Click **Add** to create a new profile -5. Enter the following information: - - **Profile Name**: OxiCloud Calendar (or any name you prefer) +5. Enter: + - **Profile Name**: OxiCloud Calendar (or any name) - **CalDAV URL**: `https://[your-oxicloud-server]/caldav/calendars/your-calendar-id` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password @@ -179,8 +173,6 @@ CalDAV is an extension of WebDAV specifically designed for calendar access, allo ## CardDAV Setup -CardDAV is an extension of WebDAV for address book access, allowing you to synchronize contacts between different devices and applications. - ### Apple Contacts (macOS/iOS) #### macOS: @@ -188,22 +180,22 @@ CardDAV is an extension of WebDAV for address book access, allowing you to synch 1. Open the Contacts app 2. Go to **Contacts** > **Add Account** > **Other contacts account** 3. Select **CardDAV account** -4. Enter the following information: +4. Enter: - **Server**: `https://[your-oxicloud-server]/carddav` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password - - **Description**: OxiCloud Contacts (or any name you prefer) + - **Description**: OxiCloud Contacts (or any name) 5. Click **Sign In** #### iOS: 1. Go to **Settings** > **Contacts** > **Accounts** > **Add Account** > **Other** 2. Tap **Add CardDAV Account** -3. Enter the following information: +3. Enter: - **Server**: `https://[your-oxicloud-server]/carddav` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password - - **Description**: OxiCloud Contacts (or any name you prefer) + - **Description**: OxiCloud Contacts (or any name) 4. Tap **Next** 5. Turn on **Contacts** and tap **Save** @@ -212,18 +204,18 @@ CardDAV is an extension of WebDAV for address book access, allowing you to synch 1. Open Thunderbird and go to the **Address Book** 2. Click on **Tools** > **Address Book** 3. Go to **File** > **New** > **Remote Address Book** -4. Enter the following information: - - **Name**: OxiCloud Contacts (or any name you prefer) +4. Enter: + - **Name**: OxiCloud Contacts (or any name) - **URL**: `https://[your-oxicloud-server]/carddav/address-books/your-address-book-id` 5. Click **OK** -6. When prompted, enter your OxiCloud username and password +6. When prompted, enter your username and password -### Android (DAVx⁵) +### Android (DAVx5) -1. Install [DAVx⁵](https://play.google.com/store/apps/details?id=at.bitfire.davdroid) from Google Play Store -2. Open DAVx⁵ and tap the **+** button +1. Install [DAVx5](https://play.google.com/store/apps/details?id=at.bitfire.davdroid) from Google Play Store +2. Open DAVx5 and tap the **+** button 3. Select **Login with URL and username** -4. Enter the following information: +4. Enter: - **Base URL**: `https://[your-oxicloud-server]/carddav` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password @@ -233,13 +225,13 @@ CardDAV is an extension of WebDAV for address book access, allowing you to synch ### Windows (Outlook) -1. Download and install [CardDAV Synchronizer](https://caldavsynchronizer.org/) (same tool as for CalDAV) +1. Download and install [CardDAV Synchronizer](https://caldavsynchronizer.org/) (same tool as CalDAV) 2. Open Outlook and navigate to the **CardDAV Synchronizer** tab 3. Click **Synchronization Profiles** 4. Click **Add** to create a new profile 5. Select **CardDAV** as the synchronization resource -6. Enter the following information: - - **Profile Name**: OxiCloud Contacts (or any name you prefer) +6. Enter: + - **Profile Name**: OxiCloud Contacts (or any name) - **CardDAV URL**: `https://[your-oxicloud-server]/carddav/address-books/your-address-book-id` - **Username**: Your OxiCloud username - **Password**: Your OxiCloud password @@ -255,70 +247,63 @@ CardDAV is an extension of WebDAV for address book access, allowing you to synch #### WebDAV Connection Issues -- Verify the server URL is correct and includes the `/webdav/` path -- Ensure your username and password are entered correctly -- Check if your network blocks WebDAV connections (ports 80/443) -- Verify that your OxiCloud server has WebDAV enabled +- Verify the server URL includes the `/webdav/` path +- Double-check username and password +- Check if your network blocks ports 80/443 +- Confirm WebDAV is enabled on the server #### Calendar/Contact Sync Issues -- Verify the server URL is correct and includes the full path (`/caldav` or `/carddav`) -- Check that your OxiCloud server is accessible from your network -- Verify that your username and password are correct -- Check that the calendar or address book ID is correct -- Verify you have proper permissions to access the resource +- Verify the server URL includes the full path (`/caldav` or `/carddav`) +- Confirm the server is reachable from your network +- Verify the calendar or address book ID is correct +- Check you have proper permissions for the resource #### Calendar Not Showing -- Verify the calendar is enabled in your client +- Confirm the calendar is enabled in your client - Check if the calendar is shared with your account -- Ensure your client supports the CalDAV protocol version +- Ensure your client supports the CalDAV protocol version in use #### Contact Photos Not Syncing -- Some clients have limitations with contact photo syncing +- Some clients have photo sync limitations - Verify the photo is in a supported format (usually JPEG) -- Check the size of the photo (some clients limit photo size) +- Check photo size limits on the client side ### Client-Specific Issues #### Windows File Explorer -For WebDAV issues on Windows: -- Make sure the WebClient service is running +- Make sure the **WebClient** service is running - Increase timeout values in the registry -- Try using a third-party WebDAV client like Cyberduck +- Try a third-party WebDAV client like Cyberduck #### iOS Devices -- If you're having trouble connecting, try going to **Settings** > **Accounts & Passwords** and manually add the account from there -- For persistent issues, remove the account and add it again +- If connection fails, try **Settings** > **Accounts & Passwords** and add the account manually +- For persistent issues, remove the account and re-add it #### Android -- DAVx⁵ requires battery optimization to be disabled for reliable background sync -- Go to **Settings** > **Apps** > **DAVx⁵** > **Battery** > **Unrestricted** +- DAVx5 requires battery optimization to be disabled for reliable background sync +- Go to **Settings** > **Apps** > **DAVx5** > **Battery** > **Unrestricted** #### Outlook - Make sure you have the latest version of CalDAV/CardDAV Synchronizer -- The plugin might need to be reactivated after Outlook updates +- The plugin may need reactivation after Outlook updates -### Performance Considerations +### Performance Tips -For optimal performance when using WebDAV: - -1. **Large Files**: When working with files larger than 100MB, consider downloading them locally before editing -2. **Slow Connections**: Enable offline caching in your client when available -3. **File Locking**: Some clients support WebDAV locking to prevent conflicts +1. **Large Files** -- for files over 100MB, download locally before editing +2. **Slow Connections** -- enable offline caching in your client when available +3. **File Locking** -- some clients support WebDAV locking to prevent conflicts ### Getting Help -If you continue to experience issues, please: +If issues persist: -1. Check the OxiCloud logs for error messages -2. Capture screenshots of the error messages -3. Contact support with details about: - - Your client application and version - - Steps to reproduce the issue - - Any error messages displayed \ No newline at end of file +1. Check the server logs for error messages +2. Capture screenshots of any errors +3. Contact support with details about your client application, version, steps to reproduce, and error messages diff --git a/doc/dav-implementation-plan.md b/doc/dav-implementation-plan.md new file mode 100644 index 00000000..1ecbbebd --- /dev/null +++ b/doc/dav-implementation-plan.md @@ -0,0 +1,286 @@ +# 24 - DAV Implementation Plan + +A phased plan for adding WebDAV, CalDAV, and CardDAV support. The order is WebDAV first (file access), then CalDAV (calendars), then CardDAV (contacts). Each phase builds on the shared infrastructure from phase 1. + +## Executive Summary + +DAV protocol support lets the platform interoperate with a wide range of clients and devices. The plan uses an incremental approach -- common infrastructure first, then each protocol in turn. + +## Implementation Phases + +### Phase 1: Common DAV Infrastructure (Est. 2-3 weeks) + +**Goals:** +- Build the shared infrastructure used by all DAV protocols +- Implement XML request/response handling +- Create adapters for basic DAV operations + +**Tasks:** +1. **Week 1: Design and Architecture** + - Design DAV component architecture + - Define interfaces for DAV adapters + - Select libraries for XML processing and RFC 4918 support + +2. **Week 2: Base Implementation** + - Implement XML serialization/deserialization handlers + - Develop middleware for DAV request processing + - Create shared structures (properties, namespaces) + - Implement DAV request validation + +3. **Week 3: Test Framework** + - Set up test environment for DAV protocols + - Implement automated test clients + - Create test cases for basic DAV operations + +**Deliverables:** +- XML processing framework for DAV requests/responses +- Base adapters for existing entities +- Test suite for DAV operations + +### Phase 2: WebDAV (Est. 3-4 weeks) + +**Goals:** +- Implement the full WebDAV protocol (RFC 4918) +- Enable file and folder access via WebDAV +- Ensure compatibility with common WebDAV clients + +**Tasks:** +1. **Week 1: Basic Operations** + - Implement **PROPFIND** and **PROPPATCH** methods + - Develop **OPTIONS** endpoint (capability discovery) + - Implement **GET**, **HEAD**, **PUT** (read/write) + +2. **Week 2: Advanced Operations** + - Implement **MKCOL** (directory creation) + - Develop **DELETE** for WebDAV resources + - Implement **COPY** and **MOVE** for files and directories + +3. **Week 3: Locking and Extended Features** + - Implement **LOCK** and **UNLOCK** for resources + - Add support for custom properties + - Develop extended WebDAV features as needed + +4. **Week 4: Testing and Optimization** + - Test with real clients (Windows, macOS, Linux) + - Optimize performance for large transfers + - Document WebDAV API and behavior + +**Deliverables:** +- Full WebDAV implementation (RFC 4918) +- Usage documentation +- Compatibility with the most common WebDAV clients + +### Phase 3: CalDAV (Est. 4-5 weeks) + +**Goals:** +- Implement the CalDAV protocol (RFC 4791) +- Create entities and repositories for calendars and events +- Support calendar operations with common clients + +**Tasks:** +1. **Week 1: Data Model** + - Implement **Calendar** and **CalendarEvent** entities + - Develop storage repositories + - Create CalDAV DTOs and adapters + +2. **Week 2: Basic Endpoints** + - Implement **PROPFIND** for calendar discovery + - Develop **MKCALENDAR** for calendar creation + - Implement **GET**/**PUT** for individual events + +3. **Week 3: Advanced Queries** + - Implement **REPORT** for calendar queries + - Develop date-range search support + - Add recurrence handling (**RRULE** rules) + +4. **Week 4: Interoperability** + - Implement efficient sync (**collection-sync**) + - Add timezone support + - Develop alarm and notification handling + +5. **Week 5: Testing and Refinement** + - Test with popular CalDAV clients + - Optimize performance for large calendars + - Document CalDAV API and behavior + +**Deliverables:** +- Full CalDAV implementation (RFC 4791) +- Calendar creation and management support +- Compatibility with popular CalDAV clients +- CalDAV usage documentation + +### Phase 4: CardDAV (Est. 3-4 weeks) + +**Goals:** +- Implement the CardDAV protocol (RFC 6352) +- Create entities and repositories for address books and contacts +- Support contact operations with common clients + +**Tasks:** +1. **Week 1: Data Model** + - Implement **AddressBook** and **Contact** entities + - Develop storage repositories + - Create CardDAV DTOs and adapters + +2. **Week 2: Basic Endpoints** + - Implement **PROPFIND** for address book discovery + - Develop **MKCOL** for address book creation + - Implement **GET**/**PUT** for individual contacts + +3. **Week 3: Queries and Search** + - Implement **REPORT** for contact queries + - Develop criteria-based contact search + - Add support for contact groups + +4. **Week 4: Testing and Refinement** + - Test with popular CardDAV clients + - Optimize performance for large address books + - Document CardDAV API and behavior + +**Deliverables:** +- Full CardDAV implementation (RFC 6352) +- Address book creation and management support +- Compatibility with popular CardDAV clients +- CardDAV usage documentation + +### Phase 5: Integration and Release (Est. 2-3 weeks) + +**Goals:** +- Integrate all DAV protocols into a cohesive solution +- Ensure cross-protocol compatibility +- Prepare documentation and release materials + +**Tasks:** +1. **Week 1: Integration** + - Consolidate shared code across protocols + - Ensure behavioral consistency + - Refine error handling and recovery + +2. **Week 2: System Testing** + - Run end-to-end integration tests + - Validate performance under load + - Verify security and permissions + +3. **Week 3: Documentation and Release** + - Finalize user guides for DAV clients + - Create developer documentation + - Prepare release package + +**Deliverables:** +- Complete, integrated DAV solution +- User and developer documentation +- Release-ready deployment package + +## Infrastructure Requirements + +### Library Dependencies + +```toml +# Add to Cargo.toml +[dependencies] +# XML processing +quick-xml = "0.30.0" +xml-rs = "0.8.14" + +# iCalendar support +icalendar = "0.15.0" + +# vCard support +vcard = "0.2.0" + +# DAV utilities +http-multipart = "0.3.0" +``` + +### Database Schema + +New tables for CalDAV and CardDAV are created during their respective phases. See `dav-integration.md` for the full schema. + +## Testing Strategy + +### Unit Tests + +- XML serialization/deserialization tests +- Input validation tests +- Business logic tests for each DAV operation + +### Integration Tests + +- End-to-end tests with simulated clients +- Full-flow tests (create, update, delete) +- Concurrency and conflict handling tests + +### Compatibility Tests + +- Test matrix with real clients (at least 3 per protocol) +- Testing across different operating systems +- RFC conformance verification + +## Performance Considerations + +1. **Query Optimization** + - Implement pagination for large result sets + - Optimize SQL queries for calendars and contacts + - Use proper indexes for fast lookups + +2. **Caching** + - Cache properties for **PROPFIND** responses + - Use ETags for cache validation + - Apply query caching for frequent reports + +3. **Efficient Processing** + - Efficient XML processing for large requests + - Data streaming for large files + - Async processing for expensive operations + +## Risks and Mitigation + +| Risk | Impact | Likelihood | Mitigation Strategy | +|------|--------|------------|---------------------| +| Client compatibility issues | High | Medium | Early testing with a variety of clients, strict adherence to specs | +| Insufficient performance | Medium | Low | Load testing from the start, design for scalability | +| Excessive complexity | Medium | Medium | Modular approach, clear abstractions, frequent code reviews | +| Security vulnerabilities | High | Low | Security reviews, strict input validation, penetration testing | +| Schedule delays | Medium | Medium | Conservative planning, clear milestones, iterative approach | + +## Success Criteria + +1. **Compatibility** + - All protocols comply with their respective RFCs + - Verified compatibility with at least 3 major clients per protocol + - Works on all major operating systems + +2. **Performance** + - Typical operation response time < 500ms + - Supports calendars with 1000+ events without significant degradation + - Supports address books with 1000+ contacts without significant degradation + +3. **Usability** + - Simple, documented client setup process + - Clear and specific error messages + - Full documentation for users and developers + +## Required Resources + +1. **Development Team** + - 1-2 backend developers (Rust) + - 1 frontend developer (for UI integration if needed) + - 1 tester + +2. **Infrastructure** + - Multi-OS test environment + - Assorted DAV clients for testing + - CI/CD server for automated tests + +3. **Skills** + - Experience with advanced HTTP protocols + - XML processing knowledge + - Familiarity with WebDAV, CalDAV, and CardDAV standards + +## Next Steps + +1. Assign resources to the project +2. Set up code repository and initial structure +3. Start Phase 1 (Common DAV Infrastructure) +4. Configure CI/CD environment for testing +5. Review and refine the plan as needed during implementation diff --git a/doc/DAV-INTEGRATION.md b/doc/dav-integration.md similarity index 58% rename from doc/DAV-INTEGRATION.md rename to doc/dav-integration.md index e4d5959d..e5717934 100644 --- a/doc/DAV-INTEGRATION.md +++ b/doc/dav-integration.md @@ -1,31 +1,31 @@ -# Integración de WebDAV, CalDAV y CardDAV en OxiCloud +# 23 - DAV Integration -Este documento describe el diseño e implementación de los protocolos WebDAV, CalDAV y CardDAV en OxiCloud, extendiendo la plataforma para soportar clientes y dispositivos que utilizan estos estándares. +WebDAV, CalDAV, and CardDAV extend the platform to support clients and devices that speak these standard protocols. The implementation follows the existing hexagonal architecture -- each protocol gets its own adapter, service, and domain layer. -## Tabla de Contenidos +## Table of Contents -1. [Introducción](#introducción) -2. [Arquitectura de la Implementación](#arquitectura-de-la-implementación) +1. [Introduction](#introduction) +2. [Implementation Architecture](#implementation-architecture) 3. [WebDAV](#webdav) 4. [CalDAV](#caldav) 5. [CardDAV](#carddav) -6. [Consideraciones de Seguridad](#consideraciones-de-seguridad) -7. [Pruebas y Compatibilidad](#pruebas-y-compatibilidad) +6. [Security Considerations](#security-considerations) +7. [Testing and Compatibility](#testing-and-compatibility) -## Introducción +## Introduction ### WebDAV (Web Distributed Authoring and Versioning) -WebDAV es una extensión del protocolo HTTP que permite a los clientes realizar operaciones sobre archivos en un servidor remoto, como crear, modificar, mover y eliminar archivos y directorios. +An HTTP extension that lets clients create, modify, move, and delete files and directories on a remote server. ### CalDAV (Calendaring Extensions to WebDAV) -CalDAV es un protocolo basado en WebDAV que permite a los clientes acceder y gestionar datos de calendario, como eventos y tareas. +A WebDAV-based protocol for accessing and managing calendar data (events and tasks). ### CardDAV (vCard Extensions to WebDAV) -CardDAV es un protocolo que extiende WebDAV para permitir el acceso y gestión de datos de contactos en formato vCard. +Extends WebDAV to allow access and management of contact data in vCard format. -## Arquitectura de la Implementación +## Implementation Architecture -La implementación de los protocolos DAV se integra en la arquitectura hexagonal existente de OxiCloud: +DAV protocols plug into the existing hexagonal architecture: ``` ┌────────────────────────────────────────────────────────────────────┐ @@ -38,22 +38,22 @@ La implementación de los protocolos DAV se integra en la arquitectura hexagonal │ └───────┬───────┘ └───────┬───────┘ └───────────┬───────────┘ │ │ │ │ │ │ └──────────┼──────────────────┼──────────────────────┼──────────────┘ - │ │ │ - ▼ ▼ ▼ + │ │ │ + ▼ ▼ ▼ ┌──────────────────────────────────────────────────────────────────┐ -│ APLICACIÓN │ +│ APPLICATION │ │ │ │ ┌───────────┐ ┌────────────┐ ┌───────────┐ ┌──────────────┐ │ │ │ │ │ │ │ │ │ │ │ -│ │FileService│ │FolderService│ │CalService │ │ContactService│ │ +│ │FileService│ │FolderService│ │CalendarSvc│ │ContactService│ │ │ │ │ │ │ │ │ │ │ │ │ └─────┬─────┘ └──────┬─────┘ └─────┬─────┘ └──────┬───────┘ │ │ │ │ │ │ │ └────────┼───────────────┼──────────────┼───────────────┼─────────┘ - │ │ │ │ - ▼ ▼ ▼ ▼ + │ │ │ │ + ▼ ▼ ▼ ▼ ┌────────────────────────────────────────────────────────────────┐ -│ DOMINIO │ +│ DOMAIN │ │ │ │ ┌─────────┐ ┌──────────┐ ┌────────────┐ ┌───────────────┐ │ │ │ │ │ │ │ │ │ │ │ @@ -64,41 +64,41 @@ La implementación de los protocolos DAV se integra en la arquitectura hexagonal └────────────────────────────────────────────────────────────────┘ ``` -### Componentes Principales +### Main Components -1. **Adaptadores DAV**: Convertirán entre las especificaciones DAV y los modelos de OxiCloud -2. **Servicios de Aplicación**: Se extenderán para incluir funcionalidades específicas DAV -3. **Modelos de Dominio**: Se añadirán nuevas entidades para Calendar y Contact -4. **Repositorios**: Implementaciones de almacenamiento para calendarios y contactos +1. **DAV Adapters** -- convert between DAV specs and internal models +2. **Application Services** -- extended to include DAV-specific functionality +3. **Domain Models** -- new entities for **Calendar** and **Contact** +4. **Repositories** -- storage implementations for calendars and contacts ## WebDAV -### Endpoints Requeridos +### Required Endpoints -| Método HTTP | Endpoint | Descripción | +| HTTP Method | Endpoint | Description | |-------------|----------|-------------| -| OPTIONS | /webdav/{path} | Indica las capacidades WebDAV soportadas | -| PROPFIND | /webdav/{path} | Recupera propiedades de recursos | -| PROPPATCH | /webdav/{path} | Modifica propiedades de recursos | -| MKCOL | /webdav/{path} | Crea colecciones (directorios) | -| GET | /webdav/{path} | Recupera contenido de recursos | -| HEAD | /webdav/{path} | Recupera metadatos de recursos | -| PUT | /webdav/{path} | Crea o actualiza recursos | -| DELETE | /webdav/{path} | Elimina recursos | -| COPY | /webdav/{path} | Copia recursos | -| MOVE | /webdav/{path} | Mueve recursos | -| LOCK | /webdav/{path} | Bloquea recursos | -| UNLOCK | /webdav/{path} | Desbloquea recursos | +| OPTIONS | /webdav/{path} | Reports supported WebDAV capabilities | +| PROPFIND | /webdav/{path} | Retrieves resource properties | +| PROPPATCH | /webdav/{path} | Modifies resource properties | +| MKCOL | /webdav/{path} | Creates collections (directories) | +| GET | /webdav/{path} | Retrieves resource content | +| HEAD | /webdav/{path} | Retrieves resource metadata | +| PUT | /webdav/{path} | Creates or updates resources | +| DELETE | /webdav/{path} | Deletes resources | +| COPY | /webdav/{path} | Copies resources | +| MOVE | /webdav/{path} | Moves resources | +| LOCK | /webdav/{path} | Locks resources | +| UNLOCK | /webdav/{path} | Unlocks resources | -### Implementación +### Implementation -1. **Manejador WebDAV**: +1. **WebDAV Handler**: ```rust // src/interfaces/api/handlers/webdav_handler.rs use std::sync::Arc; use axum::{ - Router, + Router, routing::get, extract::{Path, State, Request, Extension}, http::StatusCode, @@ -129,10 +129,10 @@ pub fn webdav_routes() -> Router> { )) } -// Implementar funciones para cada método WebDAV... +// Implement functions for each WebDAV method... ``` -2. **Adaptador WebDAV**: +2. **WebDAV Adapter**: ```rust // src/application/adapters/webdav_adapter.rs @@ -142,45 +142,45 @@ use std::io::{Read, Write}; use crate::application::dtos::file_dto::FileDto; use crate::application::dtos::folder_dto::FolderDto; -/// Convierte entre objetos de OxiCloud y representaciones WebDAV +/// Converts between internal objects and WebDAV representations pub struct WebDavAdapter; impl WebDavAdapter { - /// Convierte una propiedad PROPFIND en XML a un objeto de solicitud + /// Parses a PROPFIND XML property into a request object pub fn parse_propfind(reader: R) -> Result { - // Implementación... + // Implementation... } - - /// Genera respuesta XML para PROPFIND basada en archivos y carpetas + + /// Generates PROPFIND XML response from files and folders pub fn generate_propfind_response( writer: W, files: &[FileDto], folders: &[FolderDto], base_url: &str, ) -> Result<(), Error> { - // Implementación... + // Implementation... } - - // Otros métodos para manejar diferentes operaciones WebDAV... + + // Other methods for different WebDAV operations... } ``` ## CalDAV -### Endpoints Requeridos +### Required Endpoints -| Método HTTP | Endpoint | Descripción | +| HTTP Method | Endpoint | Description | |-------------|----------|-------------| -| PROPFIND | /caldav/{calendar} | Recupera propiedades del calendario | -| REPORT | /caldav/{calendar} | Consulta eventos del calendario | -| MKCALENDAR | /caldav/{calendar} | Crea un nuevo calendario | -| PUT | /caldav/{calendar}/{event}.ics | Crea o actualiza un evento | -| GET | /caldav/{calendar}/{event}.ics | Recupera un evento | -| DELETE | /caldav/{calendar}/{event}.ics | Elimina un evento | +| PROPFIND | /caldav/{calendar} | Retrieves calendar properties | +| REPORT | /caldav/{calendar} | Queries calendar events | +| MKCALENDAR | /caldav/{calendar} | Creates a new calendar | +| PUT | /caldav/{calendar}/{event}.ics | Creates or updates an event | +| GET | /caldav/{calendar}/{event}.ics | Retrieves an event | +| DELETE | /caldav/{calendar}/{event}.ics | Deletes an event | -### Implementación +### Implementation -1. **Nuevas Entidades de Dominio**: +1. **Domain Entities**: ```rust // src/domain/entities/calendar.rs @@ -212,14 +212,14 @@ pub struct CalendarEvent { start_time: DateTime, end_time: DateTime, all_day: bool, - rrule: Option, // Regla de recurrencia - ical_data: String, // Datos iCalendar completos + rrule: Option, // Recurrence rule + ical_data: String, // Full iCalendar data created_at: DateTime, updated_at: DateTime, } ``` -2. **Repositorios**: +2. **Repositories**: ```rust // src/domain/repositories/calendar_repository.rs @@ -260,47 +260,34 @@ pub trait CalendarEventRepository: Send + Sync { } ``` -3. **Servicio CalDAV**: +3. **CalDAV Service**: ```rust -// src/application/services/caldav_service.rs +// src/application/services/calendar_service.rs use std::sync::Arc; -use uuid::Uuid; -use chrono::{DateTime, Utc}; -use crate::domain::repositories::calendar_repository::CalendarRepository; -use crate::domain::repositories::calendar_event_repository::CalendarEventRepository; -use crate::domain::entities::calendar::Calendar; -use crate::domain::entities::calendar_event::CalendarEvent; -use crate::application::dtos::calendar_dto::{CalendarDto, CalendarEventDto}; -use crate::common::errors::{Result, DomainError}; +use crate::application::ports::calendar_ports::CalendarStoragePort; +use crate::application::dtos::calendar_dto::*; -pub struct CalDavService { - calendar_repository: Arc, - event_repository: Arc, +pub struct CalendarService { + storage: Arc, } -impl CalDavService { - pub fn new( - calendar_repository: Arc, - event_repository: Arc, - ) -> Self { - Self { - calendar_repository, - event_repository, - } +impl CalendarService { + pub fn new(storage: Arc) -> Self { + Self { storage } } - - // Implementar métodos para operaciones CalDAV... + + // Implements CalendarUseCase for calendar and event operations... } ``` -4. **Manejador CalDAV**: +4. **CalDAV Handler**: ```rust // src/interfaces/api/handlers/caldav_handler.rs use std::sync::Arc; use axum::{ - Router, + Router, routing::{get, put, delete}, extract::{Path, State, Request, Extension}, http::StatusCode, @@ -327,30 +314,28 @@ pub fn caldav_routes() -> Router> { .route("/caldav/:calendar/:event", delete(delete_event)) } -// Implementar funciones para cada método CalDAV... +// Implement functions for each CalDAV method... ``` ## CardDAV -### Endpoints Requeridos +### Required Endpoints -| Método HTTP | Endpoint | Descripción | +| HTTP Method | Endpoint | Description | |-------------|----------|-------------| -| PROPFIND | /carddav/addressbooks/{addressbook} | Recupera propiedades de la libreta de direcciones | -| REPORT | /carddav/addressbooks/{addressbook} | Consulta contactos | -| MKCOL | /carddav/addressbooks/{addressbook} | Crea una nueva libreta de direcciones | -| PUT | /carddav/addressbooks/{addressbook}/{contact}.vcf | Crea o actualiza un contacto | -| GET | /carddav/addressbooks/{addressbook}/{contact}.vcf | Recupera un contacto | -| DELETE | /carddav/addressbooks/{addressbook}/{contact}.vcf | Elimina un contacto | +| PROPFIND | /carddav/addressbooks/{addressbook} | Retrieves address book properties | +| REPORT | /carddav/addressbooks/{addressbook} | Queries contacts | +| MKCOL | /carddav/addressbooks/{addressbook} | Creates a new address book | +| PUT | /carddav/addressbooks/{addressbook}/{contact}.vcf | Creates or updates a contact | +| GET | /carddav/addressbooks/{addressbook}/{contact}.vcf | Retrieves a contact | +| DELETE | /carddav/addressbooks/{addressbook}/{contact}.vcf | Deletes a contact | -### Implementación +### Implementation -1. **Nuevas Entidades de Dominio**: +1. **Domain Entities**: ```rust -// src/domain/entities/address_book.rs -use uuid::Uuid; -use chrono::{DateTime, Utc}; +// Note: AddressBook and Contact are both defined in src/domain/entities/contact.rs #[derive(Debug, Clone)] pub struct AddressBook { @@ -358,32 +343,38 @@ pub struct AddressBook { name: String, owner_id: String, description: Option, + color: Option, + is_public: bool, created_at: DateTime, updated_at: DateTime, } -// src/domain/entities/contact.rs -use uuid::Uuid; -use chrono::{DateTime, Utc}; - #[derive(Debug, Clone)] pub struct Contact { id: Uuid, address_book_id: Uuid, - full_name: String, + uid: String, + full_name: Option, first_name: Option, last_name: Option, - email: Option, - phone: Option, - address: Option, + nickname: Option, + email: Vec, // Struct with email, type, is_primary + phone: Vec, // Struct with number, type, is_primary + address: Vec
, // Struct with street, city, state, postal_code, country, type, is_primary organization: Option, - vcard_data: String, // Datos vCard completos + title: Option, + notes: Option, + photo_url: Option, + birthday: Option, + anniversary: Option, + vcard: String, // Full vCard data + etag: String, created_at: DateTime, updated_at: DateTime, } ``` -2. **Repositorios**: +2. **Repositories**: ```rust // src/domain/repositories/address_book_repository.rs @@ -418,46 +409,31 @@ pub trait ContactRepository: Send + Sync { } ``` -3. **Servicio CardDAV**: +3. **CardDAV Service**: ```rust -// src/application/services/carddav_service.rs +// src/application/services/contact_service.rs use std::sync::Arc; -use uuid::Uuid; -use crate::domain::repositories::address_book_repository::AddressBookRepository; -use crate::domain::repositories::contact_repository::ContactRepository; -use crate::domain::entities::address_book::AddressBook; -use crate::domain::entities::contact::Contact; -use crate::application::dtos::address_book_dto::{AddressBookDto, ContactDto}; -use crate::common::errors::{Result, DomainError}; +use crate::application::dtos::contact_dto::*; +use crate::application::dtos::address_book_dto::*; -pub struct CardDavService { - address_book_repository: Arc, - contact_repository: Arc, +pub struct ContactService { + // Implements AddressBookUseCase and ContactUseCase + // Uses ContactStorageAdapter as infrastructure } -impl CardDavService { - pub fn new( - address_book_repository: Arc, - contact_repository: Arc, - ) -> Self { - Self { - address_book_repository, - contact_repository, - } - } - - // Implementar métodos para operaciones CardDAV... +impl ContactService { + // Implements methods for CardDAV operations... } ``` -4. **Manejador CardDAV**: +4. **CardDAV Handler**: ```rust // src/interfaces/api/handlers/carddav_handler.rs use std::sync::Arc; use axum::{ - Router, + Router, routing::{get, put, delete}, extract::{Path, State, Request, Extension}, http::StatusCode, @@ -484,26 +460,30 @@ pub fn carddav_routes() -> Router> { .route("/carddav/addressbooks/:addressbook/:contact", delete(delete_contact)) } -// Implementar funciones para cada método CardDAV... +// Implement functions for each CardDAV method... ``` -## Esquema de Base de Datos +## Database Schema ```sql --- Esquema para CalDAV -CREATE TABLE calendar ( +-- CalDAV schema +CREATE SCHEMA IF NOT EXISTS caldav; + +CREATE TABLE IF NOT EXISTS caldav.calendars ( id UUID PRIMARY KEY, name VARCHAR(255) NOT NULL, - owner_id VARCHAR(255) NOT NULL, + owner_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, description TEXT, color VARCHAR(50), + is_public BOOLEAN NOT NULL DEFAULT FALSE, + ctag VARCHAR(255), created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); -CREATE TABLE calendar_event ( +CREATE TABLE IF NOT EXISTS caldav.calendar_events ( id UUID PRIMARY KEY, - calendar_id UUID NOT NULL REFERENCES calendar(id) ON DELETE CASCADE, + calendar_id UUID NOT NULL REFERENCES caldav.calendars(id) ON DELETE CASCADE, summary VARCHAR(255) NOT NULL, description TEXT, location TEXT, @@ -513,91 +493,145 @@ CREATE TABLE calendar_event ( rrule TEXT, ical_uid VARCHAR(255) NOT NULL, ical_data TEXT NOT NULL, + etag VARCHAR(255), created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); --- Esquema para CardDAV -CREATE TABLE address_book ( +CREATE TABLE IF NOT EXISTS caldav.calendar_shares ( + id SERIAL PRIMARY KEY, + calendar_id UUID NOT NULL REFERENCES caldav.calendars(id) ON DELETE CASCADE, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + access_level VARCHAR(20) NOT NULL DEFAULT 'read', + UNIQUE(calendar_id, user_id) +); + +CREATE TABLE IF NOT EXISTS caldav.calendar_properties ( + id SERIAL PRIMARY KEY, + calendar_id UUID NOT NULL REFERENCES caldav.calendars(id) ON DELETE CASCADE, + property_name VARCHAR(255) NOT NULL, + property_value TEXT, + UNIQUE(calendar_id, property_name) +); + +-- CardDAV schema +CREATE SCHEMA IF NOT EXISTS carddav; + +CREATE TABLE IF NOT EXISTS carddav.address_books ( id UUID PRIMARY KEY, name VARCHAR(255) NOT NULL, - owner_id VARCHAR(255) NOT NULL, + owner_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, description TEXT, + color VARCHAR(50), + is_public BOOLEAN NOT NULL DEFAULT FALSE, + ctag VARCHAR(255), created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(owner_id, name) ); -CREATE TABLE contact ( +CREATE TABLE IF NOT EXISTS carddav.contacts ( id UUID PRIMARY KEY, - address_book_id UUID NOT NULL REFERENCES address_book(id) ON DELETE CASCADE, - full_name VARCHAR(255) NOT NULL, + address_book_id UUID NOT NULL REFERENCES carddav.address_books(id) ON DELETE CASCADE, + uid VARCHAR(255) NOT NULL, + full_name VARCHAR(255), first_name VARCHAR(255), last_name VARCHAR(255), - email VARCHAR(255), - phone VARCHAR(100), - address TEXT, + nickname VARCHAR(255), organization VARCHAR(255), - vcard_uid VARCHAR(255) NOT NULL, - vcard_data TEXT NOT NULL, + title VARCHAR(255), + notes TEXT, + photo_url TEXT, + birthday DATE, + anniversary DATE, + email JSONB, + phone JSONB, + address JSONB, + vcard TEXT NOT NULL, + etag VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(address_book_id, uid) +); + +CREATE TABLE IF NOT EXISTS carddav.address_book_shares ( + id SERIAL PRIMARY KEY, + address_book_id UUID NOT NULL REFERENCES carddav.address_books(id) ON DELETE CASCADE, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + can_write BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE(address_book_id, user_id) +); + +CREATE TABLE IF NOT EXISTS carddav.contact_groups ( + id UUID PRIMARY KEY, + address_book_id UUID NOT NULL REFERENCES carddav.address_books(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); --- Índices para búsqueda eficiente -CREATE INDEX idx_calendar_owner ON calendar(owner_id); -CREATE INDEX idx_calendar_event_calendar ON calendar_event(calendar_id); -CREATE INDEX idx_address_book_owner ON address_book(owner_id); -CREATE INDEX idx_contact_address_book ON contact(address_book_id); -CREATE INDEX idx_contact_name ON contact(full_name); +CREATE TABLE IF NOT EXISTS carddav.group_memberships ( + id SERIAL PRIMARY KEY, + group_id UUID NOT NULL REFERENCES carddav.contact_groups(id) ON DELETE CASCADE, + contact_id UUID NOT NULL REFERENCES carddav.contacts(id) ON DELETE CASCADE, + UNIQUE(group_id, contact_id) +); + +-- Indexes for efficient lookup +CREATE INDEX IF NOT EXISTS idx_calendars_owner ON caldav.calendars(owner_id); +CREATE INDEX IF NOT EXISTS idx_calendar_events_calendar ON caldav.calendar_events(calendar_id); +CREATE INDEX IF NOT EXISTS idx_address_books_owner ON carddav.address_books(owner_id); +CREATE INDEX IF NOT EXISTS idx_contacts_address_book ON carddav.contacts(address_book_id); +CREATE INDEX IF NOT EXISTS idx_contacts_full_name ON carddav.contacts(full_name); ``` -## Consideraciones de Seguridad +## Security Considerations -1. **Autenticación** - - Utilizar la autenticación existente de OxiCloud - - Soportar autenticación HTTP Basic para clientes DAV - - Implementar el esquema de autenticación Digest si es necesario +1. **Authentication** + - Uses existing authentication + - Supports HTTP Basic Authentication for DAV clients + - Digest authentication can be added if needed -2. **Autorización** - - Verificar permisos de usuario para acceder a recursos - - Implementar control de acceso basado en propietario y permisos compartidos - - Asegurar que los usuarios solo puedan acceder a sus propios calendarios y libretas de direcciones +2. **Authorization** + - Verify user permissions before granting resource access + - Owner-based and shared-permission access control + - Users can only access their own calendars and address books -3. **Prevención de Ataques** - - Validar y sanitizar todas las entradas XML - - Limitar tamaño máximo de carga útil - - Implementar rate limiting en endpoints DAV +3. **Attack Prevention** + - Validate and sanitize all XML input + - Limit maximum payload size + - Rate limiting on DAV endpoints -## Pruebas y Compatibilidad +## Testing and Compatibility -### Clientes a Probar +### Clients to Test 1. **WebDAV** - Windows Explorer - macOS Finder - Cyberduck - - FileZilla (con extensión WebDAV) + - FileZilla (with WebDAV extension) 2. **CalDAV** - Apple Calendar - Mozilla Thunderbird (Lightning) - - Microsoft Outlook (con complemento CalDAV) - - Google Calendar (mediante sincronización) + - Microsoft Outlook (with CalDAV add-in) + - Google Calendar (via sync) 3. **CardDAV** - Apple Contacts - Mozilla Thunderbird - - Microsoft Outlook (con complemento CardDAV) - - Google Contacts (mediante sincronización) + - Microsoft Outlook (with CardDAV add-in) + - Google Contacts (via sync) -### Pruebas de Cumplimiento +### Compliance Testing -- Utilizar la suite de pruebas CalDAVTester para verificar la conformidad con el estándar -- Validar cumplimiento de RFC para cada protocolo -- Pruebas de stress para evaluar rendimiento bajo carga +- Use the **CalDAVTester** suite to verify standards conformance +- Validate RFC compliance for each protocol +- Stress tests to evaluate performance under load -### Depuración +### Debugging -- Implementar logging detallado para operaciones DAV -- Crear herramientas de diagnóstico para depurar solicitudes DAV complejas -- Proporcionar mensajes de error claros para ayudar en la resolución de problemas \ No newline at end of file +- Detailed logging for DAV operations +- Diagnostic tools for debugging complex DAV requests +- Clear error messages for troubleshooting diff --git a/doc/deduplication.md b/doc/deduplication.md new file mode 100644 index 00000000..2025a65f --- /dev/null +++ b/doc/deduplication.md @@ -0,0 +1,334 @@ +# 06 - Deduplication + +OxiCloud uses **content-addressable deduplication** via SHA-256 hashing. Uploaded file content is hashed and stored in a central blob store. Identical files share the same blob, tracked by a reference counter. Disk savings scale with the number of duplicates. + +Deduplication is always enabled and non-fatal -- if dedup fails, file operations proceed normally with a warning log. + +## Architecture + +``` +User Files (references) ──▶ Dedup Index (hash→metadata) ──▶ Blob Store (actual data) +``` + +### Storage Layout + +``` +/ + .blobs/ + 00/ .. ff/ ← 256 prefix directories (hex) for FS distribution + .blob ← actual blob files + .dedup_temp/ ← temp staging directory for atomic writes + .dedup_index.json ← persistent JSON index (hash → metadata) +``` + +### Layer Placement + +| Layer | Component | File | +|---|---|---| +| Application Port | **DedupPort** trait + DTOs | `src/application/ports/dedup_ports.rs` | +| Infrastructure | **DedupService** implementation | `src/infrastructure/services/dedup_service.rs` | +| Interfaces | **DedupHandler** REST endpoints | `src/interfaces/api/handlers/dedup_handler.rs` | +| Integration | **FileUploadService** (dedup on upload) | `src/application/services/file_upload_service.rs` | +| Integration | **FileManagementService** (ref-count on delete) | `src/application/services/file_management_service.rs` | + +## Constants + +| Constant | Value | Description | +|---|---|---| +| `HASH_CHUNK_SIZE` | 256 KB (`256 * 1024`) | Chunk size for streaming SHA-256 computation | +| `MIN_DEDUP_SIZE` | 4 KB (`4096`) | Files below this size skip deduplication | + +Hardcoded in `dedup_service.rs`. No runtime configuration beyond **storage_path**. + +## Port: DedupPort Trait + +Defined in `src/application/ports/dedup_ports.rs`: + +```rust +#[async_trait] +pub trait DedupPort: Send + Sync + 'static { + /// Store content from bytes, returning dedup result + async fn store_bytes(&self, content: &[u8], content_type: Option) -> Result; + + /// Store content from an existing file path + async fn store_from_file(&self, source_path: &Path, content_type: Option) -> Result; + + /// Check if a blob exists by hash + async fn blob_exists(&self, hash: &str) -> bool; + + /// Get metadata for a blob + async fn get_blob_metadata(&self, hash: &str) -> Option; + + /// Read blob content as Vec + async fn read_blob(&self, hash: &str) -> Result, DomainError>; + + /// Read blob content as Bytes + async fn read_blob_bytes(&self, hash: &str) -> Result; + + /// Increment reference count for a blob + async fn add_reference(&self, hash: &str) -> Result<(), DomainError>; + + /// Decrement reference count; deletes blob if it reaches 0. Returns true if deleted. + async fn remove_reference(&self, hash: &str) -> Result; + + /// Compute SHA-256 hash of bytes (synchronous) + fn hash_bytes(&self, content: &[u8]) -> String; + + /// Compute SHA-256 hash of file (streaming) + async fn hash_file(&self, path: &Path) -> Result; + + /// Get deduplication statistics + async fn get_stats(&self) -> DedupStatsDto; + + /// Persist index to disk + async fn flush(&self) -> Result<(), DomainError>; + + /// Verify integrity of all blobs (existence, hash, size) + async fn verify_integrity(&self) -> Result, DomainError>; +} +``` + +### Port DTOs + +```rust +/// Result of a dedup store operation +pub enum DedupResultDto { + NewBlob { hash: String, size: u64, blob_path: PathBuf }, + ExistingBlob { hash: String, size: u64, blob_path: PathBuf, saved_bytes: u64 }, +} +// Methods: hash(), size(), blob_path(), was_deduplicated() + +/// Metadata for a stored blob +pub struct BlobMetadataDto { + pub hash: String, // SHA-256 hex string + pub size: u64, + pub ref_count: u32, + pub content_type: Option, +} + +/// Aggregate dedup statistics +pub struct DedupStatsDto { + pub total_blobs: u64, + pub total_bytes_stored: u64, + pub total_bytes_referenced: u64, + pub bytes_saved: u64, + pub dedup_hits: u64, + pub dedup_ratio: f64, +} +``` + +## Infrastructure: DedupService + +Implemented in `src/infrastructure/services/dedup_service.rs`. + +### Struct + +```rust +pub struct DedupService { + blob_root: PathBuf, // /.blobs + temp_root: PathBuf, // /.dedup_temp + index: Arc>>, // in-memory index + index_path: PathBuf, // /.dedup_index.json + stats: Arc>, +} +``` + +### Key Methods + +| Method | Description | +|---|---| +| `new(storage_root: &Path)` | Constructs paths, initializes empty index | +| `initialize()` | Creates `.blobs/` (256 prefix dirs), `.dedup_temp/`, loads index JSON | +| `blob_path(hash: &str)` | Returns `//.blob` | +| `hash_bytes(content: &[u8])` | Static SHA-256 → hex string | +| `hash_file(path: &Path)` | Streaming SHA-256 of file (256 KB chunks) | +| `store_bytes(content, content_type)` | Hash → check existing → increment ref or write new blob atomically | +| `store_from_file(source_path, content_type)` | Hash file → dedup check → move file to blob store | +| `add_reference(hash)` | Increments **ref_count** + updates stats | +| `remove_reference(hash)` | Decrements **ref_count**. If 0, deletes blob file + removes from index | +| `read_blob(hash)` | Reads blob file content | +| `get_stats()` | Returns current dedup statistics | +| `flush()` | Saves index to JSON atomically (write to `.json.tmp` then rename) | +| `verify_integrity()` | Checks every blob: file exists, hash matches, size matches | +| `garbage_collect()` | Removes blobs with `ref_count == 0`. Returns `(deleted_count, deleted_bytes)` | + +### Key Behaviors + +- **Atomic writes**: new blobs are written to `.dedup_temp/.tmp` then renamed into `.blobs//.blob` +- **Index persistence**: auto-saved every 100 new blobs, also saved explicitly via `flush()` +- **Small file bypass**: files < 4 KB skip deduplication +- **File move optimization**: `store_from_file` uses `fs::rename` to move the source file into the blob store (zero-copy on same filesystem) +- **Thread safety**: index and stats are protected by `Arc>` + +## REST API Endpoints + +All routes under `/api/dedup`, authentication required. + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/dedup/check/{hash}` | `DedupHandler::check_hash` | Check if a blob exists by SHA-256 hash | +| `POST` | `/api/dedup/upload` | `DedupHandler::upload_with_dedup` | Multipart upload with automatic dedup | +| `GET` | `/api/dedup/stats` | `DedupHandler::get_stats` | Get deduplication statistics | +| `GET` | `/api/dedup/blob/{hash}` | `DedupHandler::get_blob` | Retrieve raw blob content by hash | +| `DELETE` | `/api/dedup/blob/{hash}` | `DedupHandler::remove_reference` | Decrement ref-count (deletes blob if 0) | +| `POST` | `/api/dedup/recalculate` | `DedupHandler::recalculate_stats` | Run integrity verification + refresh stats | + +### API Response Types + +**Hash Check** (`GET /api/dedup/check/{hash}`): +```json +{ + "exists": true, + "hash": "a1b2c3d4...", + "existing_size": 1048576, + "ref_count": 3 +} +``` +- Validates 64-character hex format for the hash parameter +- `existing_size` and `ref_count` are omitted when `exists` is `false` + +**Dedup Upload** (`POST /api/dedup/upload`): +```json +{ + "is_new": false, + "hash": "a1b2c3d4...", + "size": 1048576, + "bytes_saved": 1048576, + "ref_count": 2 +} +``` +- Returns `201 Created` for new blobs, `200 OK` for deduplicated content +- Accepts multipart form data + +**Stats** (`GET /api/dedup/stats`): +```json +{ + "unique_blobs": 150, + "total_references": 300, + "bytes_saved": 524288000, + "total_logical_bytes": 1073741824, + "total_physical_bytes": 549453824, + "dedup_ratio": 2.0, + "savings_percentage": 48.8 +} +``` + +**Get Blob** (`GET /api/dedup/blob/{hash}`): +- Returns raw blob content with `Content-Type` from metadata +- Adds `X-Dedup-Hash` response header + +**Remove Reference** (`DELETE /api/dedup/blob/{hash}`): +```json +{ + "success": true, + "deleted": true, + "message": "Blob deleted (ref count reached 0)" +} +``` + +## Integration with File Upload + +**FileUploadService** holds `dedup: Option>`. + +During `smart_upload()`, dedup runs for **all upload tiers** (write-behind, buffered, streaming): + +```rust +// Inside smart_upload() — dedup runs after data is collected +{ + let dedup_data: Vec = { /* combine all chunks */ }; + self.run_dedup(&dedup_data, &content_type).await; +} +``` + +The private `run_dedup` method: + +```rust +async fn run_dedup(&self, data: &[u8], content_type: &str) { + let Some(dedup) = &self.dedup else { return }; + match dedup.store_bytes(data, Some(content_type.to_string())).await { + Ok(result) => { /* log new or dedup hit */ } + Err(e) => { warn!("DEDUP: Failed to store in blob store: {}", e); } + } +} +``` + +Dedup is non-fatal -- failures are only logged as warnings. The file upload always completes regardless of dedup outcome. + +## Integration with File Deletion + +**FileManagementService** holds `dedup_service: Option>`. + +In `delete_with_cleanup()`: + +1. **Compute content hash** -- reads file via **FileReadPort**, calls `dedup.hash_bytes(&content)` +2. **Delete file** -- tries trash (soft delete) first, falls back to permanent delete +3. **Decrement dedup ref-count** -- calls `dedup.remove_reference(hash)` which may delete the blob if **ref_count** reaches 0 + +```rust +// Private helpers in FileManagementService +async fn compute_content_hash(&self, id: &str) -> Option +async fn decrement_dedup_ref(&self, hash: &str) +``` + +## DI Wiring + +In `src/common/di.rs`: + +```rust +// Initialization (in create_core_services) +let dedup_service = Arc::new(DedupService::new(&self.storage_path)); +dedup_service.initialize().await?; + +// Stored in CoreServices as: +pub struct CoreServices { + pub dedup_service: Arc, + // ... +} + +// Injected into application services: +FileUploadService::new_full(... core.dedup_service.clone()) +FileManagementService::new_full(... core.dedup_service.clone()) +``` + +## Persistence + +Deduplication uses a **file-based JSON index** (`/.dedup_index.json`), NOT a database table. The index is loaded into memory at startup and flushed to disk: + +- Automatically every 100 new blobs +- Explicitly via `flush()` +- Uses atomic write (write to `.json.tmp` then rename) for crash safety + +## Tests + +Located at the bottom of `src/infrastructure/services/dedup_service.rs`: + +| Test | Description | +|---|---| +| `test_dedup_identical_content` | Stores same content (>4KB) twice. Verifies second is deduplicated, hashes match, `stats.dedup_hits == 1` | +| `test_reference_counting` | Stores twice (ref_count=2), removes one ref (not deleted), removes second (blob deleted) | + +## Client Usage Example + +```bash +# 1. Check if file already exists by hash +HASH=$(sha256sum myfile.txt | cut -d' ' -f1) +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/dedup/check/$HASH" + +# 2. Upload with dedup (if not exists) +curl -X POST -H "Authorization: Bearer $TOKEN" \ + -F "file=@myfile.txt" \ + "https://oxicloud.example.com/api/dedup/upload" + +# 3. Get dedup statistics +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/dedup/stats" + +# 4. Retrieve blob content +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/dedup/blob/$HASH" -o output.bin + +# 5. Recalculate stats with integrity check +curl -X POST -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/dedup/recalculate" +``` diff --git a/doc/deployment.md b/doc/deployment.md new file mode 100644 index 00000000..327beb85 --- /dev/null +++ b/doc/deployment.md @@ -0,0 +1,192 @@ +# 02 - Deployment + +OxiCloud is deployed as a containerized application with PostgreSQL. + +--- + +## Docker Setup + +### Docker Compose + +```yaml +# docker-compose.yml +version: '3' +services: + postgres: + image: postgres:17.4-alpine + environment: + POSTGRES_DB: oxicloud + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - pg_data:/var/lib/postgresql/data + - ./db/schema.sql:/docker-entrypoint-initdb.d/schema.sql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + + oxicloud: + image: oxicloud:latest + ports: + - "8086:8086" + environment: + OXICLOUD_DB_CONNECTION_STRING: "postgres://postgres:postgres@postgres/oxicloud" + volumes: + - storage_data:/app/storage + depends_on: + postgres: + condition: service_healthy + +volumes: + pg_data: + storage_data: +``` + +### Dockerfile + +3-stage Alpine-based build: +1. **Cacher** -- pre-builds dependency layer +2. **Builder** -- compiles OxiCloud (`rust:1.93.0-alpine3.23`) +3. **Runtime** -- minimal Alpine image (`alpine:3.23.3`) with `libgcc`, `ca-certificates`, `libpq`, `tzdata`, `su-exec` + +Non-root user: `oxicloud` (UID/GID 1001). Exposed port: `8086`. Entrypoint: `entrypoint.sh` (chown storage + drop privileges via `su-exec`). + +--- + +## Environment Variables + +### Server + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_STORAGE_PATH` | `./storage` | Root storage directory | +| `OXICLOUD_STATIC_PATH` | `./static` | Static files directory | +| `OXICLOUD_SERVER_PORT` | `8085` | Server port (note: `main.rs` hardcodes `8086`) | +| `OXICLOUD_SERVER_HOST` | `127.0.0.1` | Server bind address | + +### Database + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_DB_CONNECTION_STRING` | `postgres://postgres:postgres@localhost:5432/oxicloud` | PostgreSQL connection string | +| `OXICLOUD_DB_MAX_CONNECTIONS` | `20` | Max pool connections | +| `OXICLOUD_DB_MIN_CONNECTIONS` | `5` | Min pool connections | + +### Authentication + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_JWT_SECRET` | (auto-generated) | JWT signing secret. If empty, a random 32-byte hex secret is generated per session | +| `OXICLOUD_ACCESS_TOKEN_EXPIRY_SECS` | `3600` (1h) | Access token lifetime | +| `OXICLOUD_REFRESH_TOKEN_EXPIRY_SECS` | `2592000` (30d) | Refresh token lifetime | + +### Feature Flags + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_ENABLE_AUTH` | `true` | Enable authentication system | +| `OXICLOUD_ENABLE_USER_STORAGE_QUOTAS` | `false` | Enable per-user storage quotas | +| `OXICLOUD_ENABLE_FILE_SHARING` | `true` | Enable file/folder sharing | +| `OXICLOUD_ENABLE_TRASH` | `true` | Enable trash/recycle bin | +| `OXICLOUD_ENABLE_SEARCH` | `true` | Enable search functionality | + +### OIDC / SSO + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_OIDC_ENABLED` | `false` | Enable OIDC authentication | +| `OXICLOUD_OIDC_ISSUER_URL` | (empty) | OIDC provider issuer URL | +| `OXICLOUD_OIDC_CLIENT_ID` | (empty) | OIDC client ID | +| `OXICLOUD_OIDC_CLIENT_SECRET` | (empty) | OIDC client secret | +| `OXICLOUD_OIDC_REDIRECT_URI` | `http://localhost:8086/api/auth/oidc/callback` | Callback URL | +| `OXICLOUD_OIDC_SCOPES` | `openid profile email` | Requested OIDC scopes | +| `OXICLOUD_OIDC_FRONTEND_URL` | `http://localhost:8086` | Frontend URL for redirects | +| `OXICLOUD_OIDC_AUTO_PROVISION` | `true` | Auto-create users on first login | +| `OXICLOUD_OIDC_ADMIN_GROUPS` | (empty) | OIDC groups that grant admin role | +| `OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN` | `false` | Disable password login when OIDC is active | +| `OXICLOUD_OIDC_PROVIDER_NAME` | `SSO` | Display name for the OIDC provider | + +### OIDC Validation + +If **OXICLOUD_OIDC_ENABLED** is `true` but **issuer_url**, **client_id**, or **client_secret** are empty, OIDC is automatically disabled with an error log. + +--- + +## Internal Configuration (Not Environment-Configurable) + +Hardcoded defaults in `src/common/config.rs`: + +### Cache + +| Parameter | Default | +|---|---| +| File cache TTL | 60,000 ms (1 min) | +| Directory cache TTL | 120,000 ms (2 min) | +| Max cache entries | 10,000 | + +### Timeouts + +| Parameter | Default | +|---|---| +| File operation | 10,000 ms | +| Directory operation | 30,000 ms | +| Lock acquisition | 5,000 ms | +| Network operation | 15,000 ms | + +### Resources + +| Parameter | Default | +|---|---| +| Large file threshold | 100 MB | +| Large directory threshold | 1,000 entries | +| Streaming chunk size | 1 MB | +| Max in-memory file size | 50 MB | + +### Concurrency + +| Parameter | Default | +|---|---| +| Max concurrent files | 10 | +| Max concurrent dirs | 5 | +| Max concurrent I/O | 20 | +| Max parallel chunks | 8 | +| Min size for parallel chunks | 200 MB | +| Parallel chunk size | 8 MB | + +### Storage + +| Parameter | Default | +|---|---| +| Trash retention | 30 days | + +### Auth Hashing (Argon2id) + +| Parameter | Default | +|---|---| +| Memory cost | 65,536 KB (64 MB) | +| Time cost | 3 iterations | + +--- + +## Feature Dependency Matrix + +| Feature | Requires DB | Requires Auth | Feature Flag | +|---|---|---|---| +| File storage | No | No | Always on | +| Authentication | Yes | -- | `OXICLOUD_ENABLE_AUTH` | +| OIDC / SSO | Yes | Yes | `OXICLOUD_OIDC_ENABLED` | +| File sharing | Yes | Yes | `OXICLOUD_ENABLE_FILE_SHARING` | +| Trash | No | No | `OXICLOUD_ENABLE_TRASH` | +| Search | No | No | `OXICLOUD_ENABLE_SEARCH` | +| Favorites | Yes | Yes | Always on (when DB available) | +| Recent items | Yes | Yes | Always on (when DB available) | +| Storage quotas | Yes | Yes | `OXICLOUD_ENABLE_USER_STORAGE_QUOTAS` | +| Admin panel | Yes | Yes | Always on (when auth enabled) | +| WebDAV | No | Optional | Always on | +| CalDAV | Yes | Yes | Always on (when DB available) | +| CardDAV | Yes | Yes | Always on (when DB available) | +| Deduplication | No | No | Always on | +| Thumbnails | No | No | Always on | +| Chunked uploads | No | No | Always on | diff --git a/doc/favorites-and-recent.md b/doc/favorites-and-recent.md new file mode 100644 index 00000000..7180c0dc --- /dev/null +++ b/doc/favorites-and-recent.md @@ -0,0 +1,171 @@ +# 13 - Favorites and Recent Items + +Two per-user item tracking features: + +- **Favorites** -- users mark files and folders for quick access. +- **Recent Items** -- automatically tracks recently accessed files and folders. + +Both require PostgreSQL and are only available when a database connection is configured. + +--- + +## Favorites + +### Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **FavoritesUseCase**, **FavoritesRepositoryPort** | `src/application/ports/favorites_ports.rs` | +| Application Service | **FavoritesService** | `src/application/services/favorites_service.rs` | +| Application DTO | **FavoriteItemDto** | `src/application/dtos/favorites_dto.rs` | +| Infrastructure | **FavoritesPgRepository** | `src/infrastructure/repositories/pg/favorites_pg_repository.rs` | +| Interfaces | `favorites_handler` (free functions) | `src/interfaces/api/handlers/favorites_handler.rs` | + +### DTO + +```rust +pub struct FavoriteItemDto { + pub id: String, + pub user_id: String, + pub item_id: String, + pub item_type: String, // "file" | "folder" + pub created_at: DateTime, +} +``` + +### REST API + +All routes under `/api/favorites`, require authentication. User ID comes from the JWT token. + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/favorites/` | `get_favorites` | List all favorites for current user | +| `POST` | `/api/favorites/{item_type}/{item_id}` | `add_favorite` | Add a file or folder to favorites | +| `DELETE` | `/api/favorites/{item_type}/{item_id}` | `remove_favorite` | Remove from favorites | + +- `item_type` must be `"file"` or `"folder"` (validated by service) +- Adding a duplicate is idempotent (`ON CONFLICT DO NOTHING`) +- Results ordered by `created_at DESC` + +### Database Schema + +```sql +CREATE TABLE IF NOT EXISTS auth.user_favorites ( + id SERIAL PRIMARY KEY, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + item_id TEXT NOT NULL, + item_type TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, item_id, item_type) +); +``` + +Indexes: `user_id`, `item_id`, `item_type`, `created_at`, composite `(user_id, item_type)`. + +### Example + +```bash +# Add file to favorites +curl -X POST -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/favorites/file/abc-123" + +# List favorites +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/favorites/" + +# Remove from favorites +curl -X DELETE -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/favorites/folder/def-456" +``` + +--- + +## Recent Items + +### Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **RecentItemsUseCase**, **RecentItemsRepositoryPort** | `src/application/ports/recent_ports.rs` | +| Application Service | **RecentService** | `src/application/services/recent_service.rs` | +| Application DTO | **RecentItemDto** | `src/application/dtos/recent_dto.rs` | +| Infrastructure | **RecentItemsPgRepository** | `src/infrastructure/repositories/pg/recent_items_pg_repository.rs` | +| Interfaces | `recent_handler` (free functions) | `src/interfaces/api/handlers/recent_handler.rs` | + +### DTO + +```rust +pub struct RecentItemDto { + pub id: String, + pub user_id: String, + pub item_id: String, + pub item_type: String, // "file" | "folder" + pub accessed_at: DateTime, +} +``` + +### REST API + +All routes under `/api/recent`, require authentication. + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/recent/` | `get_recent_items` | List recent items (optional `?limit=N`) | +| `POST` | `/api/recent/{item_type}/{item_id}` | `record_item_access` | Record an access (upsert) | +| `DELETE` | `/api/recent/{item_type}/{item_id}` | `remove_from_recent` | Remove specific item | +| `DELETE` | `/api/recent/clear` | `clear_recent_items` | Clear all recent items | + +### Behavior + +- **Max items per user**: 50 (configured in DI, clamped to 1-100) +- **Upsert**: re-accessing an item updates its `accessed_at` timestamp +- **Auto-prune**: after recording access, old items beyond the limit are automatically pruned +- **Ordering**: results ordered by `accessed_at DESC` +- **Limit parameter**: `?limit=N` caps results (defaults to and cannot exceed **max_recent_items**) + +### Database Schema + +```sql +CREATE TABLE IF NOT EXISTS auth.user_recent_files ( + id SERIAL PRIMARY KEY, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + item_id TEXT NOT NULL, + item_type TEXT NOT NULL, + accessed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, item_id, item_type) +); +``` + +Indexes: `user_id`, `item_id`, `item_type`, `accessed_at`, composite `(user_id, accessed_at DESC)`. + +### Example + +```bash +# Record file access +curl -X POST -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/recent/file/abc-123" + +# Get recent items (last 10) +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/recent/?limit=10" + +# Clear history +curl -X DELETE -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/recent/clear" +``` + +## DI Wiring + +Both services require PostgreSQL: + +```rust +// Favorites +let repo = Arc::new(FavoritesPgRepository::new(db_pool.clone())); +let favorites_service = Arc::new(FavoritesService::new(repo)); + +// Recent +let repo = Arc::new(RecentItemsPgRepository::new(db_pool.clone())); +let recent_service = Arc::new(RecentService::new(repo, 50)); // max 50 items +``` + +Stored as `Option>` in **AppState** -- only available when DB is connected. diff --git a/doc/file-system-safety.md b/doc/file-system-safety.md new file mode 100644 index 00000000..5fe9cf60 --- /dev/null +++ b/doc/file-system-safety.md @@ -0,0 +1,156 @@ +# 03 - File System Safety + +OxiCloud ensures data integrity and durability during file operations through atomic writes, fsync, and directory synchronization. The goal: writes either complete fully or not at all, data reaches persistent storage, and the system recovers from crashes or power loss. + +--- + +## The Problem: Buffered I/O + +Standard filesystem operations use buffered I/O by default: + +```rust +// This operation may not immediately persist to disk +fs::write(path, content) +``` + +When an application writes data, the OS typically: + +1. Accepts the write into memory buffers +2. Acknowledges completion to the application +3. Schedules the actual disk write for later + +A crash during that window means data loss -- the data exists only in memory buffers that haven't been flushed. + +--- + +## OxiCloud's Approach + +All safety mechanisms live in the **FileSystemUtils** service. + +### Atomic Write Pattern + +Files are written using write-then-rename: + +```rust +/// Writes data to a file with fsync to ensure durability +/// Uses a safe atomic write pattern: write to temp file, fsync, rename +pub async fn atomic_write>(path: P, contents: &[u8]) -> Result<(), IoError> +``` + +Steps: +1. Write to a temporary file in the same directory +2. Call `fsync` to ensure data is on disk +3. Atomically rename the temp file to the target file +4. Sync the parent directory to ensure the rename is persisted + +### Directory Synchronization + +```rust +/// Creates directories with fsync +pub async fn create_dir_with_sync>(path: P) -> Result<(), IoError> +``` + +Directories are created, their entries persisted to disk, and parent directories synchronized too. + +### Rename and Delete Operations + +```rust +/// Renames a file or directory with proper syncing +pub async fn rename_with_sync, Q: AsRef>(from: P, to: Q) -> Result<(), IoError> + +/// Removes a file with directory syncing +pub async fn remove_file_with_sync>(path: P) -> Result<(), IoError> +``` + +Both complete the operation itself, then update and sync the parent directory entry. + +--- + +## Implementation Details + +### fsync on Files + +```rust +// Write file content +file.write_all(contents).await?; + +// Ensure data is synced to disk +file.flush().await?; +file.sync_all().await?; +``` + +`sync_all()` instructs the OS to flush data and metadata to the physical storage device. + +### fsync on Directories + +```rust +// Sync a directory to ensure its contents (entries) are durable +async fn sync_directory>(path: P) -> Result<(), IoError> { + let dir_file = OpenOptions::new().read(true).open(path).await?; + dir_file.sync_all().await +} +``` + +Required after any operation that modifies directory entries (create, rename, delete). + +--- + +## Usage in the Codebase + +### File Write Repository + +```rust +// Write the file to disk using atomic write with fsync +tokio::time::timeout( + self.config.timeouts.file_write_timeout(), + FileSystemUtils::atomic_write(&abs_path, &content) +).await +``` + +### File Move Operations + +```rust +// Move the file physically with fsync +time::timeout( + self.config.timeouts.file_timeout(), + FileSystemUtils::rename_with_sync(&old_abs_path, &new_abs_path) +).await +``` + +### Directory Creation + +```rust +// Ensure the parent directory exists with proper syncing +self.ensure_parent_directory(&abs_path).await?; + +// Implementation uses FileSystemUtils +async fn ensure_parent_directory(&self, abs_path: &PathBuf) -> FileRepositoryResult<()> { + if let Some(parent) = abs_path.parent() { + time::timeout( + self.config.timeouts.dir_timeout(), + FileSystemUtils::create_dir_with_sync(parent) + ).await + } +} +``` + +--- + +## Benefits + +1. **Data durability** -- critical data is synced to persistent storage +2. **Crash resilience** -- recovery from unexpected failures without data loss +3. **Consistency** -- file operations maintain a consistent filesystem state +4. **Atomic operations** -- file writes appear as all-or-nothing + +--- + +## Performance Considerations + +Syncing to disk costs more than buffered writes. OxiCloud mitigates this by: + +1. Applying these measures only to critical operations +2. Using timeouts to prevent indefinite blocking +3. Implementing parallel processing for large files + +The tradeoff favors safety for critical data while maintaining good performance for most operations. diff --git a/doc/i18n.md b/doc/i18n.md new file mode 100644 index 00000000..669b4c89 --- /dev/null +++ b/doc/i18n.md @@ -0,0 +1,99 @@ +# 16 - Internationalization + +JSON-based translation system. Translations are loaded from static files, cached in memory, and served via a public REST API (no auth required). + +## Supported Languages + +| Locale | Code | File | +|---|---|---| +| English | `en` | `static/locales/en.json` | +| Spanish | `es` | `static/locales/es.json` | +| French | `fr` | `static/locales/fr.json` | +| German | `de` | `static/locales/de.json` | +| Portuguese | `pt` | `static/locales/pt.json` | +| Persian | `fa` | `static/locales/fa.json` | +| Chinese | `zh` | `static/locales/zh.json` | + +Default locale: **en** (English). + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Domain Port | **I18nService** trait, **Locale** enum | `src/domain/services/i18n_service.rs` | +| Application Service | **I18nApplicationService** | `src/application/services/i18n_application_service.rs` | +| Application DTOs | **LocaleDto**, **TranslationRequestDto**, etc. | `src/application/dtos/i18n_dto.rs` | +| Infrastructure | **FileSystemI18nService** | `src/infrastructure/services/file_system_i18n_service.rs` | +| Interfaces | **I18nHandler** | `src/interfaces/api/handlers/i18n_handler.rs` | + +## REST API + +Public endpoints (no authentication), under `/api/i18n`: + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/i18n/locales` | `get_locales` | List available locales | +| `GET` | `/api/i18n/translate` | `translate` | Translate a key (`?key=...&locale=...`) | +| `GET` | `/api/i18n/locales/{locale_code}` | `get_translations_by_locale` | Get all translations for a locale | + +### Examples + +```bash +# List available locales +curl "https://oxicloud.example.com/api/i18n/locales" +# [{"code":"en","name":"English"},{"code":"es","name":"Spanish"}, ...] + +# Translate a key +curl "https://oxicloud.example.com/api/i18n/translate?key=app.title&locale=es" +# {"key":"app.title","locale":"es","text":"OxiCloud"} + +# Get all translations for a locale +curl "https://oxicloud.example.com/api/i18n/locales/en" +# { "app": { "title": "OxiCloud", ... }, "nav": { ... }, ... } +``` + +## Translation File Format + +Nested JSON with dot-delimited key lookups: + +```json +{ + "app": { + "title": "OxiCloud", + "description": "Your personal cloud storage" + }, + "nav": { + "files": "Files", + "shared": "Shared", + "recent": "Recent", + "favorites": "Favorites", + "trash": "Trash" + }, + "actions": { + "search": "Search files...", + "new_folder": "New folder", + "upload": "Upload", + "download": "Download", + "delete": "Delete" + }, + "share": { ... }, + "user_menu": { ... } +} +``` + +Key lookup: `"nav.files"` resolves to `"Files"`. + +## Fallback Behavior + +If a key is missing in the requested locale, the system falls back to English (`en`). If still not found, returns an `I18nError::KeyNotFound`. + +## Caching + +Translations are cached in-memory via `RwLock>`. Loaded lazily on first request per locale. + +## Frontend Integration + +The frontend uses `static/js/i18n.js` and `static/js/languageSelector.js` to: +1. Detect the user's preferred language +2. Load translations via `/api/i18n/locales/{code}` +3. Apply translations to DOM elements diff --git a/doc/IMPORTANT-DELTA_SYNC_IMPLEMENTATION.md b/doc/important-delta-sync-implementation.md similarity index 79% rename from doc/IMPORTANT-DELTA_SYNC_IMPLEMENTATION.md rename to doc/important-delta-sync-implementation.md index 079a4031..0b5de672 100644 --- a/doc/IMPORTANT-DELTA_SYNC_IMPLEMENTATION.md +++ b/doc/important-delta-sync-implementation.md @@ -1,1161 +1,1152 @@ -# Delta Sync (rsync-like) - Guía de Implementación - -> **Estado**: Pendiente de implementación -> **Prioridad**: Media -> **Ahorro estimado**: 10-100x menos transferencia de datos -> **Fecha de creación**: 2026-02-03 - -## Índice - -1. [Resumen ejecutivo](#resumen-ejecutivo) -2. [Problema que resuelve](#problema-que-resuelve) -3. [Cómo funciona](#cómo-funciona) -4. [Algoritmos clave](#algoritmos-clave) -5. [Arquitectura propuesta](#arquitectura-propuesta) -6. [Estructuras de datos](#estructuras-de-datos) -7. [API Endpoints](#api-endpoints) -8. [Implementación paso a paso](#implementación-paso-a-paso) -9. [Integración con sistema existente](#integración-con-sistema-existente) -10. [Casos de uso y efectividad](#casos-de-uso-y-efectividad) -11. [Consideraciones de rendimiento](#consideraciones-de-rendimiento) -12. [Testing](#testing) -13. [Dependencias necesarias](#dependencias-necesarias) - ---- - -## Resumen ejecutivo - -Delta Sync es una técnica de sincronización que **transfiere solo las partes modificadas** de un archivo en lugar del archivo completo. Inspirado en el algoritmo de `rsync`, permite ahorros de ancho de banda del 90-99% en escenarios comunes. - -### Beneficios principales - -| Métrica | Sin Delta Sync | Con Delta Sync | -|---------|----------------|----------------| -| Editar 1 línea en 100MB | 100MB transferidos | ~1KB transferido | -| Tiempo de sync (conexión lenta) | 4+ minutos | <1 segundo | -| Consumo de ancho de banda | 100% | 0.1-10% | - ---- - -## Problema que resuelve - -### Escenario actual (sin Delta Sync) - -``` -Usuario tiene documento.docx (50MB) en OxiCloud - │ - ▼ -Descarga completo (50MB) ────────────────────────► 50MB ↓ - │ - ▼ -Edita una palabra - │ - ▼ -Sube completo de nuevo (50MB) ──────────────────► 50MB ↑ - │ - ▼ -TOTAL: 100MB transferidos por cambiar una palabra 😱 -``` - -### Escenario objetivo (con Delta Sync) - -``` -Usuario tiene documento.docx (50MB) en OxiCloud - │ - ▼ -Descarga completo (50MB) ────────────────────────► 50MB ↓ (primera vez) - │ - ▼ -Edita una palabra - │ - ▼ -Sube SOLO los bloques modificados ──────────────► ~50KB ↑ - │ - ▼ -TOTAL: 50.05MB (ahorro del 99.9% en subida) ✅ -``` - ---- - -## Cómo funciona - -### Concepto de bloques (chunks) - -El archivo se divide en bloques de tamaño fijo (típicamente 4KB-64KB): - -``` -Archivo original (servidor): -┌────────┬────────┬────────┬────────┬────────┐ -│ Bloque │ Bloque │ Bloque │ Bloque │ Bloque │ -│ 0 │ 1 │ 2 │ 3 │ 4 │ -│ 4KB │ 4KB │ 4KB │ 4KB │ 4KB │ -│ │ │ │ │ │ -│ weak:A │ weak:B │ weak:C │ weak:D │ weak:E │ -│ sha:X1 │ sha:X2 │ sha:X3 │ sha:X4 │ sha:X5 │ -└────────┴────────┴────────┴────────┴────────┘ - -Archivo modificado (cliente): -┌────────┬────────┬────────┬────────┬────────┐ -│ Bloque │ Bloque │ Bloque │ Bloque │ Bloque │ -│ 0 │ 1 │ 2 │ 3 │ 4 │ -│ 4KB │ 4KB │ 4KB │ 4KB │ 4KB │ -│ │ │ │ │ │ -│ weak:A │ weak:B │ weak:F │ weak:D │ weak:E │ ← Bloque 2 cambió -│ sha:X1 │ sha:X2 │ sha:Y3 │ sha:X4 │ sha:X5 │ -└────────┴────────┴───▲────┴────────┴────────┘ - │ - SOLO ESTE SE TRANSFIERE -``` - -### Proceso de sincronización - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ FLUJO DE DELTA SYNC │ -├─────────────────────────────────────────────────────────────────────┤ -│ │ -│ CLIENTE SERVIDOR │ -│ │ -│ 1. Tiene archivo 1. Tiene archivo original │ -│ modificado + índice de bloques │ -│ │ -│ 2. Solicita firmas ──────────────────► │ -│ GET /files/{id}/signatures │ -│ │ -│ ◄────────────── 3. Retorna lista de firmas │ -│ [(weak, strong), ...] │ -│ │ -│ 4. Compara bloques │ -│ locales con firmas │ -│ del servidor │ -│ │ -│ 5. Genera delta ─────────────────────► │ -│ POST /files/{id}/delta │ -│ [Referencias + Datos nuevos] │ -│ │ -│ 6. Reconstruye archivo │ -│ aplicando delta │ -│ │ -│ ◄────────────── 7. Confirma actualización │ -│ │ -└─────────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Algoritmos clave - -### 1. Rolling Checksum (Adler-32 modificado) - -El "rolling checksum" permite calcular el hash de una ventana deslizante en O(1): - -```rust -/// Rolling checksum para búsqueda rápida de bloques coincidentes -/// Similar al usado por rsync (Adler-32 modificado) -pub struct RollingChecksum { - a: u32, // Suma simple de bytes - b: u32, // Suma ponderada - window_size: usize, - buffer: VecDeque, -} - -impl RollingChecksum { - pub fn new(window_size: usize) -> Self { - Self { - a: 0, - b: 0, - window_size, - buffer: VecDeque::with_capacity(window_size), - } - } - - /// Añadir un byte y calcular nuevo checksum - /// Complejidad: O(1) - pub fn roll(&mut self, new_byte: u8) -> u32 { - if self.buffer.len() >= self.window_size { - // Remover byte antiguo - let old_byte = self.buffer.pop_front().unwrap() as u32; - self.a = self.a.wrapping_sub(old_byte).wrapping_add(new_byte as u32); - self.b = self.b.wrapping_sub(old_byte * self.window_size as u32) - .wrapping_add(self.a); - } else { - // Ventana no llena todavía - self.a = self.a.wrapping_add(new_byte as u32); - self.b = self.b.wrapping_add(self.a); - } - - self.buffer.push_back(new_byte); - self.checksum() - } - - /// Calcular checksum actual - pub fn checksum(&self) -> u32 { - (self.b << 16) | (self.a & 0xFFFF) - } - - /// Reset para nuevo archivo - pub fn reset(&mut self) { - self.a = 0; - self.b = 0; - self.buffer.clear(); - } -} -``` - -### 2. Firma de bloque (Block Signature) - -Cada bloque tiene dos firmas: -- **Weak checksum** (32-bit): Búsqueda rápida O(1) -- **Strong hash** (SHA-256): Verificación definitiva - -```rust -/// Firma de un bloque para identificación -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BlockSignature { - /// Índice del bloque en el archivo - pub index: u32, - /// Offset en bytes desde el inicio del archivo - pub offset: u64, - /// Tamaño del bloque (puede ser menor para el último) - pub size: u32, - /// Rolling checksum (32-bit) - búsqueda rápida - pub weak_checksum: u32, - /// SHA-256 hash (256-bit) - verificación definitiva - pub strong_hash: [u8; 32], -} - -/// Genera firmas para todos los bloques de un archivo -pub fn generate_signatures(data: &[u8], block_size: usize) -> Vec { - let mut signatures = Vec::new(); - let mut offset = 0u64; - let mut index = 0u32; - - for chunk in data.chunks(block_size) { - // Weak checksum (rolling) - let weak = adler32_checksum(chunk); - - // Strong hash (SHA-256) - let mut hasher = Sha256::new(); - hasher.update(chunk); - let strong: [u8; 32] = hasher.finalize().into(); - - signatures.push(BlockSignature { - index, - offset, - size: chunk.len() as u32, - weak_checksum: weak, - strong_hash: strong, - }); - - offset += chunk.len() as u64; - index += 1; - } - - signatures -} - -fn adler32_checksum(data: &[u8]) -> u32 { - let mut a: u32 = 1; - let mut b: u32 = 0; - - for &byte in data { - a = (a + byte as u32) % 65521; - b = (b + a) % 65521; - } - - (b << 16) | a -} -``` - -### 3. Generación de Delta - -```rust -/// Instrucción de delta -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum DeltaInstruction { - /// Copiar bloque existente del archivo original - Copy { - /// Índice del bloque en el archivo original - block_index: u32, - }, - /// Insertar datos literales nuevos - Literal { - /// Datos nuevos a insertar - data: Vec, - }, -} - -/// Delta completo para actualizar un archivo -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FileDelta { - /// ID del archivo base - pub base_file_id: String, - /// Hash del archivo base (para verificación) - pub base_file_hash: String, - /// Nuevo tamaño del archivo - pub new_size: u64, - /// Instrucciones de delta - pub instructions: Vec, - /// Hash del archivo resultante (para verificación) - pub result_hash: String, -} - -/// Genera delta comparando archivo local con firmas remotas -pub fn generate_delta( - local_data: &[u8], - remote_signatures: &[BlockSignature], - block_size: usize, -) -> FileDelta { - // Crear índice de weak checksums para búsqueda O(1) - let mut weak_index: HashMap> = HashMap::new(); - for sig in remote_signatures { - weak_index.entry(sig.weak_checksum) - .or_default() - .push(sig); - } - - let mut instructions = Vec::new(); - let mut rolling = RollingChecksum::new(block_size); - let mut pos = 0; - let mut literal_buffer = Vec::new(); - - while pos < local_data.len() { - // Calcular rolling checksum de la ventana actual - let end = (pos + block_size).min(local_data.len()); - let window = &local_data[pos..end]; - - let weak = if window.len() == block_size { - rolling.reset(); - for &b in window { - rolling.roll(b); - } - rolling.checksum() - } else { - adler32_checksum(window) - }; - - // Buscar coincidencia - let mut found_match = false; - - if let Some(candidates) = weak_index.get(&weak) { - // Verificar con strong hash - let mut hasher = Sha256::new(); - hasher.update(window); - let strong: [u8; 32] = hasher.finalize().into(); - - for sig in candidates { - if sig.strong_hash == strong && sig.size as usize == window.len() { - // ¡Coincidencia encontrada! - - // Flush literal buffer si hay datos pendientes - if !literal_buffer.is_empty() { - instructions.push(DeltaInstruction::Literal { - data: std::mem::take(&mut literal_buffer), - }); - } - - // Añadir instrucción de copia - instructions.push(DeltaInstruction::Copy { - block_index: sig.index, - }); - - pos += window.len(); - found_match = true; - break; - } - } - } - - if !found_match { - // No hay coincidencia, añadir byte a literal buffer - literal_buffer.push(local_data[pos]); - pos += 1; - } - } - - // Flush remaining literal buffer - if !literal_buffer.is_empty() { - instructions.push(DeltaInstruction::Literal { - data: literal_buffer, - }); - } - - // Calcular hash del resultado - let mut hasher = Sha256::new(); - hasher.update(local_data); - let result_hash = hex::encode(hasher.finalize()); - - FileDelta { - base_file_id: String::new(), // Se llena al enviar - base_file_hash: String::new(), // Se llena al enviar - new_size: local_data.len() as u64, - instructions, - result_hash, - } -} -``` - -### 4. Aplicación de Delta - -```rust -/// Aplica delta a un archivo base para obtener el nuevo archivo -pub fn apply_delta( - base_data: &[u8], - signatures: &[BlockSignature], - delta: &FileDelta, - block_size: usize, -) -> Result, DeltaSyncError> { - let mut result = Vec::with_capacity(delta.new_size as usize); - - for instruction in &delta.instructions { - match instruction { - DeltaInstruction::Copy { block_index } => { - // Copiar bloque del archivo base - let sig = signatures.get(*block_index as usize) - .ok_or(DeltaSyncError::InvalidBlockIndex(*block_index))?; - - let start = sig.offset as usize; - let end = start + sig.size as usize; - - if end > base_data.len() { - return Err(DeltaSyncError::InvalidBlockRange); - } - - result.extend_from_slice(&base_data[start..end]); - } - DeltaInstruction::Literal { data } => { - // Insertar datos literales - result.extend_from_slice(data); - } - } - } - - // Verificar hash del resultado - let mut hasher = Sha256::new(); - hasher.update(&result); - let actual_hash = hex::encode(hasher.finalize()); - - if actual_hash != delta.result_hash { - return Err(DeltaSyncError::HashMismatch { - expected: delta.result_hash.clone(), - actual: actual_hash, - }); - } - - Ok(result) -} -``` - ---- - -## Arquitectura propuesta - -### Estructura de archivos - -``` -src/ -├── infrastructure/ -│ └── services/ -│ ├── mod.rs # Añadir: pub mod delta_sync_service; -│ └── delta_sync_service.rs # NUEVO: Servicio principal -│ -├── interfaces/ -│ └── api/ -│ └── handlers/ -│ ├── mod.rs # Añadir: pub mod delta_sync_handler; -│ └── delta_sync_handler.rs # NUEVO: Endpoints API -│ -└── common/ - └── di.rs # Añadir: delta_sync_service a CoreServices -``` - -### Servicio principal (delta_sync_service.rs) - -```rust -//! Delta Sync Service - Sincronización eficiente por diferencias -//! -//! Implementa algoritmo similar a rsync para transferir solo -//! las partes modificadas de los archivos. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::fs; -use tokio::sync::RwLock; -use sha2::{Sha256, Digest}; -use serde::{Deserialize, Serialize}; - -/// Tamaño de bloque por defecto (16KB - buen balance) -pub const DEFAULT_BLOCK_SIZE: usize = 16 * 1024; - -/// Tamaño mínimo de archivo para usar delta sync -pub const MIN_DELTA_SYNC_SIZE: u64 = 64 * 1024; // 64KB - -/// Errores del servicio Delta Sync -#[derive(Debug, thiserror::Error)] -pub enum DeltaSyncError { - #[error("Archivo no encontrado: {0}")] - FileNotFound(String), - - #[error("Firmas no encontradas para archivo: {0}")] - SignaturesNotFound(String), - - #[error("Índice de bloque inválido: {0}")] - InvalidBlockIndex(u32), - - #[error("Rango de bloque inválido")] - InvalidBlockRange, - - #[error("Hash no coincide: esperado {expected}, actual {actual}")] - HashMismatch { expected: String, actual: String }, - - #[error("Error de I/O: {0}")] - IoError(#[from] std::io::Error), - - #[error("Error de serialización: {0}")] - SerializationError(String), -} - -/// Servicio de Delta Sync -pub struct DeltaSyncService { - /// Directorio para almacenar índices de firmas - signatures_dir: PathBuf, - /// Cache en memoria de firmas recientes - signature_cache: Arc>>>, - /// Tamaño de bloque configurado - block_size: usize, - /// Máximo de entradas en cache - max_cache_entries: usize, -} - -impl DeltaSyncService { - pub fn new(storage_root: &Path) -> Self { - Self { - signatures_dir: storage_root.join(".delta_signatures"), - signature_cache: Arc::new(RwLock::new(HashMap::new())), - block_size: DEFAULT_BLOCK_SIZE, - max_cache_entries: 1000, - } - } - - pub fn with_block_size(mut self, block_size: usize) -> Self { - self.block_size = block_size; - self - } - - /// Inicializar servicio (crear directorios) - pub async fn initialize(&self) -> std::io::Result<()> { - fs::create_dir_all(&self.signatures_dir).await?; - tracing::info!("Delta Sync service initialized with block size: {}KB", - self.block_size / 1024); - Ok(()) - } - - /// Generar y almacenar firmas para un archivo - pub async fn index_file( - &self, - file_id: &str, - file_path: &Path - ) -> Result, DeltaSyncError> { - let data = fs::read(file_path).await?; - - // No indexar archivos pequeños - if data.len() < MIN_DELTA_SYNC_SIZE as usize { - return Ok(Vec::new()); - } - - let signatures = generate_signatures(&data, self.block_size); - - // Guardar en disco - let sig_path = self.signature_path(file_id); - let sig_json = serde_json::to_vec(&signatures) - .map_err(|e| DeltaSyncError::SerializationError(e.to_string()))?; - fs::write(&sig_path, sig_json).await?; - - // Actualizar cache - { - let mut cache = self.signature_cache.write().await; - if cache.len() >= self.max_cache_entries { - // LRU simple: eliminar primera entrada - if let Some(key) = cache.keys().next().cloned() { - cache.remove(&key); - } - } - cache.insert(file_id.to_string(), signatures.clone()); - } - - tracing::debug!("Indexed file {} with {} blocks", file_id, signatures.len()); - Ok(signatures) - } - - /// Obtener firmas de un archivo - pub async fn get_signatures( - &self, - file_id: &str - ) -> Result, DeltaSyncError> { - // Buscar en cache primero - { - let cache = self.signature_cache.read().await; - if let Some(sigs) = cache.get(file_id) { - return Ok(sigs.clone()); - } - } - - // Cargar de disco - let sig_path = self.signature_path(file_id); - if !sig_path.exists() { - return Err(DeltaSyncError::SignaturesNotFound(file_id.to_string())); - } - - let sig_json = fs::read(&sig_path).await?; - let signatures: Vec = serde_json::from_slice(&sig_json) - .map_err(|e| DeltaSyncError::SerializationError(e.to_string()))?; - - // Actualizar cache - { - let mut cache = self.signature_cache.write().await; - cache.insert(file_id.to_string(), signatures.clone()); - } - - Ok(signatures) - } - - /// Aplicar delta a un archivo - pub async fn apply_delta( - &self, - file_id: &str, - base_path: &Path, - delta: &FileDelta, - ) -> Result, DeltaSyncError> { - let base_data = fs::read(base_path).await?; - let signatures = self.get_signatures(file_id).await?; - - apply_delta(&base_data, &signatures, delta, self.block_size) - } - - /// Eliminar firmas de un archivo (cuando se borra) - pub async fn remove_signatures(&self, file_id: &str) -> Result<(), DeltaSyncError> { - // Eliminar de cache - { - let mut cache = self.signature_cache.write().await; - cache.remove(file_id); - } - - // Eliminar de disco - let sig_path = self.signature_path(file_id); - if sig_path.exists() { - fs::remove_file(&sig_path).await?; - } - - Ok(()) - } - - /// Estadísticas del servicio - pub async fn get_stats(&self) -> DeltaSyncStats { - let cache = self.signature_cache.read().await; - DeltaSyncStats { - cached_files: cache.len() as u64, - block_size: self.block_size, - } - } - - fn signature_path(&self, file_id: &str) -> PathBuf { - // Usar primeros 2 chars del ID para subdirectorio - let prefix = &file_id[..2.min(file_id.len())]; - self.signatures_dir.join(prefix).join(format!("{}.sig", file_id)) - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct DeltaSyncStats { - pub cached_files: u64, - pub block_size: usize, -} -``` - ---- - -## API Endpoints - -### Handler (delta_sync_handler.rs) - -```rust -use axum::{ - extract::{Path, State, Json}, - http::StatusCode, - response::IntoResponse, -}; -use crate::common::di::AppState; -use crate::infrastructure::services::delta_sync_service::*; - -pub struct DeltaSyncHandler; - -impl DeltaSyncHandler { - /// GET /api/files/{id}/signatures - /// - /// Obtiene las firmas de bloques de un archivo para calcular delta - pub async fn get_signatures( - State(state): State, - Path(file_id): Path, - ) -> impl IntoResponse { - let delta_service = &state.core.delta_sync_service; - - match delta_service.get_signatures(&file_id).await { - Ok(signatures) => { - Json(SignaturesResponse { - file_id, - block_size: delta_service.block_size, - block_count: signatures.len() as u32, - signatures, - }).into_response() - } - Err(DeltaSyncError::SignaturesNotFound(_)) => { - // Archivo no indexado - cliente debe hacer upload completo - (StatusCode::NOT_FOUND, Json(serde_json::json!({ - "error": "Signatures not found", - "hint": "File not indexed for delta sync, use full upload" - }))).into_response() - } - Err(e) => { - (StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ - "error": e.to_string() - }))).into_response() - } - } - } - - /// POST /api/files/{id}/delta - /// - /// Aplica un delta para actualizar un archivo - pub async fn apply_delta( - State(state): State, - Path(file_id): Path, - Json(delta): Json, - ) -> impl IntoResponse { - let delta_service = &state.core.delta_sync_service; - let file_service = &state.applications.file_service; - - // Obtener path del archivo actual - let file = match file_service.get_file(&file_id).await { - Ok(f) => f, - Err(_) => { - return (StatusCode::NOT_FOUND, Json(serde_json::json!({ - "error": "File not found" - }))).into_response(); - } - }; - - // Aplicar delta - let file_path = state.core.path_service.resolve_path(file.path()); - match delta_service.apply_delta(&file_id, &file_path, &delta).await { - Ok(new_data) => { - // Guardar nuevo contenido - if let Err(e) = tokio::fs::write(&file_path, &new_data).await { - return (StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ - "error": format!("Failed to write file: {}", e) - }))).into_response(); - } - - // Re-indexar archivo - if let Err(e) = delta_service.index_file(&file_id, &file_path).await { - tracing::warn!("Failed to re-index file after delta: {}", e); - } - - // Calcular estadísticas - let delta_size: usize = delta.instructions.iter() - .filter_map(|i| match i { - DeltaInstruction::Literal { data } => Some(data.len()), - _ => None, - }) - .sum(); - - Json(DeltaApplyResponse { - success: true, - new_size: new_data.len() as u64, - delta_size: delta_size as u64, - savings_percent: if new_data.len() > 0 { - ((1.0 - (delta_size as f64 / new_data.len() as f64)) * 100.0) as u32 - } else { 0 }, - }).into_response() - } - Err(e) => { - (StatusCode::BAD_REQUEST, Json(serde_json::json!({ - "error": e.to_string() - }))).into_response() - } - } - } - - /// POST /api/files/{id}/index - /// - /// Fuerza la indexación de un archivo para delta sync - pub async fn index_file( - State(state): State, - Path(file_id): Path, - ) -> impl IntoResponse { - let delta_service = &state.core.delta_sync_service; - let file_service = &state.applications.file_service; - - // Obtener path del archivo - let file = match file_service.get_file(&file_id).await { - Ok(f) => f, - Err(_) => { - return (StatusCode::NOT_FOUND, Json(serde_json::json!({ - "error": "File not found" - }))).into_response(); - } - }; - - let file_path = state.core.path_service.resolve_path(file.path()); - match delta_service.index_file(&file_id, &file_path).await { - Ok(signatures) => { - Json(serde_json::json!({ - "success": true, - "blocks_indexed": signatures.len() - })).into_response() - } - Err(e) => { - (StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ - "error": e.to_string() - }))).into_response() - } - } - } - - /// GET /api/delta/stats - /// - /// Estadísticas del servicio delta sync - pub async fn get_stats( - State(state): State, - ) -> impl IntoResponse { - let delta_service = &state.core.delta_sync_service; - Json(delta_service.get_stats().await) - } -} - -#[derive(Serialize)] -struct SignaturesResponse { - file_id: String, - block_size: usize, - block_count: u32, - signatures: Vec, -} - -#[derive(Serialize)] -struct DeltaApplyResponse { - success: bool, - new_size: u64, - delta_size: u64, - savings_percent: u32, -} -``` - -### Rutas a añadir en routes.rs - -```rust -// Delta Sync routes -let delta_sync_router = Router::new() - .route("/files/:id/signatures", get(DeltaSyncHandler::get_signatures)) - .route("/files/:id/delta", post(DeltaSyncHandler::apply_delta)) - .route("/files/:id/index", post(DeltaSyncHandler::index_file)) - .route("/delta/stats", get(DeltaSyncHandler::get_stats)) - .with_state(app_state.clone()); - -// Añadir a router principal -router = router.nest("/api", delta_sync_router); -``` - ---- - -## Integración con sistema existente - -### 1. Modificar upload para indexar automáticamente - -En `file_handler.rs`, después de un upload exitoso: - -```rust -// Después de guardar el archivo... - -// Indexar para delta sync (archivos >64KB) -if total_size >= 64 * 1024 { - let delta_service = &state.core.delta_sync_service; - if let Err(e) = delta_service.index_file(&file.id, &file_path).await { - tracing::warn!("Failed to index file for delta sync: {}", e); - // No es error fatal, el archivo se subió correctamente - } -} -``` - -### 2. Modificar delete para limpiar firmas - -En `file_handler.rs`, al eliminar archivo: - -```rust -// Limpiar firmas de delta sync -let delta_service = &state.core.delta_sync_service; -if let Err(e) = delta_service.remove_signatures(&id).await { - tracing::warn!("Failed to remove delta signatures: {}", e); -} -``` - -### 3. Integración con Dedup Service - -Delta Sync y Dedup son complementarios: - -``` -┌──────────────────────────────────────────────────────────────┐ -│ UPLOAD CON DELTA + DEDUP │ -├──────────────────────────────────────────────────────────────┤ -│ │ -│ 1. Cliente tiene archivo.txt modificado │ -│ │ -│ 2. GET /files/{id}/signatures │ -│ → Servidor retorna firmas de bloques │ -│ │ -│ 3. Cliente calcula delta localmente │ -│ → Solo 3 bloques de 100 cambiaron │ -│ │ -│ 4. POST /files/{id}/delta │ -│ → Envía solo los 3 bloques nuevos │ -│ │ -│ 5. Servidor aplica delta │ -│ → Reconstruye archivo completo │ -│ │ -│ 6. Servidor ejecuta dedup en archivo resultante │ -│ → Si otro usuario tiene mismo contenido, se deduplica │ -│ │ -│ RESULTADO: │ -│ ├── Delta sync: 97% menos transferencia │ -│ └── Dedup: 30-50% menos almacenamiento │ -│ │ -└──────────────────────────────────────────────────────────────┘ -``` - ---- - -## Casos de uso y efectividad - -| Tipo de archivo | Escenario | Sin Delta | Con Delta | Ahorro | -|-----------------|-----------|-----------|-----------|--------| -| `.txt` / `.md` | Editar párrafo | 1MB | ~4KB | **99.6%** | -| `.json` / `.xml` | Cambiar valor | 500KB | ~1KB | **99.8%** | -| `.rs` / `.js` | Modificar función | 100KB | ~2KB | **98%** | -| `.docx` | Editar página | 5MB | ~100KB | **98%** | -| `.xlsx` | Cambiar celdas | 2MB | ~50KB | **97.5%** | -| `.pdf` | Editar texto | 10MB | ~2MB | **80%** | -| `.psd` | Editar capa | 100MB | ~5MB | **95%** | -| `.zip` | Añadir archivo | 50MB | ~5MB | **90%** | -| `.mp4` | Re-encode | 500MB | 450MB | **10%** ❌ | -| `.jpg` | Editar imagen | 5MB | 4MB | **20%** ❌ | - -**Nota**: Para archivos muy comprimidos o re-encodeados, delta sync es menos efectivo. - ---- - -## Consideraciones de rendimiento - -### Tamaño de bloque óptimo - -| Tamaño | Pros | Cons | Mejor para | -|--------|------|------|------------| -| 4KB | Más granular, mejor ahorro | Más overhead de firmas | Archivos pequeños | -| 16KB | Buen balance | - | **Uso general** ✅ | -| 64KB | Menos overhead | Menos granular | Archivos grandes | -| 256KB | Mínimo overhead | Poco ahorro | Archivos enormes | - -### Memoria - -```rust -// Estimación de memoria por archivo indexado -// -// BlockSignature size ≈ 48 bytes (4 + 8 + 4 + 4 + 32 - con padding) -// -// Archivo 100MB con bloques de 16KB: -// - 100MB / 16KB = 6,400 bloques -// - 6,400 × 48 bytes = ~300KB de firmas -// -// Cache de 1000 archivos ≈ 300MB máximo -``` - -### CPU - -```rust -// Operaciones costosas: -// -// 1. generate_signatures(): O(n) donde n = tamaño archivo -// - SHA-256: ~500MB/s en CPU moderna -// - Adler32: ~2GB/s -// -// 2. generate_delta(): O(n × m) peor caso, O(n) típico -// - n = tamaño archivo nuevo -// - m = número de bloques originales -// - HashMap lookup: O(1) promedio -// -// 3. apply_delta(): O(n) donde n = tamaño resultado -// - Mayormente copias de memoria -``` - ---- - -## Testing - -### Tests unitarios - -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_rolling_checksum() { - let mut rc = RollingChecksum::new(4); - - // Alimentar bytes - for b in b"test" { - rc.roll(*b); - } - let checksum1 = rc.checksum(); - - // Rolling: quitar 't', añadir 'X' - rc.roll(b'X'); - let checksum2 = rc.checksum(); - - // Checksums deben ser diferentes - assert_ne!(checksum1, checksum2); - } - - #[test] - fn test_generate_signatures() { - let data = b"Hello, World! This is a test file for delta sync."; - let sigs = generate_signatures(data, 16); - - assert_eq!(sigs.len(), 4); // 50 bytes / 16 = 3.125 → 4 bloques - assert_eq!(sigs[0].offset, 0); - assert_eq!(sigs[1].offset, 16); - } - - #[test] - fn test_delta_identical_files() { - let data = b"Hello, World!"; - let sigs = generate_signatures(data, 8); - let delta = generate_delta(data, &sigs, 8); - - // Solo instrucciones Copy, sin Literal - for instr in &delta.instructions { - assert!(matches!(instr, DeltaInstruction::Copy { .. })); - } - } - - #[test] - fn test_delta_small_change() { - let original = b"Hello, World! This is original."; - let modified = b"Hello, World! This is MODIFIED."; - - let sigs = generate_signatures(original, 8); - let delta = generate_delta(modified, &sigs, 8); - - // Debería haber algunas instrucciones Copy y algunas Literal - let copies = delta.instructions.iter() - .filter(|i| matches!(i, DeltaInstruction::Copy { .. })) - .count(); - let literals = delta.instructions.iter() - .filter(|i| matches!(i, DeltaInstruction::Literal { .. })) - .count(); - - assert!(copies > 0, "Should reuse some blocks"); - assert!(literals > 0, "Should have some new data"); - } - - #[test] - fn test_apply_delta_roundtrip() { - let original = b"The quick brown fox jumps over the lazy dog."; - let modified = b"The quick brown cat jumps over the lazy dog."; - - let sigs = generate_signatures(original, 8); - let delta = generate_delta(modified, &sigs, 8); - let reconstructed = apply_delta(original, &sigs, &delta, 8).unwrap(); - - assert_eq!(reconstructed, modified); - } -} -``` - -### Tests de integración - -```rust -#[tokio::test] -async fn test_delta_sync_service_workflow() { - let temp_dir = tempfile::tempdir().unwrap(); - let service = DeltaSyncService::new(temp_dir.path()); - service.initialize().await.unwrap(); - - // Crear archivo original - let file_path = temp_dir.path().join("test.txt"); - tokio::fs::write(&file_path, b"Original content here").await.unwrap(); - - // Indexar - let sigs = service.index_file("file123", &file_path).await.unwrap(); - assert!(!sigs.is_empty()); - - // Recuperar firmas - let retrieved = service.get_signatures("file123").await.unwrap(); - assert_eq!(sigs.len(), retrieved.len()); - - // Simular modificación y delta - let modified = b"Modified content here!"; - let delta = generate_delta(modified, &sigs, service.block_size); - - // Aplicar delta - let result = service.apply_delta("file123", &file_path, &delta).await.unwrap(); - assert_eq!(result, modified); -} -``` - ---- - -## Dependencias necesarias - -Añadir a `Cargo.toml`: - -```toml -[dependencies] -# Ya existentes - verificar versiones -sha2 = "0.10" -hex = "0.4" - -# Nuevas dependencias para delta sync -thiserror = "1.0" # Para errores tipados (probablemente ya existe) -``` - ---- - -## Checklist de implementación - -- [ ] Crear `delta_sync_service.rs` con estructuras básicas -- [ ] Implementar `RollingChecksum` -- [ ] Implementar `generate_signatures()` -- [ ] Implementar `generate_delta()` -- [ ] Implementar `apply_delta()` -- [ ] Crear handler y endpoints API -- [ ] Integrar en DI (`CoreServices`) -- [ ] Añadir rutas en `routes.rs` -- [ ] Integrar con upload (indexación automática) -- [ ] Integrar con delete (limpieza de firmas) -- [ ] Tests unitarios -- [ ] Tests de integración -- [ ] Documentar API endpoints -- [ ] Métricas y logging - ---- - -## Referencias - -- [rsync algorithm](https://rsync.samba.org/tech_report/) -- [Rolling hash - Wikipedia](https://en.wikipedia.org/wiki/Rolling_hash) -- [Adler-32 checksum](https://en.wikipedia.org/wiki/Adler-32) -- [librsync](https://github.com/librsync/librsync) - ---- - -*Documento creado: 2026-02-03* -*Última actualización: 2026-02-03* +# 20 - Delta Sync Implementation + +Delta sync transfers only the modified parts of a file instead of the whole thing. Based on the rsync algorithm, it can save 90-99% bandwidth in common scenarios. + +**Status**: pending implementation +**Priority**: medium +**Estimated savings**: 10-100x less data transfer + +## Contents + +1. [Problem Statement](#problem-statement) +2. [How It Works](#how-it-works) +3. [Key Algorithms](#key-algorithms) +4. [Proposed Architecture](#proposed-architecture) +5. [Data Structures](#data-structures) +6. [API Endpoints](#api-endpoints) +7. [Step-by-Step Implementation](#step-by-step-implementation) +8. [Integration with Existing System](#integration-with-existing-system) +9. [Use Cases and Effectiveness](#use-cases-and-effectiveness) +10. [Performance Considerations](#performance-considerations) +11. [Testing](#testing) +12. [Required Dependencies](#required-dependencies) + +--- + +## Key Benefits + +| Metric | Without Delta Sync | With Delta Sync | +|---------|----------------|----------------| +| Edit 1 line in 100MB | 100MB transferred | ~1KB transferred | +| Sync time (slow connection) | 4+ minutes | <1 second | +| Bandwidth consumption | 100% | 0.1-10% | + +--- + +## Problem Statement + +### Current scenario (no delta sync) + +``` +User has document.docx (50MB) on OxiCloud + │ + ▼ +Downloads full file (50MB) ──────────────────────► 50MB ↓ + │ + ▼ +Edits one word + │ + ▼ +Uploads full file again (50MB) ──────────────────► 50MB ↑ + │ + ▼ +TOTAL: 100MB transferred to change one word +``` + +### Target scenario (with delta sync) + +``` +User has document.docx (50MB) on OxiCloud + │ + ▼ +Downloads full file (50MB) ──────────────────────► 50MB ↓ (first time) + │ + ▼ +Edits one word + │ + ▼ +Uploads ONLY modified blocks ───────────────────► ~50KB ↑ + │ + ▼ +TOTAL: 50.05MB (99.9% savings on upload) +``` + +--- + +## How It Works + +### Block (chunk) concept + +The file gets divided into fixed-size blocks (typically 4KB-64KB): + +``` +Original file (server): +┌────────┬────────┬────────┬────────┬────────┐ +│ Bloque │ Bloque │ Bloque │ Bloque │ Bloque │ +│ 0 │ 1 │ 2 │ 3 │ 4 │ +│ 4KB │ 4KB │ 4KB │ 4KB │ 4KB │ +│ │ │ │ │ │ +│ weak:A │ weak:B │ weak:C │ weak:D │ weak:E │ +│ sha:X1 │ sha:X2 │ sha:X3 │ sha:X4 │ sha:X5 │ +└────────┴────────┴────────┴────────┴────────┘ + +Modified file (client): +┌────────┬────────┬────────┬────────┬────────┐ +│ Bloque │ Bloque │ Bloque │ Bloque │ Bloque │ +│ 0 │ 1 │ 2 │ 3 │ 4 │ +│ 4KB │ 4KB │ 4KB │ 4KB │ 4KB │ +│ │ │ │ │ │ +│ weak:A │ weak:B │ weak:F │ weak:D │ weak:E │ ← Block 2 changed +│ sha:X1 │ sha:X2 │ sha:Y3 │ sha:X4 │ sha:X5 │ +└────────┴────────┴───▲────┴────────┴────────┘ + │ + ONLY THIS ONE GETS TRANSFERRED +``` + +### Sync flow + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ DELTA SYNC FLOW │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ CLIENT SERVER │ +│ │ +│ 1. Has modified 1. Has original file │ +│ file + block index │ +│ │ +│ 2. Requests signatures ─────────────► │ +│ GET /files/{id}/signatures │ +│ │ +│ ◄────────────── 3. Returns signature list │ +│ [(weak, strong), ...] │ +│ │ +│ 4. Compares local blocks │ +│ against server │ +│ signatures │ +│ │ +│ 5. Generates delta ──────────────────► │ +│ POST /files/{id}/delta │ +│ [References + New data] │ +│ │ +│ 6. Reconstructs file │ +│ by applying delta │ +│ │ +│ ◄────────────── 7. Confirms update │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Key Algorithms + +### 1. Rolling Checksum (modified Adler-32) + +The rolling checksum computes the hash of a sliding window in O(1): + +```rust +/// Rolling checksum para búsqueda rápida de bloques coincidentes +/// Similar al usado por rsync (Adler-32 modificado) +pub struct RollingChecksum { + a: u32, // Suma simple de bytes + b: u32, // Suma ponderada + window_size: usize, + buffer: VecDeque, +} + +impl RollingChecksum { + pub fn new(window_size: usize) -> Self { + Self { + a: 0, + b: 0, + window_size, + buffer: VecDeque::with_capacity(window_size), + } + } + + /// Añadir un byte y calcular nuevo checksum + /// Complejidad: O(1) + pub fn roll(&mut self, new_byte: u8) -> u32 { + if self.buffer.len() >= self.window_size { + // Remover byte antiguo + let old_byte = self.buffer.pop_front().unwrap() as u32; + self.a = self.a.wrapping_sub(old_byte).wrapping_add(new_byte as u32); + self.b = self.b.wrapping_sub(old_byte * self.window_size as u32) + .wrapping_add(self.a); + } else { + // Ventana no llena todavía + self.a = self.a.wrapping_add(new_byte as u32); + self.b = self.b.wrapping_add(self.a); + } + + self.buffer.push_back(new_byte); + self.checksum() + } + + /// Calcular checksum actual + pub fn checksum(&self) -> u32 { + (self.b << 16) | (self.a & 0xFFFF) + } + + /// Reset para nuevo archivo + pub fn reset(&mut self) { + self.a = 0; + self.b = 0; + self.buffer.clear(); + } +} +``` + +### 2. Block Signature + +Each block carries two signatures: +- **Weak checksum** (32-bit) -- fast O(1) lookup +- **Strong hash** (SHA-256) -- definitive verification + +```rust +/// Firma de un bloque para identificación +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BlockSignature { + /// Índice del bloque en el archivo + pub index: u32, + /// Offset en bytes desde el inicio del archivo + pub offset: u64, + /// Tamaño del bloque (puede ser menor para el último) + pub size: u32, + /// Rolling checksum (32-bit) - búsqueda rápida + pub weak_checksum: u32, + /// SHA-256 hash (256-bit) - verificación definitiva + pub strong_hash: [u8; 32], +} + +/// Genera firmas para todos los bloques de un archivo +pub fn generate_signatures(data: &[u8], block_size: usize) -> Vec { + let mut signatures = Vec::new(); + let mut offset = 0u64; + let mut index = 0u32; + + for chunk in data.chunks(block_size) { + // Weak checksum (rolling) + let weak = adler32_checksum(chunk); + + // Strong hash (SHA-256) + let mut hasher = Sha256::new(); + hasher.update(chunk); + let strong: [u8; 32] = hasher.finalize().into(); + + signatures.push(BlockSignature { + index, + offset, + size: chunk.len() as u32, + weak_checksum: weak, + strong_hash: strong, + }); + + offset += chunk.len() as u64; + index += 1; + } + + signatures +} + +fn adler32_checksum(data: &[u8]) -> u32 { + let mut a: u32 = 1; + let mut b: u32 = 0; + + for &byte in data { + a = (a + byte as u32) % 65521; + b = (b + a) % 65521; + } + + (b << 16) | a +} +``` + +### 3. Delta Generation + +```rust +/// Instrucción de delta +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum DeltaInstruction { + /// Copiar bloque existente del archivo original + Copy { + /// Índice del bloque en el archivo original + block_index: u32, + }, + /// Insertar datos literales nuevos + Literal { + /// Datos nuevos a insertar + data: Vec, + }, +} + +/// Delta completo para actualizar un archivo +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileDelta { + /// ID del archivo base + pub base_file_id: String, + /// Hash del archivo base (para verificación) + pub base_file_hash: String, + /// Nuevo tamaño del archivo + pub new_size: u64, + /// Instrucciones de delta + pub instructions: Vec, + /// Hash del archivo resultante (para verificación) + pub result_hash: String, +} + +/// Genera delta comparando archivo local con firmas remotas +pub fn generate_delta( + local_data: &[u8], + remote_signatures: &[BlockSignature], + block_size: usize, +) -> FileDelta { + // Crear índice de weak checksums para búsqueda O(1) + let mut weak_index: HashMap> = HashMap::new(); + for sig in remote_signatures { + weak_index.entry(sig.weak_checksum) + .or_default() + .push(sig); + } + + let mut instructions = Vec::new(); + let mut rolling = RollingChecksum::new(block_size); + let mut pos = 0; + let mut literal_buffer = Vec::new(); + + while pos < local_data.len() { + // Calcular rolling checksum de la ventana actual + let end = (pos + block_size).min(local_data.len()); + let window = &local_data[pos..end]; + + let weak = if window.len() == block_size { + rolling.reset(); + for &b in window { + rolling.roll(b); + } + rolling.checksum() + } else { + adler32_checksum(window) + }; + + // Buscar coincidencia + let mut found_match = false; + + if let Some(candidates) = weak_index.get(&weak) { + // Verificar con strong hash + let mut hasher = Sha256::new(); + hasher.update(window); + let strong: [u8; 32] = hasher.finalize().into(); + + for sig in candidates { + if sig.strong_hash == strong && sig.size as usize == window.len() { + // ¡Coincidencia encontrada! + + // Flush literal buffer si hay datos pendientes + if !literal_buffer.is_empty() { + instructions.push(DeltaInstruction::Literal { + data: std::mem::take(&mut literal_buffer), + }); + } + + // Añadir instrucción de copia + instructions.push(DeltaInstruction::Copy { + block_index: sig.index, + }); + + pos += window.len(); + found_match = true; + break; + } + } + } + + if !found_match { + // No hay coincidencia, añadir byte a literal buffer + literal_buffer.push(local_data[pos]); + pos += 1; + } + } + + // Flush remaining literal buffer + if !literal_buffer.is_empty() { + instructions.push(DeltaInstruction::Literal { + data: literal_buffer, + }); + } + + // Calcular hash del resultado + let mut hasher = Sha256::new(); + hasher.update(local_data); + let result_hash = hex::encode(hasher.finalize()); + + FileDelta { + base_file_id: String::new(), // Se llena al enviar + base_file_hash: String::new(), // Se llena al enviar + new_size: local_data.len() as u64, + instructions, + result_hash, + } +} +``` + +### 4. Delta Application + +```rust +/// Aplica delta a un archivo base para obtener el nuevo archivo +pub fn apply_delta( + base_data: &[u8], + signatures: &[BlockSignature], + delta: &FileDelta, + block_size: usize, +) -> Result, DeltaSyncError> { + let mut result = Vec::with_capacity(delta.new_size as usize); + + for instruction in &delta.instructions { + match instruction { + DeltaInstruction::Copy { block_index } => { + // Copiar bloque del archivo base + let sig = signatures.get(*block_index as usize) + .ok_or(DeltaSyncError::InvalidBlockIndex(*block_index))?; + + let start = sig.offset as usize; + let end = start + sig.size as usize; + + if end > base_data.len() { + return Err(DeltaSyncError::InvalidBlockRange); + } + + result.extend_from_slice(&base_data[start..end]); + } + DeltaInstruction::Literal { data } => { + // Insertar datos literales + result.extend_from_slice(data); + } + } + } + + // Verificar hash del resultado + let mut hasher = Sha256::new(); + hasher.update(&result); + let actual_hash = hex::encode(hasher.finalize()); + + if actual_hash != delta.result_hash { + return Err(DeltaSyncError::HashMismatch { + expected: delta.result_hash.clone(), + actual: actual_hash, + }); + } + + Ok(result) +} +``` + +--- + +## Proposed Architecture + +### File structure + +``` +src/ +├── infrastructure/ +│ └── services/ +│ ├── mod.rs # Añadir: pub mod delta_sync_service; +│ └── delta_sync_service.rs # NUEVO: Servicio principal +│ +├── interfaces/ +│ └── api/ +│ └── handlers/ +│ ├── mod.rs # Añadir: pub mod delta_sync_handler; +│ └── delta_sync_handler.rs # NUEVO: Endpoints API +│ +└── common/ + └── di.rs # Añadir: delta_sync_service a CoreServices +``` + +### Main service (delta_sync_service.rs) + +```rust +//! Delta Sync Service - Sincronización eficiente por diferencias +//! +//! Implementa algoritmo similar a rsync para transferir solo +//! las partes modificadas de los archivos. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::fs; +use tokio::sync::RwLock; +use sha2::{Sha256, Digest}; +use serde::{Deserialize, Serialize}; + +/// Tamaño de bloque por defecto (16KB - buen balance) +pub const DEFAULT_BLOCK_SIZE: usize = 16 * 1024; + +/// Tamaño mínimo de archivo para usar delta sync +pub const MIN_DELTA_SYNC_SIZE: u64 = 64 * 1024; // 64KB + +/// Errores del servicio Delta Sync +#[derive(Debug, thiserror::Error)] +pub enum DeltaSyncError { + #[error("Archivo no encontrado: {0}")] + FileNotFound(String), + + #[error("Firmas no encontradas para archivo: {0}")] + SignaturesNotFound(String), + + #[error("Índice de bloque inválido: {0}")] + InvalidBlockIndex(u32), + + #[error("Rango de bloque inválido")] + InvalidBlockRange, + + #[error("Hash no coincide: esperado {expected}, actual {actual}")] + HashMismatch { expected: String, actual: String }, + + #[error("Error de I/O: {0}")] + IoError(#[from] std::io::Error), + + #[error("Error de serialización: {0}")] + SerializationError(String), +} + +/// Servicio de Delta Sync +pub struct DeltaSyncService { + /// Directorio para almacenar índices de firmas + signatures_dir: PathBuf, + /// Cache en memoria de firmas recientes + signature_cache: Arc>>>, + /// Tamaño de bloque configurado + block_size: usize, + /// Máximo de entradas en cache + max_cache_entries: usize, +} + +impl DeltaSyncService { + pub fn new(storage_root: &Path) -> Self { + Self { + signatures_dir: storage_root.join(".delta_signatures"), + signature_cache: Arc::new(RwLock::new(HashMap::new())), + block_size: DEFAULT_BLOCK_SIZE, + max_cache_entries: 1000, + } + } + + pub fn with_block_size(mut self, block_size: usize) -> Self { + self.block_size = block_size; + self + } + + /// Inicializar servicio (crear directorios) + pub async fn initialize(&self) -> std::io::Result<()> { + fs::create_dir_all(&self.signatures_dir).await?; + tracing::info!("Delta Sync service initialized with block size: {}KB", + self.block_size / 1024); + Ok(()) + } + + /// Generar y almacenar firmas para un archivo + pub async fn index_file( + &self, + file_id: &str, + file_path: &Path + ) -> Result, DeltaSyncError> { + let data = fs::read(file_path).await?; + + // No indexar archivos pequeños + if data.len() < MIN_DELTA_SYNC_SIZE as usize { + return Ok(Vec::new()); + } + + let signatures = generate_signatures(&data, self.block_size); + + // Guardar en disco + let sig_path = self.signature_path(file_id); + let sig_json = serde_json::to_vec(&signatures) + .map_err(|e| DeltaSyncError::SerializationError(e.to_string()))?; + fs::write(&sig_path, sig_json).await?; + + // Actualizar cache + { + let mut cache = self.signature_cache.write().await; + if cache.len() >= self.max_cache_entries { + // LRU simple: eliminar primera entrada + if let Some(key) = cache.keys().next().cloned() { + cache.remove(&key); + } + } + cache.insert(file_id.to_string(), signatures.clone()); + } + + tracing::debug!("Indexed file {} with {} blocks", file_id, signatures.len()); + Ok(signatures) + } + + /// Obtener firmas de un archivo + pub async fn get_signatures( + &self, + file_id: &str + ) -> Result, DeltaSyncError> { + // Buscar en cache primero + { + let cache = self.signature_cache.read().await; + if let Some(sigs) = cache.get(file_id) { + return Ok(sigs.clone()); + } + } + + // Cargar de disco + let sig_path = self.signature_path(file_id); + if !sig_path.exists() { + return Err(DeltaSyncError::SignaturesNotFound(file_id.to_string())); + } + + let sig_json = fs::read(&sig_path).await?; + let signatures: Vec = serde_json::from_slice(&sig_json) + .map_err(|e| DeltaSyncError::SerializationError(e.to_string()))?; + + // Actualizar cache + { + let mut cache = self.signature_cache.write().await; + cache.insert(file_id.to_string(), signatures.clone()); + } + + Ok(signatures) + } + + /// Aplicar delta a un archivo + pub async fn apply_delta( + &self, + file_id: &str, + base_path: &Path, + delta: &FileDelta, + ) -> Result, DeltaSyncError> { + let base_data = fs::read(base_path).await?; + let signatures = self.get_signatures(file_id).await?; + + apply_delta(&base_data, &signatures, delta, self.block_size) + } + + /// Eliminar firmas de un archivo (cuando se borra) + pub async fn remove_signatures(&self, file_id: &str) -> Result<(), DeltaSyncError> { + // Eliminar de cache + { + let mut cache = self.signature_cache.write().await; + cache.remove(file_id); + } + + // Eliminar de disco + let sig_path = self.signature_path(file_id); + if sig_path.exists() { + fs::remove_file(&sig_path).await?; + } + + Ok(()) + } + + /// Estadísticas del servicio + pub async fn get_stats(&self) -> DeltaSyncStats { + let cache = self.signature_cache.read().await; + DeltaSyncStats { + cached_files: cache.len() as u64, + block_size: self.block_size, + } + } + + fn signature_path(&self, file_id: &str) -> PathBuf { + // Usar primeros 2 chars del ID para subdirectorio + let prefix = &file_id[..2.min(file_id.len())]; + self.signatures_dir.join(prefix).join(format!("{}.sig", file_id)) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct DeltaSyncStats { + pub cached_files: u64, + pub block_size: usize, +} +``` + +--- + +## API Endpoints + +### Handler (delta_sync_handler.rs) + +```rust +use axum::{ + extract::{Path, State, Json}, + http::StatusCode, + response::IntoResponse, +}; +use crate::common::di::AppState; +use crate::infrastructure::services::delta_sync_service::*; + +pub struct DeltaSyncHandler; + +impl DeltaSyncHandler { + /// GET /api/files/{id}/signatures + /// + /// Obtiene las firmas de bloques de un archivo para calcular delta + pub async fn get_signatures( + State(state): State, + Path(file_id): Path, + ) -> impl IntoResponse { + let delta_service = &state.core.delta_sync_service; + + match delta_service.get_signatures(&file_id).await { + Ok(signatures) => { + Json(SignaturesResponse { + file_id, + block_size: delta_service.block_size, + block_count: signatures.len() as u32, + signatures, + }).into_response() + } + Err(DeltaSyncError::SignaturesNotFound(_)) => { + // Archivo no indexado - cliente debe hacer upload completo + (StatusCode::NOT_FOUND, Json(serde_json::json!({ + "error": "Signatures not found", + "hint": "File not indexed for delta sync, use full upload" + }))).into_response() + } + Err(e) => { + (StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ + "error": e.to_string() + }))).into_response() + } + } + } + + /// POST /api/files/{id}/delta + /// + /// Aplica un delta para actualizar un archivo + pub async fn apply_delta( + State(state): State, + Path(file_id): Path, + Json(delta): Json, + ) -> impl IntoResponse { + let delta_service = &state.core.delta_sync_service; + let file_service = &state.applications.file_service; + + // Obtener path del archivo actual + let file = match file_service.get_file(&file_id).await { + Ok(f) => f, + Err(_) => { + return (StatusCode::NOT_FOUND, Json(serde_json::json!({ + "error": "File not found" + }))).into_response(); + } + }; + + // Aplicar delta + let file_path = state.core.path_service.resolve_path(file.path()); + match delta_service.apply_delta(&file_id, &file_path, &delta).await { + Ok(new_data) => { + // Guardar nuevo contenido + if let Err(e) = tokio::fs::write(&file_path, &new_data).await { + return (StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ + "error": format!("Failed to write file: {}", e) + }))).into_response(); + } + + // Re-indexar archivo + if let Err(e) = delta_service.index_file(&file_id, &file_path).await { + tracing::warn!("Failed to re-index file after delta: {}", e); + } + + // Calcular estadísticas + let delta_size: usize = delta.instructions.iter() + .filter_map(|i| match i { + DeltaInstruction::Literal { data } => Some(data.len()), + _ => None, + }) + .sum(); + + Json(DeltaApplyResponse { + success: true, + new_size: new_data.len() as u64, + delta_size: delta_size as u64, + savings_percent: if new_data.len() > 0 { + ((1.0 - (delta_size as f64 / new_data.len() as f64)) * 100.0) as u32 + } else { 0 }, + }).into_response() + } + Err(e) => { + (StatusCode::BAD_REQUEST, Json(serde_json::json!({ + "error": e.to_string() + }))).into_response() + } + } + } + + /// POST /api/files/{id}/index + /// + /// Fuerza la indexación de un archivo para delta sync + pub async fn index_file( + State(state): State, + Path(file_id): Path, + ) -> impl IntoResponse { + let delta_service = &state.core.delta_sync_service; + let file_service = &state.applications.file_service; + + // Obtener path del archivo + let file = match file_service.get_file(&file_id).await { + Ok(f) => f, + Err(_) => { + return (StatusCode::NOT_FOUND, Json(serde_json::json!({ + "error": "File not found" + }))).into_response(); + } + }; + + let file_path = state.core.path_service.resolve_path(file.path()); + match delta_service.index_file(&file_id, &file_path).await { + Ok(signatures) => { + Json(serde_json::json!({ + "success": true, + "blocks_indexed": signatures.len() + })).into_response() + } + Err(e) => { + (StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ + "error": e.to_string() + }))).into_response() + } + } + } + + /// GET /api/delta/stats + /// + /// Estadísticas del servicio delta sync + pub async fn get_stats( + State(state): State, + ) -> impl IntoResponse { + let delta_service = &state.core.delta_sync_service; + Json(delta_service.get_stats().await) + } +} + +#[derive(Serialize)] +struct SignaturesResponse { + file_id: String, + block_size: usize, + block_count: u32, + signatures: Vec, +} + +#[derive(Serialize)] +struct DeltaApplyResponse { + success: bool, + new_size: u64, + delta_size: u64, + savings_percent: u32, +} +``` + +### Routes to add in routes.rs + +```rust +// Delta Sync routes +let delta_sync_router = Router::new() + .route("/files/:id/signatures", get(DeltaSyncHandler::get_signatures)) + .route("/files/:id/delta", post(DeltaSyncHandler::apply_delta)) + .route("/files/:id/index", post(DeltaSyncHandler::index_file)) + .route("/delta/stats", get(DeltaSyncHandler::get_stats)) + .with_state(app_state.clone()); + +// Añadir a router principal +router = router.nest("/api", delta_sync_router); +``` + +--- + +## Integration with Existing System + +### 1. Auto-index on upload + +In `file_handler.rs`, after a successful upload: + +```rust +// Después de guardar el archivo... + +// Indexar para delta sync (archivos >64KB) +if total_size >= 64 * 1024 { + let delta_service = &state.core.delta_sync_service; + if let Err(e) = delta_service.index_file(&file.id, &file_path).await { + tracing::warn!("Failed to index file for delta sync: {}", e); + // No es error fatal, el archivo se subió correctamente + } +} +``` + +### 2. Clean up signatures on delete + +In `file_handler.rs`, when deleting a file: + +```rust +// Limpiar firmas de delta sync +let delta_service = &state.core.delta_sync_service; +if let Err(e) = delta_service.remove_signatures(&id).await { + tracing::warn!("Failed to remove delta signatures: {}", e); +} +``` + +### 3. Integration with Dedup Service + +Delta sync and dedup are complementary: + +``` +┌──────────────────────────────────────────────────────────────┐ +│ UPLOAD WITH DELTA + DEDUP │ +├──────────────────────────────────────────────────────────────┤ +│ │ +│ 1. Client has modified file.txt │ +│ │ +│ 2. GET /files/{id}/signatures │ +│ → Server returns block signatures │ +│ │ +│ 3. Client computes delta locally │ +│ → Only 3 of 100 blocks changed │ +│ │ +│ 4. POST /files/{id}/delta │ +│ → Sends only the 3 new blocks │ +│ │ +│ 5. Server applies delta │ +│ → Reconstructs complete file │ +│ │ +│ 6. Server runs dedup on resulting file │ +│ → If another user has same content, it deduplicates │ +│ │ +│ RESULT: │ +│ ├── Delta sync: 97% less transfer │ +│ └── Dedup: 30-50% less storage │ +│ │ +└──────────────────────────────────────────────────────────────┘ +``` + +--- + +## Use Cases and Effectiveness + +| File type | Scenario | Without Delta | With Delta | Savings | +|-----------------|-----------|-----------|-----------|--------| +| `.txt` / `.md` | Edit paragraph | 1MB | ~4KB | **99.6%** | +| `.json` / `.xml` | Change value | 500KB | ~1KB | **99.8%** | +| `.rs` / `.js` | Modify function | 100KB | ~2KB | **98%** | +| `.docx` | Edit page | 5MB | ~100KB | **98%** | +| `.xlsx` | Change cells | 2MB | ~50KB | **97.5%** | +| `.pdf` | Edit text | 10MB | ~2MB | **80%** | +| `.psd` | Edit layer | 100MB | ~5MB | **95%** | +| `.zip` | Add file | 50MB | ~5MB | **90%** | +| `.mp4` | Re-encode | 500MB | 450MB | **10%** | +| `.jpg` | Edit image | 5MB | 4MB | **20%** | + +For highly compressed or re-encoded files, delta sync is less effective. + +--- + +## Performance Considerations + +### Optimal block size + +| Size | Pros | Cons | Best for | +|--------|------|------|------------| +| 4KB | More granular, better savings | More signature overhead | Small files | +| 16KB | Good balance | - | **General use** | +| 64KB | Less overhead | Less granular | Large files | +| 256KB | Minimal overhead | Little savings | Very large files | + +### Memory + +```rust +// Estimación de memoria por archivo indexado +// +// BlockSignature size ≈ 48 bytes (4 + 8 + 4 + 4 + 32 - con padding) +// +// Archivo 100MB con bloques de 16KB: +// - 100MB / 16KB = 6,400 bloques +// - 6,400 × 48 bytes = ~300KB de firmas +// +// Cache de 1000 archivos ≈ 300MB máximo +``` + +### CPU + +```rust +// Operaciones costosas: +// +// 1. generate_signatures(): O(n) donde n = tamaño archivo +// - SHA-256: ~500MB/s en CPU moderna +// - Adler32: ~2GB/s +// +// 2. generate_delta(): O(n × m) peor caso, O(n) típico +// - n = tamaño archivo nuevo +// - m = número de bloques originales +// - HashMap lookup: O(1) promedio +// +// 3. apply_delta(): O(n) donde n = tamaño resultado +// - Mayormente copias de memoria +``` + +--- + +## Testing + +### Unit tests + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rolling_checksum() { + let mut rc = RollingChecksum::new(4); + + // Alimentar bytes + for b in b"test" { + rc.roll(*b); + } + let checksum1 = rc.checksum(); + + // Rolling: quitar 't', añadir 'X' + rc.roll(b'X'); + let checksum2 = rc.checksum(); + + // Checksums deben ser diferentes + assert_ne!(checksum1, checksum2); + } + + #[test] + fn test_generate_signatures() { + let data = b"Hello, World! This is a test file for delta sync."; + let sigs = generate_signatures(data, 16); + + assert_eq!(sigs.len(), 4); // 50 bytes / 16 = 3.125 → 4 bloques + assert_eq!(sigs[0].offset, 0); + assert_eq!(sigs[1].offset, 16); + } + + #[test] + fn test_delta_identical_files() { + let data = b"Hello, World!"; + let sigs = generate_signatures(data, 8); + let delta = generate_delta(data, &sigs, 8); + + // Solo instrucciones Copy, sin Literal + for instr in &delta.instructions { + assert!(matches!(instr, DeltaInstruction::Copy { .. })); + } + } + + #[test] + fn test_delta_small_change() { + let original = b"Hello, World! This is original."; + let modified = b"Hello, World! This is MODIFIED."; + + let sigs = generate_signatures(original, 8); + let delta = generate_delta(modified, &sigs, 8); + + // Debería haber algunas instrucciones Copy y algunas Literal + let copies = delta.instructions.iter() + .filter(|i| matches!(i, DeltaInstruction::Copy { .. })) + .count(); + let literals = delta.instructions.iter() + .filter(|i| matches!(i, DeltaInstruction::Literal { .. })) + .count(); + + assert!(copies > 0, "Should reuse some blocks"); + assert!(literals > 0, "Should have some new data"); + } + + #[test] + fn test_apply_delta_roundtrip() { + let original = b"The quick brown fox jumps over the lazy dog."; + let modified = b"The quick brown cat jumps over the lazy dog."; + + let sigs = generate_signatures(original, 8); + let delta = generate_delta(modified, &sigs, 8); + let reconstructed = apply_delta(original, &sigs, &delta, 8).unwrap(); + + assert_eq!(reconstructed, modified); + } +} +``` + +### Integration tests + +```rust +#[tokio::test] +async fn test_delta_sync_service_workflow() { + let temp_dir = tempfile::tempdir().unwrap(); + let service = DeltaSyncService::new(temp_dir.path()); + service.initialize().await.unwrap(); + + // Crear archivo original + let file_path = temp_dir.path().join("test.txt"); + tokio::fs::write(&file_path, b"Original content here").await.unwrap(); + + // Indexar + let sigs = service.index_file("file123", &file_path).await.unwrap(); + assert!(!sigs.is_empty()); + + // Recuperar firmas + let retrieved = service.get_signatures("file123").await.unwrap(); + assert_eq!(sigs.len(), retrieved.len()); + + // Simular modificación y delta + let modified = b"Modified content here!"; + let delta = generate_delta(modified, &sigs, service.block_size); + + // Aplicar delta + let result = service.apply_delta("file123", &file_path, &delta).await.unwrap(); + assert_eq!(result, modified); +} +``` + +--- + +## Required Dependencies + +Add to `Cargo.toml`: + +```toml +[dependencies] +# Ya existentes - verificar versiones +sha2 = "0.10" +hex = "0.4" + +# Nuevas dependencias para delta sync +thiserror = "1.0" # Para errores tipados (probablemente ya existe) +``` + +--- + +## Implementation Checklist + +- [ ] Create `delta_sync_service.rs` with basic structures +- [ ] Implement **RollingChecksum** +- [ ] Implement **generate_signatures()** +- [ ] Implement **generate_delta()** +- [ ] Implement **apply_delta()** +- [ ] Create handler and API endpoints +- [ ] Integrate into DI (**CoreServices**) +- [ ] Add routes in `routes.rs` +- [ ] Integrate with upload (automatic indexing) +- [ ] Integrate with delete (signature cleanup) +- [ ] Unit tests +- [ ] Integration tests +- [ ] Document API endpoints +- [ ] Metrics and logging + +--- + +## References + +- [rsync algorithm](https://rsync.samba.org/tech_report/) +- [Rolling hash - Wikipedia](https://en.wikipedia.org/wiki/Rolling_hash) +- [Adler-32 checksum](https://en.wikipedia.org/wiki/Adler-32) +- [librsync](https://github.com/librsync/librsync) diff --git a/doc/internal-architecture.md b/doc/internal-architecture.md new file mode 100644 index 00000000..55e26d96 --- /dev/null +++ b/doc/internal-architecture.md @@ -0,0 +1,477 @@ +# 01 - Internal Architecture + +OxiCloud follows a **hexagonal (ports & adapters) architecture** organized in four layers: + +``` +Domain → Application → Infrastructure → Interfaces +``` + +All cross-layer dependencies point inward via trait-based ports. The DI container (**AppServiceFactory**) wires concrete implementations at startup. + +--- + +## Dependency Injection Container + +### AppServiceFactory + +**File:** `src/common/di.rs` + +```rust +pub struct AppServiceFactory { + storage_path: PathBuf, + locales_path: PathBuf, + config: AppConfig, +} +``` + +Initialization order in `build_app_state()`: + +1. **Core services** -- path, caches, ID mapping, thumbnail, write-behind, chunked upload, transcode, dedup, compression +2. **Repository services** -- folder repo (stub mediator first), then **FileSystemStorageMediator** (real), file repos, metadata cache, buffer pool +3. **Trash service** (if **enable_trash** enabled) +4. **Application services** -- folder, file upload/retrieval/management, search, i18n +5. **Share service** (if **enable_file_sharing** enabled) +6. **DB-dependent services** -- favorites, recent, storage usage, auth (via **auth_factory**) +7. **Preload** translations + metadata cache +8. **ZIP service** (needs file retrieval + folder service, wired last) +9. **Assemble AppState** + admin settings + CalDAV/CardDAV + +### AppState (Global State) + +```rust +pub struct AppState { + pub core: CoreServices, + pub repositories: RepositoryServices, + pub applications: ApplicationServices, + pub db_pool: Option>, + pub auth_service: Option, + pub admin_settings_service: Option>, + pub trash_service: Option>, + pub share_service: Option>, + pub favorites_service: Option>, + pub recent_service: Option>, + pub storage_usage_service: Option>, + pub calendar_service: Option>, + pub contact_service: Option>, + pub calendar_use_case: Option>, + pub addressbook_use_case: Option>, + pub contact_use_case: Option>, +} +``` + +Builder pattern: `new()` → `with_database()` → `with_auth_services()` → `with_trash_service()` → ... → `for_routing()`. The `Default` impl uses stubs from `crate::common::stubs`. + +### Service Groups + +```rust +pub struct CoreServices { + pub path_service: Arc, + pub file_content_cache: Arc, + pub id_mapping_service: Arc, // folder IDs + pub file_id_mapping_service: Arc, // file IDs (concrete) + pub id_mapping_optimizer: Arc, + pub thumbnail_service: Arc, + pub write_behind_cache: Arc, + pub chunked_upload_service: Arc, + pub image_transcode_service: Arc, + pub dedup_service: Arc, + pub compression_service: Arc, + pub zip_service: Arc, + pub config: AppConfig, +} + +pub struct RepositoryServices { + pub folder_repository: Arc, + pub file_read_repository: Arc, + pub file_write_repository: Arc, + pub i18n_repository: Arc, + pub storage_mediator: Arc, + pub metadata_cache: Arc, + pub trash_repository: Option>, +} + +pub struct ApplicationServices { + pub folder_service_concrete: Arc, + pub folder_service: Arc, + pub file_upload_service: Arc, + pub file_retrieval_service: Arc, + pub file_management_service: Arc, + pub file_use_case_factory: Arc, + pub i18n_service: Arc, + pub trash_service: Option>, + pub search_service: Option>, + pub share_service: Option>, + pub favorites_service: Option>, + pub recent_service: Option>, +} + +pub struct AuthServices { + pub token_service: Arc, + pub auth_application_service: Arc, +} +``` + +--- + +## ID Mapping System + +Maps bidirectionally between **filesystem StoragePaths** and **UUID identifiers**. Two separate instances exist: one for folders (`folder_ids.json`), one for files (`file_ids.json`). + +### StoragePath (Domain Value Object) + +**File:** `src/domain/services/path_service.rs` + +```rust +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct StoragePath { + segments: Vec, // e.g., ["Mi Carpeta - admin", "file.txt"] +} +``` + +| Method | Description | +|---|---| +| `root()` | Empty path (storage root) | +| `from_string(path)` | Parse from `/`-delimited string | +| `join(segment)` | Append a segment | +| `file_name()` | Last segment | +| `parent()` | All segments except last | +| `to_string()` | Join segments with `/` | + +### IdMappingPort (Application Port) + +**File:** `src/application/ports/outbound.rs` + +```rust +#[async_trait] +pub trait IdMappingPort: Send + Sync + 'static { + async fn get_or_create_id(&self, path: &StoragePath) -> Result; + async fn get_path_by_id(&self, id: &str) -> Result; + async fn update_path(&self, id: &str, new_path: &StoragePath) -> Result<(), DomainError>; + async fn remove_id(&self, id: &str) -> Result<(), DomainError>; + async fn save_changes(&self) -> Result<(), DomainError>; + // Default impls for PathBuf variants: + async fn get_file_path(&self, file_id: &str) -> Result; + async fn update_file_path(&self, file_id: &str, new_path: &PathBuf) -> Result<(), DomainError>; +} +``` + +### IdMappingService (Base Implementation) + +**File:** `src/infrastructure/services/id_mapping_service.rs` + +```rust +pub struct IdMappingService { + map_path: PathBuf, // e.g., storage/file_ids.json + id_map: RwLock, + save_mutex: Mutex<()>, + timeouts: TimeoutConfig, + pending_save: RwLock, +} + +struct IdMap { + path_to_id: HashMap, + id_to_path: HashMap, + version: u32, +} +``` + +Operations: +- `get_or_create_id(path)` -- Read-lock first (cache hit). Write-lock on miss, generates `Uuid::new_v4()`. +- `save_pending_changes()` -- Atomic write: serialize → write `.tmp` file → rename over original (with retry). +- `new(map_path)` -- Loads from JSON, rebuilds inverse map if inconsistent. + +Persistence format (`storage/file_ids.json`): +```json +{ + "path_to_id": { "/Mi Carpeta - admin/doc.pdf": "a1b2c3d4-..." }, + "id_to_path": { "a1b2c3d4-...": "/Mi Carpeta - admin/doc.pdf" }, + "version": 42 +} +``` + +### IdMappingOptimizer (Cache Layer) + +**File:** `src/infrastructure/services/id_mapping_optimizer.rs` + +Wraps **IdMappingService** with an in-memory TTL cache: + +| Parameter | Value | +|---|---| +| Max cache entries | 10,000 | +| TTL | 300 s (5 min) | +| Cleanup interval | 150 s (2.5 min) | +| Batch threshold | ≥ 20 queued items | +| Max concurrent batches | 2 (semaphore) | + +```rust +pub struct IdMappingOptimizer { + base_service: Arc, + path_to_id_cache: RwLock>, + id_to_path_cache: RwLock>, + stats: RwLock, + batch_limiter: Semaphore, + pending_batch: Mutex, +} +``` + +Lookup flow: check cache → if miss, queue request → trigger batch if ≥ 20 pending → fallback to **base_service** → update cache. + +On `update_path` / `remove_id`, cache entries are invalidated first, then delegated. + +Used only for folder ID mapping. File ID mapping uses the base **IdMappingService** directly. + +--- + +## Path Service + +**File:** `src/infrastructure/services/path_service.rs` + +```rust +pub struct PathService { + root_path: PathBuf, // e.g., ./storage +} +``` + +### Path Resolution + +| Method | Description | +|---|---| +| `resolve_path(storage_path)` | Appends **StoragePath** segments to **root_path** → absolute `PathBuf` | +| `to_storage_path(physical_path)` | Strips **root_path** prefix → **StoragePath** (returns `None` if outside root) | +| `create_file_path(folder, name)` | Combines folder path + filename | +| `is_direct_child(parent, child)` | Check parent-child relationship | +| `is_in_root(path)` | Verify path is within storage root | + +### Path Validation + +`validate_path(path)` rejects: +- Empty path segments +- Segments containing dangerous characters: `\`, `:`, `*`, `?`, `"`, `<`, `>`, `|` +- Segments starting with `.` (exception: `.well-known` for WebDAV/CalDAV/CardDAV) + +### Trait Implementations + +- **StoragePort** -- `resolve_path()`, `ensure_directory()` (validates first, then `fs::create_dir_all`), `file_exists()`, `directory_exists()` +- **StorageMediator** -- Simplified stub variant. Folder lookup methods return `NotFound`. + +--- + +## Storage Mediator + +Bridges folder IDs to filesystem paths by combining folder repository, path service, and ID mapping. + +### StorageMediator Trait + +**File:** `src/application/services/storage_mediator.rs` + +```rust +#[async_trait] +pub trait StorageMediator: Send + Sync + 'static { + async fn get_folder_path(&self, folder_id: &str) -> StorageMediatorResult; + async fn get_folder_storage_path(&self, folder_id: &str) -> StorageMediatorResult; + async fn get_folder(&self, folder_id: &str) -> StorageMediatorResult; + async fn file_exists_at_path(&self, path: &Path) -> StorageMediatorResult; + async fn file_exists_at_storage_path(&self, storage_path: &StoragePath) -> StorageMediatorResult; + async fn folder_exists_at_path(&self, path: &Path) -> StorageMediatorResult; + async fn folder_exists_at_storage_path(&self, storage_path: &StoragePath) -> StorageMediatorResult; + fn resolve_path(&self, relative_path: &Path) -> PathBuf; + fn resolve_storage_path(&self, storage_path: &StoragePath) -> PathBuf; + async fn ensure_directory(&self, path: &Path) -> StorageMediatorResult<()>; + async fn ensure_storage_directory(&self, storage_path: &StoragePath) -> StorageMediatorResult<()>; +} +``` + +### FileSystemStorageMediator + +```rust +pub struct FileSystemStorageMediator { + pub folder_storage_port: Arc, + pub path_service: Arc, + pub id_mapping: Arc, +} +``` + +Folder ID → filesystem path resolution: + +``` +folder_id ──► FolderStoragePort.get_folder(id) + ──► Folder entity + ──► IdMappingPort.get_path_by_id(folder.id()) + ──► StoragePath + ──► StoragePort.resolve_path(storage_path) + ──► PathBuf (absolute) +``` + +A **StubStorageMediator** also exists (returns `/tmp` paths) for DI bootstrap before the real mediator is available. + +--- + +## Session Management + +### Session Entity + +**File:** `src/domain/entities/session.rs` + +```rust +pub struct Session { + id: String, // UUID v4 + user_id: String, + refresh_token: String, + expires_at: DateTime, + ip_address: Option, + user_agent: Option, + created_at: DateTime, + revoked: bool, +} +``` + +Constructors: +- `Session::new(user_id, refresh_token, ip_address, user_agent, expires_in_days)` -- generates UUID, panics if **user_id** or **refresh_token** empty +- `Session::from_raw(...)` -- for DB reconstruction + +### SessionRepository (Domain Port) + +**File:** `src/domain/repositories/session_repository.rs` + +```rust +#[async_trait] +pub trait SessionRepository: Send + Sync + 'static { + async fn create_session(&self, session: Session) -> SessionRepositoryResult; + async fn get_session_by_id(&self, id: &str) -> SessionRepositoryResult; + async fn get_session_by_refresh_token(&self, token: &str) -> SessionRepositoryResult; + async fn get_sessions_by_user_id(&self, user_id: &str) -> SessionRepositoryResult>; + async fn revoke_session(&self, session_id: &str) -> SessionRepositoryResult<()>; + async fn revoke_all_user_sessions(&self, user_id: &str) -> SessionRepositoryResult; + async fn delete_expired_sessions(&self) -> SessionRepositoryResult; +} +``` + +### SessionStoragePort (Application Port) + +**File:** `src/application/ports/auth_ports.rs` + +```rust +#[async_trait] +pub trait SessionStoragePort: Send + Sync + 'static { + async fn create_session(&self, session: Session) -> Result; + async fn get_session_by_refresh_token(&self, token: &str) -> Result; + async fn revoke_session(&self, session_id: &str) -> Result<(), DomainError>; + async fn revoke_all_user_sessions(&self, user_id: &str) -> Result; +} +``` + +### SessionPgRepository (Infrastructure) + +**File:** `src/infrastructure/repositories/pg/session_pg_repository.rs` + +```rust +pub struct SessionPgRepository { + pool: Arc, +} +``` + +Implements both **SessionRepository** and **SessionStoragePort**. Uses `with_transaction()` helper for write operations. `create_session` also updates `auth.users.last_login_at` within the same transaction. + +### Database Schema + +```sql +CREATE TABLE IF NOT EXISTS auth.sessions ( + id VARCHAR(36) PRIMARY KEY, + user_id VARCHAR(36) NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + refresh_token TEXT NOT NULL UNIQUE, + expires_at TIMESTAMP WITH TIME ZONE NOT NULL, + ip_address TEXT, + user_agent TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + revoked BOOLEAN NOT NULL DEFAULT FALSE +); + +-- Indexes +CREATE INDEX idx_sessions_user_id ON auth.sessions(user_id); +CREATE INDEX idx_sessions_refresh_token ON auth.sessions(refresh_token); +CREATE INDEX idx_sessions_expires_at ON auth.sessions(expires_at); +CREATE INDEX idx_sessions_active ON auth.sessions(user_id, revoked) + WHERE NOT revoked AND is_session_active(expires_at); +``` + +### Auth Service + +**File:** `src/application/services/auth_application_service.rs` + +**AuthApplicationService** orchestrates authentication using: +- **UserStoragePort** -- user CRUD +- **SessionStoragePort** -- session lifecycle +- **PasswordHasherPort** -- Argon2id hashing +- **TokenServicePort** -- JWT generation/validation +- `RwLock` -- hot-reloadable OIDC configuration +- `Mutex>` -- in-flight OIDC login states + +Wired by `auth_factory.rs`: **UserPgRepository** + **SessionPgRepository** + **Argon2PasswordHasher** + **JwtTokenService** → **AuthApplicationService**. + +--- + +## File Use Case Factory + +**File:** `src/application/services/file_use_case_factory.rs` + +```rust +pub trait FileUseCaseFactory: Send + Sync + 'static { + fn create_file_upload_use_case(&self) -> Arc; + fn create_file_retrieval_use_case(&self) -> Arc; + fn create_file_management_use_case(&self) -> Arc; +} +``` + +**AppFileUseCaseFactory** creates lightweight service instances with only **FileReadPort** / **FileWritePort**. The main DI-wired services use `*_full()` constructors that inject write-behind cache, dedup, content cache, and transcode ports for full optimization. + +### File Operation Port Hierarchy + +| Port | Key Methods | +|---|---| +| **FileUploadUseCase** | `upload_file()`, `smart_upload()` (returns **UploadStrategy**: `WriteBehind` <256KB, `Buffered` 256KB-1MB, `Streaming` ≥1MB), `create_file()`, `update_file()` | +| **FileRetrievalUseCase** | `get_file()`, `get_file_content()`, `get_file_stream()`, `get_file_optimized()` (write-behind → content-cache → WebP transcode → mmap → streaming), `get_file_range_stream()` | +| **FileManagementUseCase** | `move_file()`, `rename_file()`, `delete_file()`, `delete_with_cleanup()` (trash-first with dedup reference cleanup) | + +--- + +## Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Interfaces Layer │ +│ Axum Router → API Routes + Middleware (Auth, Compress) │ +└─────────────────────┬───────────────────────────────────────┘ + │ Arc +┌─────────────────────▼───────────────────────────────────────┐ +│ Application Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌─────────────────────┐ │ +│ │ FileUpload │ │ FolderService│ │ AuthApplication │ │ +│ │ FileRetrieval│ │ SearchService│ │ AdminSettings │ │ +│ │ FileMgmt │ │ I18nService │ │ TrashService │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────────┬──────────┘ │ +│ │ Ports (traits) │ │ │ +└─────────┼────────────────┼─────────────────────┼────────────┘ + │ │ │ +┌─────────▼────────────────▼─────────────────────▼────────────┐ +│ Infrastructure Layer │ +│ ┌────────────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ FileFsRead/ │ │ IdMapping │ │ SessionPg │ │ +│ │ FileFsWrite │ │ + Optimizer │ │ UserPg │ │ +│ │ FolderFs │ │ PathService │ │ JwtTokenService │ │ +│ │ TrashFs │ │ StorageMed. │ │ Argon2Hasher │ │ +│ └────────────────┘ └──────────────┘ └──────────────────┘ │ +│ ┌────────────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ ContentCache │ │ Thumbnail │ │ WriteBehind │ │ +│ │ MetadataCache │ │ Transcode │ │ BufferPool │ │ +│ │ BufferPool │ │ Dedup │ │ Compression │ │ +│ └────────────────┘ └──────────────┘ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + │ +┌─────────────────────▼───────────────────────────────────────┐ +│ Domain Layer │ +│ Entities: File, Folder, Session, User, Calendar, Contact │ +│ Value Objects: StoragePath │ +│ Repository Traits: SessionRepository, ... │ +│ Domain Errors │ +└─────────────────────────────────────────────────────────────┘ +``` diff --git a/doc/lto-optimizations.md b/doc/lto-optimizations.md new file mode 100644 index 00000000..2267eb9d --- /dev/null +++ b/doc/lto-optimizations.md @@ -0,0 +1,82 @@ +# 05 - LTO Optimizations + +OxiCloud uses Link Time Optimization (LTO) to improve runtime performance. LTO allows the compiler to optimize across module boundaries during linking -- better inlining, dead code elimination, and more efficient binaries. + +--- + +## Implemented Optimizations + +### Release Profile +```toml +[profile.release] +lto = "fat" # Full cross-module optimization +codegen-units = 1 # Maximum optimization but slower compile time +opt-level = 3 # Maximum optimization level +panic = "abort" # Smaller binary size by removing panic unwinding +strip = true # Removes debug symbols for smaller binary +``` + +### Development Profile +```toml +[profile.dev] +opt-level = 1 # Light optimization for faster build time +debug = true # Keep debug information for development +``` + +### Benchmark Profile +```toml +[profile.bench] +lto = "fat" # Full optimization for benchmarks +codegen-units = 1 # Maximum optimization +opt-level = 3 # Maximum optimization level +``` + +--- + +## Performance Effects + +1. **Smaller binary size** -- unused code and metadata removed +2. **Faster execution** -- better inlining and code optimizations +3. **Reduced memory usage** -- more efficient code layout + +--- + +## LTO Options + +- **fat**: Full LTO across all crate boundaries. Maximum optimization, longest compile time. +- **thin**: Faster LTO that trades some optimization for compile speed. Good for development. +- **off**: No cross-module optimization. + +--- + +## Build Time Impact + +LTO increases compilation time. The tradeoff: + +- Development builds: minimal LTO (`opt-level = 1`) for faster iteration +- Release builds: full LTO for maximum runtime performance +- Benchmark builds: full LTO to measure actual optimized performance + +--- + +## Measuring Impact + +```bash +# Run benchmarks with all optimizations +cargo bench + +# Compare with non-optimized build (remove for comparison only) +RUSTFLAGS="-C lto=off" cargo bench +``` + +--- + +## When to Adjust + +Consider changing these settings if: + +1. You need faster compile times during development +2. You're experiencing unexpected runtime behavior +3. You want to experiment with optimization vs. binary size tradeoffs + +The defaults work well for most cases. diff --git a/doc/OIDC-ARCHITECTURE.md b/doc/oidc-architecture.md similarity index 62% rename from doc/OIDC-ARCHITECTURE.md rename to doc/oidc-architecture.md index 25075e22..9539ee44 100644 --- a/doc/OIDC-ARCHITECTURE.md +++ b/doc/oidc-architecture.md @@ -1,13 +1,13 @@ -# Arquitectura de Integración OIDC en OxiCloud +# 29 - OIDC Architecture -Este documento describe la arquitectura y el flujo de autenticación OpenID Connect (OIDC) en OxiCloud. +OpenID Connect (OIDC) authentication follows the Authorization Code Flow. The system supports multiple identity providers (Authentik, Authelia, KeyCloak) through a single configurable integration point. -## Diagrama de Arquitectura +## Architecture Diagram ``` ┌─────────────────────────────────────────────────────────────────────────┐ │ │ -│ PROVEEDOR DE IDENTIDAD │ +│ IDENTITY PROVIDER │ │ │ │ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ │ │ │ │ │ │ │ │ @@ -16,12 +16,12 @@ Este documento describe la arquitectura y el flujo de autenticación OpenID Conn │ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │ │ │ │ │ │ └────────────┼──────────────────────┼──────────────────────┼─────────────┘ - │ │ │ - │ │ │ - │ │ │ - │ OIDC │ - │ │ │ - │ │ │ + │ │ │ + │ │ │ + │ │ │ + │ OIDC │ + │ │ │ + │ │ │ ┌────────────┼──────────────────────┼──────────────────────┼─────────────┐ │ │ │ │ │ │ ▼ ▼ ▼ │ @@ -59,15 +59,15 @@ Este documento describe la arquitectura y el flujo de autenticación OpenID Conn │ ┌────────────────────────────────────────────────────────────────────────┐ │ │ -│ NAVEGADOR WEB │ +│ WEB BROWSER │ │ │ │ ┌───────────────────────────────────────────────────────────────┐ │ │ │ │ │ -│ │ Interfaz de Usuario │ │ +│ │ User Interface │ │ │ │ │ │ │ │ ┌──────────────┐ ┌──────────────┐ │ │ │ │ │ │ │ │ │ │ -│ │ │ Login.html │ │ oidcAuth.js │ │ │ +│ │ │ Login.html │ │ auth.js │ │ │ │ │ │ │ │ │ │ │ │ │ └──────────────┘ └──────────────┘ │ │ │ │ │ │ @@ -76,87 +76,74 @@ Este documento describe la arquitectura y el flujo de autenticación OpenID Conn └────────────────────────────────────────────────────────────────────────┘ ``` -## Flujo de Autenticación OIDC +## OIDC Authentication Flow -El flujo de autenticación OIDC en OxiCloud sigue el flujo de código de autorización (Authorization Code Flow): +The flow follows the standard Authorization Code Flow: -1. **Inicio de la Autenticación**: - - El usuario hace clic en "Login con [Proveedor]" en la página de inicio de sesión. - - El frontend genera un estado aleatorio para protección CSRF. - - El frontend solicita a OxiCloud una URL de autorización. +1. **Authentication Start** -- the user clicks "Login with [Provider]" on the login page. The frontend generates a random state for CSRF protection and requests an authorization URL from the backend. -2. **Redirección al Proveedor de Identidad**: - - OxiCloud genera una URL de autorización y la devuelve al frontend. - - El navegador redirige al usuario a la página de inicio de sesión del proveedor de identidad. +2. **Redirect to Identity Provider** -- the backend generates and returns the authorization URL. The browser redirects the user to the provider's login page. -3. **Autenticación en el Proveedor**: - - El usuario se autentica en el proveedor de identidad (con contraseña, 2FA, etc.). - - El proveedor redirige al usuario de vuelta a OxiCloud con un código de autorización. +3. **Authentication at the Provider** -- the user authenticates (password, 2FA, etc.). The provider redirects back with an authorization code. -4. **Intercambio del Código de Autorización**: - - El frontend de OxiCloud recibe el código de autorización y lo envía al backend. - - OxiCloud intercambia el código por tokens de acceso e ID con el proveedor de identidad. - - OxiCloud verifica el token de ID y extrae la información del usuario. +4. **Authorization Code Exchange** -- the frontend sends the authorization code to the backend. The backend exchanges it for access and ID tokens with the provider, then verifies the ID token and extracts user info. -5. **Creación/Recuperación de Usuario**: - - OxiCloud busca un usuario existente con el ID externo del proveedor. - - Si no existe y la creación automática está habilitada, se crea un nuevo usuario. - - Si no existe y la creación automática está deshabilitada, se devuelve un error. +5. **User Creation/Retrieval** -- the backend looks up an existing user by the provider's external ID. If none exists and auto-provisioning is enabled, a new user is created. If disabled, an error is returned. -6. **Generación de Tokens de Sesión**: - - OxiCloud genera sus propios tokens de acceso y actualización para el usuario. - - Estos tokens se utilizan para autenticar las solicitudes subsiguientes a la API de OxiCloud. +6. **Session Token Generation** -- the backend generates its own access and refresh tokens for the user. These tokens authenticate subsequent API requests. -7. **Respuesta al Cliente**: - - OxiCloud devuelve los tokens y la información del usuario al frontend. - - El frontend almacena los tokens y redirige al usuario a la página principal. +7. **Response to Client** -- tokens and user info are returned to the frontend. The frontend stores them and redirects to the main page. -## Componentes Principales +## Main Components -### 1. OidcService +### OidcService -Este servicio gestiona la comunicación con los proveedores OIDC: -- Descubre los endpoints OIDC de los proveedores -- Genera URLs de autorización -- Intercambia códigos de autorización por tokens -- Verifica tokens y extrae información de usuario +Handles communication with OIDC providers: +- Discovers provider OIDC endpoints +- Generates authorization URLs +- Exchanges authorization codes for tokens +- Verifies tokens and extracts user info -### 2. AuthApplicationService +### AuthApplicationService -Coordina el proceso de autenticación: -- Proporciona una interfaz entre la capa de API y los servicios de dominio -- Gestiona el proceso de creación/recuperación de usuarios -- Coordina la generación de tokens de acceso para OxiCloud +Coordinates the authentication process: +- Acts as interface between the API layer and domain services +- Manages user creation/retrieval +- Coordinates access token generation -### 3. Auth Handler +### Auth Handler -Expone endpoints HTTP para el flujo de autenticación OIDC: -- `/api/auth/oidc/providers` - Lista los proveedores OIDC disponibles -- `/api/auth/oidc/auth` - Genera una URL de autorización para un proveedor -- `/api/auth/oidc/callback` - Procesa la respuesta del proveedor y completa la autenticación +Exposes HTTP endpoints for the OIDC auth flow: +- `GET /api/auth/oidc/providers` -- lists available OIDC providers +- `GET /api/auth/oidc/authorize` -- generates an authorization URL for the OIDC provider +- `GET /api/auth/oidc/callback` -- receives the redirect from the provider with the authorization code +- `POST /api/auth/oidc/exchange` -- exchanges the authorization code for session tokens -### 4. Frontend (oidcAuth.js) +### Frontend (login.html + auth.js) -Gestiona la parte del cliente del flujo de autenticación: -- Muestra botones para los proveedores OIDC -- Inicia el flujo de autenticación -- Maneja la redirección de retorno del proveedor -- Procesa y almacena los tokens de sesión +Handles the client-side of the auth flow: +- Shows SSO button for the configured OIDC provider in `login.html` +- Initiates the authentication flow via `auth.js` +- Handles the return redirect from the provider +- Processes and stores session tokens -## Configuración Multi-Proveedor +## Provider Configuration -OxiCloud permite configurar múltiples proveedores OIDC simultáneamente: +One OIDC provider is configured per instance via environment variables prefixed with **OXICLOUD_OIDC_***: -1. **Configuración Separada**: Cada proveedor tiene su propia configuración independiente. -2. **Selección de Proveedor**: Los usuarios pueden elegir con qué proveedor autenticarse. -3. **Mapeo de Identidades**: OxiCloud mapea identidades de diferentes proveedores a usuarios internos. +1. **Single provider** per instance. +2. **Environment variables**: **OXICLOUD_OIDC_ENABLED**, **OXICLOUD_OIDC_ISSUER_URL**, **OXICLOUD_OIDC_CLIENT_ID**, **OXICLOUD_OIDC_CLIENT_SECRET**, etc. +3. **Auto-provisioning**: users can be created automatically on first OIDC login (**OXICLOUD_OIDC_AUTO_PROVISION**). +4. **Role mapping**: admin groups are configured via **OXICLOUD_OIDC_ADMIN_GROUPS**. -## Seguridad +See `oidc-config-examples.md` for provider-specific configuration examples. -La implementación OIDC en OxiCloud incluye varias medidas de seguridad: +## Security -1. **Protección CSRF**: Utiliza un estado aleatorio para prevenir ataques CSRF. -2. **Validación de Tokens**: Verifica firmas y vigencia de los tokens JWT. -3. **Código de Autorización**: Utiliza el flujo de código de autorización, que es más seguro que el flujo implícito. -4. **HTTPS**: Requiere conexiones HTTPS para todas las comunicaciones OIDC. -5. **Secretos del Cliente**: Los secretos del cliente se almacenan de forma segura y nunca se exponen al frontend. \ No newline at end of file +The OIDC implementation includes several security measures: + +1. **CSRF protection** -- random state parameter prevents CSRF attacks. +2. **Token validation** -- JWT signatures and expiration are verified. +3. **Authorization Code Flow** -- more secure than the implicit flow. +4. **HTTPS** -- required for all OIDC communications. +5. **Client secrets** -- stored securely, never exposed to the frontend. diff --git a/doc/oidc-config-examples.md b/doc/oidc-config-examples.md new file mode 100644 index 00000000..f4fd4388 --- /dev/null +++ b/doc/oidc-config-examples.md @@ -0,0 +1,215 @@ +# 31 - OIDC Config Examples + +Configuration examples for integrating with different OIDC (OpenID Connect) providers. One OIDC provider per instance. + +## Table of Contents + +1. [General OIDC Configuration](#general-oidc-configuration) +2. [Authentik](#authentik) +3. [Authelia](#authelia) +4. [KeyCloak](#keycloak) +5. [Troubleshooting](#troubleshooting) + +## General OIDC Configuration + +To enable OIDC, set these environment variables: + +```bash +# Enable OIDC +OXICLOUD_OIDC_ENABLED=true + +# OIDC provider configuration +OXICLOUD_OIDC_PROVIDER_NAME="Display Name" +OXICLOUD_OIDC_ISSUER_URL="https://provider.example.com/realms/your-realm" +OXICLOUD_OIDC_CLIENT_ID="your-client-id" +OXICLOUD_OIDC_CLIENT_SECRET="your-client-secret" +OXICLOUD_OIDC_REDIRECT_URI="https://your-oxicloud.example.com/api/auth/oidc/callback" +OXICLOUD_OIDC_SCOPES="openid profile email" +OXICLOUD_OIDC_FRONTEND_URL="https://your-oxicloud.example.com" +OXICLOUD_OIDC_AUTO_PROVISION="true" +OXICLOUD_OIDC_ADMIN_GROUPS="admin-group" +OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN="false" +``` + +## Authentik + +[Authentik](https://goauthentik.io/) is an open-source identity platform providing authentication, authorization, and user management. + +### 1. Create an Application in Authentik + +1. Log into the Authentik admin panel +2. Go to "Applications" -> "Create" +3. Enter a name for the application (e.g. "OxiCloud") +4. Select "OAuth2/OpenID Provider" as the provider type +5. In the OAuth2 configuration: + - **Redirect URI/Callback URL**: `https://your-oxicloud.example.com/api/auth/oidc/callback` + - **Client Type**: Confidential + - **Client ID**: auto-generated (note it down) + - **Client Secret**: auto-generated (note it down) + - **Scopes**: openid, email, profile +6. In the UI configuration: + - **Launch URL**: `https://your-oxicloud.example.com/` + - **Icon**: optional + +### 2. Configure for Authentik + +```yaml +# docker-compose.yml +version: '3' +services: + oxicloud: + image: oxicloud:latest + environment: + OXICLOUD_OIDC_ENABLED: "true" + OXICLOUD_OIDC_PROVIDER_NAME: "Authentik" + OXICLOUD_OIDC_ISSUER_URL: "https://authentik.example.com/application/o/oxicloud" + OXICLOUD_OIDC_CLIENT_ID: "your-authentik-client-id" + OXICLOUD_OIDC_CLIENT_SECRET: "your-authentik-client-secret" + OXICLOUD_OIDC_REDIRECT_URI: "https://oxicloud.example.com/api/auth/oidc/callback" + OXICLOUD_OIDC_SCOPES: "openid profile email" + OXICLOUD_OIDC_FRONTEND_URL: "https://oxicloud.example.com" + OXICLOUD_OIDC_AUTO_PROVISION: "true" + OXICLOUD_OIDC_ADMIN_GROUPS: "" + OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN: "false" + ports: + - "8086:8086" + volumes: + - ./storage:/app/storage +``` + +## Authelia + +[Authelia](https://www.authelia.com/) is an open-source multi-factor authentication solution. + +### 1. Configure Authelia + +Edit your Authelia configuration (`configuration.yml`): + +```yaml +identity_providers: + oidc: + hmac_secret: your-secure-secret # Change to a secure random value + issuer_private_key: /config/private.pem # Path to your private key + cors: + endpoints: ['authorization', 'token', 'revocation', 'introspection'] + allowed_origins: + - https://oxicloud.example.com + clients: + - id: oxicloud + description: OxiCloud + secret: your-secure-client-secret # Change this + public: false + authorization_policy: two_factor + redirect_uris: + - https://oxicloud.example.com/api/auth/oidc/callback + scopes: ['openid', 'profile', 'email', 'groups'] + userinfo_signing_algorithm: none +``` + +### 2. Configure for Authelia + +```yaml +# docker-compose.yml +version: '3' +services: + oxicloud: + image: oxicloud:latest + environment: + OXICLOUD_OIDC_ENABLED: "true" + OXICLOUD_OIDC_PROVIDER_NAME: "Authelia" + OXICLOUD_OIDC_ISSUER_URL: "https://authelia.example.com" + OXICLOUD_OIDC_CLIENT_ID: "oxicloud" + OXICLOUD_OIDC_CLIENT_SECRET: "your-secure-client-secret" + OXICLOUD_OIDC_REDIRECT_URI: "https://oxicloud.example.com/api/auth/oidc/callback" + OXICLOUD_OIDC_SCOPES: "openid profile email groups" + OXICLOUD_OIDC_FRONTEND_URL: "https://oxicloud.example.com" + OXICLOUD_OIDC_AUTO_PROVISION: "true" + OXICLOUD_OIDC_ADMIN_GROUPS: "" + OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN: "false" + ports: + - "8086:8086" + volumes: + - ./storage:/app/storage +``` + +## KeyCloak + +[KeyCloak](https://www.keycloak.org/) is an open-source identity and access management solution. + +### 1. Create a Client in KeyCloak + +1. Log into the KeyCloak admin console +2. Select your Realm +3. Go to "Clients" -> "Create" +4. Fill in the form: + - **Client ID**: `oxicloud` + - **Client Protocol**: `openid-connect` + - **Root URL**: `https://oxicloud.example.com` +5. In the client configuration: + - **Access Type**: `confidential` + - **Valid Redirect URIs**: `https://oxicloud.example.com/api/auth/oidc/callback` + - **Web Origins**: `https://oxicloud.example.com` (or `+` to allow all origins) +6. Save the configuration +7. Go to the "Credentials" tab and copy the generated "Secret" + +### 2. Configure for KeyCloak + +```yaml +# docker-compose.yml +version: '3' +services: + oxicloud: + image: oxicloud:latest + environment: + OXICLOUD_OIDC_ENABLED: "true" + OXICLOUD_OIDC_PROVIDER_NAME: "KeyCloak" + OXICLOUD_OIDC_ISSUER_URL: "https://keycloak.example.com/realms/your-realm" + OXICLOUD_OIDC_CLIENT_ID: "oxicloud" + OXICLOUD_OIDC_CLIENT_SECRET: "your-keycloak-client-secret" + OXICLOUD_OIDC_REDIRECT_URI: "https://oxicloud.example.com/api/auth/oidc/callback" + OXICLOUD_OIDC_SCOPES: "openid profile email" + OXICLOUD_OIDC_FRONTEND_URL: "https://oxicloud.example.com" + OXICLOUD_OIDC_AUTO_PROVISION: "true" + OXICLOUD_OIDC_ADMIN_GROUPS: "oxicloud-admins" + OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN: "false" + ports: + - "8086:8086" + volumes: + - ./storage:/app/storage +``` + +## Troubleshooting + +### Error: "Failed to discover OIDC provider" + +The backend cannot reach the provider's discovery endpoint. + +**Fixes:** +1. Verify the discovery URL is correct +2. Check that the backend can reach the URL (firewalls, DNS, etc.) +3. If using a self-signed certificate, configure the appropriate trust + +### Error: "Invalid redirect URI" + +The OIDC provider is rejecting the redirect URI. + +**Fixes:** +1. Make sure the redirect URI configured in the backend matches exactly what is registered in the provider +2. Check for protocol differences (http vs https), port, or path mismatches + +### Error: "User does not exist and auto-creation is disabled" + +**Fixes:** +1. Enable auto-provisioning: `OXICLOUD_OIDC_AUTO_PROVISION="true"` +2. Or manually create the user before attempting OIDC login + +### Error: "Could not extract user ID from claim" + +The backend cannot find the user ID attribute in the token claims. + +**Fixes:** +1. Verify the provider returns the `sub` claim in tokens +2. Make sure scopes in **OXICLOUD_OIDC_SCOPES** include `openid` +3. Configure the provider to include the required claims in tokens + +See `oidc-architecture.md` and `oidc-integration.md` for deeper technical details. diff --git a/doc/oidc-integration.md b/doc/oidc-integration.md new file mode 100644 index 00000000..986d8c71 --- /dev/null +++ b/doc/oidc-integration.md @@ -0,0 +1,266 @@ +# 30 - OIDC Integration + +OpenID Connect (OIDC) is an identity layer on top of OAuth 2.0. It lets clients verify user identity based on authentication performed by an authorization server and obtain basic profile information. Adding OIDC enables SSO with providers like Authentik, Authelia, and KeyCloak. + +What it gives us: +1. Users authenticate with their existing IdP credentials +2. No need for separate username/password management +3. Modern auth best practices baked in +4. Seamless experience for users already on SSO + +## OIDC Configuration + +OIDC is configured separately from **AuthConfig** via **OidcConfig** in `src/common/config.rs`. This is a single-provider model -- one OIDC provider per instance: + +```rust +/// OpenID Connect (OIDC) configuration +pub struct OidcConfig { + pub enabled: bool, // Whether OIDC is enabled + pub issuer_url: String, // OIDC Issuer URL + pub client_id: String, // OIDC Client ID + pub client_secret: String, // OIDC Client Secret + pub redirect_uri: String, // Redirect URI (default: http://localhost:8086/api/auth/oidc/callback) + pub scopes: String, // Scopes to request (default: "openid profile email") + pub frontend_url: String, // Frontend URL for post-login redirect + pub auto_provision: bool, // Auto-create users on first login (JIT provisioning) + pub admin_groups: String, // Comma-separated OIDC groups that map to admin role + pub disable_password_login: bool, // Disable password-based login entirely + pub provider_name: String, // Display name (default: "SSO") +} +``` + +Environment variables use the **OXICLOUD_OIDC_*** prefix: + +```bash +OXICLOUD_OIDC_ENABLED=true +OXICLOUD_OIDC_ISSUER_URL="https://authentik.example.com/application/o/oxicloud/" +OXICLOUD_OIDC_CLIENT_ID="your-client-id" +OXICLOUD_OIDC_CLIENT_SECRET="your-client-secret" +OXICLOUD_OIDC_REDIRECT_URI="https://oxicloud.example.com/api/auth/oidc/callback" +OXICLOUD_OIDC_SCOPES="openid profile email" +OXICLOUD_OIDC_FRONTEND_URL="https://oxicloud.example.com" +OXICLOUD_OIDC_AUTO_PROVISION=true +OXICLOUD_OIDC_ADMIN_GROUPS="oxicloud-admins" +OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN=false +OXICLOUD_OIDC_PROVIDER_NAME="Authentik" +``` + +## OIDC Service Implementation + +The OIDC service lives in the infrastructure layer at `src/infrastructure/services/oidc_service.rs` and implements the **OidcServicePort** trait defined in `src/application/ports/auth_ports.rs`: + +```rust +// src/application/ports/auth_ports.rs — Port trait +#[async_trait] +pub trait OidcServicePort: Send + Sync + 'static { + fn enabled(&self) -> bool; + fn provider_name(&self) -> &str; + fn generate_auth_url(&self, state: &str) -> Result; + async fn exchange_code(&self, code: &str) -> Result; + async fn get_user_info(&self, token_set: &OidcTokenSet) -> Result; +} + +// src/infrastructure/services/oidc_service.rs — Implementation +pub struct OidcService { + config: OidcConfig, + http_client: reqwest::Client, + // Discovery metadata cached after initialization +} + +impl OidcService { + pub async fn new(config: OidcConfig) -> Result { + // Discovers OIDC endpoints from issuer_url + // ... + } +} +``` + +Follows hexagonal architecture: the port (**OidcServicePort**) is in the application layer, and the implementation (**OidcService**) is in the infrastructure layer. + +## User Entity OIDC Support + +The **User** entity in `src/domain/entities/user.rs` supports OIDC users via two fields: + +```rust +#[derive(Debug, Clone)] +pub struct User { + // ... standard fields ... + oidc_provider: Option, // OIDC provider name (e.g., "authentik") + oidc_subject: Option, // OIDC subject identifier (unique ID from provider) +} + +impl User { + pub fn oidc_provider(&self) -> Option<&str> { + self.oidc_provider.as_deref() + } + + pub fn oidc_subject(&self) -> Option<&str> { + self.oidc_subject.as_deref() + } + + // Constructor for OIDC users + pub fn new_oidc(username, email, role, quota, oidc_provider, oidc_subject) -> Self; +} +``` + +## Database Schema + +The **auth.users** table includes OIDC columns: + +```sql +ALTER TABLE auth.users ADD COLUMN IF NOT EXISTS oidc_provider VARCHAR(255); +ALTER TABLE auth.users ADD COLUMN IF NOT EXISTS oidc_subject VARCHAR(255); +CREATE UNIQUE INDEX IF NOT EXISTS idx_users_oidc ON auth.users(oidc_provider, oidc_subject) WHERE oidc_provider IS NOT NULL; +``` + +Users are matched by **oidc_provider** + **oidc_subject** combination. The **UserRepository** trait includes `get_user_by_oidc_subject()` for lookups. + +## Auth Application Service + +**AuthApplicationService** in `src/application/services/auth_application_service.rs` coordinates OIDC authentication: + +```rust +impl AuthApplicationService { + // Initialize with OIDC support + pub fn with_oidc(self, oidc_service: Arc, oidc_config: OidcConfig) -> Self; + + // Reload OIDC configuration (for admin settings changes) + pub async fn reload_oidc(&self, config: OidcConfig) -> Result<(), DomainError>; + + // Disable OIDC + pub fn disable_oidc(&self); + + // Check if OIDC is enabled + pub fn oidc_enabled(&self) -> bool; + + // Get OIDC config + pub fn oidc_config(&self) -> Option; + + // Get OIDC service reference + pub fn oidc_service(&self) -> Option>; + + // Check if password login is disabled + pub fn password_login_disabled(&self) -> bool; + + // Prepare OIDC authorization URL + pub fn prepare_oidc_authorize(&self) -> Result; + + // Handle OIDC callback (exchange code for tokens) + pub async fn oidc_callback(&self, code: &str, state: &str) -> Result; +} +``` + +## Auth Handler Routes + +OIDC endpoints in `src/interfaces/api/handlers/auth_handler.rs`: + +```rust +// Public OIDC routes (no auth required) — nested under /api/auth/ +.route("/status", get(get_system_status)) +.route("/oidc/providers", get(oidc_providers)) +.route("/oidc/authorize", get(oidc_authorize)) +.route("/oidc/callback", get(oidc_callback)) +.route("/oidc/exchange", post(oidc_exchange)) +``` + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/auth/oidc/providers` | Returns OIDC provider info (name, enabled state) | +| GET | `/api/auth/oidc/authorize` | Returns authorization URL for redirect to IdP | +| GET | `/api/auth/oidc/callback` | Receives callback redirect from IdP with auth code | +| POST | `/api/auth/oidc/exchange` | Exchanges auth code for JWT tokens | + +## DTOs + +DTOs in `src/application/dtos/user_dto.rs`: + +```rust +// Response with authorization URL +#[derive(Debug, Clone, Serialize)] +pub struct OidcAuthorizeResponseDto { + pub authorize_url: String, + pub state: String, +} + +// Query params received from IdP callback +#[derive(Debug, Clone, Deserialize)] +pub struct OidcCallbackQueryDto { + pub code: String, + pub state: String, +} + +// Request to exchange code for tokens +#[derive(Debug, Clone, Deserialize)] +pub struct OidcExchangeDto { + pub code: String, + pub state: String, +} + +// Provider info response +#[derive(Debug, Clone, Serialize)] +pub struct OidcProviderInfoDto { + pub enabled: bool, + pub provider_name: String, + pub disable_password_login: bool, +} + +// User info from OIDC claims +#[derive(Debug, Clone, Serialize)] +pub struct OidcUserInfoDto { + pub subject: String, + pub email: Option, + pub name: Option, + pub preferred_username: Option, + pub groups: Vec, +} +``` + +## Frontend Integration + +OIDC login is built directly into `static/login.html` and handled by `static/js/auth.js`. There is no separate `oidcAuth.js` file. The login page checks the system status endpoint to see if OIDC is enabled, then shows an SSO button accordingly. + +```html + + +``` + +## Configuration Example + +KeyCloak setup via docker-compose: + +```yaml +# docker-compose.yml +version: '3' +services: + oxicloud: + image: oxicloud:latest + environment: + OXICLOUD_OIDC_ENABLED: "true" + OXICLOUD_OIDC_ISSUER_URL: "https://keycloak.example.com/realms/your-realm" + OXICLOUD_OIDC_CLIENT_ID: "oxicloud" + OXICLOUD_OIDC_CLIENT_SECRET: "your-client-secret" + OXICLOUD_OIDC_REDIRECT_URI: "https://oxicloud.example.com/api/auth/oidc/callback" + OXICLOUD_OIDC_SCOPES: "openid profile email" + OXICLOUD_OIDC_FRONTEND_URL: "https://oxicloud.example.com" + OXICLOUD_OIDC_AUTO_PROVISION: "true" + OXICLOUD_OIDC_ADMIN_GROUPS: "oxicloud-admins" + OXICLOUD_OIDC_DISABLE_PASSWORD_LOGIN: "false" + OXICLOUD_OIDC_PROVIDER_NAME: "KeyCloak" + ports: + - "8086:8086" + volumes: + - ./storage:/app/storage +``` + +See `oidc-config-examples.md` for more provider-specific configurations. + +## Additional Notes + +1. **Security** -- always use HTTPS for OIDC connections. Ensure proper TLS configuration. +2. **User mapping** -- OIDC users are identified by **oidc_provider** + **oidc_subject** in the **auth.users** table. Groups from OIDC can map to admin role via **OXICLOUD_OIDC_ADMIN_GROUPS**. +3. **Single provider** -- one OIDC provider per instance. Managed via admin settings UI or environment variables. +4. **Session management** -- after OIDC authentication, the backend generates its own JWT access/refresh tokens. Sessions work identically to password-based login from that point. +5. **Access control** -- OIDC users share the same permissions model as local users. Admin role can be auto-assigned based on OIDC group membership. +6. **Testing** -- use the admin settings UI (`/admin.html`) to configure and test OIDC connections. diff --git a/doc/README-AUTH.md b/doc/readme-auth.md similarity index 50% rename from doc/README-AUTH.md rename to doc/readme-auth.md index 2e85d136..90e45296 100644 --- a/doc/README-AUTH.md +++ b/doc/readme-auth.md @@ -1,28 +1,22 @@ -# OxiCloud Authentication System +# 32 - Authentication -This document describes the authentication system for OxiCloud, a file storage system built with Rust and PostgreSQL. - -## Overview - -OxiCloud uses a standard JWT (JSON Web Token) authentication system with the following features: - -- User registration and login -- Role-based access control (Admin/User) -- JWT token with refresh capabilities -- Secure password hashing with Argon2id -- User storage quotas -- File and folder ownership +The auth system uses JWT (JSON Web Tokens) with Argon2id password hashing. Features include registration, login, role-based access control (Admin/User), token refresh, storage quotas, and file/folder ownership. ## API Endpoints -The authentication API is available at the `/api/auth` endpoint: +All auth endpoints live under `/api/auth`: -- **POST /api/auth/register** - Register a new user -- **POST /api/auth/login** - Login and get tokens -- **POST /api/auth/refresh** - Refresh access token -- **GET /api/auth/me** - Get current user information -- **PUT /api/auth/change-password** - Change user password -- **POST /api/auth/logout** - Logout and invalidate refresh token +- **POST /api/auth/register** -- register a new user +- **POST /api/auth/login** -- login and get tokens +- **POST /api/auth/refresh** -- refresh access token +- **GET /api/auth/me** -- get current user info +- **PUT /api/auth/change-password** -- change user password +- **POST /api/auth/logout** -- logout and invalidate refresh token +- **GET /api/auth/status** -- system status (auth enabled, OIDC enabled, etc.) +- **GET /api/auth/oidc/providers** -- list available OIDC providers +- **GET /api/auth/oidc/authorize** -- generate OIDC authorization URL +- **GET /api/auth/oidc/callback** -- receive OIDC callback redirect +- **POST /api/auth/oidc/exchange** -- exchange authorization code for tokens ## Request/Response Examples @@ -141,7 +135,7 @@ Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... 200 OK ``` -## Testing the Authentication System +## Testing 1. Start PostgreSQL and create the database: ```bash @@ -149,52 +143,55 @@ Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... psql -d oxicloud -f db/schema.sql ``` -2. Set environment variables for authentication: +2. Set environment variables: ```bash source test-auth-env.sh ``` -3. Start the OxiCloud server: +3. Start the server: ```bash cargo run ``` -4. Run the authentication test script: +4. Run the auth test script: ```bash ./test-auth-api.sh ``` ## Database Schema -The authentication system uses the following tables: +The auth system uses these tables in the **auth** schema: -- `users` - Store user information -- `sessions` - Store refresh token sessions -- `file_ownership` - Track file ownership -- `folder_ownership` - Track folder ownership +- **auth.users** -- user info (includes **oidc_provider** and **oidc_subject** columns for OIDC users) +- **auth.sessions** -- refresh token sessions +- **auth.user_files** -- file ownership (user_id, file_path, file_id, size_bytes) +- **auth.user_favorites** -- user favorites (user_id, item_id, item_type) +- **auth.user_recent_files** -- recently accessed files (user_id, item_id, item_type, accessed_at) +- **auth.admin_settings** -- admin settings (key-value with category and secret flag) ## Implementation Details -- **Password Hashing**: Argon2id with memory cost of 65536 (64MB), time cost of 3, and 4 parallelism -- **JWT Secret**: Configured via environment variable `OXICLOUD_JWT_SECRET` -- **Token Expiry**: Access token expires in 1 hour, refresh token in 30 days (configurable) -- **Database Connection**: PostgreSQL with connection pooling -- **Middleware**: Auth middleware for protected routes +- **Password hashing**: Argon2id with memory cost 65536 (64MB), time cost 3, parallelism 4 +- **JWT secret**: configured via **OXICLOUD_JWT_SECRET** environment variable +- **Token expiry**: access token 1 hour, refresh token 30 days (configurable) +- **Database connection**: PostgreSQL with connection pooling +- **Middleware**: auth middleware for protected routes -## Security Considerations +## Security -- Passwords are never stored in plain text, only as Argon2id hashes -- JWT tokens are signed with a secret key +- Passwords stored only as Argon2id hashes, never in plain text +- JWT tokens signed with a secret key - Refresh tokens can be revoked to force logout -- Rate limiting should be implemented for login attempts +- Rate limiting should be applied to login attempts - Password policy requires at least 8 characters -- Regular security audits recommended -## Future Improvements +See `oidc-integration.md` for OIDC/SSO authentication details. + +## Future Work - Email verification for new registrations - Password reset functionality - Enhanced password policy - Two-factor authentication - OAuth integration for social logins -- Session management UI \ No newline at end of file +- Session management UI diff --git a/doc/search.md b/doc/search.md new file mode 100644 index 00000000..34e45d7c --- /dev/null +++ b/doc/search.md @@ -0,0 +1,120 @@ +# 12 - Search + +File and folder search with multi-criteria filtering, recursive traversal, pagination, and in-memory result caching. Two modes: a simple `GET` with query parameters, or an advanced `POST` with a full criteria body. + +Controlled by the feature flag **OXICLOUD_ENABLE_SEARCH** (default: `true`). + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **SearchUseCase** trait | `src/application/ports/inbound.rs` | +| Application Service | **SearchService** | `src/application/services/search_service.rs` | +| Application DTOs | **SearchCriteriaDto**, **SearchResultsDto** | `src/application/dtos/search_dto.rs` | +| Interfaces | **SearchHandler** | `src/interfaces/api/handlers/search_handler.rs` | + +## Search Criteria + +```rust +pub struct SearchCriteriaDto { + pub name_contains: Option, // text to search in file/folder names + pub file_types: Option>, // extensions, e.g. ["pdf", "jpg"] + pub created_after: Option, // epoch seconds + pub created_before: Option, + pub modified_after: Option, + pub modified_before: Option, + pub min_size: Option, // bytes + pub max_size: Option, + pub folder_id: Option, // scope to a specific folder + pub recursive: bool, // default: true + pub limit: usize, // default: 100 + pub offset: usize, // default: 0 +} +``` + +## Search Results + +```rust +pub struct SearchResultsDto { + pub files: Vec, + pub folders: Vec, + pub total_count: Option, + pub limit: usize, + pub offset: usize, + pub has_more: bool, +} +``` + +## REST API Endpoints + +All routes under `/api/search`, require authentication. + +| Method | Path | Handler | Description | +|---|---|---|---| +| `GET` | `/api/search/` | `SearchHandler::search_files_get` | Simple search via query params | +| `POST` | `/api/search/advanced` | `SearchHandler::search_files_post` | Advanced search via JSON body | +| `DELETE` | `/api/search/cache` | `SearchHandler::clear_search_cache` | Clear the result cache | + +### GET Query Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `String` | Text to search in names | +| `type` | `String` | File extension filter (comma-separated) | +| `created_after` | `u64` | Epoch seconds | +| `created_before` | `u64` | Epoch seconds | +| `modified_after` | `u64` | Epoch seconds | +| `modified_before` | `u64` | Epoch seconds | +| `min_size` | `u64` | Minimum size in bytes | +| `max_size` | `u64` | Maximum size in bytes | +| `folder_id` | `String` | Scope to folder | +| `recursive` | `bool` | Recursive search (default: true) | +| `limit` | `usize` | Max results (default: 100) | +| `offset` | `usize` | Pagination offset | + +### Example + +```bash +# Simple search +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/search/?query=report&type=pdf&limit=20" + +# Advanced search +curl -X POST -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"name_contains":"report","file_types":["pdf","docx"],"min_size":1024,"recursive":true,"limit":50}' \ + "https://oxicloud.example.com/api/search/advanced" +``` + +## Implementation Details + +### Search Algorithm + +1. If `folder_id` is provided, start from that folder. Otherwise start from root. +2. When `recursive` is true, traverse all subfolders depth-first via `search_recursive()`. +3. Files are filtered by name match, extension, date range, and size range. +4. Folders are filtered by name match and date range. +5. Results are paginated with `offset`/`limit`. + +### Caching + +Results are cached in-memory using a hash of the search criteria + user ID as the key. + +- **Cache TTL**: 5 minutes (300 seconds) +- **Max entries**: 1000 +- **Cleanup**: background `tokio` task runs periodically, evicts expired entries +- **Manual clear**: `DELETE /api/search/cache` + +## DI Wiring + +```rust +// In di.rs — SearchService creation +let search_service = SearchService::new( + repos.file_read_repository.clone(), + repos.folder_repository.clone(), + 300, // cache TTL seconds + 1000, // max cache entries +); +``` + +Stored in `AppState.applications.search_service: Option>`. diff --git a/doc/share-integration.md b/doc/share-integration.md new file mode 100644 index 00000000..1b00efcf --- /dev/null +++ b/doc/share-integration.md @@ -0,0 +1,360 @@ +# 15 - Share Integration + +File and folder sharing via public links. Users generate access links that work even for people without accounts. Supports optional password protection, expiration, and granular permissions. Follows hexagonal architecture throughout. + +## Domain Entities + +**Share** (`src/domain/entities/share.rs`) -- the core entity representing a shared resource: + +```rust +pub struct Share { + pub id: String, // unique link identifier + pub item_id: String, // ID of shared file or folder + pub item_type: ShareItemType, // File or Folder + pub token: String, // unique token for public access + pub password_hash: Option, // optional password hash + pub expires_at: Option, // optional expiration timestamp + pub permissions: SharePermissions, // granted permissions + pub created_at: u64, // creation timestamp + pub created_by: String, // creator user ID + pub access_count: u64, // access counter +} + +pub enum ShareItemType { + File, + Folder +} + +pub struct SharePermissions { + pub read: bool, // read permission + pub write: bool, // write permission + pub reshare: bool, // re-share permission +} +``` + +The entity has methods to validate expiration, verify passwords, increment the access counter, and modify properties (permissions, password, expiration). + +## Repository Interface + +**ShareRepository** (`src/domain/repositories/share_repository.rs`) defines persistence operations: + +```rust +#[async_trait] +pub trait ShareRepository: Send + Sync + 'static { + async fn save(&self, share: &Share) -> Result; + async fn find_by_id(&self, id: &str) -> Result; + async fn find_by_token(&self, token: &str) -> Result; + async fn find_by_item(&self, item_id: &str, item_type: &ShareItemType) -> Result, ShareRepositoryError>; + async fn update(&self, share: &Share) -> Result; + async fn delete(&self, id: &str) -> Result<(), ShareRepositoryError>; + async fn find_by_user(&self, user_id: &str, offset: usize, limit: usize) -> Result<(Vec, usize), ShareRepositoryError>; +} +``` + +## Application Ports + +**ShareUseCase** and **ShareStoragePort** (`src/application/ports/share_ports.rs`): + +```rust +#[async_trait] +pub trait ShareUseCase: Send + Sync + 'static { + async fn create_shared_link(&self, user_id: &str, dto: CreateShareDto) -> Result; + async fn get_shared_link(&self, id: &str) -> Result; + async fn get_shared_link_by_token(&self, token: &str) -> Result; + async fn get_shared_links_for_item(&self, item_id: &str, item_type: &ShareItemType) -> Result, DomainError>; + async fn update_shared_link(&self, id: &str, dto: UpdateShareDto) -> Result; + async fn delete_shared_link(&self, id: &str) -> Result<(), DomainError>; + async fn get_user_shared_links(&self, user_id: &str, page: usize, per_page: usize) -> Result, DomainError>; + async fn verify_shared_link_password(&self, token: &str, password: &str) -> Result; + async fn register_shared_link_access(&self, token: &str) -> Result<(), DomainError>; +} + +#[async_trait] +pub trait ShareStoragePort: Send + Sync + 'static { + async fn save_share(&self, share: &Share) -> Result; + async fn find_share_by_id(&self, id: &str) -> Result; + // ... other methods +} +``` + +## DTOs + +**DTOs** (`src/application/dtos/share_dto.rs`): + +```rust +pub struct CreateShareDto { + pub item_id: String, + pub item_type: String, + pub password: Option, + pub expires_at: Option, + pub permissions: Option, +} + +pub struct UpdateShareDto { + pub password: Option, + pub expires_at: Option, + pub permissions: Option, +} + +pub struct SharePermissionsDto { + pub read: bool, + pub write: bool, + pub reshare: bool, +} + +pub struct ShareDto { + pub id: String, + pub item_id: String, + pub item_type: String, + pub token: String, + pub url: String, + pub password_protected: bool, + pub expires_at: Option, + pub permissions: SharePermissionsDto, + pub created_at: u64, + pub created_by: String, + pub access_count: u64, +} +``` + +## Application Service + +**ShareService** (`src/application/services/share_service.rs`) implements the business logic: + +```rust +pub struct ShareService { + config: Arc, + share_repository: Arc, + file_repository: Arc, + folder_repository: Arc, +} +``` + +Handles: shared element validation, permission management, unique link/token generation, password protection, expiration control, and access tracking. + +## Infrastructure + +**ShareFsRepository** (`src/infrastructure/repositories/share_fs_repository.rs`) persists share links to the filesystem: + +```rust +pub struct ShareFsRepository { + config: Arc, +} + +struct ShareRecord { + id: String, + item_id: String, + item_type: String, + token: String, + password_hash: Option, + expires_at: Option, + permissions_read: bool, + permissions_write: bool, + permissions_reshare: bool, + created_at: u64, + created_by: String, + access_count: u64, +} +``` + +Stores shared links in a JSON file. Supports queries and updates, search by ID/token/user, and pagination. + +## API Handlers and Routes + +**Handlers** (`src/interfaces/api/handlers/share_handler.rs`): + +```rust +pub async fn create_shared_link( + State(share_use_case): State>, + Json(dto): Json, +) -> impl IntoResponse { + // ... +} + +pub async fn get_shared_link( + State(share_use_case): State>, + Path(id): Path, +) -> impl IntoResponse { + // ... +} + +pub async fn get_user_shares( + State(share_use_case): State>, + Query(query): Query, +) -> impl IntoResponse { + // ... +} + +pub async fn update_shared_link( + State(share_use_case): State>, + Path(id): Path, + Json(dto): Json, +) -> impl IntoResponse { + // ... +} + +pub async fn delete_shared_link( + State(share_use_case): State>, + Path(id): Path, +) -> impl IntoResponse { + // ... +} + +pub async fn access_shared_item( + State(share_use_case): State>, + Path(token): Path, +) -> impl IntoResponse { + // ... +} + +pub async fn verify_shared_item_password( + State(share_use_case): State>, + Path(token): Path, + Json(req): Json, +) -> impl IntoResponse { + // ... +} +``` + +**Routes** (`src/interfaces/api/routes.rs`): + +```rust +// Private routes for managing shared links +let share_router = Router::new() + .route("/", post(share_handler::create_shared_link)) + .route("/", get(share_handler::get_user_shares)) + .route("/{id}", get(share_handler::get_shared_link)) + .route("/{id}", put(share_handler::update_shared_link)) + .route("/{id}", delete(share_handler::delete_shared_link)); + +// Public routes for accessing shared links +let public_share_router = Router::new() + .route("/{token}", get(share_handler::access_shared_item)) + .route("/{token}/verify", post(share_handler::verify_shared_item_password)); + +// Main router configuration +router + .nest("/shares", share_router) // private API: /api/shares/... + .nest("/s", public_share_router); // public API: /api/s/... +``` + +## System Integration + +### Configuration + +```rust +pub struct FeaturesConfig { + // ... + pub enable_file_sharing: bool, + // ... +} +``` + +### Dependency Injection + +The service is instantiated via **AppServiceFactory** in `src/common/di.rs` and injected into **AppState**: + +```rust +// In AppServiceFactory::create_share_service() +let share_service: Option> = if config.features.enable_file_sharing { + let share_repository = Arc::new(ShareFsRepository::new(Arc::new(config.clone()))); + let share_service = Arc::new(ShareService::new( + Arc::new(config.clone()), + share_repository, + file_read_repository.clone(), + folder_repository.clone(), + password_hasher.clone(), + )); + Some(share_service) +} else { + None +}; + +// Add to AppState +let app_state = AppState { + // ... + share_service: share_service.clone(), + // ... +}; +``` + +## Workflows + +### Creating a Shared Link + +1. User selects a file or folder to share. +2. Frontend sends a POST to `/api/shares/` with details (optional password, expiration, permissions). +3. `ShareService.create_shared_link()` validates data and verifies the item exists. +4. A unique token and access URL are generated. +5. The link is saved to the repository. +6. The URL and link details are returned. + +### Accessing a Shared Resource + +1. Someone opens a shared link (e.g., `http://oxicloud.example/api/s/{token}`). +2. Backend checks: valid token, not expired, password-protected or not. +3. If password-protected, the user is prompted. +4. Access counter increments. +5. Resource metadata is returned for display in the UI. +6. The user can access content according to the granted permissions. + +## Security + +**Password Protection** -- passwords are stored as hashes, not plaintext. Currently uses a simple hash but the design supports stronger algorithms like bcrypt. + +**Expiration Control** -- links can be configured to expire automatically. The system checks expiration before granting access. + +**Permission Control** -- granular permission model (read, write, reshare). Each operation validates permissions before allowing the action. + +## Error Handling + +```rust +pub enum ShareServiceError { + #[error("Share not found: {0}")] + NotFound(String), + + #[error("Item not found: {0}")] + ItemNotFound(String), + + #[error("Access denied: {0}")] + AccessDenied(String), + + #[error("Invalid password: {0}")] + InvalidPassword(String), + + #[error("Share expired")] + Expired, + + #[error("Repository error: {0}")] + Repository(String), + + #[error("Invalid item type: {0}")] + InvalidItemType(String), + + #[error("Validation error: {0}")] + Validation(String), +} +``` + +HTTP status code mapping: +- `NotFound` -> HTTP 404 +- `PasswordRequired` -> HTTP 401 + metadata +- `Expired` -> HTTP 410 Gone +- `AccessDenied` -> HTTP 403 +- `ValidationError` -> HTTP 400 + +## Future Enhancements + +1. **Notifications** -- alert users when their shared resources are accessed +2. **Activity Log** -- detailed audit trail of who accessed what and when +3. **Usage Limits** -- max access count or bandwidth per shared link +4. **Advanced Statistics** -- detailed metrics on shared resource usage +5. **Alternative Persistence** -- database or cloud storage backends (same interface) + +## Technical Notes + +- **Performance**: JSON file-based storage works for moderate volumes. For higher load, migrate to a database. +- **Scalability**: the design supports horizontal scaling via distributed or cloud-based repositories. +- **Maintenance**: clear separation of concerns makes testing and maintenance straightforward. + +The sharing feature is enabled by default in the current configuration. diff --git a/doc/storage-quotas.md b/doc/storage-quotas.md new file mode 100644 index 00000000..789fba4e --- /dev/null +++ b/doc/storage-quotas.md @@ -0,0 +1,74 @@ +# 11 - Storage Quotas + +Per-user storage quotas track disk usage and can limit how much storage each user consumes. Controlled by the feature flag **OXICLOUD_ENABLE_USER_STORAGE_QUOTAS** (default: `false`). + +## Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **StorageUsagePort** trait | `src/application/ports/storage_ports.rs` | +| Application Service | **StorageUsageService** | `src/application/services/storage_usage_service.rs` | +| Admin API | `/api/admin/users/{id}/quota` | `src/interfaces/api/handlers/admin_handler.rs` | + +## Port Trait + +```rust +#[async_trait] +pub trait StorageUsagePort: Send + Sync + 'static { + async fn update_user_storage_usage(&self, user_id: &str) -> Result; + async fn update_all_users_storage_usage(&self) -> Result<(), DomainError>; +} +``` + +## How Usage is Calculated + +1. Look up the user's username by ID. +2. Find the user's home folder: `"Mi Carpeta - {username}"` (naming convention). +3. Recursively traverse all subfolders, summing file sizes. +4. Skip directory entries (`mime_type = "directory"` or `"application/directory"`). +5. Update `auth.users.storage_used` via **UserStoragePort**. + +`update_all_users_storage_usage()` processes all users concurrently via `tokio::spawn`. + +## Admin Quota Management + +Admins set per-user quotas through the admin API: + +```bash +# Set 10 GB quota for a user +curl -X PUT -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"quota_bytes": 10737418240}' \ + "https://oxicloud.example.com/api/admin/users/{user_id}/quota" +``` + +## Dashboard Stats + +The admin dashboard (`GET /api/admin/dashboard`) includes quota-related metrics: + +```json +{ + "quotas_enabled": true, + "total_quota_bytes": 107374182400, + "total_used_bytes": 53687091200, + "storage_usage_percent": 50.0, + "users_over_80_percent": 5, + "users_over_quota": 1 +} +``` + +## Configuration + +```bash +# Enable storage quotas (default: false) +OXICLOUD_ENABLE_USER_STORAGE_QUOTAS=true +``` + +Part of **FeaturesConfig** in `src/common/config.rs`: + +```rust +pub struct FeaturesConfig { + pub enable_user_storage_quotas: bool, // default: false + // ... +} +``` diff --git a/doc/thumbnails-and-transcoding.md b/doc/thumbnails-and-transcoding.md new file mode 100644 index 00000000..469555ad --- /dev/null +++ b/doc/thumbnails-and-transcoding.md @@ -0,0 +1,191 @@ +# 08 - Thumbnails and Transcoding + +OxiCloud provides two image optimization features: + +- **Thumbnails**: on-demand generation of WebP thumbnails in 3 sizes, with background pre-generation on upload +- **Image Transcoding**: automatic JPEG/PNG/GIF → WebP conversion based on browser `Accept` header + +Both features use multi-level caching (memory LRU + disk) and are non-blocking. + +--- + +## Thumbnails + +### Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **ThumbnailPort** trait, **ThumbnailSize** enum | `src/application/ports/thumbnail_ports.rs` | +| Infrastructure | **ThumbnailService** | `src/infrastructure/services/thumbnail_service.rs` | +| Interfaces | Integrated in **FileHandler** | `src/interfaces/api/handlers/file_handler.rs` | + +### Thumbnail Sizes + +| Size | Dimensions | Directory Name | +|---|---|---| +| `Icon` | 150x150 | `icon` | +| `Preview` | 400x400 | `preview` | +| `Large` | 800x800 | `large` | + +### Supported Formats + +Input: `image/jpeg`, `image/jpg`, `image/png`, `image/gif`, `image/webp` + +Output: always **WebP** (using Lanczos3 resize filter) + +### Storage Layout + +``` +/ + .thumbnails/ + icon/ + .webp + preview/ + .webp + large/ + .webp +``` + +### REST API + +| Method | Path | Description | +|---|---|---| +| `GET` | `/api/files/{id}/thumbnail/{size}` | Get thumbnail (`size` = `icon` / `preview` / `large`) | +| `POST` | `/api/files/upload` | Upload file + auto-generate thumbnails for images | + +Response headers for thumbnail GET: +- `Content-Type: image/webp` +- `Cache-Control: public, max-age=31536000, immutable` +- `ETag: "thumb-{id}-{size}"` + +### Generation Flow + +1. **On upload** (`upload_file_with_thumbnails`): after successful file upload, if the MIME type is a supported image, thumbnails for all 3 sizes are generated in a background `tokio::spawn` task +2. **On GET** (lazy): if a thumbnail doesn't exist on disk, it's generated on-demand, cached in memory and on disk + +### Caching + +- **Memory LRU cache**: configurable max entries and max bytes +- **Disk cache**: persistent WebP files in `.thumbnails/` +- Cache lookup order: memory → disk → generate + +### Port Trait + +```rust +#[async_trait] +pub trait ThumbnailPort: Send + Sync + 'static { + fn is_supported_image(&self, mime_type: &str) -> bool; + async fn get_thumbnail(&self, file_id: &str, size: ThumbnailSize, original_path: &Path) -> Result; + fn generate_all_sizes_background(self: Arc, file_id: String, original_path: PathBuf); + async fn delete_thumbnails(&self, file_id: &str) -> Result<(), DomainError>; + async fn get_stats(&self) -> ThumbnailStatsDto; +} +``` + +--- + +## Image Transcoding + +### Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **ImageTranscodePort** trait, **OutputFormat**, **BrowserCapabilities** | `src/application/ports/transcode_ports.rs` | +| Infrastructure | **ImageTranscodeService** | `src/infrastructure/services/image_transcode_service.rs` | +| Interfaces | Integrated in download handler | `src/interfaces/api/handlers/file_handler.rs` | + +### How It Works + +1. Download handler reads the `Accept` header from the HTTP request +2. `BrowserCapabilities::from_accept_header()` determines if the browser supports WebP +3. If the file is a transcodable image (JPEG/PNG/GIF, ≤5 MB) and the browser supports WebP: check memory cache → check disk cache → transcode on miss +4. If WebP output is **larger** than the original, serve the original instead (smart skip) + +### Constants + +| Constant | Value | Description | +|---|---|---| +| `MAX_TRANSCODE_SIZE` | 5 MB | Files above this skip transcoding | + +### Supported Input Formats + +`image/jpeg`, `image/jpg`, `image/png`, `image/gif` + +Not transcoded: `image/webp` (already optimal), `image/svg+xml`, `image/bmp` + +### Storage Layout + +``` +/ + .transcoded/ + webp/ + .webp +``` + +### Caching + +- **Memory LRU cache**: configurable max entries and max bytes +- **Disk cache**: persistent WebP files in `.transcoded/webp/` +- Disk writes are fire-and-forget via `tokio::spawn` (non-blocking) +- `invalidate(file_id)` evicts from both memory and disk + +### Port Trait + +```rust +#[async_trait] +pub trait ImageTranscodePort: Send + Sync + 'static { + fn can_transcode(&self, mime_type: &str) -> bool; + fn should_transcode(&self, mime_type: &str, file_size: u64) -> bool; + async fn get_transcoded(&self, file_id: &str, original_content: &[u8], original_mime: &str, target_format: OutputFormat) + -> Result<(Bytes, String, bool), DomainError>; + async fn invalidate(&self, file_id: &str); + async fn get_stats(&self) -> TranscodeStatsDto; + async fn clear_cache(&self) -> Result<(), DomainError>; +} +``` + +### Browser Detection + +```rust +pub struct BrowserCapabilities { + pub supports_webp: bool, + pub supports_avif: bool, // reserved for future +} + +impl BrowserCapabilities { + pub fn from_accept_header(accept: Option<&str>) -> Self; + pub fn best_format(&self) -> Option; +} +``` + +### Statistics + +```rust +pub struct TranscodeStatsDto { + pub cache_hits: u64, + pub disk_hits: u64, + pub transcodes: u64, + pub bytes_saved: u64, + pub transcode_errors: u64, +} +``` + +### Example Flow + +``` +Client: GET /api/files/abc-123/download + Accept: image/webp, image/png, */* + +Server: 1. File is "photo.jpg" (800KB) → can transcode yes, should transcode yes + 2. Check memory cache → miss + 3. Check disk cache (.transcoded/webp/abc-123.webp) → miss + 4. Transcode JPEG → WebP (600KB) → smaller yes + 5. Cache in memory + async write to disk + 6. Respond with WebP (saves 200KB, 25% reduction) +``` + +## Dependencies + +- `image = "0.25"` (with `jpeg`, `png`, `gif`, `webp` features) -- used by both thumbnail and transcode +- Thumbnail resize: `imageops::resize` with `FilterType::Lanczos3` +- WebP encoding: via `image` crate's WebP encoder diff --git a/doc/trash-feature-summary.md b/doc/trash-feature-summary.md new file mode 100644 index 00000000..30a2dff7 --- /dev/null +++ b/doc/trash-feature-summary.md @@ -0,0 +1,79 @@ +# 14 - Trash Feature + +Soft-delete for files and folders. Items go to a per-user trash bin instead of being permanently removed. Configurable retention period with automatic cleanup. + +## Architecture + +Follows the hexagonal architecture: + +1. **Domain Layer** (`/src/domain/`): + - Entities: **TrashedItem** representing files and folders in the trash + - Repository interfaces: **TrashRepository** defining trash management operations + +2. **Application Layer** (`/src/application/`): + - DTOs: **TrashedItemDto** for data transfer between layers + - Ports: **TrashUseCase** defining available operations + - Services: **TrashService** implementing the trash use cases + +3. **Infrastructure Layer** (`/src/infrastructure/`): + - Repositories: **TrashFsRepository** for filesystem-based trash storage + - Trash-related methods in existing repositories: `FileWriteRepository::move_to_trash()`, `FolderRepository::move_to_trash()`, etc. + - Services: **TrashCleanupService** for automatic cleanup of expired items + +4. **Interface Layer** (`/src/interfaces/`): + - API handlers: `trash_handler.rs` with HTTP endpoints for trash operations + - Routes: updated `routes.rs` to include trash endpoints + +## Key Features + +1. **Soft Deletion** -- files and folders move to trash, not immediately deleted +2. **Per-User Trash** -- each user has an isolated trash bin +3. **Retention Policy** -- items auto-delete after a configurable period +4. **Restoration** -- items can be restored to their original location +5. **Permanent Deletion** -- items can be permanently deleted before retention expires +6. **Empty Trash** -- wipe everything in the trash at once + +## API Endpoints + +- `GET /api/trash` or `GET /api/trash/` -- list all items in the user's trash +- `DELETE /api/trash/files/:id` -- move a file to trash +- `DELETE /api/trash/folders/:id` -- move a folder to trash +- `POST /api/trash/:id/restore` -- restore an item to its original location +- `DELETE /api/trash/:id` -- permanently delete an item from trash +- `DELETE /api/trash/empty` -- empty the entire trash bin + +## Testing + +1. **Unit Tests** -- testing **TrashService**: + - Move files and folders to trash + - Restore items from trash + - Permanent deletion + - Empty trash operation + +2. **Integration Tests** -- Python script hitting the API endpoints: + - End-to-end testing of all trash operations + - Verification of move, list, restore, and delete behavior + +3. **Shell Script** -- for manual testing and demonstration + +## Configuration + +- **OXICLOUD_ENABLE_TRASH**: enable/disable the trash feature via **FeaturesConfig** (default: true) +- **OXICLOUD_TRASH_RETENTION_DAYS**: days to keep items before automatic deletion (default: 30, via **StorageConfig**) + +## Implementation Details + +1. **Physical File Storage** -- when items are trashed, they physically move to a `.trash` directory +2. **Metadata Storage** -- trashed item info stored in a separate database table or file +3. **User Isolation** -- trash items are isolated by user ID +4. **Automatic Cleanup** -- a background job runs periodically to clean up expired items +5. **Transaction Safety** -- operations are atomic with proper error handling + +## Future Enhancements + +1. **Trash Quotas** -- limit trash storage per user +2. **Batch Operations** -- trash, restore, or delete multiple items at once +3. **Storage Optimization** -- deduplication for trashed items +4. **Version Control** -- track file versions when moving to trash +5. **Scheduled Cleanup** -- let users configure custom retention periods +6. **Trash Monitoring** -- metrics and alerts for trash usage and cleanup diff --git a/doc/WEBDAV-INTEGRATION-GUIDE.md b/doc/webdav-integration-guide.md similarity index 81% rename from doc/WEBDAV-INTEGRATION-GUIDE.md rename to doc/webdav-integration-guide.md index 49fc6323..500c0ab6 100644 --- a/doc/WEBDAV-INTEGRATION-GUIDE.md +++ b/doc/webdav-integration-guide.md @@ -1,6 +1,6 @@ -# WebDAV Integration Guide for OxiCloud +# 22 - WebDAV Integration Guide -This guide provides developers with information on how to interact with OxiCloud's WebDAV interface programmatically and how to extend the WebDAV functionality. +The WebDAV interface exposes file operations over HTTP at a single base path. All standard WebDAV methods are supported: **PROPFIND**, **GET**, **PUT**, **MKCOL**, **MOVE**, **COPY**, **DELETE**. Authentication is HTTP Basic over TLS. ## Table of Contents @@ -20,13 +20,13 @@ This guide provides developers with information on how to interact with OxiCloud ## Base URL and Endpoints -The WebDAV interface is available at: +The WebDAV interface lives at: ``` https://[your-oxicloud-server]/webdav/ ``` -All file and folder operations are performed under this base path. Resource paths are appended to this URL. +All file and folder operations hang off this base path. Append the resource path to the URL. Examples: - Root folder: `https://[your-oxicloud-server]/webdav/` @@ -36,23 +36,23 @@ Examples: ## Authentication -OxiCloud's WebDAV interface supports HTTP Basic Authentication. When making requests, include the `Authorization` header with base64-encoded credentials: +WebDAV uses HTTP Basic Authentication. Include the `Authorization` header with base64-encoded credentials: ``` Authorization: Basic base64(username:password) ``` -For security reasons, always use HTTPS when connecting to WebDAV. +Always use HTTPS. ## Common Operations ### Listing Directories -To list the contents of a directory, use the `PROPFIND` method with an appropriate `Depth` header: +Use the **PROPFIND** method with a **Depth** header: -- `Depth: 0` - Returns information about the resource itself -- `Depth: 1` - Returns information about the resource and its immediate children (recommended) -- `Depth: infinity` - Returns information about the resource and all descendants (use carefully with large directories) +- `Depth: 0` -- info about the resource itself +- `Depth: 1` -- the resource and its immediate children (recommended) +- `Depth: infinity` -- the resource and all descendants (careful with large trees) Request: ```http @@ -92,7 +92,7 @@ Content-Type: application/xml; charset=utf-8 ### Downloading Files -To download a file, use the standard HTTP `GET` method: +Standard HTTP **GET**: ```http GET /webdav/projects/document.pdf HTTP/1.1 @@ -100,7 +100,7 @@ Host: your-oxicloud-server Authorization: Basic [credentials] ``` -The server will respond with the file content and appropriate headers: +Response: ```http HTTP/1.1 200 OK @@ -114,7 +114,7 @@ ETag: "abc123" ### Uploading Files -To upload or update a file, use the HTTP `PUT` method: +Use HTTP **PUT** to upload or update a file: ```http PUT /webdav/projects/document.pdf HTTP/1.1 @@ -126,21 +126,11 @@ Authorization: Basic [credentials] [File content] ``` -For new files, the server responds with: - -```http -HTTP/1.1 201 Created -``` - -For updated files, the server responds with: - -```http -HTTP/1.1 204 No Content -``` +New files return `201 Created`. Updates return `204 No Content`. ### Creating Folders -To create a folder, use the WebDAV `MKCOL` method: +Use the **MKCOL** method: ```http MKCOL /webdav/projects/new-folder HTTP/1.1 @@ -148,15 +138,11 @@ Host: your-oxicloud-server Authorization: Basic [credentials] ``` -Successful response: - -```http -HTTP/1.1 201 Created -``` +Returns `201 Created` on success. ### Moving and Copying -To move resources, use the WebDAV `MOVE` method: +**MOVE** a resource: ```http MOVE /webdav/old-location.pdf HTTP/1.1 @@ -165,7 +151,7 @@ Destination: https://your-oxicloud-server/webdav/new-location.pdf Authorization: Basic [credentials] ``` -To copy resources, use the WebDAV `COPY` method: +**COPY** a resource: ```http COPY /webdav/original.pdf HTTP/1.1 @@ -174,15 +160,11 @@ Destination: https://your-oxicloud-server/webdav/copy.pdf Authorization: Basic [credentials] ``` -For both operations, a successful response is: - -```http -HTTP/1.1 204 No Content -``` +Both return `204 No Content` on success. ### Deleting Resources -To delete a file or folder, use the HTTP `DELETE` method: +Use HTTP **DELETE**: ```http DELETE /webdav/projects/document.pdf HTTP/1.1 @@ -190,11 +172,7 @@ Host: your-oxicloud-server Authorization: Basic [credentials] ``` -Successful response: - -```http -HTTP/1.1 204 No Content -``` +Returns `204 No Content` on success. ## XML Schemas @@ -278,10 +256,10 @@ body = ''' ''' response = requests.request( - 'PROPFIND', - f'{base_url}/projects/', - headers=headers, - data=body, + 'PROPFIND', + f'{base_url}/projects/', + headers=headers, + data=body, auth=auth ) @@ -291,17 +269,17 @@ if response.status_code == 207: # Multi-Status for response_elem in root.findall('.//{DAV:}response'): href = response_elem.find('.//{DAV:}href').text print(f"Resource: {href}") - + # Get displayname if available displayname = response_elem.find('.//{DAV:}displayname') if displayname is not None and displayname.text: print(f" Name: {displayname.text}") - + # Check if it's a collection (folder) resourcetype = response_elem.find('.//{DAV:}resourcetype') is_collection = resourcetype is not None and resourcetype.find('.//{DAV:}collection') is not None print(f" Type: {'Folder' if is_collection else 'File'}") - + # Get size if it's a file if not is_collection: contentlength = response_elem.find('.//{DAV:}getcontentlength') @@ -311,7 +289,7 @@ if response.status_code == 207: # Multi-Status # 2. Upload a file with open('local-file.pdf', 'rb') as f: file_content = f.read() - + response = requests.put( f'{base_url}/projects/document.pdf', data=file_content, @@ -368,7 +346,7 @@ if response.status_code == 204: ### JavaScript Example -Using browser's `fetch` API: +Using the browser `fetch` API: ```javascript // Base configuration @@ -392,29 +370,29 @@ async function listDirectory(path) { ` }); - + if (response.status === 207) { const text = await response.text(); const parser = new DOMParser(); const xmlDoc = parser.parseFromString(text, 'text/xml'); - + const responses = xmlDoc.getElementsByTagNameNS('DAV:', 'response'); const resources = []; - + for (let i = 0; i < responses.length; i++) { const response = responses[i]; const href = response.getElementsByTagNameNS('DAV:', 'href')[0].textContent; - + let displayName = ''; const displayNameElems = response.getElementsByTagNameNS('DAV:', 'displayname'); if (displayNameElems.length > 0) { displayName = displayNameElems[0].textContent; } - + // Check if resource is a collection (folder) const resourceTypeElem = response.getElementsByTagNameNS('DAV:', 'resourcetype')[0]; const isCollection = resourceTypeElem.getElementsByTagNameNS('DAV:', 'collection').length > 0; - + // Get file size if it's a file let size = null; if (!isCollection) { @@ -423,7 +401,7 @@ async function listDirectory(path) { size = parseInt(contentLengthElems[0].textContent, 10); } } - + resources.push({ href, displayName, @@ -431,7 +409,7 @@ async function listDirectory(path) { size }); } - + return resources; } else { throw new Error(`Failed to list directory: ${response.status}`); @@ -448,7 +426,7 @@ async function uploadFile(path, fileContent) { }, body: fileContent }); - + return response.status === 201 || response.status === 204; } @@ -468,7 +446,7 @@ async function downloadFile(path) { method: 'GET', headers }); - + if (response.status === 200) { return await response.blob(); } else { @@ -481,13 +459,13 @@ async function downloadAndSave(path, filename) { try { const blob = await downloadFile(path); const url = URL.createObjectURL(blob); - + const a = document.createElement('a'); a.href = url; a.download = filename; document.body.appendChild(a); a.click(); - + // Clean up document.body.removeChild(a); URL.revokeObjectURL(url); @@ -502,7 +480,7 @@ async function createFolder(path) { method: 'MKCOL', headers }); - + return response.status === 201; } @@ -515,7 +493,7 @@ async function moveResource(fromPath, toPath) { 'Destination': `${baseUrl}${toPath}` } }); - + return response.status === 204; } @@ -525,7 +503,7 @@ async function deleteResource(path) { method: 'DELETE', headers }); - + return response.status === 204; } ``` @@ -544,18 +522,18 @@ class WebDavClient { private readonly HttpClient _httpClient; private readonly string _baseUrl; - + public WebDavClient(string baseUrl, string username, string password) { _baseUrl = baseUrl.TrimEnd('/') + "/webdav"; _httpClient = new HttpClient(); - + // Set Basic Authentication var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}")); - _httpClient.DefaultRequestHeaders.Authorization = + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials); } - + public async Task ListDirectoryAsync(string path) { var request = new HttpRequestMessage(new HttpMethod("PROPFIND"), $"{_baseUrl}/{path.TrimStart('/')}"); @@ -568,63 +546,63 @@ class WebDavClient Encoding.UTF8, "application/xml" ); - + var response = await _httpClient.SendAsync(request); - + if (response.StatusCode == System.Net.HttpStatusCode.MultiStatus) { var content = await response.Content.ReadAsStringAsync(); return XDocument.Parse(content); } - + throw new Exception($"Failed to list directory: {response.StatusCode}"); } - + public async Task UploadFileAsync(string path, byte[] content) { var request = new HttpRequestMessage(HttpMethod.Put, $"{_baseUrl}/{path.TrimStart('/')}"); request.Content = new ByteArrayContent(content); - + var response = await _httpClient.SendAsync(request); - - return response.StatusCode == System.Net.HttpStatusCode.Created || + + return response.StatusCode == System.Net.HttpStatusCode.Created || response.StatusCode == System.Net.HttpStatusCode.NoContent; } - + public async Task DownloadFileAsync(string path) { var response = await _httpClient.GetAsync($"{_baseUrl}/{path.TrimStart('/')}"); - + if (response.IsSuccessStatusCode) { return await response.Content.ReadAsByteArrayAsync(); } - + throw new Exception($"Failed to download file: {response.StatusCode}"); } - + public async Task CreateFolderAsync(string path) { var request = new HttpRequestMessage(new HttpMethod("MKCOL"), $"{_baseUrl}/{path.TrimStart('/')}"); var response = await _httpClient.SendAsync(request); - + return response.StatusCode == System.Net.HttpStatusCode.Created; } - + public async Task MoveResourceAsync(string fromPath, string toPath) { var request = new HttpRequestMessage(new HttpMethod("MOVE"), $"{_baseUrl}/{fromPath.TrimStart('/')}"); request.Headers.Add("Destination", $"{_baseUrl}/{toPath.TrimStart('/')}"); - + var response = await _httpClient.SendAsync(request); - + return response.StatusCode == System.Net.HttpStatusCode.NoContent; } - + public async Task DeleteResourceAsync(string path) { var response = await _httpClient.DeleteAsync($"{_baseUrl}/{path.TrimStart('/')}"); - + return response.StatusCode == System.Net.HttpStatusCode.NoContent; } } @@ -633,7 +611,7 @@ class WebDavClient async Task RunExampleAsync() { var client = new WebDavClient("https://your-oxicloud-server", "username", "password"); - + // List directory try { @@ -645,7 +623,7 @@ async Task RunExampleAsync() { Console.WriteLine($"Error listing directory: {ex.Message}"); } - + // Upload a file try { @@ -657,7 +635,7 @@ async Task RunExampleAsync() { Console.WriteLine($"Error uploading file: {ex.Message}"); } - + // Download a file try { @@ -678,9 +656,9 @@ async Task RunExampleAsync() To support custom WebDAV properties: -1. Define your XML namespace for custom properties -2. Implement storage for these properties (database table recommended) -3. Update the WebDAV adapter to handle these properties +1. Define your XML namespace for custom properties. +2. Implement storage for them (database table recommended). +3. Update the WebDAV adapter to handle these properties. Example adapter code for custom properties: @@ -692,7 +670,7 @@ fn handle_custom_property(name: &QualifiedName, value: Option<&str>) -> Result) -> Result` of processed folder IDs +- ZIP is built entirely in-memory (`ZipWriter>>`) +- UNIX permissions: `0o755` for all entries + +### Port Trait + +```rust +#[async_trait] +pub trait ZipPort: Send + Sync + 'static { + async fn create_folder_zip(&self, folder_id: &str, folder_name: &str) -> Result, DomainError>; +} +``` + +### Example + +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "https://oxicloud.example.com/api/folders/abc-123/download" \ + -o my-folder.zip +``` + +--- + +## Gzip Compression + +### Architecture + +| Layer | Component | File | +|---|---|---| +| Application Port | **CompressionPort** trait | `src/application/ports/compression_ports.rs` | +| Infrastructure | **GzipCompressionService** | `src/infrastructure/services/compression_service.rs` | + +### Configuration + +| Constant | Value | Description | +|---|---|---| +| `COMPRESSION_SIZE_THRESHOLD` | 50 KB | Files below this are never compressed | + +### Compression Levels + +```rust +pub enum CompressionLevel { + None = 0, + Fast = 1, + Default = 6, + Best = 9, +} +``` + +### Port Trait + +```rust +#[async_trait] +pub trait CompressionPort: Send + Sync + 'static { + async fn compress_data(&self, data: &[u8], level: CompressionLevel) -> Result, DomainError>; + async fn decompress_data(&self, compressed_data: &[u8]) -> Result, DomainError>; + fn should_compress(&self, mime_type: &str, size: u64) -> bool; +} +``` + +### Skip List + +These MIME types are **never compressed** (already compressed or binary): +- `image/*` (except `svg`, `bmp`) +- `audio/*`, `video/*` +- `application/zip`, `application/gzip`, `application/x-compressed` +- `application/x-7z-compressed`, `application/x-rar-compressed` +- `application/x-bzip2`, `application/x-xz` + +### Implementation Details + +- Uses the `flate2` crate (`GzEncoder` / `GzDecoder`) +- Compress/decompress run inside `spawn_blocking` to avoid blocking the async runtime +- Optional **BufferPool** integration for buffer reuse +- Buffer pool estimates: 80% of input size for compression, 5x for decompression