Merge pull request #191 from jaredwolff/feat/sql-migration-system

feat: add SQL migration system using sqlx::migrate!()

.github/workflows/ci.yml

@@ -56,7 +56,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: Initialize test database
-        run: psql -h localhost -U postgres -d oxicloud_test -f db/schema.sql
+        run: psql -h localhost -U postgres -d oxicloud_test -f migrations/20260307000000_initial_schema.sql
         env:
           PGPASSWORD: postgres
 

.github/workflows/docker-build.yml

@@ -70,7 +70,4 @@ jobs:
           echo "=== Static files present ==="
           docker run --rm --entrypoint sh test/oxicloud:test -c "ls /app/static/index.html && echo 'OK: static files present'"
 
-          echo "=== DB schema present ==="
-          docker run --rm --entrypoint sh test/oxicloud:test -c "ls /app/db/schema.sql && echo 'OK: schema present'"
-
           echo "✅ Docker build and test completed successfully"

Cargo.toml

@@ -28,7 +28,7 @@ mime_guess = "2.0.5"
 uuid = { version = "1.21.0", features = ["v4", "serde"] }
 thiserror = "2.0.18"
 mockall = { version = "0.14.0", optional = true }
-sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json"] }
+sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json", "migrate"] }
 jsonwebtoken = { version = "10.3.0", features = ["rust_crypto"] }
 argon2 = "0.5.3"
 rand_core = { version = "0.6", features = ["std", "getrandom"] }

Dockerfile

@@ -24,7 +24,7 @@ COPY --from=cacher /usr/local/cargo/registry /usr/local/cargo/registry
 COPY Cargo.toml Cargo.lock build.rs ./
 COPY src src
 COPY static static
-COPY db db
+COPY migrations migrations
 # Build with all optimizations (DATABASE_URL only needed at compile-time for sqlx)
 ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud"
 RUN DATABASE_URL="${DATABASE_URL}" RUSTFLAGS="-C target-cpu=native" cargo build --release
@@ -59,8 +59,6 @@ RUN chmod +x /usr/local/bin/entrypoint.sh
 
 # Copy processed static files (bundled/minified by build.rs in release)
 COPY --from=builder --chown=oxicloud:oxicloud /app/static-dist /app/static
-COPY --chown=oxicloud:oxicloud db /app/db
-
 # Create storage directory with proper permissions
 RUN mkdir -p /app/storage && chown -R oxicloud:oxicloud /app/storage
 

docker-compose.yml

@@ -12,7 +12,6 @@ services:
       - oxicloud
     volumes:
       - pg_data:/var/lib/postgresql/data
-      - ./db/schema.sql:/docker-entrypoint-initdb.d/10-schema.sql
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s

migrations/20260307000000_initial_schema.sql

@@ -1,6 +1,6 @@
 -- ============================================================
 -- OxiCloud Unified Database Schema
--- For clean installations: psql -f db/schema.sql
+-- Applied automatically via sqlx migrations on startup.
 -- ============================================================
 -- Order: auth (base) → caldav → carddav
 -- All tables use IF NOT EXISTS for idempotent re-runs.

src/infrastructure/db.rs

@@ -25,8 +25,9 @@ pub struct DbPools {
 
 /// Create both the primary and maintenance database pools.
 ///
-/// The schema is applied once via the primary pool.  The maintenance pool
+/// Pending migrations are applied via the primary pool on startup.
-/// shares the same connection string but has its own, smaller budget.
+/// The maintenance pool shares the same connection string but has its
+/// own, smaller budget.
 pub async fn create_database_pools(config: &AppConfig) -> Result<DbPools> {
     tracing::info!(
         "Initializing PostgreSQL connections with URL: {}",
@@ -48,16 +49,16 @@ pub async fn create_database_pools(config: &AppConfig) -> Result<DbPools> {
     )
     .await?;
 
-    // Apply schema through the primary pool (idempotent)
+    // Run pending migrations (idempotent, tracked in _sqlx_migrations table)
-    tracing::info!("Applying database schema...");
+    tracing::info!("Running database migrations...");
-    if let Err(e) = apply_schema(&primary).await {
+    if let Err(e) = run_migrations(&primary).await {
         return Err(DbError(format!(
-            "Database schema could not be applied: {}. \
+            "Database migrations failed: {}. \
-             Run manually: psql -f db/schema.sql",
+             Check the migrations/ directory for issues.",
             e
         )));
     }
-    tracing::info!("Database schema applied successfully");
+    tracing::info!("Database migrations complete");
 
     // --- maintenance pool ---
     let maintenance = create_pool_with_retries(
@@ -156,172 +157,14 @@ async fn create_pool_with_retries(
     )))
 }
 
-/// Apply the embedded schema.sql to the database.
+/// Run pending migrations from the `migrations/` directory.
-/// First tries `raw_sql` (simple query protocol). If that fails, falls back
-/// to splitting the SQL into individual statements and executing them one by one.
-async fn apply_schema(pool: &PgPool) -> Result<()> {
-    let schema_sql = include_str!("../../db/schema.sql");
-
-    // Attempt 1: raw_sql sends the entire script via the simple query protocol
-    match sqlx::raw_sql(schema_sql).execute(pool).await {
-        Ok(_) => return Ok(()),
-        Err(e) => {
-            tracing::warn!(
-                "raw_sql failed ({}), falling back to statement-by-statement execution",
-                e
-            );
-        }
-    }
-
-    // Attempt 2: split into individual statements respecting dollar-quoting
-    let statements = split_sql_statements(schema_sql);
-    for (i, stmt) in statements.iter().enumerate() {
-        let trimmed = stmt.trim();
-        if trimmed.is_empty() || trimmed == ";" {
-            continue;
-        }
-        if let Err(e) = sqlx::raw_sql(trimmed).execute(pool).await {
-            let preview = if trimmed.len() > 200 {
-                &trimmed[..200]
-            } else {
-                trimmed
-            };
-            tracing::error!(
-                "Schema statement {} failed: {}\n--- SQL ---\n{}\n-----------",
-                i + 1,
-                e,
-                preview
-            );
-            return Err(DbError(format!("Schema statement {} failed: {}", i + 1, e)));
-        }
-    }
-
-    Ok(())
-}
-
-/// Split a SQL script into individual statements, correctly handling:
-/// - Dollar-quoted blocks (`$BODY$...$BODY$`, `$$...$$`)
-/// - Single-quoted strings (`'...'`)
-/// - Line comments (`-- ...`)
-/// - Block comments (`/* ... */`)
 ///
-/// Uses byte-level iteration over the `&str` directly — no intermediate
+/// Uses sqlx's built-in migration system which tracks applied migrations
-/// `Vec<char>` allocation (saves ~4× the input size in heap memory).
+/// in a `_sqlx_migrations` table. Each migration runs in its own transaction.
-/// SQL is ASCII-safe, so byte comparison is sufficient for all delimiters.
+/// Migration files are embedded at compile time via `sqlx::migrate!()`.
-fn split_sql_statements(sql: &str) -> Vec<String> {
+async fn run_migrations(pool: &PgPool) -> Result<()> {
-    let mut statements = Vec::new();
+    match sqlx::migrate!().run(pool).await {
-    let mut current = String::new();
+        Ok(()) => Ok(()),
-    let bytes = sql.as_bytes();
+        Err(e) => Err(DbError(format!("Migration error: {}", e))),
-    let len = bytes.len();
-    let mut i = 0;
-
-    while i < len {
-        let b = bytes[i];
-
-        // Line comment
-        if b == b'-' && i + 1 < len && bytes[i + 1] == b'-' {
-            while i < len && bytes[i] != b'\n' {
-                current.push(bytes[i] as char);
-                i += 1;
-            }
-            continue;
-        }
-
-        // Block comment
-        if b == b'/' && i + 1 < len && bytes[i + 1] == b'*' {
-            current.push('/');
-            current.push('*');
-            i += 2;
-            while i + 1 < len && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
-                current.push(bytes[i] as char);
-                i += 1;
-            }
-            if i + 1 < len {
-                current.push('*');
-                current.push('/');
-                i += 2;
-            }
-            continue;
-        }
-
-        // Single-quoted string
-        if b == b'\'' {
-            current.push('\'');
-            i += 1;
-            while i < len {
-                current.push(bytes[i] as char);
-                if bytes[i] == b'\'' {
-                    if i + 1 < len && bytes[i + 1] == b'\'' {
-                        current.push('\'');
-                        i += 2;
-                    } else {
-                        i += 1;
-                        break;
-                    }
-                } else {
-                    i += 1;
     }
 }
-            continue;
-        }
-
-        // Dollar-quoted string ($tag$...$tag$ or $$...$$)
-        if b == b'$' {
-            i += 1;
-            let mut tag = String::from("$");
-            while i < len && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') {
-                tag.push(bytes[i] as char);
-                i += 1;
-            }
-            if i < len && bytes[i] == b'$' {
-                tag.push('$');
-                i += 1;
-                // We have a dollar-quote tag, find the closing tag
-                current.push_str(&tag);
-                let tag_bytes = tag.as_bytes();
-                loop {
-                    if i >= len {
-                        break;
-                    }
-                    if bytes[i] == b'$'
-                        && i + tag_bytes.len() <= len
-                        && &bytes[i..i + tag_bytes.len()] == tag_bytes
-                    {
-                        current.push_str(&tag);
-                        i += tag_bytes.len();
-                        break;
-                    }
-                    current.push(bytes[i] as char);
-                    i += 1;
-                }
-            } else {
-                // Not a valid dollar-quote, push what we consumed
-                current.push_str(&tag);
-            }
-            continue;
-        }
-
-        // Statement separator
-        if b == b';' {
-            current.push(';');
-            let trimmed = current.trim().to_string();
-            if !trimmed.is_empty() && trimmed != ";" {
-                statements.push(trimmed);
-            }
-            current.clear();
-            i += 1;
-            continue;
-        }
-
-        current.push(b as char);
-        i += 1;
-    }
-
-    // Trailing statement without semicolon
-    let trimmed = current.trim().to_string();
-    if !trimmed.is_empty() && trimmed != ";" {
-        statements.push(trimmed);
-    }
-
-    statements
-}