From 036390a242b68c4adbf789f1b3d5856ca0f64b2c Mon Sep 17 00:00:00 2001 From: Jared Wolff Date: Mon, 9 Mar 2026 15:29:24 -0400 Subject: [PATCH 1/2] feat: add SQL migration system using sqlx::migrate!() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace manual schema.sql application with sqlx's built-in migration system. Migrations are embedded at compile time and tracked in the _sqlx_migrations table. Pending migrations run automatically on startup. - Move db/schema.sql → migrations/20260307000000_initial_schema.sql - Remove apply_schema() and split_sql_statements() from db.rs - Add run_migrations() using sqlx::migrate!() macro - Remove docker-compose schema.sql mount (app handles it now) - Enable sqlx "migrate" feature in Cargo.toml Future schema changes: add a new timestamped .sql in migrations/. Closes #190 Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 2 +- docker-compose.yml | 1 - .../20260307000000_initial_schema.sql | 2 +- src/infrastructure/db.rs | 191 ++---------------- 4 files changed, 19 insertions(+), 177 deletions(-) rename db/schema.sql => migrations/20260307000000_initial_schema.sql (99%) diff --git a/Cargo.toml b/Cargo.toml index f6aac2b7..b54799b1 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ mime_guess = "2.0.5" uuid = { version = "1.21.0", features = ["v4", "serde"] } thiserror = "2.0.18" mockall = { version = "0.14.0", optional = true } -sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json"] } +sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json", "migrate"] } jsonwebtoken = { version = "10.3.0", features = ["rust_crypto"] } argon2 = "0.5.3" rand_core = { version = "0.6", features = ["std", "getrandom"] } diff --git a/docker-compose.yml b/docker-compose.yml index bb06e18b..15f0a9a3 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,6 @@ services: - oxicloud volumes: - pg_data:/var/lib/postgresql/data - - ./db/schema.sql:/docker-entrypoint-initdb.d/10-schema.sql healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s diff --git a/db/schema.sql b/migrations/20260307000000_initial_schema.sql similarity index 99% rename from db/schema.sql rename to migrations/20260307000000_initial_schema.sql index 618db650..4bcff713 100755 --- a/db/schema.sql +++ b/migrations/20260307000000_initial_schema.sql @@ -1,6 +1,6 @@ -- ============================================================ -- OxiCloud Unified Database Schema --- For clean installations: psql -f db/schema.sql +-- Applied automatically via sqlx migrations on startup. -- ============================================================ -- Order: auth (base) → caldav → carddav -- All tables use IF NOT EXISTS for idempotent re-runs. diff --git a/src/infrastructure/db.rs b/src/infrastructure/db.rs index b7d1c1d5..13c10042 100755 --- a/src/infrastructure/db.rs +++ b/src/infrastructure/db.rs @@ -25,8 +25,9 @@ pub struct DbPools { /// Create both the primary and maintenance database pools. /// -/// The schema is applied once via the primary pool. The maintenance pool -/// shares the same connection string but has its own, smaller budget. +/// Pending migrations are applied via the primary pool on startup. +/// The maintenance pool shares the same connection string but has its +/// own, smaller budget. pub async fn create_database_pools(config: &AppConfig) -> Result { tracing::info!( "Initializing PostgreSQL connections with URL: {}", @@ -48,16 +49,16 @@ pub async fn create_database_pools(config: &AppConfig) -> Result { ) .await?; - // Apply schema through the primary pool (idempotent) - tracing::info!("Applying database schema..."); - if let Err(e) = apply_schema(&primary).await { + // Run pending migrations (idempotent, tracked in _sqlx_migrations table) + tracing::info!("Running database migrations..."); + if let Err(e) = run_migrations(&primary).await { return Err(DbError(format!( - "Database schema could not be applied: {}. \ - Run manually: psql -f db/schema.sql", + "Database migrations failed: {}. \ + Check the migrations/ directory for issues.", e ))); } - tracing::info!("Database schema applied successfully"); + tracing::info!("Database migrations complete"); // --- maintenance pool --- let maintenance = create_pool_with_retries( @@ -156,172 +157,14 @@ async fn create_pool_with_retries( ))) } -/// Apply the embedded schema.sql to the database. -/// First tries `raw_sql` (simple query protocol). If that fails, falls back -/// to splitting the SQL into individual statements and executing them one by one. -async fn apply_schema(pool: &PgPool) -> Result<()> { - let schema_sql = include_str!("../../db/schema.sql"); - - // Attempt 1: raw_sql sends the entire script via the simple query protocol - match sqlx::raw_sql(schema_sql).execute(pool).await { - Ok(_) => return Ok(()), - Err(e) => { - tracing::warn!( - "raw_sql failed ({}), falling back to statement-by-statement execution", - e - ); - } - } - - // Attempt 2: split into individual statements respecting dollar-quoting - let statements = split_sql_statements(schema_sql); - for (i, stmt) in statements.iter().enumerate() { - let trimmed = stmt.trim(); - if trimmed.is_empty() || trimmed == ";" { - continue; - } - if let Err(e) = sqlx::raw_sql(trimmed).execute(pool).await { - let preview = if trimmed.len() > 200 { - &trimmed[..200] - } else { - trimmed - }; - tracing::error!( - "Schema statement {} failed: {}\n--- SQL ---\n{}\n-----------", - i + 1, - e, - preview - ); - return Err(DbError(format!("Schema statement {} failed: {}", i + 1, e))); - } - } - - Ok(()) -} - -/// Split a SQL script into individual statements, correctly handling: -/// - Dollar-quoted blocks (`$BODY$...$BODY$`, `$$...$$`) -/// - Single-quoted strings (`'...'`) -/// - Line comments (`-- ...`) -/// - Block comments (`/* ... */`) +/// Run pending migrations from the `migrations/` directory. /// -/// Uses byte-level iteration over the `&str` directly — no intermediate -/// `Vec` allocation (saves ~4× the input size in heap memory). -/// SQL is ASCII-safe, so byte comparison is sufficient for all delimiters. -fn split_sql_statements(sql: &str) -> Vec { - let mut statements = Vec::new(); - let mut current = String::new(); - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - - while i < len { - let b = bytes[i]; - - // Line comment - if b == b'-' && i + 1 < len && bytes[i + 1] == b'-' { - while i < len && bytes[i] != b'\n' { - current.push(bytes[i] as char); - i += 1; - } - continue; - } - - // Block comment - if b == b'/' && i + 1 < len && bytes[i + 1] == b'*' { - current.push('/'); - current.push('*'); - i += 2; - while i + 1 < len && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { - current.push(bytes[i] as char); - i += 1; - } - if i + 1 < len { - current.push('*'); - current.push('/'); - i += 2; - } - continue; - } - - // Single-quoted string - if b == b'\'' { - current.push('\''); - i += 1; - while i < len { - current.push(bytes[i] as char); - if bytes[i] == b'\'' { - if i + 1 < len && bytes[i + 1] == b'\'' { - current.push('\''); - i += 2; - } else { - i += 1; - break; - } - } else { - i += 1; - } - } - continue; - } - - // Dollar-quoted string ($tag$...$tag$ or $$...$$) - if b == b'$' { - i += 1; - let mut tag = String::from("$"); - while i < len && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') { - tag.push(bytes[i] as char); - i += 1; - } - if i < len && bytes[i] == b'$' { - tag.push('$'); - i += 1; - // We have a dollar-quote tag, find the closing tag - current.push_str(&tag); - let tag_bytes = tag.as_bytes(); - loop { - if i >= len { - break; - } - if bytes[i] == b'$' - && i + tag_bytes.len() <= len - && &bytes[i..i + tag_bytes.len()] == tag_bytes - { - current.push_str(&tag); - i += tag_bytes.len(); - break; - } - current.push(bytes[i] as char); - i += 1; - } - } else { - // Not a valid dollar-quote, push what we consumed - current.push_str(&tag); - } - continue; - } - - // Statement separator - if b == b';' { - current.push(';'); - let trimmed = current.trim().to_string(); - if !trimmed.is_empty() && trimmed != ";" { - statements.push(trimmed); - } - current.clear(); - i += 1; - continue; - } - - current.push(b as char); - i += 1; +/// Uses sqlx's built-in migration system which tracks applied migrations +/// in a `_sqlx_migrations` table. Each migration runs in its own transaction. +/// Migration files are embedded at compile time via `sqlx::migrate!()`. +async fn run_migrations(pool: &PgPool) -> Result<()> { + match sqlx::migrate!().run(pool).await { + Ok(()) => Ok(()), + Err(e) => Err(DbError(format!("Migration error: {}", e))), } - - // Trailing statement without semicolon - let trimmed = current.trim().to_string(); - if !trimmed.is_empty() && trimmed != ";" { - statements.push(trimmed); - } - - statements } From f6e2b3068864cb8a923b45d8c6305bc0439050b2 Mon Sep 17 00:00:00 2001 From: Jared Wolff Date: Mon, 9 Mar 2026 15:43:15 -0400 Subject: [PATCH 2/2] fix(ci): update references from db/schema.sql to sqlx migrations CI and Docker builds broke after removing db/ in favor of migrations/. - ci.yml: point psql init at migrations/20260307000000_initial_schema.sql - Dockerfile: COPY migrations instead of db; remove unused db copy in final stage - docker-build.yml: remove db/schema.sql presence check Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 2 +- .github/workflows/docker-build.yml | 3 --- Dockerfile | 4 +--- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e3d941f..8b7f0e2f 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Initialize test database - run: psql -h localhost -U postgres -d oxicloud_test -f db/schema.sql + run: psql -h localhost -U postgres -d oxicloud_test -f migrations/20260307000000_initial_schema.sql env: PGPASSWORD: postgres diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0b347043..1a2cfc30 100755 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -70,7 +70,4 @@ jobs: echo "=== Static files present ===" docker run --rm --entrypoint sh test/oxicloud:test -c "ls /app/static/index.html && echo 'OK: static files present'" - echo "=== DB schema present ===" - docker run --rm --entrypoint sh test/oxicloud:test -c "ls /app/db/schema.sql && echo 'OK: schema present'" - echo "✅ Docker build and test completed successfully" diff --git a/Dockerfile b/Dockerfile index 4c899176..4c5b81db 100755 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY --from=cacher /usr/local/cargo/registry /usr/local/cargo/registry COPY Cargo.toml Cargo.lock build.rs ./ COPY src src COPY static static -COPY db db +COPY migrations migrations # Build with all optimizations (DATABASE_URL only needed at compile-time for sqlx) ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud" RUN DATABASE_URL="${DATABASE_URL}" RUSTFLAGS="-C target-cpu=native" cargo build --release @@ -59,8 +59,6 @@ RUN chmod +x /usr/local/bin/entrypoint.sh # Copy processed static files (bundled/minified by build.rs in release) COPY --from=builder --chown=oxicloud:oxicloud /app/static-dist /app/static -COPY --chown=oxicloud:oxicloud db /app/db - # Create storage directory with proper permissions RUN mkdir -p /app/storage && chown -R oxicloud:oxicloud /app/storage