cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #191 from jaredwolff/feat/sql-migration-system

Browse Source 
feat: add SQL migration system using sqlx::migrate!()
This commit is contained in:
Dionisio Pozo
2026-03-09 20:54:42 +01:00
committed by GitHub
parent 47638224f3 f6e2b30688
commit f1cfed248a
7 changed files with 21 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+1 -1
View File
@@ -56,7 +56,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
- name: Initialize test database
run: psql -h localhost -U postgres -d oxicloud_test -f db/schema.sql
run: psql -h localhost -U postgres -d oxicloud_test -f migrations/20260307000000_initial_schema.sql
env:
PGPASSWORD: postgres
.github/workflows/docker-build.yml
-3
View File
@@ -70,7 +70,4 @@ jobs:
echo "=== Static files present ==="
docker run --rm --entrypoint sh test/oxicloud:test -c "ls /app/static/index.html && echo 'OK: static files present'"
echo "=== DB schema present ==="
docker run --rm --entrypoint sh test/oxicloud:test -c "ls /app/db/schema.sql && echo 'OK: schema present'"
echo "✅ Docker build and test completed successfully"
Cargo.toml
+1 -1
View File
@@ -28,7 +28,7 @@ mime_guess = "2.0.5"
uuid = { version = "1.21.0", features = ["v4", "serde"] }
thiserror = "2.0.18"
mockall = { version = "0.14.0", optional = true }
sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json"] }
sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json", "migrate"] }
jsonwebtoken = { version = "10.3.0", features = ["rust_crypto"] }
argon2 = "0.5.3"
rand_core = { version = "0.6", features = ["std", "getrandom"] }
Dockerfile
+1 -3
View File
@@ -24,7 +24,7 @@ COPY --from=cacher /usr/local/cargo/registry /usr/local/cargo/registry
COPY Cargo.toml Cargo.lock build.rs ./
COPY src src
COPY static static
COPY db db
COPY migrations migrations
# Build with all optimizations (DATABASE_URL only needed at compile-time for sqlx)
ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud"
RUN DATABASE_URL="${DATABASE_URL}" RUSTFLAGS="-C target-cpu=native" cargo build --release
@@ -59,8 +59,6 @@ RUN chmod +x /usr/local/bin/entrypoint.sh
# Copy processed static files (bundled/minified by build.rs in release)
COPY --from=builder --chown=oxicloud:oxicloud /app/static-dist /app/static
COPY --chown=oxicloud:oxicloud db /app/db
# Create storage directory with proper permissions
RUN mkdir -p /app/storage && chown -R oxicloud:oxicloud /app/storage
docker-compose.yml
-1
View File
@@ -12,7 +12,6 @@ services:
- oxicloud
volumes:
- pg_data:/var/lib/postgresql/data
- ./db/schema.sql:/docker-entrypoint-initdb.d/10-schema.sql
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
db/schema.sql → migrations/20260307000000_initial_schema.sql
+1 -1
View File
@@ -1,6 +1,6 @@
-- ============================================================
-- OxiCloud Unified Database Schema
-- For clean installations: psql -f db/schema.sql
-- Applied automatically via sqlx migrations on startup.
-- ============================================================
-- Order: auth (base) → caldav → carddav
-- All tables use IF NOT EXISTS for idempotent re-runs.
src/infrastructure/db.rs
+17 -174
View File
@@ -25,8 +25,9 @@ pub struct DbPools {
/// Create both the primary and maintenance database pools.
///
/// The schema is applied once via the primary pool. The maintenance pool
/// shares the same connection string but has its own, smaller budget.
/// Pending migrations are applied via the primary pool on startup.
/// The maintenance pool shares the same connection string but has its
/// own, smaller budget.
pub async fn create_database_pools(config: &AppConfig) -> Result<DbPools> {
tracing::info!(
"Initializing PostgreSQL connections with URL: {}",
@@ -48,16 +49,16 @@ pub async fn create_database_pools(config: &AppConfig) -> Result<DbPools> {
)
.await?;
// Apply schema through the primary pool (idempotent)
tracing::info!("Applying database schema...");
if let Err(e) = apply_schema(&primary).await {
// Run pending migrations (idempotent, tracked in _sqlx_migrations table)
tracing::info!("Running database migrations...");
if let Err(e) = run_migrations(&primary).await {
return Err(DbError(format!(
"Database schema could not be applied: {}. \
Run manually: psql -f db/schema.sql",
"Database migrations failed: {}. \
Check the migrations/ directory for issues.",
e
)));
}
tracing::info!("Database schema applied successfully");
tracing::info!("Database migrations complete");
// --- maintenance pool ---
let maintenance = create_pool_with_retries(
@@ -156,172 +157,14 @@ async fn create_pool_with_retries(
)))
}
/// Apply the embedded schema.sql to the database.
/// First tries `raw_sql` (simple query protocol). If that fails, falls back
/// to splitting the SQL into individual statements and executing them one by one.
async fn apply_schema(pool: &PgPool) -> Result<()> {
let schema_sql = include_str!("../../db/schema.sql");
// Attempt 1: raw_sql sends the entire script via the simple query protocol
match sqlx::raw_sql(schema_sql).execute(pool).await {
Ok(_) => return Ok(()),
Err(e) => {
tracing::warn!(
"raw_sql failed ({}), falling back to statement-by-statement execution",
e
);
}
}
// Attempt 2: split into individual statements respecting dollar-quoting
let statements = split_sql_statements(schema_sql);
for (i, stmt) in statements.iter().enumerate() {
let trimmed = stmt.trim();
if trimmed.is_empty() || trimmed == ";" {
continue;
}
if let Err(e) = sqlx::raw_sql(trimmed).execute(pool).await {
let preview = if trimmed.len() > 200 {
&trimmed[..200]
} else {
trimmed
};
tracing::error!(
"Schema statement {} failed: {}\n--- SQL ---\n{}\n-----------",
i + 1,
e,
preview
);
return Err(DbError(format!("Schema statement {} failed: {}", i + 1, e)));
}
}
Ok(())
}
/// Split a SQL script into individual statements, correctly handling:
/// - Dollar-quoted blocks (`$BODY$...$BODY$`, `$$...$$`)
/// - Single-quoted strings (`'...'`)
/// - Line comments (`-- ...`)
/// - Block comments (`/* ... */`)
/// Run pending migrations from the `migrations/` directory.
///
/// Uses byte-level iteration over the `&str` directly — no intermediate
/// `Vec<char>` allocation (saves ~4× the input size in heap memory).
/// SQL is ASCII-safe, so byte comparison is sufficient for all delimiters.
fn split_sql_statements(sql: &str) -> Vec<String> {
let mut statements = Vec::new();
let mut current = String::new();
let bytes = sql.as_bytes();
let len = bytes.len();
let mut i = 0;
while i < len {
let b = bytes[i];
// Line comment
if b == b'-' && i + 1 < len && bytes[i + 1] == b'-' {
while i < len && bytes[i] != b'\n' {
current.push(bytes[i] as char);
i += 1;
}
continue;
}
// Block comment
if b == b'/' && i + 1 < len && bytes[i + 1] == b'*' {
current.push('/');
current.push('*');
i += 2;
while i + 1 < len && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
current.push(bytes[i] as char);
i += 1;
}
if i + 1 < len {
current.push('*');
current.push('/');
i += 2;
}
continue;
}
// Single-quoted string
if b == b'\'' {
current.push('\'');
i += 1;
while i < len {
current.push(bytes[i] as char);
if bytes[i] == b'\'' {
if i + 1 < len && bytes[i + 1] == b'\'' {
current.push('\'');
i += 2;
} else {
i += 1;
break;
}
} else {
i += 1;
}
}
continue;
}
// Dollar-quoted string ($tag$...$tag$ or $$...$$)
if b == b'$' {
i += 1;
let mut tag = String::from("$");
while i < len && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') {
tag.push(bytes[i] as char);
i += 1;
}
if i < len && bytes[i] == b'$' {
tag.push('$');
i += 1;
// We have a dollar-quote tag, find the closing tag
current.push_str(&tag);
let tag_bytes = tag.as_bytes();
loop {
if i >= len {
break;
}
if bytes[i] == b'$'
&& i + tag_bytes.len() <= len
&& &bytes[i..i + tag_bytes.len()] == tag_bytes
{
current.push_str(&tag);
i += tag_bytes.len();
break;
}
current.push(bytes[i] as char);
i += 1;
}
} else {
// Not a valid dollar-quote, push what we consumed
current.push_str(&tag);
}
continue;
}
// Statement separator
if b == b';' {
current.push(';');
let trimmed = current.trim().to_string();
if !trimmed.is_empty() && trimmed != ";" {
statements.push(trimmed);
}
current.clear();
i += 1;
continue;
}
current.push(b as char);
i += 1;
/// Uses sqlx's built-in migration system which tracks applied migrations
/// in a `_sqlx_migrations` table. Each migration runs in its own transaction.
/// Migration files are embedded at compile time via `sqlx::migrate!()`.
async fn run_migrations(pool: &PgPool) -> Result<()> {
match sqlx::migrate!().run(pool).await {
Ok(()) => Ok(()),
Err(e) => Err(DbError(format!("Migration error: {}", e))),
}
// Trailing statement without semicolon
let trimmed = current.trim().to_string();
if !trimmed.is_empty() && trimmed != ";" {
statements.push(trimmed);
}
statements
}