From c893f9ae2908fdcf8ca712614e3b366c3181e851 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Thu, 18 Jun 2026 01:52:47 +0200 Subject: [PATCH] chore(roles): remove deprecated tools this tools are were heere to be tracked, no more necessary once migration is done --- tools/audit-grants-bundle-shape.sql | 215 ------------------ tools/d-prep-real-db-smoke.sh | 328 ---------------------------- 2 files changed, 543 deletions(-) delete mode 100644 tools/audit-grants-bundle-shape.sql delete mode 100755 tools/d-prep-real-db-smoke.sh diff --git a/tools/audit-grants-bundle-shape.sql b/tools/audit-grants-bundle-shape.sql deleted file mode 100644 index fb3d24de..00000000 --- a/tools/audit-grants-bundle-shape.sql +++ /dev/null @@ -1,215 +0,0 @@ --- ════════════════════════════════════════════════════════════════════════════ --- audit-grants-bundle-shape.sql — Pre-D-Prep diagnostic --- ════════════════════════════════════════════════════════════════════════════ --- Purpose: confirm the assumption that >99% of existing `storage.access_grants` --- rows already cluster into the standard role bundles --- (viewer / commenter / contributor / editor / owner). The answer determines --- whether the access_grants → role_grants migration is fully mechanical (just --- run the backfill) or whether per-row decisions are needed for some edge --- cases. --- --- READ-ONLY. Safe to run against any environment including production. --- --- Usage: --- psql "$DATABASE_URL" -f tools/audit-grants-bundle-shape.sql --- --- Reference role bundles (mirror `Role::expand()` in --- `src/application/dtos/grant_dto.rs`): --- viewer = {read} --- commenter = {read, comment} ← reserved variant --- contributor = {read, create} ← new in D-Prep --- editor = {read, comment, create, update} --- owner = {read, comment, create, update, share, delete} --- (post-D-Prep: + manage, when Group-as-Resource lands) --- ════════════════════════════════════════════════════════════════════════════ - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' 0. Total population' -\echo '─────────────────────────────────────────────────────────────────────────' - -SELECT count(*) AS total_grant_rows, - count(DISTINCT (subject_type, subject_id, resource_type, resource_id)) - AS distinct_clusters, - ROUND( - count(*)::numeric - / NULLIF(count(DISTINCT (subject_type, subject_id, resource_type, resource_id)), 0), - 2 - ) AS avg_rows_per_cluster -FROM storage.access_grants; - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' 1. Bundle distribution — which permission sets exist, how popular?' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' Each row = a unique permission set; cluster_count = how many (subject,' -\echo ' resource) pairs have exactly that set.' - -WITH cluster AS ( - SELECT subject_type, - subject_id, - resource_type, - resource_id, - array_agg(permission ORDER BY permission) AS perms - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -), -known_bundles AS ( - SELECT ARRAY['read']::text[] AS perms, 'viewer' AS role - UNION ALL SELECT ARRAY['comment','read']::text[], 'commenter' - UNION ALL SELECT ARRAY['create','read']::text[], 'contributor' - UNION ALL SELECT ARRAY['comment','create','read','update']::text[], 'editor' - UNION ALL SELECT ARRAY['comment','create','delete','read','share','update']::text[], 'owner' -) -SELECT cluster.perms, - count(*) AS cluster_count, - ROUND(100.0 * count(*) / SUM(count(*)) OVER (), 2) AS pct, - COALESCE(known_bundles.role, '(non-bundle)') AS maps_to_role -FROM cluster -LEFT JOIN known_bundles ON known_bundles.perms = cluster.perms -GROUP BY cluster.perms, known_bundles.role -ORDER BY cluster_count DESC; - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' 2. Bundle-shaped vs not — the headline number' -\echo '─────────────────────────────────────────────────────────────────────────' - -WITH cluster AS ( - SELECT subject_type, - subject_id, - resource_type, - resource_id, - array_agg(permission ORDER BY permission) AS perms - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -), -known_bundles AS ( - SELECT ARRAY['read']::text[] AS perms - UNION ALL SELECT ARRAY['comment','read']::text[] - UNION ALL SELECT ARRAY['create','read']::text[] - UNION ALL SELECT ARRAY['comment','create','read','update']::text[] - UNION ALL SELECT ARRAY['comment','create','delete','read','share','update']::text[] -), -shape AS ( - SELECT CASE WHEN cluster.perms = ANY(SELECT kb.perms FROM known_bundles kb) - THEN 'bundle-shaped' - ELSE 'NON-bundle (needs per-row decision)' - END AS shape - FROM cluster -) -SELECT shape, - count(*) AS clusters, - ROUND(100.0 * count(*) / SUM(count(*)) OVER (), 2) AS pct -FROM shape -GROUP BY shape -ORDER BY clusters DESC; - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' 3. Non-bundle clusters in detail (the <1% — investigate each)' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' If this returns 0 rows, the migration is FULLY mechanical. Otherwise' -\echo ' eyeball each row and decide: closest role + audit log entry, or' -\echo ' refuse-to-migrate (rare).' - -WITH cluster AS ( - SELECT subject_type, - subject_id, - resource_type, - resource_id, - array_agg(permission ORDER BY permission) AS perms - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -), -known_bundles AS ( - SELECT ARRAY['read']::text[] AS perms - UNION ALL SELECT ARRAY['comment','read']::text[] - UNION ALL SELECT ARRAY['create','read']::text[] - UNION ALL SELECT ARRAY['comment','create','read','update']::text[] - UNION ALL SELECT ARRAY['comment','create','delete','read','share','update']::text[] -) -SELECT cluster.subject_type, - cluster.subject_id, - cluster.resource_type, - cluster.resource_id, - cluster.perms -FROM cluster -WHERE NOT (cluster.perms = ANY(SELECT kb.perms FROM known_bundles kb)) -ORDER BY cluster.resource_type, cluster.resource_id -LIMIT 200; - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' 4. Sanity checks — invariants that should be true today' -\echo '─────────────────────────────────────────────────────────────────────────' - -\echo '' -\echo ' 4a. Clusters with no Read permission (broken? if >0, investigate)' -WITH cluster AS ( - SELECT subject_type, subject_id, resource_type, resource_id, - array_agg(permission ORDER BY permission) AS perms - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -) -SELECT count(*) AS clusters_without_read -FROM cluster -WHERE NOT ('read' = ANY(perms)); - -\echo '' -\echo ' 4b. Subject-type distribution (sanity check on token / external counts)' -SELECT subject_type, - count(*) AS rows, - count(DISTINCT subject_id) AS distinct_subjects, - count(DISTINCT (resource_type, resource_id)) AS distinct_resources -FROM storage.access_grants -GROUP BY subject_type -ORDER BY subject_type; - -\echo '' -\echo ' 4c. Resource-type distribution' -SELECT resource_type, count(*) AS rows -FROM storage.access_grants -GROUP BY resource_type -ORDER BY rows DESC; - -\echo '' -\echo ' 4d. Permission distribution (total rows per permission value)' -SELECT permission, count(*) AS rows -FROM storage.access_grants -GROUP BY permission -ORDER BY rows DESC; - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' 5. Migration cost estimate' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' role_grants will have ~ (distinct clusters) rows, replacing ~ (total)' -\echo ' rows in access_grants. The ratio is the per-row reduction factor.' - -WITH cluster AS ( - SELECT subject_type, subject_id, resource_type, resource_id - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -) -SELECT (SELECT count(*) FROM storage.access_grants) AS access_grants_rows_today, - (SELECT count(*) FROM cluster) AS role_grants_rows_after, - ROUND( - (SELECT count(*) FROM storage.access_grants)::numeric - / NULLIF((SELECT count(*) FROM cluster), 0), - 2 - ) AS reduction_factor; - - -\echo '' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo ' DONE — share section 2 (headline) and section 3 (non-bundle detail)' -\echo ' to make the D-Prep PR scope decision.' -\echo '─────────────────────────────────────────────────────────────────────────' -\echo '' diff --git a/tools/d-prep-real-db-smoke.sh b/tools/d-prep-real-db-smoke.sh deleted file mode 100755 index 6ded56ac..00000000 --- a/tools/d-prep-real-db-smoke.sh +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env bash -# ════════════════════════════════════════════════════════════════════════════ -# d-prep-real-db-smoke.sh — Validate D-Prep migration against sandbox data -# ════════════════════════════════════════════════════════════════════════════ -# Reads from your sandbox DB (real data — but it's a sandbox, not prod), dumps -# it, restores into an isolated parallel DB, applies the D-Prep migration -# (`20260730000000_role_grants.sql`), and verifies the backfill against the -# pre-recorded audit numbers from `tools/audit-grants-bundle-shape.sql`. -# -# The sandbox DB itself is NEVER MODIFIED — everything happens in the parallel -# `oxicloud_dprep_smoke` DB which is dropped at the start of each run. -# -# Usage: -# tools/d-prep-real-db-smoke.sh # uses $DATABASE_URL -# tools/d-prep-real-db-smoke.sh --keep # leave smoke DB for poking after -# tools/d-prep-real-db-smoke.sh --help -# -# Requirements: -# - pg_dump / pg_restore / psql (PostgreSQL 14+ should be fine; sandbox version-matched) -# - DATABASE_URL pointing at the sandbox DB -# -# PATH gotcha (Mac brew): `brew install postgresql@18` is keg-only; if pg_dump -# isn't on PATH, run: -# export PATH="$(brew --prefix postgresql@18)/bin:$PATH" -# -# Expected audit numbers (from your audit run on 2026-06-17): -# - 73 access_grants rows clustering into 38 (subject, resource) pairs -# - 29 viewer / 5 editor / 4 owner -# These are encoded as assertions below. If your sandbox data has shifted -# since the audit, update EXPECTED_* variables OR re-run the audit script -# first. -# -# Memory cross-refs: -# - bug_pg_dump_folders_circular_fk.md — circular FK on `folders` forces -# `pg_restore --disable-triggers` -# - project_drive_sequence_a.md — D-Prep scope + audit data -# ════════════════════════════════════════════════════════════════════════════ - -set -euo pipefail - - -# ── Config ────────────────────────────────────────────────────────────────── - -SMOKE_DB_NAME="oxicloud_dprep_smoke" -DUMP_FILE="${TMPDIR:-/tmp}/oxicloud-sandbox-${SMOKE_DB_NAME}.dump" -MIGRATION_FILE="migrations/20260730000000_role_grants.sql" - -# Expected values from the audit (tools/audit-grants-bundle-shape.sql run -# on 2026-06-17). Bump these if you re-run the audit and see different -# numbers — they are intentionally HARDCODED so a silent drift gets caught. -EXPECTED_DISTINCT_CLUSTERS=38 -EXPECTED_VIEWER=29 -EXPECTED_EDITOR=5 -EXPECTED_OWNER=4 - -KEEP_DB=0 - - -# ── Arg parsing ───────────────────────────────────────────────────────────── - -usage() { - sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//' - exit "${1:-0}" -} - -for arg in "$@"; do - case "$arg" in - --keep|--keep-db) KEEP_DB=1 ;; - -h|--help) usage 0 ;; - *) echo "Unknown arg: $arg" >&2 ; usage 2 ;; - esac -done - - -# ── Pre-flight ────────────────────────────────────────────────────────────── - -if [[ -z "${DATABASE_URL:-}" ]]; then - echo "ERROR: \$DATABASE_URL must be set and point at the sandbox DB." >&2 - exit 2 -fi - -for bin in pg_dump pg_restore psql; do - if ! command -v "$bin" >/dev/null 2>&1; then - echo "ERROR: '$bin' not found on PATH." >&2 - echo "Hint: brew-installed postgresql is keg-only. Run:" >&2 - echo " export PATH=\"\$(brew --prefix postgresql@18)/bin:\$PATH\"" >&2 - exit 2 - fi -done - -if [[ ! -f "$MIGRATION_FILE" ]]; then - echo "ERROR: migration file not found: $MIGRATION_FILE" >&2 - echo "Run this script from the repo root." >&2 - exit 2 -fi - -# Derive connection URLs for the maintenance DB (where we'll issue -# DROP/CREATE DATABASE) and the smoke DB itself, by splicing -# $SMOKE_DB_NAME into $DATABASE_URL. Avoids relying on libpq's -# default local Unix socket — `dropdb`/`createdb`/`psql` without an -# explicit -d won't read $DATABASE_URL and try /tmp/.s.PGSQL.5432 -# (which fails on Mac brew where the server runs on a non-default -# socket / port). -URL_NO_QUERY="${DATABASE_URL%%\?*}" -URL_QUERY="${DATABASE_URL:${#URL_NO_QUERY}}" # "" or "?…" -SOURCE_DB_NAME="${URL_NO_QUERY##*/}" -URL_BASE="${URL_NO_QUERY%/"$SOURCE_DB_NAME"}" -MAINTENANCE_URL="${URL_BASE}/postgres${URL_QUERY}" -SMOKE_URL="${URL_BASE}/${SMOKE_DB_NAME}${URL_QUERY}" - -if [[ "$SOURCE_DB_NAME" == "$SMOKE_DB_NAME" ]]; then - echo "ERROR: DATABASE_URL appears to point at '$SMOKE_DB_NAME' — refusing" >&2 - echo "to dump-and-restore on top of itself. Set DATABASE_URL to the sandbox." >&2 - exit 2 -fi - - -# ── Helpers ───────────────────────────────────────────────────────────────── - -log() { echo "[smoke] $*"; } -fail() { echo "[smoke] FAIL: $*" >&2 ; exit 1; } -pass() { echo "[smoke] PASS: $*"; } - -# Run a query against the smoke DB and capture a single scalar. -scalar() { - psql -tAX -d "$SMOKE_URL" -c "$1" -} - -# Run a query and check the result equals an expected scalar. -expect_scalar() { - local query="$1" expected="$2" label="$3" - local actual - actual=$(scalar "$query") - if [[ "$actual" == "$expected" ]]; then - pass "$label: $actual" - else - fail "$label: expected $expected, got '$actual'" - fi -} - - -# ── 1. Snapshot the sandbox ───────────────────────────────────────────────── - -log "Dumping sandbox DB → $DUMP_FILE" -log "(custom format, so pg_restore --disable-triggers can side-step the" -log " circular FK on storage.folders.parent_id — see bug memory.)" -pg_dump "$DATABASE_URL" --format=custom --no-owner --no-privileges \ - --file="$DUMP_FILE" -log "Dumped $(du -h "$DUMP_FILE" | awk '{print $1}')" - - -# ── 2. Recreate the smoke DB ──────────────────────────────────────────────── - -log "Dropping smoke DB '$SMOKE_DB_NAME' if it exists" -psql -d "$MAINTENANCE_URL" -c "DROP DATABASE IF EXISTS \"$SMOKE_DB_NAME\"" - -log "Creating fresh smoke DB '$SMOKE_DB_NAME'" -psql -d "$MAINTENANCE_URL" -c "CREATE DATABASE \"$SMOKE_DB_NAME\"" - - -# ── 3. Restore sandbox into smoke DB ──────────────────────────────────────── - -log "Restoring sandbox dump into smoke DB" -# --disable-triggers handles the storage.folders parent_id circular FK without -# wrapping every COPY in SET CONSTRAINTS ALL DEFERRED. -# --no-owner --no-privileges already on dump side; --single-transaction makes -# the restore atomic so a mid-flight failure leaves no half-state. -pg_restore --dbname="$SMOKE_URL" --disable-triggers \ - --single-transaction --no-owner --no-privileges \ - "$DUMP_FILE" - - -# ── 4. Wipe any prior role_grants state from the smoke DB ────────────────── -# The sandbox may already have role_grants if the server has been booted -# with the D-Prep migration applied (auto-migrate on startup). For the -# smoke we want to exercise the migration FRESHLY — same as a brand-new -# install — so drop both the migration's tables and let it recreate them -# from scratch against the access_grants snapshot. - -log "Wiping any prior role_grants state from smoke DB (so we test the migration freshly)" -psql -d "$SMOKE_URL" --set ON_ERROR_STOP=1 -c " -DROP TABLE IF EXISTS storage.role_grants_migration_log CASCADE; -DROP TABLE IF EXISTS storage.role_grants CASCADE; -" - -# ── 5. Pre-migration sanity ──────────────────────────────────────────────── - -log "Pre-migration sanity checks" -expect_scalar "SELECT count(*) FROM storage.access_grants" \ - "73" "access_grants row count matches audit" - -if [[ $(scalar "SELECT to_regclass('storage.role_grants') IS NULL") != "t" ]]; then - fail "storage.role_grants survived the drop — something is wrong." -fi -pass "storage.role_grants does not exist (clean slate)" - - -# ── 5. Apply the D-Prep migration ─────────────────────────────────────────── - -log "Applying D-Prep migration: $MIGRATION_FILE" -# Wrap in a transaction so a constraint failure rolls back cleanly; the -# migration itself has BEGIN/COMMIT semantics via -1 to psql. -psql -d "$SMOKE_URL" --set ON_ERROR_STOP=1 -1 -f "$MIGRATION_FILE" -log "Migration applied" - - -# ── 6. Post-migration assertions ──────────────────────────────────────────── - -log "Post-migration verification" - -# Row count: should match `distinct_clusters` from the audit -expect_scalar "SELECT count(*) FROM storage.role_grants" \ - "$EXPECTED_DISTINCT_CLUSTERS" "role_grants row count" - -# Role distribution -expect_scalar "SELECT count(*) FROM storage.role_grants WHERE role = 'viewer'" \ - "$EXPECTED_VIEWER" "viewer count" -expect_scalar "SELECT count(*) FROM storage.role_grants WHERE role = 'editor'" \ - "$EXPECTED_EDITOR" "editor count" -expect_scalar "SELECT count(*) FROM storage.role_grants WHERE role = 'owner'" \ - "$EXPECTED_OWNER" "owner count" - -# Zero NULL roles, zero non-bundle roles (the CHECK constraint should -# already enforce this, but proving it here too) -expect_scalar "SELECT count(*) FROM storage.role_grants WHERE role IS NULL" \ - "0" "no NULL roles" -expect_scalar "SELECT count(*) FROM storage.role_grants WHERE role NOT IN ('viewer','commenter','contributor','editor','owner')" \ - "0" "no unknown roles" - -# access_grants is untouched -expect_scalar "SELECT count(*) FROM storage.access_grants" \ - "73" "access_grants row count unchanged (dual-write safety net intact)" - - -# ── 7. Equivalence check (strongest assertion) ────────────────────────────── -# For every role_grants row, the corresponding (subject, resource) cluster -# in access_grants must have exactly the role's bundle as its permission -# set. If any row's bundle doesn't match what `Role::expand()` says, the -# backfill mis-mapped and the equivalence count goes non-zero. - -log "Bundle equivalence check: role_grants ↔ access_grants" - -EQUIVALENCE_MISMATCHES=$(scalar " -WITH expected AS ( - SELECT 'viewer'::text AS role, ARRAY['read']::text[] AS perms - UNION ALL SELECT 'commenter', ARRAY['comment','read']::text[] - UNION ALL SELECT 'contributor', ARRAY['create','read']::text[] - UNION ALL SELECT 'editor', ARRAY['comment','create','read','update']::text[] - UNION ALL SELECT 'owner', ARRAY['comment','create','delete','read','share','update']::text[] -), -actual AS ( - SELECT subject_type, subject_id, resource_type, resource_id, - array_agg(permission ORDER BY permission) AS perms - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -) -SELECT count(*) -FROM storage.role_grants rg -JOIN actual a USING (subject_type, subject_id, resource_type, resource_id) -JOIN expected e ON e.role = rg.role -WHERE a.perms <> e.perms -") - -if [[ "$EQUIVALENCE_MISMATCHES" == "0" ]]; then - pass "every role_grants row's bundle matches access_grants exactly" -else - log "" - log "MISMATCH DETAIL (first 10):" - psql -d "$SMOKE_URL" -c " -WITH expected AS ( - SELECT 'viewer'::text AS role, ARRAY['read']::text[] AS perms - UNION ALL SELECT 'commenter', ARRAY['comment','read']::text[] - UNION ALL SELECT 'contributor', ARRAY['create','read']::text[] - UNION ALL SELECT 'editor', ARRAY['comment','create','read','update']::text[] - UNION ALL SELECT 'owner', ARRAY['comment','create','delete','read','share','update']::text[] -), -actual AS ( - SELECT subject_type, subject_id, resource_type, resource_id, - array_agg(permission ORDER BY permission) AS perms - FROM storage.access_grants - GROUP BY 1, 2, 3, 4 -) -SELECT rg.subject_type, rg.subject_id, rg.resource_type, rg.resource_id, - rg.role, - a.perms AS access_grants_perms, - e.perms AS expected_perms -FROM storage.role_grants rg -JOIN actual a USING (subject_type, subject_id, resource_type, resource_id) -JOIN expected e ON e.role = rg.role -WHERE a.perms <> e.perms -LIMIT 10 -" - fail "$EQUIVALENCE_MISMATCHES role_grants rows have a bundle that diverges from their access_grants cluster" -fi - - -# ── 8. Migration audit log shape ──────────────────────────────────────────── -# `role_grants_migration_log` is the one-shot table created by the migration -# to record any demotions. With 100%-bundle-shaped data (the audit says so), -# it should be EMPTY — nothing was demoted. - -if [[ $(scalar "SELECT to_regclass('storage.role_grants_migration_log') IS NOT NULL") == "t" ]]; then - expect_scalar "SELECT count(*) FROM storage.role_grants_migration_log" \ - "0" "migration audit log empty (no demotions, as expected for 100% bundle-shaped data)" -fi - - -# ── 9. Cleanup ────────────────────────────────────────────────────────────── - -log "" -log "─────────────────────────────────────────────────────────" -log " ALL ASSERTIONS PASSED" -log "─────────────────────────────────────────────────────────" -log "" - -if [[ "$KEEP_DB" == "1" ]]; then - log "Leaving smoke DB '$SMOKE_DB_NAME' for inspection." - log "Poke at it with:" - log " psql \"$SMOKE_URL\"" - log "" - log "Drop when done:" - log " psql -d \"$MAINTENANCE_URL\" -c 'DROP DATABASE \"$SMOKE_DB_NAME\"'" -else - log "Dropping smoke DB '$SMOKE_DB_NAME'" - psql -d "$MAINTENANCE_URL" -c "DROP DATABASE \"$SMOKE_DB_NAME\"" -fi - -log "Dump file kept at $DUMP_FILE (delete with: rm '$DUMP_FILE')"