feat(recoverable-job): wire API (cancel, view run, ...)
This commit is contained in:
@@ -34,7 +34,7 @@ pub use handler::JobHandler;
|
||||
pub use pg_job_store::{PgJobStore, PgJobStoreProvider};
|
||||
pub use recoverable::{
|
||||
JobStore, JobStoreProvider, OpenedRun, RecoverableAdapter, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, run_or_resume,
|
||||
RunStatus, RunSummary, run_or_resume,
|
||||
};
|
||||
pub use registry::{JobEntry, JobRegistry, JobSummary, RegisterError};
|
||||
pub use types::{ErrCause, JobOutcome, JobRunArgs};
|
||||
|
||||
@@ -16,7 +16,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::common::errors::DomainError;
|
||||
|
||||
use super::recoverable::{JobStore, JobStoreProvider, OpenedRun, RunStatus};
|
||||
use super::recoverable::{JobStore, JobStoreProvider, OpenedRun, RunStatus, RunSummary};
|
||||
|
||||
// ─── PgJobStore — bound to one run ──────────────────────────────────────────
|
||||
|
||||
@@ -239,8 +239,132 @@ impl JobStoreProvider for PgJobStoreProvider {
|
||||
.map_err(|e| map_sqlx_err("boot_recovery_sweep", e))?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
async fn list_runs(&self, job_name: &str, limit: u32) -> Result<Vec<RunSummary>, DomainError> {
|
||||
// Cap the limit at 100 defensively — the API layer should
|
||||
// also clamp, but a broken caller shouldn't tank the DB.
|
||||
let capped = limit.min(100) as i64;
|
||||
let rows: Vec<RunSummaryRow> = sqlx::query_as(RUN_SUMMARY_SELECT_LIST)
|
||||
.bind(job_name)
|
||||
.bind(capped)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("list_runs", e))?;
|
||||
rows.into_iter().map(row_to_summary).collect()
|
||||
}
|
||||
|
||||
async fn get_run_by_id(&self, run_id: Uuid) -> Result<Option<RunSummary>, DomainError> {
|
||||
let row: Option<RunSummaryRow> = sqlx::query_as(RUN_SUMMARY_SELECT_BY_ID)
|
||||
.bind(run_id)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("get_run_by_id", e))?;
|
||||
row.map(row_to_summary).transpose()
|
||||
}
|
||||
|
||||
async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError> {
|
||||
// Only Running → CancelRequested flips. `Paused` can be
|
||||
// cancelled by not resuming — no need for a state change.
|
||||
// `CancelRequested` already is what it is.
|
||||
// Multiple Running rows shouldn't exist (partial unique index),
|
||||
// but LIMIT 1 is defensive.
|
||||
let flipped: Option<(Uuid,)> = sqlx::query_as(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'CancelRequested',
|
||||
last_progress_at = NOW()
|
||||
WHERE id = (
|
||||
SELECT id FROM jobs.recoverable_runs
|
||||
WHERE job_name = $1
|
||||
AND status = 'Running'
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id
|
||||
"#,
|
||||
)
|
||||
.bind(job_name)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("request_cancel", e))?;
|
||||
Ok(flipped.map(|(id,)| id))
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Shared row → RunSummary decoder ────────────────────────────────────────
|
||||
|
||||
/// Row shape returned by the run-summary SELECTs. Kept as a distinct
|
||||
/// type so both `list_runs` and `get_run_by_id` share the projection
|
||||
/// (SQL column list + decoder). Order matches the SELECT below.
|
||||
type RunSummaryRow = (
|
||||
Uuid, // id
|
||||
String, // job_name
|
||||
String, // status
|
||||
DateTime<Utc>, // started_at
|
||||
DateTime<Utc>, // last_progress_at
|
||||
Option<DateTime<Utc>>, // completed_at
|
||||
Option<Vec<u8>>, // cursor
|
||||
serde_json::Value, // stats
|
||||
serde_json::Value, // params
|
||||
Option<String>, // error_message
|
||||
);
|
||||
|
||||
const RUN_SUMMARY_COLUMNS: &str = "id, job_name, status, started_at, last_progress_at, completed_at, cursor, stats, params, error_message";
|
||||
|
||||
// `format!` isn't const, but `concat!` gives us a &'static str at compile
|
||||
// time — worth it so the SELECT strings show up in tracing / SQL logs
|
||||
// as one contiguous line instead of a runtime string build.
|
||||
const RUN_SUMMARY_SELECT_LIST: &str = concat!(
|
||||
"SELECT id, job_name, status, started_at, last_progress_at, completed_at, cursor, stats, params, error_message ",
|
||||
"FROM jobs.recoverable_runs ",
|
||||
"WHERE job_name = $1 ",
|
||||
"ORDER BY started_at DESC ",
|
||||
"LIMIT $2"
|
||||
);
|
||||
|
||||
const RUN_SUMMARY_SELECT_BY_ID: &str = concat!(
|
||||
"SELECT id, job_name, status, started_at, last_progress_at, completed_at, cursor, stats, params, error_message ",
|
||||
"FROM jobs.recoverable_runs ",
|
||||
"WHERE id = $1"
|
||||
);
|
||||
|
||||
fn row_to_summary(row: RunSummaryRow) -> Result<RunSummary, DomainError> {
|
||||
let (
|
||||
id,
|
||||
job_name,
|
||||
status_str,
|
||||
started_at,
|
||||
last_progress_at,
|
||||
completed_at,
|
||||
cursor,
|
||||
stats,
|
||||
params,
|
||||
error_message,
|
||||
) = row;
|
||||
let status = RunStatus::parse(&status_str).ok_or_else(|| {
|
||||
DomainError::internal_error("JobStore", format!("unknown status: {status_str}"))
|
||||
})?;
|
||||
Ok(RunSummary {
|
||||
id,
|
||||
job_name,
|
||||
status,
|
||||
started_at,
|
||||
last_progress_at,
|
||||
completed_at,
|
||||
stats,
|
||||
params,
|
||||
cursor_hex: cursor.map(hex::encode),
|
||||
error_message,
|
||||
})
|
||||
}
|
||||
|
||||
// Suppress the dead-code lint on the column list — kept as a
|
||||
// human-readable constant even though the actual SELECTs currently
|
||||
// inline it. Future rewrites of the SELECTs (e.g. adding stats
|
||||
// projection) will use it.
|
||||
#[allow(dead_code)]
|
||||
const _RUN_SUMMARY_COLUMNS_UNUSED: &str = RUN_SUMMARY_COLUMNS;
|
||||
|
||||
/// Row shape returned by `open_or_start`'s SELECT — factored out
|
||||
/// so clippy's `type_complexity` lint doesn't yell at the query.
|
||||
type ExistingRun = (Uuid, String, DateTime<Utc>, Option<Vec<u8>>);
|
||||
@@ -283,10 +407,7 @@ impl PgJobStoreProvider {
|
||||
})?;
|
||||
match status {
|
||||
RunStatus::Running | RunStatus::CancelRequested => {
|
||||
Ok(OpenedRun::AlreadyActive {
|
||||
run_id: id,
|
||||
status,
|
||||
})
|
||||
Ok(OpenedRun::AlreadyActive { run_id: id, status })
|
||||
}
|
||||
RunStatus::Paused => {
|
||||
// Flip to Running and hand back the cursor.
|
||||
@@ -313,20 +434,15 @@ impl PgJobStoreProvider {
|
||||
.bind(id)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
OpenErr::Fatal(map_sqlx_err("open_or_start.resume", e))
|
||||
})?;
|
||||
.map_err(|e| OpenErr::Fatal(map_sqlx_err("open_or_start.resume", e)))?;
|
||||
let (started_at, cursor_bytes) = row.ok_or_else(|| {
|
||||
OpenErr::Fatal(DomainError::internal_error(
|
||||
"JobStore",
|
||||
format!("run vanished during resume: {id}"),
|
||||
))
|
||||
})?;
|
||||
let store: Arc<dyn JobStore> = Arc::new(PgJobStore::new(
|
||||
self.pool.clone(),
|
||||
id,
|
||||
started_at,
|
||||
));
|
||||
let store: Arc<dyn JobStore> =
|
||||
Arc::new(PgJobStore::new(self.pool.clone(), id, started_at));
|
||||
Ok(OpenedRun::Resumed {
|
||||
store,
|
||||
cursor: cursor_bytes.unwrap_or_default(),
|
||||
|
||||
@@ -256,6 +256,54 @@ pub trait JobStoreProvider: Send + Sync {
|
||||
/// via `POST /api/admin/jobs/{name}/trigger`, which calls
|
||||
/// `open_or_start` and picks up the Paused cursor.
|
||||
async fn boot_recovery_sweep(&self) -> Result<u64, DomainError>;
|
||||
|
||||
/// Latest N runs for `job_name`, newest first, terminal + non-terminal
|
||||
/// both included. Powers `GET /api/admin/jobs/{name}/runs`. `limit`
|
||||
/// caps the return size; the API layer clamps it too.
|
||||
async fn list_runs(&self, job_name: &str, limit: u32) -> Result<Vec<RunSummary>, DomainError>;
|
||||
|
||||
/// Fetch one run by id. Powers `GET /api/admin/jobs/{name}/runs/{id}`.
|
||||
/// Returns `None` when the id doesn't exist (unknown or pruned).
|
||||
async fn get_run_by_id(&self, run_id: Uuid) -> Result<Option<RunSummary>, DomainError>;
|
||||
|
||||
/// Request cancellation of the CURRENT active run for `job_name`
|
||||
/// by flipping its status from `Running` → `CancelRequested`.
|
||||
/// Returns the run's id when a Running row was flipped, `None`
|
||||
/// when there was no Running row to cancel (nothing in flight,
|
||||
/// or the latest non-terminal row is already `Paused` /
|
||||
/// `CancelRequested`).
|
||||
///
|
||||
/// Cooperative — the handler still needs to poll `store.status()`
|
||||
/// and return `RunOutcome::Paused` at the next safe boundary. If
|
||||
/// the handler doesn't poll, cancel is a no-op until the run
|
||||
/// completes naturally.
|
||||
async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError>;
|
||||
}
|
||||
|
||||
/// Serialisable snapshot of one `jobs.recoverable_runs` row, returned
|
||||
/// by the admin listing + get-run endpoints.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct RunSummary {
|
||||
pub id: Uuid,
|
||||
pub job_name: String,
|
||||
pub status: RunStatus,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub last_progress_at: DateTime<Utc>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
/// `stats` JSONB dump — job-specific counters (scanned_count,
|
||||
/// migrated_blobs, findings_this_run, …).
|
||||
pub stats: serde_json::Value,
|
||||
/// `params` JSONB dump — per-run params captured at start
|
||||
/// (grace_window_secs, source_backend, …).
|
||||
pub params: serde_json::Value,
|
||||
/// Cursor as hex — omitted when null. Operators occasionally want
|
||||
/// to inspect this for "where did the scan get to" diagnostics;
|
||||
/// the raw bytes are opaque per-job so we render as hex.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cursor_hex: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error_message: Option<String>,
|
||||
}
|
||||
|
||||
/// Result of [`JobStoreProvider::open_or_start`].
|
||||
@@ -324,11 +372,7 @@ pub async fn run_or_resume(
|
||||
}
|
||||
RunOutcome::Paused { cursor } => {
|
||||
let cursor_hex = hex::encode(&cursor);
|
||||
log_terminal_write_err(
|
||||
"mark_paused",
|
||||
run_id,
|
||||
store.mark_paused(Some(cursor)).await,
|
||||
);
|
||||
log_terminal_write_err("mark_paused", run_id, store.mark_paused(Some(cursor)).await);
|
||||
JobOutcome::ok_with(
|
||||
0,
|
||||
serde_json::json!({
|
||||
@@ -339,11 +383,7 @@ pub async fn run_or_resume(
|
||||
)
|
||||
}
|
||||
RunOutcome::Failed { message } => {
|
||||
log_terminal_write_err(
|
||||
"mark_failed",
|
||||
run_id,
|
||||
store.mark_failed(&message).await,
|
||||
);
|
||||
log_terminal_write_err("mark_failed", run_id, store.mark_failed(&message).await);
|
||||
JobOutcome::err(format!("{message} (run_id={run_id})"))
|
||||
}
|
||||
}
|
||||
@@ -461,11 +501,7 @@ mod tests {
|
||||
async fn status(&self) -> Result<RunStatus, DomainError> {
|
||||
Ok(self.state.lock().unwrap().status)
|
||||
}
|
||||
async fn checkpoint(
|
||||
&self,
|
||||
cursor: Vec<u8>,
|
||||
delta_count: u64,
|
||||
) -> Result<(), DomainError> {
|
||||
async fn checkpoint(&self, cursor: Vec<u8>, delta_count: u64) -> Result<(), DomainError> {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
s.cursor = Some(cursor);
|
||||
s.scanned_count += delta_count;
|
||||
@@ -530,15 +566,15 @@ mod tests {
|
||||
/// assertions.
|
||||
fn last_status(&self) -> Option<RunStatus> {
|
||||
let stores = self.stores.lock().unwrap();
|
||||
stores
|
||||
.last()
|
||||
.map(|s| s.state.lock().unwrap().status)
|
||||
stores.last().map(|s| s.state.lock().unwrap().status)
|
||||
}
|
||||
|
||||
/// Test-only read — last-created run's cursor.
|
||||
fn last_cursor(&self) -> Option<Vec<u8>> {
|
||||
let stores = self.stores.lock().unwrap();
|
||||
stores.last().and_then(|s| s.state.lock().unwrap().cursor.clone())
|
||||
stores
|
||||
.last()
|
||||
.and_then(|s| s.state.lock().unwrap().cursor.clone())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -596,6 +632,68 @@ mod tests {
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
async fn list_runs(
|
||||
&self,
|
||||
job_name: &str,
|
||||
limit: u32,
|
||||
) -> Result<Vec<RunSummary>, DomainError> {
|
||||
let stores = self.stores.lock().unwrap();
|
||||
let now = Utc::now();
|
||||
let out: Vec<RunSummary> = stores
|
||||
.iter()
|
||||
.rev() // newest first — MemProvider stores in insertion order
|
||||
.take(limit as usize)
|
||||
.map(|s| {
|
||||
let state = s.state.lock().unwrap();
|
||||
RunSummary {
|
||||
id: s.run_id,
|
||||
job_name: job_name.to_string(),
|
||||
status: state.status,
|
||||
started_at: s.started_at,
|
||||
last_progress_at: now,
|
||||
completed_at: None,
|
||||
stats: serde_json::json!({ "scanned_count": state.scanned_count }),
|
||||
params: serde_json::json!({}),
|
||||
cursor_hex: state.cursor.as_ref().map(hex::encode),
|
||||
error_message: state.error_message.clone(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn get_run_by_id(&self, run_id: Uuid) -> Result<Option<RunSummary>, DomainError> {
|
||||
let stores = self.stores.lock().unwrap();
|
||||
let now = Utc::now();
|
||||
Ok(stores.iter().find(|s| s.run_id == run_id).map(|s| {
|
||||
let state = s.state.lock().unwrap();
|
||||
RunSummary {
|
||||
id: s.run_id,
|
||||
job_name: "mem".to_string(),
|
||||
status: state.status,
|
||||
started_at: s.started_at,
|
||||
last_progress_at: now,
|
||||
completed_at: None,
|
||||
stats: serde_json::json!({ "scanned_count": state.scanned_count }),
|
||||
params: serde_json::json!({}),
|
||||
cursor_hex: state.cursor.as_ref().map(hex::encode),
|
||||
error_message: state.error_message.clone(),
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async fn request_cancel(&self, _job_name: &str) -> Result<Option<Uuid>, DomainError> {
|
||||
let stores = self.stores.lock().unwrap();
|
||||
if let Some(s) = stores.last() {
|
||||
let mut state = s.state.lock().unwrap();
|
||||
if state.status == RunStatus::Running {
|
||||
state.status = RunStatus::CancelRequested;
|
||||
return Ok(Some(s.run_id));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Handlers ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -120,9 +120,7 @@ impl JobRegistry {
|
||||
cadence,
|
||||
);
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"JobRegistry::register({name}) failed — DI wiring bug: {e}"
|
||||
),
|
||||
Err(e) => panic!("JobRegistry::register({name}) failed — DI wiring bug: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user