From 8b0fb03b5ce2e1b4ef8f51e921944ff52ebaadca Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sun, 2 Aug 2026 16:55:07 +0200 Subject: [PATCH] feat(recoverable jobs): clarify life cycle pause vs cancel a job can be paused/resumed a job as an exclusibity by it's name if you want to run another job with same name: either cancel the first one, or wait of it's terminaison pause does not permit to run the other job, this can create race conditions --- frontend/src/lib/api/endpoints/adminJobs.ts | 56 +++++-- frontend/src/lib/api/types.ts | 8 +- .../src/lib/components/AdminJobsPanel.svelte | 150 +++++++++++++++--- src/infrastructure/scheduler/pg_job_store.rs | 115 ++++++++++++++ src/infrastructure/scheduler/recoverable.rs | 149 +++++++++++++++-- src/interfaces/api/handlers/admin_handler.rs | 82 +++++++++- tests/api/recoverable_jobs.hurl | 11 +- 7 files changed, 511 insertions(+), 60 deletions(-) diff --git a/frontend/src/lib/api/endpoints/adminJobs.ts b/frontend/src/lib/api/endpoints/adminJobs.ts index 067919b9..71bf7d65 100644 --- a/frontend/src/lib/api/endpoints/adminJobs.ts +++ b/frontend/src/lib/api/endpoints/adminJobs.ts @@ -31,12 +31,23 @@ export interface TriggerResponse { detached?: boolean; } -/** Envelope from `POST /api/admin/jobs/{name}/cancel`. `run_id` is - * the id of the run whose `Running` status was flipped to - * `CancelRequested` (null when nothing was in flight to cancel). */ +/** Envelope from `POST /api/admin/jobs/{name}/cancel` — terminal + * cancel. `run_id` populated iff a non-terminal row was flipped + * (Running/CancelRequested get the intent stamp; Paused gets a + * direct DB flip to Cancelled). */ export interface CancelResponse { - ok: boolean; - run_id: string | null; + cancelled: boolean; + run_id?: string; + reason?: string; + note?: string; +} + +/** Envelope from `POST /api/admin/jobs/{name}/pause` — soft pause. */ +export interface PauseResponse { + paused: boolean; + run_id?: string; + reason?: string; + note?: string; } /** @@ -92,11 +103,11 @@ export async function triggerJob( } /** - * `POST /api/admin/jobs/{name}/cancel` — cooperatively request cancel - * of the currently running instance. The handler observes it on its - * next `store.status()` poll and returns `RunOutcome::Paused` at the - * next safe boundary. If nothing is running, this is a no-op that - * returns `run_id: null`. + * `POST /api/admin/jobs/{name}/cancel` — TERMINAL cancel. Abandons + * the run: Running/CancelRequested rows get stamped with the intent + * flag and land as `Cancelled` when the handler yields; Paused rows + * get flipped directly to `Cancelled`. Not resumable. Use `pauseJob` + * for interruption-with-resume semantics. */ export async function cancelJob(name: string): Promise { const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, { @@ -117,6 +128,31 @@ export async function cancelJob(name: string): Promise { return (await res.json()) as CancelResponse; } +/** + * `POST /api/admin/jobs/{name}/pause` — cooperative pause. Row lands + * as `Paused` when the handler yields; a subsequent trigger click + * resumes from the cursor via `run_or_resume`. Use `cancelJob` to + * abandon terminally. + */ +export async function pauseJob(name: string): Promise { + const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/pause`, { + method: 'POST', + credentials: 'same-origin', + headers: { ...JSON_HEADERS, ...getCsrfHeaders() } + }); + if (!res.ok) { + let msg = `pause failed: ${res.status}`; + try { + const body = (await res.json()) as { error?: string; message?: string }; + msg = body.error ?? body.message ?? msg; + } catch { + /* no JSON body */ + } + throw new Error(msg); + } + return (await res.json()) as PauseResponse; +} + /** * `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable * runs for `name`, newest first. Backend caps `limit` at 100. diff --git a/frontend/src/lib/api/types.ts b/frontend/src/lib/api/types.ts index 8875c590..f6db9397 100644 --- a/frontend/src/lib/api/types.ts +++ b/frontend/src/lib/api/types.ts @@ -584,7 +584,13 @@ export interface JobSummary { * non-terminal set (Running / Paused / CancelRequested) is what the * DB's `one_active_run_per_job` partial unique index scopes. */ -export type RunStatus = 'Running' | 'Paused' | 'CancelRequested' | 'Completed' | 'Failed'; +export type RunStatus = + | 'Running' + | 'Paused' + | 'CancelRequested' + | 'Completed' + | 'Failed' + | 'Cancelled'; /** * `RunSummary` — one row per recoverable-job run from diff --git a/frontend/src/lib/components/AdminJobsPanel.svelte b/frontend/src/lib/components/AdminJobsPanel.svelte index 31060d7c..2d5fd49e 100644 --- a/frontend/src/lib/components/AdminJobsPanel.svelte +++ b/frontend/src/lib/components/AdminJobsPanel.svelte @@ -28,6 +28,7 @@ listJobs, listRuns, listFindings, + pauseJob, triggerJob, cancelJob, purgeJobRuns @@ -287,23 +288,71 @@ } } - async function onCancel(name: string) { - const key = `cancel:${name}`; + async function onPause(name: string) { + const key = `pause:${name}`; markBusy(key, true); try { - const res = await cancelJob(name); - if (res.run_id) { + const res = await pauseJob(name); + if (res.paused) { ui.notify( t( - 'admin.jobs.cancel_requested', + 'admin.jobs.pause_requested', { name }, - 'Cancel requested — {{name}} will pause at the next safe boundary' + 'Pause requested — {{name}} will pause at the next checkpoint (progress preserved)' ), 'info' ); } else { ui.notify( - t('admin.jobs.cancel_noop', { name }, 'Nothing to cancel — {{name}} is not running'), + t('admin.jobs.pause_noop', { name }, 'Nothing to pause — {{name}} is not running'), + 'info' + ); + } + await loadJobs(); + if (expandedJob === name) await loadRuns(name); + } catch (e) { + ui.notify(errorMessage(e), 'error'); + } finally { + markBusy(key, false); + } + } + + async function onCancel(name: string) { + // Terminal cancel confirmation — this is destructive (marks the + // run as Cancelled, cursor preserved for post-mortem but not + // resumable). Skip the confirm for non-recoverable jobs since + // there's no persistent state to lose there today. + if ( + !window.confirm( + t( + 'admin.jobs.cancel_confirm', + { name }, + 'Cancel run of {{name}}? The run will be marked as Cancelled and cannot be resumed. Progress bytes on disk stay put — this only affects the run row.' + ) + ) + ) { + return; + } + const key = `cancel:${name}`; + markBusy(key, true); + try { + const res = await cancelJob(name); + if (res.cancelled) { + ui.notify( + t( + 'admin.jobs.cancel_requested', + { name }, + 'Cancel requested — {{name}} will land in Cancelled at the next batch boundary (Paused rows flip immediately)' + ), + 'info' + ); + } else { + ui.notify( + t( + 'admin.jobs.cancel_noop', + { name }, + 'Nothing to cancel — {{name}} has no non-terminal run' + ), 'info' ); } @@ -422,11 +471,41 @@ return 'jobs-panel__pill jobs-panel__pill--ok'; case 'Failed': return 'jobs-panel__pill jobs-panel__pill--err'; + case 'Cancelled': + return 'jobs-panel__pill jobs-panel__pill--neutral'; default: return 'jobs-panel__pill jobs-panel__pill--neutral'; } } + /** + * Human-facing label for a `RunStatus`. Translates the internal + * DB status enum into text an operator can read at a glance — + * notably renders `CancelRequested` as "Pausing" for the + * recoverable-run case (the mechanism is a cancel flag, but the + * user intent is pause). Non-recoverable cancels aren't a thing + * today because non-recoverable jobs run to completion inline, + * so `CancelRequested` here is always the pause path. + */ + function statusLabel(status: RunStatus): string { + switch (status) { + case 'Running': + return t('admin.jobs.status_running', 'Running'); + case 'Paused': + return t('admin.jobs.status_paused', 'Paused'); + case 'CancelRequested': + return t('admin.jobs.status_ending', 'Ending'); + case 'Completed': + return t('admin.jobs.status_completed', 'Completed'); + case 'Failed': + return t('admin.jobs.status_failed', 'Failed'); + case 'Cancelled': + return t('admin.jobs.status_cancelled', 'Cancelled'); + default: + return status; + } + } + /** Coarse "3 min ago" / "2 h ago" — same shape as the parent * admin page's timeAgo(). Duplicated locally so the component * stays self-contained; extract if a third caller emerges. */ @@ -639,6 +718,8 @@ {#if job.paused_run} + {@const p = job.paused_run} {@const label = p.total && p.total > 0 @@ -659,6 +740,17 @@ > {label} + {:else} - {/if} - {#if supportsDeep(job.name) && !job.paused_run} - + {#if supportsDeep(job.name)} + + {/if} {/if} {#if isRunning(job) && canExpand} {#if isRecoverable(job)} - + + {:else}