cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(recoverable jobs): clarify life cycle pause vs cancel

Browse Source 
a job can be paused/resumed
    a job as an exclusibity by it's name
    if you want to run another job with same name:
    either cancel the first one, or wait of it's terminaison

    pause does not permit to run the other job, this can create
    race conditions
This commit is contained in:
Edouard Vanbelle
2026-08-02 16:55:07 +02:00
parent 5eec0fb36e
commit 8b0fb03b5c
7 changed files with 511 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/lib/api/endpoints/adminJobs.ts
+46 -10
View File
@@ -31,12 +31,23 @@ export interface TriggerResponse {
detached?: boolean;
}
/** Envelope from `POST /api/admin/jobs/{name}/cancel`. `run_id` is
* the id of the run whose `Running` status was flipped to
* `CancelRequested` (null when nothing was in flight to cancel). */
/** Envelope from `POST /api/admin/jobs/{name}/cancel` — terminal
* cancel. `run_id` populated iff a non-terminal row was flipped
* (Running/CancelRequested get the intent stamp; Paused gets a
* direct DB flip to Cancelled). */
export interface CancelResponse {
ok: boolean;
run_id: string | null;
cancelled: boolean;
run_id?: string;
reason?: string;
note?: string;
}
/** Envelope from `POST /api/admin/jobs/{name}/pause` — soft pause. */
export interface PauseResponse {
paused: boolean;
run_id?: string;
reason?: string;
note?: string;
}
/**
@@ -92,11 +103,11 @@ export async function triggerJob(
}
/**
* `POST /api/admin/jobs/{name}/cancel` — cooperatively request cancel
* of the currently running instance. The handler observes it on its
* next `store.status()` poll and returns `RunOutcome::Paused` at the
* next safe boundary. If nothing is running, this is a no-op that
* returns `run_id: null`.
* `POST /api/admin/jobs/{name}/cancel` — TERMINAL cancel. Abandons
* the run: Running/CancelRequested rows get stamped with the intent
* flag and land as `Cancelled` when the handler yields; Paused rows
* get flipped directly to `Cancelled`. Not resumable. Use `pauseJob`
* for interruption-with-resume semantics.
*/
export async function cancelJob(name: string): Promise<CancelResponse> {
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, {
@@ -117,6 +128,31 @@ export async function cancelJob(name: string): Promise<CancelResponse> {
return (await res.json()) as CancelResponse;
}
/**
* `POST /api/admin/jobs/{name}/pause` — cooperative pause. Row lands
* as `Paused` when the handler yields; a subsequent trigger click
* resumes from the cursor via `run_or_resume`. Use `cancelJob` to
* abandon terminally.
*/
export async function pauseJob(name: string): Promise<PauseResponse> {
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/pause`, {
method: 'POST',
credentials: 'same-origin',
headers: { ...JSON_HEADERS, ...getCsrfHeaders() }
});
if (!res.ok) {
let msg = `pause failed: ${res.status}`;
try {
const body = (await res.json()) as { error?: string; message?: string };
msg = body.error ?? body.message ?? msg;
} catch {
/* no JSON body */
}
throw new Error(msg);
}
return (await res.json()) as PauseResponse;
}
/**
* `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable
* runs for `name`, newest first. Backend caps `limit` at 100.
frontend/src/lib/api/types.ts
+7 -1
View File
@@ -584,7 +584,13 @@ export interface JobSummary {
* non-terminal set (Running / Paused / CancelRequested) is what the
* DB's `one_active_run_per_job` partial unique index scopes.
*/
export type RunStatus = 'Running' | 'Paused' | 'CancelRequested' | 'Completed' | 'Failed';
export type RunStatus =
| 'Running'
| 'Paused'
| 'CancelRequested'
| 'Completed'
| 'Failed'
| 'Cancelled';
/**
* `RunSummary` — one row per recoverable-job run from
frontend/src/lib/components/AdminJobsPanel.svelte
+125 -25
View File
@@ -28,6 +28,7 @@
listJobs,
listRuns,
listFindings,
pauseJob,
triggerJob,
cancelJob,
purgeJobRuns
@@ -287,23 +288,71 @@
}
}
async function onCancel(name: string) {
const key = `cancel:${name}`;
async function onPause(name: string) {
const key = `pause:${name}`;
markBusy(key, true);
try {
const res = await cancelJob(name);
if (res.run_id) {
const res = await pauseJob(name);
if (res.paused) {
ui.notify(
t(
'admin.jobs.cancel_requested',
'admin.jobs.pause_requested',
{ name },
'Cancel requested — {{name}} will pause at the next safe boundary'
'Pause requested — {{name}} will pause at the next checkpoint (progress preserved)'
),
'info'
);
} else {
ui.notify(
t('admin.jobs.cancel_noop', { name }, 'Nothing to cancel — {{name}} is not running'),
t('admin.jobs.pause_noop', { name }, 'Nothing to pause — {{name}} is not running'),
'info'
);
}
await loadJobs();
if (expandedJob === name) await loadRuns(name);
} catch (e) {
ui.notify(errorMessage(e), 'error');
} finally {
markBusy(key, false);
}
}
async function onCancel(name: string) {
// Terminal cancel confirmation — this is destructive (marks the
// run as Cancelled, cursor preserved for post-mortem but not
// resumable). Skip the confirm for non-recoverable jobs since
// there's no persistent state to lose there today.
if (
!window.confirm(
t(
'admin.jobs.cancel_confirm',
{ name },
'Cancel run of {{name}}? The run will be marked as Cancelled and cannot be resumed. Progress bytes on disk stay put — this only affects the run row.'
)
)
) {
return;
}
const key = `cancel:${name}`;
markBusy(key, true);
try {
const res = await cancelJob(name);
if (res.cancelled) {
ui.notify(
t(
'admin.jobs.cancel_requested',
{ name },
'Cancel requested — {{name}} will land in Cancelled at the next batch boundary (Paused rows flip immediately)'
),
'info'
);
} else {
ui.notify(
t(
'admin.jobs.cancel_noop',
{ name },
'Nothing to cancel — {{name}} has no non-terminal run'
),
'info'
);
}
@@ -422,11 +471,41 @@
return 'jobs-panel__pill jobs-panel__pill--ok';
case 'Failed':
return 'jobs-panel__pill jobs-panel__pill--err';
case 'Cancelled':
return 'jobs-panel__pill jobs-panel__pill--neutral';
default:
return 'jobs-panel__pill jobs-panel__pill--neutral';
}
}
/**
* Human-facing label for a `RunStatus`. Translates the internal
* DB status enum into text an operator can read at a glance —
* notably renders `CancelRequested` as "Pausing" for the
* recoverable-run case (the mechanism is a cancel flag, but the
* user intent is pause). Non-recoverable cancels aren't a thing
* today because non-recoverable jobs run to completion inline,
* so `CancelRequested` here is always the pause path.
*/
function statusLabel(status: RunStatus): string {
switch (status) {
case 'Running':
return t('admin.jobs.status_running', 'Running');
case 'Paused':
return t('admin.jobs.status_paused', 'Paused');
case 'CancelRequested':
return t('admin.jobs.status_ending', 'Ending');
case 'Completed':
return t('admin.jobs.status_completed', 'Completed');
case 'Failed':
return t('admin.jobs.status_failed', 'Failed');
case 'Cancelled':
return t('admin.jobs.status_cancelled', 'Cancelled');
default:
return status;
}
}
/** Coarse "3 min ago" / "2 h ago" — same shape as the parent
* admin page's timeAgo(). Duplicated locally so the component
* stays self-contained; extract if a third caller emerges. */
@@ -639,6 +718,8 @@
</td>
<td class="jobs-panel__actions">
{#if job.paused_run}
<!-- Paused row: [Resume (X/Y)] to continue, [Cancel]
to abandon the checkpoint (marks run Cancelled). -->
{@const p = job.paused_run}
{@const label =
p.total && p.total > 0
@@ -659,6 +740,17 @@
>
{label}
</button>
<button
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
disabled={busyKeys.has(`cancel:${job.name}`)}
onclick={() => onCancel(job.name)}
title={t(
'admin.jobs.cancel_paused_title',
'Abandon the paused run — marks it as Cancelled. Not resumable.'
)}
>
{t('admin.jobs.cancel', 'Cancel')}
</button>
{:else}
<button
class="jobs-panel__btn jobs-panel__btn--small"
@@ -667,28 +759,25 @@
>
{t('admin.jobs.run', 'Run')}
</button>
{/if}
{#if supportsDeep(job.name) && !job.paused_run}
<button
class="jobs-panel__btn jobs-panel__btn--small"
disabled={busyKeys.has(`trigger:${job.name}:deep`)}
onclick={() => onTrigger(job.name, { deep: true })}
>
{t('admin.jobs.run_deep', 'Run deep')}
</button>
{#if supportsDeep(job.name)}
<button
class="jobs-panel__btn jobs-panel__btn--small"
disabled={busyKeys.has(`trigger:${job.name}:deep`)}
onclick={() => onTrigger(job.name, { deep: true })}
>
{t('admin.jobs.run_deep', 'Run deep')}
</button>
{/if}
{/if}
{#if isRunning(job) && canExpand}
{#if isRecoverable(job)}
<!-- Recoverable jobs: the "cancel" endpoint just
flips CancelRequested → handler yields at the
next batch boundary → status=Paused (resumable
with a fresh Resume click, cursor preserved).
Label it "Pause" so admins know it's not
destructive. -->
<!-- Recoverable running: [Pause] preserves cursor
for later resume; [Cancel] abandons terminally
(engine writes Cancelled when the handler yields). -->
<button
class="jobs-panel__btn jobs-panel__btn--small"
disabled={busyKeys.has(`cancel:${job.name}`)}
onclick={() => onCancel(job.name)}
disabled={busyKeys.has(`pause:${job.name}`)}
onclick={() => onPause(job.name)}
title={t(
'admin.jobs.pause_title',
'Signal a graceful pause at the next batch boundary. Run row stays as `Paused` — Resume picks up from the checkpoint.'
@@ -696,6 +785,17 @@
>
{t('admin.jobs.pause', 'Pause')}
</button>
<button
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
disabled={busyKeys.has(`cancel:${job.name}`)}
onclick={() => onCancel(job.name)}
title={t(
'admin.jobs.cancel_running_title',
'Abandon the run — marks it as Cancelled at the next batch boundary. Not resumable.'
)}
>
{t('admin.jobs.cancel', 'Cancel')}
</button>
{:else}
<button
class="jobs-panel__btn jobs-panel__btn--small jobs-panel__btn--danger"
@@ -771,7 +871,7 @@
</td>
<td>
<span class={statusClass(run.status)}>
{run.status}
{statusLabel(run.status)}
</span>
</td>
<td class="jobs-panel__muted">