From 57952b2fdc934f13d943a51bf7f31af4fd48386a Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Mon, 7 Sep 2026 23:20:25 +0200 Subject: [PATCH] fix(jobs): a paused run must not log outcome="ok" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From Ed's local→S3 outage run, which paused correctly and then said: event="job.run" job=backend_migration outcome="ok" ... "paused":true,"retryable":true This is the State-vs-Outcome distinction Ed drew earlier, in a channel the earlier fix did not touch. The admin panel now separates the two; the scheduler's own log line only ever carried the outcome, so a migration frozen on an unreachable backend read as a clean run at INFO. `JobOutcome` has just `Ok` and `Err`, and a pause is carried as `Ok` with `paused: true` in `extra` — correct in itself: the handler did its job and stopped cleanly at a checkpoint. The persisted shape is unchanged for that reason. But projecting it to `outcome="ok"` tells an operator the opposite of what they need to know, which is that nothing will progress until the backend returns and someone resumes. Paused runs now log at WARN with `outcome="paused"`, a `retryable` field, and a message saying so. `grep 'outcome="ok"'` no longer matches a blocked migration. Co-Authored-By: Claude Opus 5 (1M context) --- src/infrastructure/scheduler/engine.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/infrastructure/scheduler/engine.rs b/src/infrastructure/scheduler/engine.rs index 58dc0b5c..88e18cf4 100644 --- a/src/infrastructure/scheduler/engine.rs +++ b/src/infrastructure/scheduler/engine.rs @@ -275,6 +275,30 @@ fn log_outcome(name: &str, outcome: &JobOutcome, cause: Option, elapse // structured log renderer to project the `elapsed_ms` field. let elapsed = format_elapsed(elapsed_ms); match outcome { + // A paused run is carried as `Ok` — the handler did its job and + // stopped cleanly at a checkpoint — but logging it as `ok` says + // the opposite of what an operator needs to know: the migration + // is blocked and will not progress until the backend returns. + // Same distinction the admin panel draws between a run's STATE + // and its OUTCOME; this line only ever showed the outcome. + JobOutcome::Ok { count, extra } + if extra.get("paused") == Some(&serde_json::Value::Bool(true)) => + { + tracing::warn!( + target: "oxicloud::scheduler", + event = "job.run", + job = %name, + outcome = "paused", + retryable = extra.get("retryable") == Some(&serde_json::Value::Bool(true)), + count = *count, + elapsed_ms = elapsed_ms, + extra = %extra, + "job {} PAUSED after {} — count={} (resume when the cause clears)", + name, + elapsed, + count, + ); + } JobOutcome::Ok { count, extra } => { tracing::info!( target: "oxicloud::scheduler",