From 9f9abdee82705c232c8a42edf37a7b12efa7b216 Mon Sep 17 00:00:00 2001 From: Agent Date: Sat, 18 Apr 2026 07:20:16 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20[nomad-step-5]=20S5.4=20=E2=80=94=20disp?= =?UTF-8?q?atcher.sh=20DISPATCHER=5FBACKEND=3Dnomad=20branch=20(nomad=20jo?= =?UTF-8?q?b=20dispatch)=20(#991)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/edge/dispatcher.sh | 189 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 181 insertions(+), 8 deletions(-) diff --git a/docker/edge/dispatcher.sh b/docker/edge/dispatcher.sh index a48abf2..d243781 100755 --- a/docker/edge/dispatcher.sh +++ b/docker/edge/dispatcher.sh @@ -560,10 +560,186 @@ _launch_runner_docker() { # _launch_runner_nomad ACTION_ID SECRETS_CSV MOUNTS_CSV # -# Nomad backend stub — will be implemented in migration Step 5. +# Dispatches a vault-runner batch job via `nomad job dispatch`. +# Polls `nomad job status` until terminal state (completed/failed). +# Reads exit code from allocation and writes .result.json. +# +# Usage: _launch_runner_nomad +# Returns: exit code of the nomad job (0=success, non-zero=failure) _launch_runner_nomad() { - echo "nomad backend not yet implemented" >&2 - return 1 + local action_id="$1" + local secrets_csv="$2" + local mounts_csv="$3" + + log "Dispatching vault-runner batch job via Nomad for action: ${action_id}" + + # Dispatch the parameterized batch job + # The vault-runner job expects meta: action_id, secrets_csv + # mounts_csv is passed as env var for the nomad task to consume + local dispatch_output + dispatch_output=$(nomad job dispatch \ + -detach \ + -meta action_id="$action_id" \ + -meta secrets_csv="$secrets_csv" \ + -meta mounts_csv="${mounts_csv:-}" \ + vault-runner 2>&1) || { + log "ERROR: Failed to dispatch vault-runner job for ${action_id}" + log "Dispatch output: ${dispatch_output}" + write_result "$action_id" 1 "Nomad dispatch failed: ${dispatch_output}" + return 1 + } + + # Extract dispatch ID from output (UUID format) + local dispatch_id + dispatch_id=$(echo "$dispatch_output" | grep -oE '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' || true) + + if [ -z "$dispatch_id" ]; then + log "ERROR: Could not extract dispatch ID from nomad output" + log "Dispatch output: ${dispatch_output}" + write_result "$action_id" 1 "Could not extract dispatch ID from nomad output" + return 1 + fi + + log "Dispatched vault-runner with ID: ${dispatch_id}" + + # Poll job status until terminal state + # Batch jobs transition: running -> completed/failed + local max_wait=300 # 5 minutes max wait + local elapsed=0 + local poll_interval=5 + local alloc_id="" + + log "Polling nomad job status for dispatch ${dispatch_id}..." + + while [ "$elapsed" -lt "$max_wait" ]; do + # Get job status with JSON output + local job_status_json + job_status_json=$(nomad job status -json "vault-runner" 2>/dev/null) || { + log "ERROR: Failed to get job status for vault-runner" + write_result "$action_id" 1 "Failed to get job status" + return 1 + } + + # Check evaluation state + local eval_status + eval_status=$(echo "$job_status_json" | jq -r '.EvalID // empty' 2>/dev/null) || eval_status="" + + if [ -z "$eval_status" ]; then + sleep "$poll_interval" + elapsed=$((elapsed + poll_interval)) + continue + fi + + # Get allocation ID from the job status + alloc_id=$(echo "$job_status_json" | jq -r '.Allocations[0]?.ID // empty' 2>/dev/null) || alloc_id="" + + # Alternative: check job status field + local job_state + job_state=$(echo "$job_status_json" | jq -r '.State // empty' 2>/dev/null) || job_state="" + + # Check allocation state directly + if [ -n "$alloc_id" ]; then + local alloc_state + alloc_state=$(nomad alloc status -short "$alloc_id" 2>/dev/null || true) + + case "$alloc_state" in + *completed*|*success*|*dead*) + log "Allocation ${alloc_id} reached terminal state: ${alloc_state}" + break + ;; + *running*|*pending*|*starting*) + log "Allocation ${alloc_id} still running (state: ${alloc_state})..." + ;; + *failed*|*crashed*) + log "Allocation ${alloc_id} failed (state: ${alloc_state})" + break + ;; + esac + fi + + # Also check job-level state + case "$job_state" in + complete|dead) + log "Job vault-runner reached terminal state: ${job_state}" + break + ;; + failed) + log "Job vault-runner failed" + break + ;; + esac + + sleep "$poll_interval" + elapsed=$((elapsed + poll_interval)) + done + + if [ "$elapsed" -ge "$max_wait" ]; then + log "ERROR: Timeout waiting for vault-runner job to complete" + write_result "$action_id" 1 "Timeout waiting for nomad job to complete" + return 1 + fi + + # Get final job status and exit code + local final_status_json + final_status_json=$(nomad job status -json "vault-runner" 2>/dev/null) || { + log "ERROR: Failed to get final job status" + write_result "$action_id" 1 "Failed to get final job status" + return 1 + } + + # Get allocation exit code + local exit_code=0 + local logs="" + + if [ -n "$alloc_id" ]; then + # Get allocation exit code + local alloc_exit_code + alloc_exit_code=$(nomad alloc status -short "$alloc_id" 2>/dev/null | grep -oE 'exit_code=[0-9]+' | cut -d= -f2 || true) + + if [ -n "$alloc_exit_code" ]; then + exit_code="$alloc_exit_code" + else + # Try JSON parsing + alloc_exit_code=$(nomad alloc status -json "$alloc_id" 2>/dev/null | jq -r '.TaskState.LastState // empty' 2>/dev/null) || alloc_exit_code="" + if [ -z "$alloc_exit_code" ]; then + alloc_exit_code=$(nomad alloc status -json "$alloc_id" 2>/dev/null | jq -r '.ExitCode // empty' 2>/dev/null) || alloc_exit_code="" + fi + if [ -n "$alloc_exit_code" ] && [ "$alloc_exit_code" != "null" ]; then + exit_code="$alloc_exit_code" + fi + fi + + # Get allocation logs + logs=$(nomad alloc logs -short "$alloc_id" 2>/dev/null || true) + fi + + # If we couldn't get exit code from alloc, check job state + if [ "$exit_code" -eq 0 ]; then + local final_state + final_state=$(echo "$final_status_json" | jq -r '.State // empty' 2>/dev/null) || final_state="" + + case "$final_state" in + failed|dead) + exit_code=1 + ;; + esac + fi + + # Truncate logs if too long + if [ ${#logs} -gt 1000 ]; then + logs="${logs: -1000}" + fi + + # Write result file + write_result "$action_id" "$exit_code" "$logs" + + if [ "$exit_code" -eq 0 ]; then + log "Vault-runner job completed successfully for action: ${action_id}" + else + log "Vault-runner job failed for action: ${action_id} (exit code: ${exit_code})" + fi + + return "$exit_code" } # Launch runner for the given action (backend-agnostic orchestrator) @@ -1051,11 +1227,8 @@ main() { # Validate backend selection at startup case "$DISPATCHER_BACKEND" in - docker) ;; - nomad) - log "ERROR: nomad backend not yet implemented" - echo "nomad backend not yet implemented" >&2 - exit 1 + docker|nomad) + log "Using ${DISPATCHER_BACKEND} backend for vault-runner dispatch" ;; *) log "ERROR: unknown DISPATCHER_BACKEND=${DISPATCHER_BACKEND}"