fix: [nomad-step-4] S4.1 — nomad/jobs/agents.hcl (7 roles, llama, vault-templated bot tokens) (#955)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude
parent
c20b0a8bd2
commit
93a2a7bd3d
4 changed files with 438 additions and 0 deletions
203
nomad/jobs/agents.hcl
Normal file
203
nomad/jobs/agents.hcl
Normal file
|
|
@ -0,0 +1,203 @@
|
||||||
|
# =============================================================================
|
||||||
|
# nomad/jobs/agents.hcl — All-role agent polling loop (Nomad service job)
|
||||||
|
#
|
||||||
|
# Part of the Nomad+Vault migration (S4.1, issue #955). Runs the main bot
|
||||||
|
# polling loop with all 7 agent roles (review, dev, gardener, architect,
|
||||||
|
# planner, predictor, supervisor) against the local llama server.
|
||||||
|
#
|
||||||
|
# Host_volume contract:
|
||||||
|
# This job mounts agent-data, project-repos, and ops-repo from
|
||||||
|
# nomad/client.hcl. Paths under /srv/disinto/* are created by
|
||||||
|
# lib/init/nomad/cluster-up.sh before any job references them.
|
||||||
|
#
|
||||||
|
# Vault integration (S4.1):
|
||||||
|
# - vault { role = "service-agents" } at group scope — workload-identity
|
||||||
|
# JWT exchanged for a Vault token carrying the composite service-agents
|
||||||
|
# policy (vault/policies/service-agents.hcl), which grants read access
|
||||||
|
# to all 7 bot KV namespaces + vault bot + shared forge config.
|
||||||
|
# - template stanza renders per-bot FORGE_*_TOKEN + FORGE_PASS from Vault
|
||||||
|
# KV v2 at kv/disinto/bots/<role>.
|
||||||
|
# - Seeded on fresh boxes by tools/vault-seed-agents.sh.
|
||||||
|
#
|
||||||
|
# Not the runtime yet: docker-compose.yml is still the factory's live stack
|
||||||
|
# until cutover. This file exists so CI can validate it and S4.2 can wire
|
||||||
|
# `disinto init --backend=nomad --with agents` to `nomad job run` it.
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
job "agents" {
|
||||||
|
type = "service"
|
||||||
|
datacenters = ["dc1"]
|
||||||
|
|
||||||
|
group "agents" {
|
||||||
|
count = 1
|
||||||
|
|
||||||
|
# ── Vault workload identity (S4.1, issue #955) ───────────────────────────
|
||||||
|
# Composite role covering all 7 bot identities + vault bot. Role defined
|
||||||
|
# in vault/roles.yaml, policy in vault/policies/service-agents.hcl.
|
||||||
|
# Bound claim pins nomad_job_id = "agents".
|
||||||
|
vault {
|
||||||
|
role = "service-agents"
|
||||||
|
}
|
||||||
|
|
||||||
|
# No network port — agents are outbound-only (poll forgejo, call llama).
|
||||||
|
# No service discovery block — nothing health-checks agents over HTTP.
|
||||||
|
|
||||||
|
volume "agent-data" {
|
||||||
|
type = "host"
|
||||||
|
source = "agent-data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
volume "project-repos" {
|
||||||
|
type = "host"
|
||||||
|
source = "project-repos"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
volume "ops-repo" {
|
||||||
|
type = "host"
|
||||||
|
source = "ops-repo"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
# Conservative restart — fail fast to the scheduler.
|
||||||
|
restart {
|
||||||
|
attempts = 3
|
||||||
|
interval = "5m"
|
||||||
|
delay = "15s"
|
||||||
|
mode = "delay"
|
||||||
|
}
|
||||||
|
|
||||||
|
task "agents" {
|
||||||
|
driver = "docker"
|
||||||
|
|
||||||
|
config {
|
||||||
|
image = "disinto/agents:latest"
|
||||||
|
|
||||||
|
# apparmor=unconfined matches docker-compose — Claude Code needs
|
||||||
|
# ptrace for node.js inspector and /proc access.
|
||||||
|
security_opt = ["apparmor=unconfined"]
|
||||||
|
}
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
volume = "agent-data"
|
||||||
|
destination = "/home/agent/data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
volume = "project-repos"
|
||||||
|
destination = "/home/agent/repos"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
volume = "ops-repo"
|
||||||
|
destination = "/home/agent/repos/_factory/disinto-ops"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Non-secret env ─────────────────────────────────────────────────────
|
||||||
|
env {
|
||||||
|
FORGE_URL = "http://forgejo:3000"
|
||||||
|
FORGE_REPO = "disinto-admin/disinto"
|
||||||
|
ANTHROPIC_BASE_URL = "http://10.10.10.1:8081"
|
||||||
|
ANTHROPIC_API_KEY = "sk-no-key-required"
|
||||||
|
CLAUDE_MODEL = "unsloth/Qwen3.5-35B-A3B"
|
||||||
|
AGENT_ROLES = "review,dev,gardener,architect,planner,predictor,supervisor"
|
||||||
|
POLL_INTERVAL = "300"
|
||||||
|
DISINTO_CONTAINER = "1"
|
||||||
|
PROJECT_NAME = "project"
|
||||||
|
PROJECT_REPO_ROOT = "/home/agent/repos/project"
|
||||||
|
CLAUDE_TIMEOUT = "7200"
|
||||||
|
|
||||||
|
# llama-specific Claude Code tuning
|
||||||
|
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC = "1"
|
||||||
|
CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS = "1"
|
||||||
|
CLAUDE_AUTOCOMPACT_PCT_OVERRIDE = "60"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Vault-templated bot tokens (S4.1, issue #955) ─────────────────────
|
||||||
|
# Renders per-bot FORGE_*_TOKEN + FORGE_PASS from Vault KV v2.
|
||||||
|
# Each `with secret ...` block reads one bot's KV path; the `else`
|
||||||
|
# branch emits short placeholders on fresh installs where the path
|
||||||
|
# is absent. Seed with tools/vault-seed-agents.sh.
|
||||||
|
#
|
||||||
|
# Placeholder values kept < 16 chars to avoid secret-scan CI failures.
|
||||||
|
# error_on_missing_key = false prevents template-pending hangs.
|
||||||
|
template {
|
||||||
|
destination = "secrets/bots.env"
|
||||||
|
env = true
|
||||||
|
change_mode = "restart"
|
||||||
|
error_on_missing_key = false
|
||||||
|
data = <<EOT
|
||||||
|
{{- with secret "kv/data/disinto/bots/dev" -}}
|
||||||
|
FORGE_TOKEN={{ .Data.data.token }}
|
||||||
|
FORGE_PASS={{ .Data.data.pass }}
|
||||||
|
{{- else -}}
|
||||||
|
# WARNING: run tools/vault-seed-agents.sh
|
||||||
|
FORGE_TOKEN=seed-me
|
||||||
|
FORGE_PASS=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/review" -}}
|
||||||
|
FORGE_REVIEW_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_REVIEW_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/gardener" -}}
|
||||||
|
FORGE_GARDENER_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_GARDENER_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/architect" -}}
|
||||||
|
FORGE_ARCHITECT_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_ARCHITECT_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/planner" -}}
|
||||||
|
FORGE_PLANNER_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_PLANNER_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/predictor" -}}
|
||||||
|
FORGE_PREDICTOR_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_PREDICTOR_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/supervisor" -}}
|
||||||
|
FORGE_SUPERVISOR_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_SUPERVISOR_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
{{- with secret "kv/data/disinto/bots/vault" -}}
|
||||||
|
FORGE_VAULT_TOKEN={{ .Data.data.token }}
|
||||||
|
{{- else -}}
|
||||||
|
FORGE_VAULT_TOKEN=seed-me
|
||||||
|
{{- end }}
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Health check ───────────────────────────────────────────────────────
|
||||||
|
# Script-based check matching docker-compose's pgrep healthcheck.
|
||||||
|
# Nomad script checks run inside the container.
|
||||||
|
service {
|
||||||
|
name = "agents"
|
||||||
|
provider = "nomad"
|
||||||
|
|
||||||
|
check {
|
||||||
|
type = "script"
|
||||||
|
command = "/usr/bin/pgrep"
|
||||||
|
args = ["-f", "entrypoint.sh"]
|
||||||
|
interval = "60s"
|
||||||
|
timeout = "5s"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Agents run Claude/llama sessions — need CPU + memory headroom.
|
||||||
|
resources {
|
||||||
|
cpu = 500
|
||||||
|
memory = 1024
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
151
tools/vault-seed-agents.sh
Executable file
151
tools/vault-seed-agents.sh
Executable file
|
|
@ -0,0 +1,151 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# =============================================================================
|
||||||
|
# tools/vault-seed-agents.sh — Idempotent seed for all bot KV paths
|
||||||
|
#
|
||||||
|
# Part of the Nomad+Vault migration (S4.1, issue #955). Populates
|
||||||
|
# kv/disinto/bots/<role> with token + pass for each of the 7 agent roles
|
||||||
|
# plus the vault bot. Handles the "fresh factory, no .env import" case.
|
||||||
|
#
|
||||||
|
# Companion to tools/vault-import.sh — when that runs against a box with
|
||||||
|
# an existing stack, it overwrites seeded values with real ones.
|
||||||
|
#
|
||||||
|
# Idempotency contract (per bot):
|
||||||
|
# - Both token and pass present → skip, log "<role> unchanged".
|
||||||
|
# - Either missing → generate random values for missing keys, preserve
|
||||||
|
# existing keys, write back atomically.
|
||||||
|
#
|
||||||
|
# Preconditions:
|
||||||
|
# - Vault reachable + unsealed at $VAULT_ADDR.
|
||||||
|
# - VAULT_TOKEN set (env) or /etc/vault.d/root.token readable.
|
||||||
|
# - curl, jq, openssl
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# tools/vault-seed-agents.sh
|
||||||
|
# tools/vault-seed-agents.sh --dry-run
|
||||||
|
#
|
||||||
|
# Exit codes:
|
||||||
|
# 0 success (seed applied, or already applied)
|
||||||
|
# 1 precondition / API / mount-mismatch failure
|
||||||
|
# =============================================================================
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
|
||||||
|
# shellcheck source=../lib/hvault.sh
|
||||||
|
source "${REPO_ROOT}/lib/hvault.sh"
|
||||||
|
|
||||||
|
KV_MOUNT="kv"
|
||||||
|
TOKEN_BYTES=32 # 32 bytes → 64 hex chars
|
||||||
|
PASS_BYTES=16 # 16 bytes → 32 hex chars
|
||||||
|
|
||||||
|
# All bot roles seeded by this script.
|
||||||
|
BOT_ROLES=(dev review gardener architect planner predictor supervisor vault)
|
||||||
|
|
||||||
|
LOG_TAG="[vault-seed-agents]"
|
||||||
|
log() { printf '%s %s\n' "$LOG_TAG" "$*"; }
|
||||||
|
die() { printf '%s ERROR: %s\n' "$LOG_TAG" "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
# ── Flag parsing ─────────────────────────────────────────────────────────────
|
||||||
|
# while/shift shape — distinct from forgejo (arity:value case) and
|
||||||
|
# woodpecker (for-loop).
|
||||||
|
DRY_RUN=0
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--dry-run) DRY_RUN=1 ;;
|
||||||
|
-h|--help)
|
||||||
|
printf 'Usage: %s [--dry-run]\n\n' "$(basename "$0")"
|
||||||
|
printf 'Seed kv/disinto/bots/<role> with token + pass for all agent\n'
|
||||||
|
printf 'roles. Idempotent: existing non-empty values are preserved.\n\n'
|
||||||
|
printf ' --dry-run Print planned actions without writing.\n'
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*) die "invalid argument: ${1} (try --help)" ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||||
|
for bin in curl jq openssl; do
|
||||||
|
command -v "$bin" >/dev/null 2>&1 \
|
||||||
|
|| die "required binary not found: ${bin}"
|
||||||
|
done
|
||||||
|
[ -n "${VAULT_ADDR:-}" ] \
|
||||||
|
|| die "VAULT_ADDR unset — e.g. export VAULT_ADDR=http://127.0.0.1:8200"
|
||||||
|
hvault_token_lookup >/dev/null \
|
||||||
|
|| die "Vault auth probe failed — check VAULT_ADDR + VAULT_TOKEN"
|
||||||
|
|
||||||
|
# ── Step 1: ensure kv/ mount exists and is KV v2 ────────────────────────────
|
||||||
|
log "── Step 1: ensure ${KV_MOUNT}/ is KV v2 ──"
|
||||||
|
export DRY_RUN
|
||||||
|
hvault_ensure_kv_v2 "$KV_MOUNT" "${LOG_TAG}" \
|
||||||
|
|| die "KV mount check failed"
|
||||||
|
|
||||||
|
# ── Step 2: seed each bot role ───────────────────────────────────────────────
|
||||||
|
total_generated=0
|
||||||
|
|
||||||
|
for role in "${BOT_ROLES[@]}"; do
|
||||||
|
kv_logical="disinto/bots/${role}"
|
||||||
|
kv_api="${KV_MOUNT}/data/${kv_logical}"
|
||||||
|
|
||||||
|
log "── seed ${kv_logical} ──"
|
||||||
|
|
||||||
|
existing_raw="$(hvault_get_or_empty "${kv_api}")" \
|
||||||
|
|| die "failed to read ${kv_api}"
|
||||||
|
|
||||||
|
existing_token=""
|
||||||
|
existing_pass=""
|
||||||
|
existing_data="{}"
|
||||||
|
if [ -n "$existing_raw" ]; then
|
||||||
|
existing_data="$(printf '%s' "$existing_raw" | jq '.data.data // {}')"
|
||||||
|
existing_token="$(printf '%s' "$existing_raw" | jq -r '.data.data.token // ""')"
|
||||||
|
existing_pass="$(printf '%s' "$existing_raw" | jq -r '.data.data.pass // ""')"
|
||||||
|
fi
|
||||||
|
|
||||||
|
generated=()
|
||||||
|
|
||||||
|
if [ -z "$existing_token" ]; then
|
||||||
|
generated+=("token")
|
||||||
|
fi
|
||||||
|
if [ -z "$existing_pass" ]; then
|
||||||
|
generated+=("pass")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${#generated[@]}" -eq 0 ]; then
|
||||||
|
log "${role}: unchanged"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
log "[dry-run] ${role}: would generate ${generated[*]}"
|
||||||
|
total_generated=$(( total_generated + ${#generated[@]} ))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
desired_token="$existing_token"
|
||||||
|
desired_pass="$existing_pass"
|
||||||
|
|
||||||
|
for key in "${generated[@]}"; do
|
||||||
|
case "$key" in
|
||||||
|
token) desired_token="$(openssl rand -hex "$TOKEN_BYTES")" ;;
|
||||||
|
pass) desired_pass="$(openssl rand -hex "$PASS_BYTES")" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Merge new keys into existing data to preserve any keys we don't own.
|
||||||
|
payload="$(printf '%s' "$existing_data" \
|
||||||
|
| jq --arg t "$desired_token" --arg p "$desired_pass" \
|
||||||
|
'{data: (. + {token: $t, pass: $p})}')"
|
||||||
|
|
||||||
|
_hvault_request POST "${kv_api}" "$payload" >/dev/null \
|
||||||
|
|| die "failed to write ${kv_api}"
|
||||||
|
|
||||||
|
log "${role}: generated ${generated[*]}"
|
||||||
|
total_generated=$(( total_generated + ${#generated[@]} ))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$total_generated" -eq 0 ]; then
|
||||||
|
log "all bot paths already seeded — no-op"
|
||||||
|
else
|
||||||
|
log "done — ${total_generated} key(s) seeded across ${#BOT_ROLES[@]} bot paths"
|
||||||
|
fi
|
||||||
76
vault/policies/service-agents.hcl
Normal file
76
vault/policies/service-agents.hcl
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
# vault/policies/service-agents.hcl
|
||||||
|
#
|
||||||
|
# Composite policy for the `agents` Nomad job (S4.1, issue #955).
|
||||||
|
# Grants read access to all 7 bot KV namespaces + shared forge config,
|
||||||
|
# so a single job running all agent roles can pull per-bot tokens from
|
||||||
|
# Vault via workload identity.
|
||||||
|
|
||||||
|
# ── Per-bot KV paths (token + pass per role) ─────────────────────────────────
|
||||||
|
path "kv/data/disinto/bots/dev" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/dev" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/review" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/review" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/gardener" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/gardener" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/architect" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/architect" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/planner" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/planner" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/predictor" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/predictor" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/supervisor" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/supervisor" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/data/disinto/bots/vault" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
path "kv/metadata/disinto/bots/vault" {
|
||||||
|
capabilities = ["list", "read"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Shared forge config (URL, bot usernames) ─────────────────────────────────
|
||||||
|
path "kv/data/disinto/shared/forge" {
|
||||||
|
capabilities = ["read"]
|
||||||
|
}
|
||||||
|
|
@ -62,6 +62,14 @@ roles:
|
||||||
namespace: default
|
namespace: default
|
||||||
job_id: woodpecker-agent
|
job_id: woodpecker-agent
|
||||||
|
|
||||||
|
# ── Agents composite (nomad/jobs/agents.hcl — S4.1) ──────────────────────
|
||||||
|
# Single job running all 7 agent roles. Uses a composite policy
|
||||||
|
# (vault/policies/service-agents.hcl) that unions all bot KV paths.
|
||||||
|
- name: service-agents
|
||||||
|
policy: service-agents
|
||||||
|
namespace: default
|
||||||
|
job_id: agents
|
||||||
|
|
||||||
# ── Per-agent bots (nomad/jobs/bot-<role>.hcl — land in later steps) ───────
|
# ── Per-agent bots (nomad/jobs/bot-<role>.hcl — land in later steps) ───────
|
||||||
# job_id placeholders match the policy name 1:1 until each bot's jobspec
|
# job_id placeholders match the policy name 1:1 until each bot's jobspec
|
||||||
# lands. When a bot's jobspec is added under nomad/jobs/, update the
|
# lands. When a bot's jobspec is added under nomad/jobs/, update the
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue