Compare commits
1 commit
main
...
fix/issue-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a17cf5ec41 |
53 changed files with 249 additions and 2142 deletions
|
|
@ -301,28 +301,6 @@ def main() -> int:
|
|||
"9a57368f3c1dfd29ec328596b86962a0": "Flag parsing loop + case start (vault-seed-woodpecker + wp-oauth-register)",
|
||||
"9d72d40ff303cbed0b7e628fc15381c3": "Case loop + dry-run handler (vault-seed-woodpecker + wp-oauth-register)",
|
||||
"5b52ddbbf47948e3cbc1b383f0909588": "Help + invalid arg handler end (vault-seed-woodpecker + wp-oauth-register)",
|
||||
# Common vault-seed script preamble + precondition patterns
|
||||
# Shared across tools/vault-seed-{forgejo,agents,woodpecker}.sh
|
||||
"dff3675c151fcdbd2fef798826ae919b": "Vault-seed preamble: set -euo + path setup + source hvault.sh + KV_MOUNT",
|
||||
"1cd9f0d083e24e6e6b2071db9b6dae09": "Vault-seed preconditions: binary check loop + VAULT_ADDR guard",
|
||||
"63bfa88d71764c95c65a9a248f3e40ab": "Vault-seed preconditions: binary check end + VAULT_ADDR die",
|
||||
"34873ad3570b211ce1d90468ab6ac94c": "Vault-seed preconditions: VAULT_ADDR die + hvault_token_lookup",
|
||||
"71a52270f249e843cda48ad896d9f781": "Vault-seed preconditions: VAULT_ADDR + hvault_token_lookup + die",
|
||||
# Common vault-seed script flag parsing patterns
|
||||
# Shared across tools/vault-seed-{forgejo,ops-repo}.sh
|
||||
"6906b7787796c2ccb8dd622e2ad4e7bf": "vault-seed DRY_RUN init + case pattern (forgejo + ops-repo)",
|
||||
"a0df5283b616b964f8bc32fd99ec1b5a": "vault-seed case pattern start (forgejo + ops-repo)",
|
||||
"e15e3272fdd9f0f46ce9e726aea9f853": "vault-seed case pattern dry-run handler (forgejo + ops-repo)",
|
||||
"c9f22385cc49a3dac1d336bc14c6315b": "vault-seed DRY_RUN assignment (forgejo + ops-repo)",
|
||||
"106f4071e88f841b3208b01144cd1c39": "vault-seed case pattern dry-run end (forgejo + ops-repo)",
|
||||
"c15506dcb6bb340b25d1c39d442dd2e6": "vault-seed help text + invalid arg handler (forgejo + ops-repo)",
|
||||
"1feecd3b3caf00045fae938ddf2811de": "vault-seed invalid arg handler (forgejo + ops-repo)",
|
||||
"919780d5e7182715344f5aa02b191294": "vault-seed invalid arg + esac pattern (forgejo + ops-repo)",
|
||||
"8dce1d292bce8e60ef4c0665b62945b0": "vault-seed esac + binary check loop (forgejo + ops-repo)",
|
||||
"ca043687143a5b47bd54e65a99ce8ee8": "vault-seed binary check loop start (forgejo + ops-repo)",
|
||||
"aefd9f655411a955395e6e5995ddbe6f": "vault-seed binary check pattern (forgejo + ops-repo)",
|
||||
"60f0c46deb5491599457efb4048918e5": "vault-seed VAULT_ADDR + hvault_token_lookup check (forgejo + ops-repo)",
|
||||
"f6838f581ef6b4d82b55268389032769": "vault-seed VAULT_ADDR + hvault_token_lookup die (forgejo + ops-repo)",
|
||||
}
|
||||
|
||||
if not sh_files:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Disinto — Agent Instructions
|
||||
|
||||
## What this repo is
|
||||
|
|
@ -37,9 +37,9 @@ disinto/ (code repo)
|
|||
│ examples/ — example vault action TOMLs (promote, publish, release, webhook-call)
|
||||
├── lib/ env.sh, agent-sdk.sh, ci-helpers.sh, ci-debug.sh, load-project.sh, parse-deps.sh, guard.sh, mirrors.sh, pr-lifecycle.sh, issue-lifecycle.sh, worktree.sh, formula-session.sh, stack-lock.sh, forge-setup.sh, forge-push.sh, ops-setup.sh, ci-setup.sh, generators.sh, hire-agent.sh, release.sh, build-graph.py, branch-protection.sh, secret-scan.sh, tea-helpers.sh, action-vault.sh, ci-log-reader.py, git-creds.sh, sprint-filer.sh, hvault.sh
|
||||
│ hooks/ — Claude Code session hooks (on-compact-reinject, on-idle-stop, on-phase-change, on-pretooluse-guard, on-session-end, on-stop-failure)
|
||||
│ init/nomad/ — cluster-up.sh, install.sh, vault-init.sh, lib-systemd.sh (Nomad+Vault Step 0 installers, #821-#825); wp-oauth-register.sh (Forgejo OAuth2 app + Vault KV seeder for Woodpecker, S3.3); deploy.sh (dependency-ordered Nomad job deploy + health-wait, S4)
|
||||
├── nomad/ server.hcl, client.hcl (allow_privileged for woodpecker-agent, S3-fix-5), vault.hcl — HCL configs deployed to /etc/nomad.d/ and /etc/vault.d/ by lib/init/nomad/cluster-up.sh
|
||||
│ jobs/ — Nomad jobspecs: forgejo.hcl (Vault secrets via template, S2.4); woodpecker-server.hcl + woodpecker-agent.hcl (host-net, docker.sock, Vault KV, S3.1-S3.2); agents.hcl (7 roles, llama, Vault-templated bot tokens, S4.1); vault-runner.hcl (parameterized batch dispatch, S5.3); staging.hcl (Caddy file-server, dynamic port — edge discovers via service registration, S5.2); chat.hcl (Claude chat UI, tmpfs via mount block, Vault OAuth secrets, S5.2); edge.hcl (Caddy proxy + dispatcher sidecar, S5.1)
|
||||
│ init/nomad/ — cluster-up.sh, install.sh, vault-init.sh, lib-systemd.sh (Nomad+Vault Step 0 installers, #821-#825)
|
||||
├── nomad/ server.hcl, client.hcl, vault.hcl — HCL configs deployed to /etc/nomad.d/ and /etc/vault.d/ by lib/init/nomad/cluster-up.sh
|
||||
│ jobs/ — Nomad jobspecs (forgejo.hcl reads Vault secrets via template stanza, S2.4)
|
||||
├── projects/ *.toml.example — templates; *.toml — local per-box config (gitignored)
|
||||
├── formulas/ Issue templates (TOML specs for multi-step agent tasks)
|
||||
├── docker/ Dockerfiles and entrypoints: reproduce, triage, edge dispatcher, chat (server.py, entrypoint-chat.sh, Dockerfile, ui/)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Architect — Agent Instructions
|
||||
|
||||
## What this agent is
|
||||
|
|
|
|||
355
bin/disinto
355
bin/disinto
|
|
@ -82,7 +82,7 @@ Init options:
|
|||
--ci-id <n> Woodpecker CI repo ID (default: 0 = no CI)
|
||||
--forge-url <url> Forge base URL (default: http://localhost:3000)
|
||||
--backend <value> Orchestration backend: docker (default) | nomad
|
||||
--with <services> (nomad) Deploy services: forgejo,woodpecker,agents,staging,chat,edge[,...] (S1.3, S3.4, S4.2, S5.2, S5.5)
|
||||
--with <services> (nomad) Deploy services: forgejo,woodpecker-server,woodpecker-agent[,...] (S1.3, S3.4)
|
||||
--empty (nomad) Bring up cluster only, no jobs (S0.4)
|
||||
--bare Skip compose generation (bare-metal setup)
|
||||
--build Use local docker build instead of registry images (dev mode)
|
||||
|
|
@ -783,37 +783,60 @@ _disinto_init_nomad() {
|
|||
fi
|
||||
|
||||
if [ -n "$with_services" ]; then
|
||||
# Interleaved seed/deploy per service (S2.6, #928, #948): match the
|
||||
# real-run path so dry-run output accurately represents execution order.
|
||||
# Build ordered deploy list: only include services present in with_services
|
||||
# Normalize services: auto-include forgejo when woodpecker is requested
|
||||
# (woodpecker without forgejo is nonsensical)
|
||||
local normalized_services="$with_services"
|
||||
if echo "$with_services" | grep -q "woodpecker" && ! echo "$with_services" | grep -q "forgejo"; then
|
||||
echo "Note: --with woodpecker implies --with forgejo (OAuth dependency)"
|
||||
normalized_services="forgejo,${with_services}"
|
||||
fi
|
||||
|
||||
# Define deployment order: forgejo -> woodpecker-server -> woodpecker-agent
|
||||
# Only include services that are requested (after normalization)
|
||||
local DEPLOY_ORDER=""
|
||||
for ordered_svc in forgejo woodpecker-server woodpecker-agent agents staging chat edge; do
|
||||
if echo ",$with_services," | grep -q ",$ordered_svc,"; then
|
||||
DEPLOY_ORDER="${DEPLOY_ORDER:+${DEPLOY_ORDER} }${ordered_svc}"
|
||||
for ordered_svc in forgejo woodpecker-server woodpecker-agent; do
|
||||
if echo ",$normalized_services," | grep -q ",$ordered_svc,"; then
|
||||
if [ -z "$DEPLOY_ORDER" ]; then
|
||||
DEPLOY_ORDER="$ordered_svc"
|
||||
else
|
||||
DEPLOY_ORDER="$DEPLOY_ORDER $ordered_svc"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
local IFS=' '
|
||||
# Vault seed plan (S2.6, #928): one line per service whose
|
||||
# tools/vault-seed-<svc>.sh ships. Services without a seeder are
|
||||
# silently skipped — the real-run loop below mirrors this,
|
||||
# making `--with woodpecker` in Step 3 auto-invoke
|
||||
# tools/vault-seed-woodpecker.sh once that file lands without
|
||||
# any further change to bin/disinto.
|
||||
local seed_hdr_printed=false
|
||||
local IFS=','
|
||||
for svc in $normalized_services; do
|
||||
svc=$(echo "$svc" | xargs) # trim whitespace
|
||||
local seed_script="${FACTORY_ROOT}/tools/vault-seed-${svc}.sh"
|
||||
if [ -x "$seed_script" ]; then
|
||||
if [ "$seed_hdr_printed" = false ]; then
|
||||
echo "── Vault seed dry-run ─────────────────────────────────"
|
||||
seed_hdr_printed=true
|
||||
fi
|
||||
echo "[seed] [dry-run] ${seed_script} --dry-run"
|
||||
fi
|
||||
done
|
||||
[ "$seed_hdr_printed" = true ] && echo ""
|
||||
|
||||
echo "── Deploy services dry-run ────────────────────────────"
|
||||
echo "[deploy] services to deploy: ${normalized_services}"
|
||||
echo "[deploy] deployment order: ${DEPLOY_ORDER}"
|
||||
for svc in $DEPLOY_ORDER; do
|
||||
# Seed this service (if seed script exists)
|
||||
local seed_name="$svc"
|
||||
# Validate known services first
|
||||
case "$svc" in
|
||||
woodpecker-server|woodpecker-agent) seed_name="woodpecker" ;;
|
||||
agents) seed_name="agents" ;;
|
||||
chat) seed_name="chat" ;;
|
||||
edge) seed_name="ops-repo" ;;
|
||||
forgejo|woodpecker-server|woodpecker-agent) ;;
|
||||
*)
|
||||
echo "Error: unknown service '${svc}' — known: forgejo, woodpecker-server, woodpecker-agent" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
local seed_script="${FACTORY_ROOT}/tools/vault-seed-${seed_name}.sh"
|
||||
if [ -x "$seed_script" ]; then
|
||||
echo "── Vault seed dry-run ─────────────────────────────────"
|
||||
echo "[seed] [dry-run] ${seed_script} --dry-run"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Deploy this service
|
||||
echo "── Deploy services dry-run ────────────────────────────"
|
||||
echo "[deploy] services to deploy: ${with_services}"
|
||||
local jobspec_path="${FACTORY_ROOT}/nomad/jobs/${svc}.hcl"
|
||||
if [ ! -f "$jobspec_path" ]; then
|
||||
echo "Error: jobspec not found: ${jobspec_path}" >&2
|
||||
|
|
@ -824,32 +847,6 @@ _disinto_init_nomad() {
|
|||
done
|
||||
echo "[deploy] dry-run complete"
|
||||
fi
|
||||
|
||||
# Dry-run vault-runner (unconditionally, not gated by --with)
|
||||
echo ""
|
||||
echo "── Vault-runner dry-run ───────────────────────────────────"
|
||||
local vault_runner_path="${FACTORY_ROOT}/nomad/jobs/vault-runner.hcl"
|
||||
if [ -f "$vault_runner_path" ]; then
|
||||
echo "[deploy] vault-runner: [dry-run] nomad job validate ${vault_runner_path}"
|
||||
echo "[deploy] vault-runner: [dry-run] nomad job run -detach ${vault_runner_path}"
|
||||
else
|
||||
echo "[deploy] vault-runner: jobspec not found, skipping"
|
||||
fi
|
||||
|
||||
# Build custom images dry-run (if agents, chat, or edge services are included)
|
||||
if echo ",$with_services," | grep -qE ",(agents|chat|edge),"; then
|
||||
echo ""
|
||||
echo "── Build images dry-run ──────────────────────────────"
|
||||
if echo ",$with_services," | grep -q ",agents,"; then
|
||||
echo "[build] [dry-run] docker build -t disinto/agents:local -f ${FACTORY_ROOT}/docker/agents/Dockerfile ${FACTORY_ROOT}"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",chat,"; then
|
||||
echo "[build] [dry-run] docker build -t disinto/chat:local -f ${FACTORY_ROOT}/docker/chat/Dockerfile ${FACTORY_ROOT}/docker/chat"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",edge,"; then
|
||||
echo "[build] [dry-run] docker build -t disinto/edge:local -f ${FACTORY_ROOT}/docker/edge/Dockerfile ${FACTORY_ROOT}/docker/edge"
|
||||
fi
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -937,127 +934,114 @@ _disinto_init_nomad() {
|
|||
echo "[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services"
|
||||
fi
|
||||
|
||||
# Build custom images required by Nomad jobs (S4.2, S5.2, S5.5) — before deploy.
|
||||
# Single-node factory dev box: no multi-node pull needed, no registry auth.
|
||||
# Can upgrade to approach B (registry push/pull) later if multi-node.
|
||||
if echo ",$with_services," | grep -qE ",(agents|chat|edge),"; then
|
||||
echo ""
|
||||
echo "── Building custom images ─────────────────────────────"
|
||||
if echo ",$with_services," | grep -q ",agents,"; then
|
||||
local tag="disinto/agents:local"
|
||||
echo "── Building $tag ─────────────────────────────"
|
||||
docker build -t "$tag" -f "${FACTORY_ROOT}/docker/agents/Dockerfile" "${FACTORY_ROOT}" 2>&1 | tail -5
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",chat,"; then
|
||||
local tag="disinto/chat:local"
|
||||
echo "── Building $tag ─────────────────────────────"
|
||||
docker build -t "$tag" -f "${FACTORY_ROOT}/docker/chat/Dockerfile" "${FACTORY_ROOT}/docker/chat" 2>&1 | tail -5
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",edge,"; then
|
||||
local tag="disinto/edge:local"
|
||||
echo "── Building $tag ─────────────────────────────"
|
||||
docker build -t "$tag" -f "${FACTORY_ROOT}/docker/edge/Dockerfile" "${FACTORY_ROOT}/docker/edge" 2>&1 | tail -5
|
||||
fi
|
||||
fi
|
||||
|
||||
# Interleaved seed/deploy per service (S2.6, #928, #948).
|
||||
# We interleave seed + deploy per service (not batch all seeds then all deploys)
|
||||
# so that OAuth-dependent services can reach their dependencies during seeding.
|
||||
# E.g., seed-forgejo → deploy-forgejo → seed-woodpecker (OAuth can now reach
|
||||
# running forgejo) → deploy-woodpecker.
|
||||
# Seed Vault for services that ship their own seeder (S2.6, #928).
|
||||
# Convention: tools/vault-seed-<svc>.sh — auto-invoked when --with <svc>
|
||||
# is requested. Runs AFTER vault-import so that real imported values
|
||||
# win over generated seeds when both are present; each seeder is
|
||||
# idempotent on a per-key basis (see vault-seed-forgejo.sh's
|
||||
# "missing → generate, present → unchanged" contract), so re-running
|
||||
# init does not rotate existing keys. Services without a seeder are
|
||||
# silently skipped — keeps this loop forward-compatible with Step 3+
|
||||
# services that may ship their own seeder without touching bin/disinto.
|
||||
#
|
||||
# VAULT_ADDR is passed explicitly because cluster-up.sh writes the
|
||||
# profile.d export *during* this same init run, so the current shell
|
||||
# hasn't sourced it yet; sibling vault-* scripts (engines/policies/
|
||||
# auth/import) default VAULT_ADDR internally via _hvault_default_env,
|
||||
# but vault-seed-forgejo.sh requires the caller to set it.
|
||||
#
|
||||
# The non-root branch invokes the seeder as `sudo -n -- env VAR=val
|
||||
# script` rather than `sudo -n VAR=val -- script`: sudo treats bare
|
||||
# `VAR=val` args as sudoers env-assignments, which the default
|
||||
# `env_reset=on` policy silently discards unless the variable is in
|
||||
# `env_keep` (VAULT_ADDR is not). Using `env` as the actual command
|
||||
# sets VAULT_ADDR in the child process regardless of sudoers policy.
|
||||
if [ -n "$with_services" ]; then
|
||||
# Normalize services: auto-include forgejo when woodpecker is requested
|
||||
local normalized_services="$with_services"
|
||||
if echo "$with_services" | grep -q "woodpecker" && ! echo "$with_services" | grep -q "forgejo"; then
|
||||
normalized_services="forgejo,${with_services}"
|
||||
fi
|
||||
|
||||
local vault_addr="${VAULT_ADDR:-http://127.0.0.1:8200}"
|
||||
|
||||
# Build ordered deploy list (S3.4, S4.2, S5.2, S5.5): forgejo → woodpecker-server → woodpecker-agent → agents → staging → chat → edge
|
||||
local DEPLOY_ORDER=""
|
||||
for ordered_svc in forgejo woodpecker-server woodpecker-agent agents staging chat edge; do
|
||||
if echo ",$with_services," | grep -q ",$ordered_svc,"; then
|
||||
DEPLOY_ORDER="${DEPLOY_ORDER:+${DEPLOY_ORDER} }${ordered_svc}"
|
||||
fi
|
||||
done
|
||||
|
||||
local IFS=' '
|
||||
for svc in $DEPLOY_ORDER; do
|
||||
# Seed this service (if seed script exists)
|
||||
local seed_name="$svc"
|
||||
case "$svc" in
|
||||
woodpecker-server|woodpecker-agent) seed_name="woodpecker" ;;
|
||||
agents) seed_name="agents" ;;
|
||||
chat) seed_name="chat" ;;
|
||||
edge) seed_name="ops-repo" ;;
|
||||
esac
|
||||
local seed_script="${FACTORY_ROOT}/tools/vault-seed-${seed_name}.sh"
|
||||
local IFS=','
|
||||
for svc in $normalized_services; do
|
||||
svc=$(echo "$svc" | xargs) # trim whitespace
|
||||
local seed_script="${FACTORY_ROOT}/tools/vault-seed-${svc}.sh"
|
||||
if [ -x "$seed_script" ]; then
|
||||
echo ""
|
||||
echo "── Seeding Vault for ${seed_name} ───────────────────────────"
|
||||
echo "── Seeding Vault for ${svc} ───────────────────────────"
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
VAULT_ADDR="$vault_addr" "$seed_script" || exit $?
|
||||
else
|
||||
if ! command -v sudo >/dev/null 2>&1; then
|
||||
echo "Error: vault-seed-${seed_name}.sh must run as root and sudo is not installed" >&2
|
||||
echo "Error: vault-seed-${svc}.sh must run as root and sudo is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n -- env "VAULT_ADDR=$vault_addr" "$seed_script" || exit $?
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Deploy this service
|
||||
echo ""
|
||||
echo "── Deploying ${svc} ───────────────────────────────────────"
|
||||
# Deploy services if requested
|
||||
if [ -n "$with_services" ]; then
|
||||
# Normalize services: auto-include forgejo when woodpecker is requested
|
||||
# (woodpecker without forgejo is nonsensical)
|
||||
local normalized_services="$with_services"
|
||||
if echo "$with_services" | grep -q "woodpecker" && ! echo "$with_services" | grep -q "forgejo"; then
|
||||
echo "Note: --with woodpecker implies --with forgejo (OAuth dependency)"
|
||||
normalized_services="forgejo,${with_services}"
|
||||
fi
|
||||
|
||||
# Seed host volumes before deployment (if needed)
|
||||
# Define deployment order: forgejo -> woodpecker-server -> woodpecker-agent
|
||||
# Only include services that are requested (after normalization)
|
||||
local DEPLOY_ORDER=""
|
||||
for ordered_svc in forgejo woodpecker-server woodpecker-agent; do
|
||||
if echo ",$normalized_services," | grep -q ",$ordered_svc,"; then
|
||||
if [ -z "$DEPLOY_ORDER" ]; then
|
||||
DEPLOY_ORDER="$ordered_svc"
|
||||
else
|
||||
DEPLOY_ORDER="$DEPLOY_ORDER $ordered_svc"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "── Deploying services ─────────────────────────────────"
|
||||
local -a deploy_cmd=("$deploy_sh")
|
||||
# Split comma-separated service list into positional args (in deploy order)
|
||||
local IFS=' '
|
||||
for svc in $DEPLOY_ORDER; do
|
||||
if ! echo "$svc" | grep -qE '^[a-zA-Z0-9_-]+$'; then
|
||||
echo "Error: invalid service name '${svc}' — must match ^[a-zA-Z0-9_-]+$" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Validate known services FIRST (before jobspec check)
|
||||
case "$svc" in
|
||||
staging)
|
||||
# Seed site-content host volume (/srv/disinto/docker) with static content
|
||||
# The staging jobspec mounts this volume read-only to /srv/site
|
||||
local site_content_src="${FACTORY_ROOT}/docker/index.html"
|
||||
local site_content_dst="/srv/disinto/docker"
|
||||
if [ -f "$site_content_src" ] && [ -d "$site_content_dst" ]; then
|
||||
if ! cmp -s "$site_content_src" "${site_content_dst}/index.html" 2>/dev/null; then
|
||||
echo "[staging] seeding site-content volume..."
|
||||
cp "$site_content_src" "${site_content_dst}/index.html"
|
||||
fi
|
||||
fi
|
||||
forgejo|woodpecker-server|woodpecker-agent) ;;
|
||||
*)
|
||||
echo "Error: unknown service '${svc}' — known: forgejo, woodpecker-server, woodpecker-agent" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Check jobspec exists
|
||||
local jobspec_path="${FACTORY_ROOT}/nomad/jobs/${svc}.hcl"
|
||||
if [ ! -f "$jobspec_path" ]; then
|
||||
echo "Error: jobspec not found: ${jobspec_path}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local -a deploy_cmd=("$deploy_sh" "$svc")
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
"${deploy_cmd[@]}" || exit $?
|
||||
else
|
||||
if ! command -v sudo >/dev/null 2>&1; then
|
||||
echo "Error: deploy.sh must run as root and sudo is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n -- "${deploy_cmd[@]}" || exit $?
|
||||
fi
|
||||
deploy_cmd+=("$svc")
|
||||
done
|
||||
|
||||
# Run vault-runner (unconditionally, not gated by --with) — infrastructure job
|
||||
# vault-runner is always present since it's needed for vault action dispatch
|
||||
echo ""
|
||||
echo "── Running vault-runner ────────────────────────────────────"
|
||||
local vault_runner_path="${FACTORY_ROOT}/nomad/jobs/vault-runner.hcl"
|
||||
if [ -f "$vault_runner_path" ]; then
|
||||
echo "[deploy] vault-runner: running Nomad job (infrastructure)"
|
||||
local -a vault_runner_cmd=("$deploy_sh" "vault-runner")
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
"${vault_runner_cmd[@]}" || exit $?
|
||||
else
|
||||
if ! command -v sudo >/dev/null 2>&1; then
|
||||
echo "Error: deploy.sh must run as root and sudo is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n -- "${vault_runner_cmd[@]}" || exit $?
|
||||
fi
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
"${deploy_cmd[@]}" || exit $?
|
||||
else
|
||||
echo "[deploy] vault-runner: jobspec not found, skipping"
|
||||
if ! command -v sudo >/dev/null 2>&1; then
|
||||
echo "Error: deploy.sh must run as root and sudo is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n -- "${deploy_cmd[@]}" || exit $?
|
||||
fi
|
||||
|
||||
# Print final summary
|
||||
|
|
@ -1074,25 +1058,16 @@ _disinto_init_nomad() {
|
|||
else
|
||||
echo "Imported: (none — seed kv/disinto/* manually before deploying secret-dependent services)"
|
||||
fi
|
||||
echo "Deployed: ${with_services}"
|
||||
if echo ",$with_services," | grep -q ",forgejo,"; then
|
||||
echo "Deployed: ${normalized_services}"
|
||||
if echo "$normalized_services" | grep -q "forgejo"; then
|
||||
echo "Ports: forgejo: 3000"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",woodpecker-server,"; then
|
||||
if echo "$normalized_services" | grep -q "woodpecker-server"; then
|
||||
echo " woodpecker-server: 8000"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",woodpecker-agent,"; then
|
||||
if echo "$normalized_services" | grep -q "woodpecker-agent"; then
|
||||
echo " woodpecker-agent: (agent connected)"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",agents,"; then
|
||||
echo " agents: (polling loop running)"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",staging,"; then
|
||||
echo " staging: (internal, no external port)"
|
||||
fi
|
||||
if echo ",$with_services," | grep -q ",chat,"; then
|
||||
echo " chat: 8080"
|
||||
fi
|
||||
echo "────────────────────────────────────────────────────────"
|
||||
fi
|
||||
|
||||
|
|
@ -1178,70 +1153,6 @@ disinto_init() {
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Normalize --with services (S3.4): expand 'woodpecker' shorthand to
|
||||
# 'woodpecker-server,woodpecker-agent', auto-include forgejo when
|
||||
# woodpecker is requested (OAuth dependency), and validate all names.
|
||||
if [ -n "$with_services" ]; then
|
||||
# Expand 'woodpecker' (bare) → 'woodpecker-server,woodpecker-agent'.
|
||||
# Must not match already-expanded 'woodpecker-server'/'woodpecker-agent'.
|
||||
local expanded=""
|
||||
local IFS=','
|
||||
for _svc in $with_services; do
|
||||
_svc=$(echo "$_svc" | xargs)
|
||||
case "$_svc" in
|
||||
woodpecker) _svc="woodpecker-server,woodpecker-agent" ;;
|
||||
agents) _svc="agents" ;;
|
||||
esac
|
||||
expanded="${expanded:+${expanded},}${_svc}"
|
||||
done
|
||||
with_services="$expanded"
|
||||
unset IFS
|
||||
|
||||
# Auto-include forgejo when woodpecker is requested
|
||||
if echo ",$with_services," | grep -q ",woodpecker-server,\|,woodpecker-agent," \
|
||||
&& ! echo ",$with_services," | grep -q ",forgejo,"; then
|
||||
echo "Note: --with woodpecker implies --with forgejo (OAuth dependency)"
|
||||
with_services="forgejo,${with_services}"
|
||||
fi
|
||||
|
||||
# Auto-include forgejo and woodpecker when agents is requested
|
||||
if echo ",$with_services," | grep -q ",agents,"; then
|
||||
if ! echo ",$with_services," | grep -q ",forgejo,"; then
|
||||
echo "Note: --with agents implies --with forgejo (agents need forge)"
|
||||
with_services="forgejo,${with_services}"
|
||||
fi
|
||||
if ! echo ",$with_services," | grep -q ",woodpecker-server,\|,woodpecker-agent,"; then
|
||||
echo "Note: --with agents implies --with woodpecker (agents need CI)"
|
||||
with_services="${with_services},woodpecker-server,woodpecker-agent"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Auto-include all dependencies when edge is requested (S5.5)
|
||||
if echo ",$with_services," | grep -q ",edge,"; then
|
||||
# Edge depends on all backend services
|
||||
for dep in forgejo woodpecker-server woodpecker-agent agents staging chat; do
|
||||
if ! echo ",$with_services," | grep -q ",${dep},"; then
|
||||
echo "Note: --with edge implies --with ${dep} (edge depends on all backend services)"
|
||||
with_services="${with_services},${dep}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Validate all service names are known
|
||||
local IFS=','
|
||||
for _svc in $with_services; do
|
||||
_svc=$(echo "$_svc" | xargs)
|
||||
case "$_svc" in
|
||||
forgejo|woodpecker-server|woodpecker-agent|agents|staging|chat|edge) ;;
|
||||
*)
|
||||
echo "Error: unknown service '${_svc}' — known: forgejo, woodpecker-server, woodpecker-agent, agents, staging, chat, edge" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
unset IFS
|
||||
fi
|
||||
|
||||
# --import-* flag validation (S2.5). These three flags form an import
|
||||
# triple and must be consistent before dispatch: sops encryption is
|
||||
# useless without the age key to decrypt it, so either both --import-sops
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Dev Agent
|
||||
|
||||
**Role**: Implement issues autonomously — write code, push branches, address
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ services:
|
|||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
|
|
@ -77,6 +78,7 @@ services:
|
|||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
|
|
@ -137,6 +139,7 @@ services:
|
|||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
|
|
|
|||
|
|
@ -1,26 +1,21 @@
|
|||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
bash curl git jq tmux nodejs npm python3 python3-pip openssh-client ca-certificates age shellcheck procps gosu \
|
||||
bash curl git jq tmux python3 python3-pip openssh-client ca-certificates age shellcheck procps gosu \
|
||||
&& pip3 install --break-system-packages networkx tomlkit \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Pre-built binaries (copied from docker/agents/bin/)
|
||||
# SOPS — encrypted data decryption tool
|
||||
# Download sops binary (replaces manual COPY of vendored binary)
|
||||
ARG SOPS_VERSION=3.9.4
|
||||
RUN curl -fsSL "https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64" \
|
||||
-o /usr/local/bin/sops && chmod +x /usr/local/bin/sops
|
||||
COPY docker/agents/bin/sops /usr/local/bin/sops
|
||||
RUN chmod +x /usr/local/bin/sops
|
||||
|
||||
# tea CLI — official Gitea/Forgejo CLI for issue/label/comment operations
|
||||
# Download tea binary (replaces manual COPY of vendored binary)
|
||||
ARG TEA_VERSION=0.9.2
|
||||
RUN curl -fsSL "https://dl.gitea.com/tea/${TEA_VERSION}/tea-${TEA_VERSION}-linux-amd64" \
|
||||
-o /usr/local/bin/tea && chmod +x /usr/local/bin/tea
|
||||
COPY docker/agents/bin/tea /usr/local/bin/tea
|
||||
RUN chmod +x /usr/local/bin/tea
|
||||
|
||||
# Install Claude Code CLI — agent runtime for all LLM backends (llama, Claude API).
|
||||
# The CLI is the execution environment; ANTHROPIC_BASE_URL selects the model provider.
|
||||
RUN npm install -g @anthropic-ai/claude-code@2.1.84
|
||||
# Claude CLI is mounted from the host via docker-compose volume.
|
||||
# No internet access to cli.anthropic.com required at build time.
|
||||
|
||||
# Non-root user
|
||||
RUN useradd -m -u 1000 -s /bin/bash agent
|
||||
|
|
|
|||
|
|
@ -1,22 +1,20 @@
|
|||
# disinto-chat — minimal HTTP backend for Claude chat UI
|
||||
#
|
||||
# Small Debian slim base with Python runtime and Node.js.
|
||||
# Small Debian slim base with Python runtime.
|
||||
# Chosen for simplicity and small image size (~100MB).
|
||||
#
|
||||
# Image size: ~100MB (well under the 200MB ceiling)
|
||||
#
|
||||
# Claude CLI is baked into the image — same pattern as the agents container.
|
||||
# The claude binary is mounted from the host at runtime via docker-compose,
|
||||
# not baked into the image — same pattern as the agents container.
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
# Install Node.js (required for Claude CLI) and Python
|
||||
# Install Python (no build-time network access needed)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
nodejs npm python3 \
|
||||
python3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Claude Code CLI — chat backend runtime
|
||||
RUN npm install -g @anthropic-ai/claude-code@2.1.84
|
||||
|
||||
# Non-root user — fixed UID 10001 for sandbox hardening (#706)
|
||||
RUN useradd -m -u 10001 -s /bin/bash chat
|
||||
|
||||
|
|
|
|||
|
|
@ -560,168 +560,10 @@ _launch_runner_docker() {
|
|||
|
||||
# _launch_runner_nomad ACTION_ID SECRETS_CSV MOUNTS_CSV
|
||||
#
|
||||
# Dispatches a vault-runner batch job via `nomad job dispatch`.
|
||||
# Polls `nomad job status` until terminal state (completed/failed).
|
||||
# Reads exit code from allocation and writes <action-id>.result.json.
|
||||
#
|
||||
# Usage: _launch_runner_nomad <action_id> <secrets_csv> <mounts_csv>
|
||||
# Returns: exit code of the nomad job (0=success, non-zero=failure)
|
||||
# Nomad backend stub — will be implemented in migration Step 5.
|
||||
_launch_runner_nomad() {
|
||||
local action_id="$1"
|
||||
local secrets_csv="$2"
|
||||
local mounts_csv="$3"
|
||||
|
||||
log "Dispatching vault-runner batch job via Nomad for action: ${action_id}"
|
||||
|
||||
# Dispatch the parameterized batch job
|
||||
# The vault-runner job expects meta: action_id, secrets_csv
|
||||
# Note: mounts_csv is not passed as meta (not declared in vault-runner.hcl)
|
||||
local dispatch_output
|
||||
dispatch_output=$(nomad job dispatch \
|
||||
-detach \
|
||||
-meta action_id="$action_id" \
|
||||
-meta secrets_csv="$secrets_csv" \
|
||||
vault-runner 2>&1) || {
|
||||
log "ERROR: Failed to dispatch vault-runner job for ${action_id}"
|
||||
log "Dispatch output: ${dispatch_output}"
|
||||
write_result "$action_id" 1 "Nomad dispatch failed: ${dispatch_output}"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Extract dispatched job ID from output (format: "vault-runner/dispatch-<timestamp>-<uuid>")
|
||||
local dispatched_job_id
|
||||
dispatched_job_id=$(echo "$dispatch_output" | grep -oP '(?<=Dispatched Job ID = ).+' || true)
|
||||
|
||||
if [ -z "$dispatched_job_id" ]; then
|
||||
log "ERROR: Could not extract dispatched job ID from nomad output"
|
||||
log "Dispatch output: ${dispatch_output}"
|
||||
write_result "$action_id" 1 "Could not extract dispatched job ID from nomad output"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Dispatched vault-runner with job ID: ${dispatched_job_id}"
|
||||
|
||||
# Poll job status until terminal state
|
||||
# Batch jobs transition: running -> completed/failed
|
||||
local max_wait=300 # 5 minutes max wait
|
||||
local elapsed=0
|
||||
local poll_interval=5
|
||||
local alloc_id=""
|
||||
|
||||
log "Polling nomad job status for ${dispatched_job_id}..."
|
||||
|
||||
while [ "$elapsed" -lt "$max_wait" ]; do
|
||||
# Get job status with JSON output for the dispatched child job
|
||||
local job_status_json
|
||||
job_status_json=$(nomad job status -json "$dispatched_job_id" 2>/dev/null) || {
|
||||
log "ERROR: Failed to get job status for ${dispatched_job_id}"
|
||||
write_result "$action_id" 1 "Failed to get job status for ${dispatched_job_id}"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Check job status field (transitions to "dead" on completion)
|
||||
local job_state
|
||||
job_state=$(echo "$job_status_json" | jq -r '.Status // empty' 2>/dev/null) || job_state=""
|
||||
|
||||
# Check allocation state directly
|
||||
alloc_id=$(echo "$job_status_json" | jq -r '.Allocations[0]?.ID // empty' 2>/dev/null) || alloc_id=""
|
||||
|
||||
if [ -n "$alloc_id" ]; then
|
||||
local alloc_state
|
||||
alloc_state=$(nomad alloc status -short "$alloc_id" 2>/dev/null || true)
|
||||
|
||||
case "$alloc_state" in
|
||||
*completed*|*success*|*dead*)
|
||||
log "Allocation ${alloc_id} reached terminal state: ${alloc_state}"
|
||||
break
|
||||
;;
|
||||
*running*|*pending*|*starting*)
|
||||
log "Allocation ${alloc_id} still running (state: ${alloc_state})..."
|
||||
;;
|
||||
*failed*|*crashed*)
|
||||
log "Allocation ${alloc_id} failed (state: ${alloc_state})"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Also check job-level state
|
||||
case "$job_state" in
|
||||
dead)
|
||||
log "Job ${dispatched_job_id} reached terminal state: ${job_state}"
|
||||
break
|
||||
;;
|
||||
failed)
|
||||
log "Job ${dispatched_job_id} failed"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
|
||||
sleep "$poll_interval"
|
||||
elapsed=$((elapsed + poll_interval))
|
||||
done
|
||||
|
||||
if [ "$elapsed" -ge "$max_wait" ]; then
|
||||
log "ERROR: Timeout waiting for vault-runner job to complete"
|
||||
write_result "$action_id" 1 "Timeout waiting for nomad job to complete"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get final job status and exit code
|
||||
local final_status_json
|
||||
final_status_json=$(nomad job status -json "$dispatched_job_id" 2>/dev/null) || {
|
||||
log "ERROR: Failed to get final job status"
|
||||
write_result "$action_id" 1 "Failed to get final job status"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Get allocation exit code
|
||||
local exit_code=0
|
||||
local logs=""
|
||||
|
||||
if [ -n "$alloc_id" ]; then
|
||||
# Get allocation logs
|
||||
logs=$(nomad alloc logs -short "$alloc_id" 2>/dev/null || true)
|
||||
|
||||
# Try to get exit code from alloc status JSON
|
||||
# Nomad alloc status -json has .TaskStates["<task_name>"].Events[].ExitCode
|
||||
local alloc_exit_code
|
||||
alloc_exit_code=$(nomad alloc status -json "$alloc_id" 2>/dev/null | jq -r '.TaskStates["runner"].Events[-1].ExitCode // empty' 2>/dev/null) || alloc_exit_code=""
|
||||
|
||||
if [ -n "$alloc_exit_code" ] && [ "$alloc_exit_code" != "null" ]; then
|
||||
exit_code="$alloc_exit_code"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If we couldn't get exit code from alloc, check job state as fallback
|
||||
# Note: "dead" = terminal state for batch jobs (includes successful completion)
|
||||
# Only "failed" indicates actual failure
|
||||
if [ "$exit_code" -eq 0 ]; then
|
||||
local final_state
|
||||
final_state=$(echo "$final_status_json" | jq -r '.Status // empty' 2>/dev/null) || final_state=""
|
||||
|
||||
case "$final_state" in
|
||||
failed)
|
||||
exit_code=1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Truncate logs if too long
|
||||
if [ ${#logs} -gt 1000 ]; then
|
||||
logs="${logs: -1000}"
|
||||
fi
|
||||
|
||||
# Write result file
|
||||
write_result "$action_id" "$exit_code" "$logs"
|
||||
|
||||
if [ "$exit_code" -eq 0 ]; then
|
||||
log "Vault-runner job completed successfully for action: ${action_id}"
|
||||
else
|
||||
log "Vault-runner job failed for action: ${action_id} (exit code: ${exit_code})"
|
||||
fi
|
||||
|
||||
return "$exit_code"
|
||||
echo "nomad backend not yet implemented" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Launch runner for the given action (backend-agnostic orchestrator)
|
||||
|
|
@ -1209,8 +1051,11 @@ main() {
|
|||
|
||||
# Validate backend selection at startup
|
||||
case "$DISPATCHER_BACKEND" in
|
||||
docker|nomad)
|
||||
log "Using ${DISPATCHER_BACKEND} backend for vault-runner dispatch"
|
||||
docker) ;;
|
||||
nomad)
|
||||
log "ERROR: nomad backend not yet implemented"
|
||||
echo "nomad backend not yet implemented" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
log "ERROR: unknown DISPATCHER_BACKEND=${DISPATCHER_BACKEND}"
|
||||
|
|
|
|||
|
|
@ -234,13 +234,6 @@ fi
|
|||
rm -f "$_fetch_log"
|
||||
done) &
|
||||
|
||||
# Nomad template renders Caddyfile to /local/Caddyfile via service discovery;
|
||||
# copy it into the expected location if present (compose uses the mounted path).
|
||||
if [ -f /local/Caddyfile ]; then
|
||||
cp /local/Caddyfile /etc/caddy/Caddyfile
|
||||
echo "edge: using Nomad-rendered Caddyfile from /local/Caddyfile" >&2
|
||||
fi
|
||||
|
||||
# Caddy as main process — run in foreground via wait so background jobs survive
|
||||
# (exec replaces the shell, which can orphan backgrounded subshells)
|
||||
caddy run --config /etc/caddy/Caddyfile --adapter caddyfile &
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Gardener Agent
|
||||
|
||||
**Role**: Backlog grooming — detect duplicate issues, missing acceptance
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"issue":915,"group":"lib/generators.sh","title":"remove no-op sed in generate_compose --build mode","reason":"sed replaces agents: with itself — no behavior change; single-line removal","ts":"2026-04-17T01:04:05Z"}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: 0bb04545d47fb43b2cab0a1f4406c2a2b57f4eba -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Shared Helpers (`lib/`)
|
||||
|
||||
All agents source `lib/env.sh` as their first action. Additional helpers are
|
||||
|
|
@ -30,9 +30,9 @@ sourced as needed.
|
|||
| `lib/git-creds.sh` | Shared git credential helper configuration. `configure_git_creds([HOME_DIR] [RUN_AS_CMD])` — writes a static credential helper script and configures git globally to use password-based HTTP auth (Forgejo 11.x rejects API tokens for `git push`, #361). **Retry on cold boot (#741)**: resolves bot username from `FORGE_TOKEN` with 5 retries (exponential backoff 1-5s); fails loudly and returns 1 if Forgejo is unreachable — never falls back to a wrong hardcoded default (exports `BOT_USER` on success). `repair_baked_cred_urls([--as RUN_AS_CMD] DIR ...)` — rewrites any git remote URLs that have credentials baked in to use clean URLs instead; uses `safe.directory` bypass for root-owned repos (#671). Requires `FORGE_PASS`, `FORGE_URL`, `FORGE_TOKEN`. | entrypoints (agents, edge) |
|
||||
| `lib/ops-setup.sh` | `setup_ops_repo()` — creates ops repo on Forgejo if it doesn't exist, configures bot collaborators, clones/initializes ops repo locally, seeds directory structure (vault, knowledge, evidence, sprints). Evidence subdirectories seeded: engagement/, red-team/, holdout/, evolution/, user-test/. Also seeds sprints/ for architect output. Exports `_ACTUAL_OPS_SLUG`. `migrate_ops_repo(ops_root, [primary_branch])` — idempotent migration helper that seeds missing directories and .gitkeep files on existing ops repos (pre-#407 deployments). | bin/disinto (init) |
|
||||
| `lib/ci-setup.sh` | `_install_cron_impl()` — installs crontab entries for bare-metal deployments (compose mode uses polling loop instead). `_create_forgejo_oauth_app()` — generic helper to create an OAuth2 app on Forgejo (shared by Woodpecker and chat). `_create_woodpecker_oauth_impl()` — creates Woodpecker OAuth2 app (thin wrapper). `_create_chat_oauth_impl()` — creates disinto-chat OAuth2 app, writes `CHAT_OAUTH_CLIENT_ID`/`CHAT_OAUTH_CLIENT_SECRET` to `.env` (#708). `_generate_woodpecker_token_impl()` — auto-generates WOODPECKER_TOKEN via OAuth2 flow. `_activate_woodpecker_repo_impl()` — activates repo in Woodpecker. All gated by `_load_ci_context()` which validates required env vars. | bin/disinto (init) |
|
||||
| `lib/generators.sh` | Template generation for `disinto init`: `generate_compose()` — docker-compose.yml (uses `codeberg.org/forgejo/forgejo:11.0` tag; `CLAUDE_BIN_DIR` volume mount removed from agents/llama services — only `reproduce` and `edge` still use the host-mounted CLI (#992); adds `security_opt: [apparmor:unconfined]` to all services for rootless container compatibility; Forgejo includes a healthcheck so dependent services use `condition: service_healthy` — fixes cold-start races, #665; adds `chat` service block with isolated `chat-config` named volume and `CHAT_HISTORY_DIR` bind-mount for per-user NDJSON history persistence (#710); injects `FORWARD_AUTH_SECRET` for Caddy↔chat defense-in-depth auth (#709); cost-cap env vars `CHAT_MAX_REQUESTS_PER_HOUR`, `CHAT_MAX_REQUESTS_PER_DAY`, `CHAT_MAX_TOKENS_PER_DAY` (#711); subdomain fallback comment for `EDGE_TUNNEL_FQDN_*` vars (#713); all `depends_on` now use `condition: service_healthy/started` instead of bare service names; all services now include `restart: unless-stopped` including the edge service — #768; agents service now uses `image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}` instead of `build:` (#429); `WOODPECKER_PLUGINS_PRIVILEGED` env var added to woodpecker service (#779); agents-llama conditional block gated on `ENABLE_LLAMA_AGENT=1` (#769); `agents-llama-all` compose service (profile `agents-llama-all`, all 7 roles: review,dev,gardener,architect,planner,predictor,supervisor) added by #801; agents service gains volume mounts for `./projects`, `./.env`, `./state`), `generate_caddyfile()` — Caddyfile (routes: `/forge/*` → forgejo:3000, `/woodpecker/*` → woodpecker:8000, `/staging/*` → staging:80; `/chat/login` and `/chat/oauth/callback` bypass `forward_auth` so unauthenticated users can reach the OAuth flow; `/chat/*` gated by `forward_auth` on `chat:8080/chat/auth/verify` which stamps `X-Forwarded-User` (#709); root `/` redirects to `/forge/`), `generate_staging_index()` — staging index, `generate_deploy_pipelines()` — Woodpecker deployment pipeline configs. Requires `FACTORY_ROOT`, `PROJECT_NAME`, `PRIMARY_BRANCH`. | bin/disinto (init) |
|
||||
| `lib/generators.sh` | Template generation for `disinto init`: `generate_compose()` — docker-compose.yml (uses `codeberg.org/forgejo/forgejo:11.0` tag; adds `security_opt: [apparmor:unconfined]` to all services for rootless container compatibility; Forgejo includes a healthcheck so dependent services use `condition: service_healthy` — fixes cold-start races, #665; adds `chat` service block with isolated `chat-config` named volume and `CHAT_HISTORY_DIR` bind-mount for per-user NDJSON history persistence (#710); injects `FORWARD_AUTH_SECRET` for Caddy↔chat defense-in-depth auth (#709); cost-cap env vars `CHAT_MAX_REQUESTS_PER_HOUR`, `CHAT_MAX_REQUESTS_PER_DAY`, `CHAT_MAX_TOKENS_PER_DAY` (#711); subdomain fallback comment for `EDGE_TUNNEL_FQDN_*` vars (#713); all `depends_on` now use `condition: service_healthy/started` instead of bare service names; all services now include `restart: unless-stopped` including the edge service — #768; agents service now uses `image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}` instead of `build:` (#429); `WOODPECKER_PLUGINS_PRIVILEGED` env var added to woodpecker service (#779); agents-llama conditional block gated on `ENABLE_LLAMA_AGENT=1` (#769); `agents-llama-all` compose service (profile `agents-llama-all`, all 7 roles: review,dev,gardener,architect,planner,predictor,supervisor) added by #801; agents service gains volume mounts for `./projects`, `./.env`, `./state`), `generate_caddyfile()` — Caddyfile (routes: `/forge/*` → forgejo:3000, `/woodpecker/*` → woodpecker:8000, `/staging/*` → staging:80; `/chat/login` and `/chat/oauth/callback` bypass `forward_auth` so unauthenticated users can reach the OAuth flow; `/chat/*` gated by `forward_auth` on `chat:8080/chat/auth/verify` which stamps `X-Forwarded-User` (#709); root `/` redirects to `/forge/`), `generate_staging_index()` — staging index, `generate_deploy_pipelines()` — Woodpecker deployment pipeline configs. Requires `FACTORY_ROOT`, `PROJECT_NAME`, `PRIMARY_BRANCH`. | bin/disinto (init) |
|
||||
| `lib/sprint-filer.sh` | Post-merge sub-issue filer for sprint PRs. Invoked by the `.woodpecker/ops-filer.yml` pipeline after a sprint PR merges to ops repo `main`. Parses `<!-- filer:begin --> ... <!-- filer:end -->` blocks from sprint PR bodies to extract sub-issue definitions, creates them on the project repo using `FORGE_FILER_TOKEN` (narrow-scope `filer-bot` identity with `issues:write` only), adds `in-progress` label to the parent vision issue, and handles vision lifecycle closure when all sub-issues are closed. Uses `filer_api_all()` for paginated fetches. Idempotent: uses `<!-- decomposed-from: #<vision>, sprint: <slug>, id: <id> -->` markers to skip already-filed issues. Requires `FORGE_FILER_TOKEN`, `FORGE_API`, `FORGE_API_BASE`, `FORGE_OPS_REPO`. | `.woodpecker/ops-filer.yml` (CI pipeline on ops repo) |
|
||||
| `lib/hire-agent.sh` | `disinto_hire_an_agent()` — user creation, `.profile` repo setup, formula copying, branch protection, and state marker creation for hiring a new agent. Requires `FORGE_URL`, `FORGE_TOKEN`, `FACTORY_ROOT`, `PROJECT_NAME`. Extracted from `bin/disinto`. | bin/disinto (hire) |
|
||||
| `lib/release.sh` | `disinto_release()` — vault TOML creation, branch setup on ops repo, PR creation, and auto-merge request for a versioned release. `_assert_release_globals()` validates required env vars. Requires `FORGE_URL`, `FORGE_TOKEN`, `FORGE_OPS_REPO`, `FACTORY_ROOT`, `PRIMARY_BRANCH`. Extracted from `bin/disinto`. | bin/disinto (release) |
|
||||
| `lib/hvault.sh` | HashiCorp Vault helper module. `hvault_kv_get(PATH, [KEY])` — read KV v2 secret, optionally extract one key. `hvault_kv_put(PATH, KEY=VAL ...)` — write KV v2 secret. `hvault_kv_list(PATH)` — list keys at a KV path. `hvault_get_or_empty(PATH)` — GET /v1/PATH; 200→raw body, 404→empty, else structured error + return 1 (used by sync scripts to distinguish "absent, create" from hard failure without tripping errexit, #881). `hvault_ensure_kv_v2(MOUNT, [LOG_PREFIX])` — idempotent KV v2 mount assertion: enables mount if absent, fails loudly if present as wrong type/version. Extracted from all `vault-seed-*.sh` scripts to eliminate dup-detector violations. Respects `DRY_RUN=1`. `hvault_policy_apply(NAME, FILE)` — idempotent policy upsert. `hvault_jwt_login(ROLE, JWT)` — exchange JWT for short-lived token. `hvault_token_lookup()` — returns TTL/policies/accessor for current token. `_hvault_seed_key(PATH, KEY, [GENERATOR])` — seed one KV key if absent; reads existing data and merges to preserve sibling keys (KV v2 replaces atomically); returns 0=created, 1=unchanged, 2=API error (#992). All functions use `VAULT_ADDR` + `VAULT_TOKEN` from env (fallback: `/etc/vault.d/root.token`), emit structured JSON errors to stderr on failure. Tests: `tests/lib-hvault.bats` (requires `vault server -dev`). | `tools/vault-apply-policies.sh`, `tools/vault-apply-roles.sh`, `lib/init/nomad/vault-nomad-auth.sh`, `tools/vault-seed-*.sh` |
|
||||
| `lib/init/nomad/` | Nomad+Vault installer scripts. `cluster-up.sh` — idempotent Step-0 orchestrator that runs all steps in order (installs packages, writes HCL, enables systemd units, unseals Vault); uses `poll_until_healthy()` helper for deduped readiness polling; `HOST_VOLUME_DIRS` array now includes `/srv/disinto/docker` (for staging file-server, S5.2, #989, #992). `install.sh` — installs pinned Nomad+Vault apt packages. `vault-init.sh` — initializes Vault (unseal keys → `/etc/vault.d/`), creates dev-persisted unseal unit. `lib-systemd.sh` — shared systemd unit helpers. `systemd-nomad.sh`, `systemd-vault.sh` — write and enable service units. `vault-nomad-auth.sh` — Step-2 script that enables Vault's JWT auth at path `jwt-nomad`, writes the JWKS/algs config pointing at Nomad's workload-identity signer, delegates role sync to `tools/vault-apply-roles.sh`, installs `/etc/nomad.d/server.hcl`, and SIGHUPs `nomad.service` if the file changed (#881). `wp-oauth-register.sh` — S3.3 script that creates the Woodpecker OAuth2 app in Forgejo and stores `forgejo_client`/`forgejo_secret` in Vault KV v2 at `kv/disinto/shared/woodpecker`; idempotent (skips if app or secrets already present); called by `bin/disinto --with woodpecker`. `deploy.sh` — S4 dependency-ordered Nomad job deploy + health-wait; takes a list of jobspec basenames, submits each to Nomad and polls until healthy before proceeding to the next; supports `--dry-run` and per-job timeout overrides via `JOB_READY_TIMEOUT_<JOBNAME>`; global default timeout `JOB_READY_TIMEOUT_SECS` is 360s (raised from 240s for chat cold-start, #1036); invoked by `bin/disinto --with <svc>` and `cluster-up.sh`; deploy order now covers staging, chat, edge (S5.5, #992). Idempotent: each step checks current state before acting. Sourced and called by `cluster-up.sh`; not sourced by agents. | `bin/disinto init --backend=nomad` |
|
||||
| `lib/hvault.sh` | HashiCorp Vault helper module. `hvault_kv_get(PATH, [KEY])` — read KV v2 secret, optionally extract one key. `hvault_kv_put(PATH, KEY=VAL ...)` — write KV v2 secret. `hvault_kv_list(PATH)` — list keys at a KV path. `hvault_get_or_empty(PATH)` — GET /v1/PATH; 200→raw body, 404→empty, else structured error + return 1 (used by sync scripts to distinguish "absent, create" from hard failure without tripping errexit, #881). `hvault_policy_apply(NAME, FILE)` — idempotent policy upsert. `hvault_jwt_login(ROLE, JWT)` — exchange JWT for short-lived token. `hvault_token_lookup()` — returns TTL/policies/accessor for current token. All functions use `VAULT_ADDR` + `VAULT_TOKEN` from env (fallback: `/etc/vault.d/root.token`), emit structured JSON errors to stderr on failure. Tests: `tests/lib-hvault.bats` (requires `vault server -dev`). | `tools/vault-apply-policies.sh`, `tools/vault-apply-roles.sh`, `lib/init/nomad/vault-nomad-auth.sh` |
|
||||
| `lib/init/nomad/` | Nomad+Vault installer scripts. `cluster-up.sh` — idempotent Step-0 orchestrator that runs all steps in order (installs packages, writes HCL, enables systemd units, unseals Vault); uses `poll_until_healthy()` helper for deduped readiness polling. `install.sh` — installs pinned Nomad+Vault apt packages. `vault-init.sh` — initializes Vault (unseal keys → `/etc/vault.d/`), creates dev-persisted unseal unit. `lib-systemd.sh` — shared systemd unit helpers. `systemd-nomad.sh`, `systemd-vault.sh` — write and enable service units. `vault-nomad-auth.sh` — Step-2 script that enables Vault's JWT auth at path `jwt-nomad`, writes the JWKS/algs config pointing at Nomad's workload-identity signer, delegates role sync to `tools/vault-apply-roles.sh`, installs `/etc/nomad.d/server.hcl`, and SIGHUPs `nomad.service` if the file changed (#881). Idempotent: each step checks current state before acting. Sourced and called by `cluster-up.sh`; not sourced by agents. | `bin/disinto init --backend=nomad` |
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ _generate_local_model_services() {
|
|||
- project-repos-${service_name}:/home/agent/repos
|
||||
- \${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:\${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- \${CLAUDE_CONFIG_FILE:-\${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- \${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- \${AGENT_SSH_DIR:-\${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ./projects:/home/agent/disinto/projects:ro
|
||||
- ./.env:/home/agent/disinto/.env:ro
|
||||
|
|
@ -381,6 +382,7 @@ services:
|
|||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
|
|
@ -634,13 +636,13 @@ COMPOSEEOF
|
|||
_generate_local_model_services "$compose_file"
|
||||
|
||||
# Resolve the Claude CLI binary path and persist as CLAUDE_BIN_DIR in .env.
|
||||
# Only used by reproduce and edge services which still use host-mounted CLI.
|
||||
# docker-compose.yml references ${CLAUDE_BIN_DIR} so the value must be set.
|
||||
local claude_bin
|
||||
claude_bin="$(command -v claude 2>/dev/null || true)"
|
||||
if [ -n "$claude_bin" ]; then
|
||||
claude_bin="$(readlink -f "$claude_bin")"
|
||||
else
|
||||
echo "Warning: claude CLI not found in PATH — reproduce/edge services will fail to start" >&2
|
||||
echo "Warning: claude CLI not found in PATH — set CLAUDE_BIN_DIR in .env manually" >&2
|
||||
claude_bin="/usr/local/bin/claude"
|
||||
fi
|
||||
# Persist CLAUDE_BIN_DIR into .env so docker-compose can resolve it.
|
||||
|
|
@ -657,6 +659,7 @@ COMPOSEEOF
|
|||
|
||||
# In build mode, replace image: with build: for locally-built images
|
||||
if [ "$use_build" = true ]; then
|
||||
sed -i 's|^\( agents:\)|\1|' "$compose_file"
|
||||
sed -i '/^ image: ghcr\.io\/disinto\/agents:/{s|image: ghcr\.io/disinto/agents:.*|build:\n context: .\n dockerfile: docker/agents/Dockerfile\n pull_policy: build|}' "$compose_file"
|
||||
sed -i '/^ image: ghcr\.io\/disinto\/edge:/{s|image: ghcr\.io/disinto/edge:.*|build: ./docker/edge\n pull_policy: build|}' "$compose_file"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -405,36 +405,3 @@ hvault_token_lookup() {
|
|||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
# _hvault_seed_key — Seed a single KV key if it doesn't exist.
|
||||
# Reads existing data and merges to preserve sibling keys (KV v2 replaces
|
||||
# .data atomically). Returns 0=created, 1=unchanged, 2=API error.
|
||||
# Args:
|
||||
# path: KV v2 logical path (e.g. "disinto/shared/chat")
|
||||
# key: key name within the path (e.g. "chat_oauth_client_id")
|
||||
# generator: shell command that outputs a random value (default: openssl rand -hex 32)
|
||||
# Usage:
|
||||
# _hvault_seed_key "disinto/shared/chat" "chat_oauth_client_id"
|
||||
# rc=$? # 0=created, 1=unchanged
|
||||
_hvault_seed_key() {
|
||||
local path="$1" key="$2" generator="${3:-openssl rand -hex 32}"
|
||||
local existing
|
||||
existing=$(hvault_kv_get "$path" "$key" 2>/dev/null) || true
|
||||
if [ -n "$existing" ]; then
|
||||
return 1 # unchanged
|
||||
fi
|
||||
|
||||
local value
|
||||
value=$(eval "$generator")
|
||||
|
||||
# Read existing data to preserve sibling keys (KV v2 replaces atomically)
|
||||
local kv_api="${VAULT_KV_MOUNT}/data/${path}"
|
||||
local raw existing_data payload
|
||||
raw="$(hvault_get_or_empty "$kv_api")" || return 2
|
||||
existing_data="{}"
|
||||
[ -n "$raw" ] && existing_data="$(printf '%s' "$raw" | jq '.data.data // {}')"
|
||||
payload="$(printf '%s' "$existing_data" \
|
||||
| jq --arg k "$key" --arg v "$value" '{data: (. + {($k): $v})}')"
|
||||
_hvault_request POST "$kv_api" "$payload" >/dev/null
|
||||
return 0 # created
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,7 +66,6 @@ HOST_VOLUME_DIRS=(
|
|||
"/srv/disinto/agent-data"
|
||||
"/srv/disinto/project-repos"
|
||||
"/srv/disinto/caddy-data"
|
||||
"/srv/disinto/docker"
|
||||
"/srv/disinto/chat-history"
|
||||
"/srv/disinto/ops-repo"
|
||||
)
|
||||
|
|
@ -117,7 +116,7 @@ if [ "$dry_run" = true ]; then
|
|||
[dry-run] Step 4/9: create host-volume dirs under /srv/disinto/
|
||||
EOF
|
||||
for d in "${HOST_VOLUME_DIRS[@]}"; do
|
||||
printf ' → install -d -m 0777 %s\n' "$d"
|
||||
printf ' → install -d -m 0755 %s\n' "$d"
|
||||
done
|
||||
cat <<EOF
|
||||
|
||||
|
|
@ -281,10 +280,8 @@ for d in "${HOST_VOLUME_DIRS[@]}"; do
|
|||
log "unchanged: ${d}"
|
||||
else
|
||||
log "creating: ${d}"
|
||||
install -d -m 0777 -o root -g root "$d"
|
||||
install -d -m 0755 -o root -g root "$d"
|
||||
fi
|
||||
# Ensure correct permissions (fixes pre-existing 0755 dirs on re-run)
|
||||
chmod 0777 "$d"
|
||||
done
|
||||
|
||||
# ── Step 5/9: /etc/nomad.d/server.hcl + client.hcl ───────────────────────────
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
# Environment:
|
||||
# REPO_ROOT — absolute path to repo root (defaults to parent of
|
||||
# this script's parent directory)
|
||||
# JOB_READY_TIMEOUT_SECS — poll timeout in seconds (default: 360)
|
||||
# JOB_READY_TIMEOUT_SECS — poll timeout in seconds (default: 240)
|
||||
# JOB_READY_TIMEOUT_<JOBNAME> — per-job timeout override (e.g.,
|
||||
# JOB_READY_TIMEOUT_FORGEJO=300)
|
||||
#
|
||||
|
|
@ -33,7 +33,7 @@ set -euo pipefail
|
|||
# ── Configuration ────────────────────────────────────────────────────────────
|
||||
SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="${REPO_ROOT:-$(cd "${SCRIPT_ROOT}/../../.." && pwd)}"
|
||||
JOB_READY_TIMEOUT_SECS="${JOB_READY_TIMEOUT_SECS:-360}"
|
||||
JOB_READY_TIMEOUT_SECS="${JOB_READY_TIMEOUT_SECS:-240}"
|
||||
|
||||
DRY_RUN=0
|
||||
|
||||
|
|
@ -177,8 +177,7 @@ for job_name in "${JOBS[@]}"; do
|
|||
fi
|
||||
|
||||
# Per-job timeout override: JOB_READY_TIMEOUT_<UPPERCASE_JOBNAME>
|
||||
# Sanitize job name: replace hyphens with underscores (bash vars can't have hyphens)
|
||||
job_upper=$(printf '%s' "$job_name" | tr '[:lower:]-' '[:upper:]_' | tr ' ' '_')
|
||||
job_upper=$(printf '%s' "$job_name" | tr '[:lower:]' '[:upper:]')
|
||||
timeout_var="JOB_READY_TIMEOUT_${job_upper}"
|
||||
job_timeout="${!timeout_var:-$JOB_READY_TIMEOUT_SECS}"
|
||||
|
||||
|
|
|
|||
|
|
@ -43,8 +43,8 @@ set -euo pipefail
|
|||
|
||||
# Source the hvault module for Vault helpers
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
# shellcheck source=../../../lib/hvault.sh
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
# shellcheck source=../../lib/hvault.sh
|
||||
source "${REPO_ROOT}/lib/hvault.sh"
|
||||
|
||||
# Configuration
|
||||
|
|
|
|||
|
|
@ -1,27 +1,21 @@
|
|||
<!-- last-reviewed: 0bb04545d47fb43b2cab0a1f4406c2a2b57f4eba -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# nomad/ — Agent Instructions
|
||||
|
||||
Nomad + Vault HCL for the factory's single-node cluster. These files are
|
||||
the source of truth that `lib/init/nomad/cluster-up.sh` copies onto a
|
||||
factory box under `/etc/nomad.d/` and `/etc/vault.d/` at init time.
|
||||
|
||||
This directory covers the **Nomad+Vault migration (Steps 0–5)** —
|
||||
see issues #821–#992 for the step breakdown.
|
||||
This directory covers the **Nomad+Vault migration (Steps 0–2)** —
|
||||
see issues #821–#884 for the step breakdown.
|
||||
|
||||
## What lives here
|
||||
|
||||
| File/Dir | Deployed to | Owned by |
|
||||
|---|---|---|
|
||||
| `server.hcl` | `/etc/nomad.d/server.hcl` | agent role, bind, ports, `data_dir` (S0.2) |
|
||||
| `client.hcl` | `/etc/nomad.d/client.hcl` | Docker driver cfg + `host_volume` declarations (S0.2); `allow_privileged = true` for woodpecker-agent Docker-in-Docker (S3-fix-5, #961) |
|
||||
| `client.hcl` | `/etc/nomad.d/client.hcl` | Docker driver cfg + `host_volume` declarations (S0.2) |
|
||||
| `vault.hcl` | `/etc/vault.d/vault.hcl` | Vault storage, listener, UI, `disable_mlock` (S0.3) |
|
||||
| `jobs/forgejo.hcl` | submitted via `lib/init/nomad/deploy.sh` | Forgejo job; reads creds from Vault via consul-template stanza (S2.4) |
|
||||
| `jobs/woodpecker-server.hcl` | submitted via `lib/init/nomad/deploy.sh` | Woodpecker CI server; host networking, Vault KV for `WOODPECKER_AGENT_SECRET` + Forgejo OAuth creds (S3.1) |
|
||||
| `jobs/woodpecker-agent.hcl` | submitted via `lib/init/nomad/deploy.sh` | Woodpecker CI agent; host networking, `docker.sock` mount, Vault KV for `WOODPECKER_AGENT_SECRET`; `WOODPECKER_SERVER` uses `${attr.unique.network.ip-address}:9000` (Nomad interpolation) — port binds to LXC alloc IP, not localhost (S3.2, S3-fix-6, #964) |
|
||||
| `jobs/agents.hcl` | submitted via `lib/init/nomad/deploy.sh` | All 7 agent roles (dev, review, gardener, planner, predictor, supervisor, architect) + llama variant; Vault-templated bot tokens via `service-agents` policy; `force_pull = false` — image is built locally by `bin/disinto --with agents`, no registry (S4.1, S4-fix-2, S4-fix-5, #955, #972, #978) |
|
||||
| `jobs/staging.hcl` | submitted via `lib/init/nomad/deploy.sh` | Caddy file-server mounting `docker/` as `/srv/site:ro`; no Vault integration; **dynamic host port** (no static 80 — edge owns 80/443, collision fixed in S5-fix-7 #1018); edge discovers via Nomad service registration (S5.2, #989) |
|
||||
| `jobs/chat.hcl` | submitted via `lib/init/nomad/deploy.sh` | Claude chat UI; custom `disinto/chat:local` image; sandbox hardening (cap_drop ALL, **tmpfs via mount block** not `tmpfs=` arg — S5-fix-5 #1012, pids_limit 128); Vault-templated OAuth secrets via `service-chat` policy (S5.2, #989) |
|
||||
| `jobs/edge.hcl` | submitted via `lib/init/nomad/deploy.sh` | Caddy reverse proxy + dispatcher sidecar; routes /forge, /woodpecker, /staging, /chat; uses `disinto/edge:local` image built by `bin/disinto --with edge`; **both Caddy and dispatcher tasks use `network_mode = "host"`** — upstreams are `127.0.0.1:<port>` (forgejo :3000, woodpecker :8000, chat :8080), not Docker hostnames (#1031, #1034); `FORGE_URL` rendered via Nomad service discovery template (not static env) to handle bridge vs. host network differences (#1034); dispatcher Vault secret path changed to `kv/data/disinto/shared/ops-repo` (#1041); Vault-templated ops-repo creds via `service-dispatcher` policy (S5.1, #988) |
|
||||
|
||||
Nomad auto-merges every `*.hcl` under `-config=/etc/nomad.d/`, so the
|
||||
split between `server.hcl` and `client.hcl` is for readability, not
|
||||
|
|
@ -36,6 +30,8 @@ convention, KV path summary, and JWT-auth role bindings (S2.1/S2.3).
|
|||
|
||||
## Not yet implemented
|
||||
|
||||
- **Additional jobspecs** (woodpecker, agents, caddy) — Step 1 brought up
|
||||
Forgejo; remaining services land in later steps.
|
||||
- **TLS, ACLs, gossip encryption** — deliberately absent for now; land
|
||||
alongside multi-node support.
|
||||
|
||||
|
|
|
|||
|
|
@ -49,12 +49,6 @@ client {
|
|||
read_only = false
|
||||
}
|
||||
|
||||
# staging static content (docker/ directory with images, HTML, etc.)
|
||||
host_volume "site-content" {
|
||||
path = "/srv/disinto/docker"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
# disinto chat transcripts + attachments.
|
||||
host_volume "chat-history" {
|
||||
path = "/srv/disinto/chat-history"
|
||||
|
|
@ -70,11 +64,11 @@ client {
|
|||
|
||||
# Docker task driver. `volumes.enabled = true` is required so jobspecs
|
||||
# can mount host_volume declarations defined above. `allow_privileged`
|
||||
# is true — woodpecker-agent requires `privileged = true` to access
|
||||
# docker.sock and spawn CI pipeline containers.
|
||||
# stays false — no factory workload needs privileged containers today,
|
||||
# and flipping it is an audit-worthy change.
|
||||
plugin "docker" {
|
||||
config {
|
||||
allow_privileged = true
|
||||
allow_privileged = false
|
||||
|
||||
volumes {
|
||||
enabled = true
|
||||
|
|
|
|||
|
|
@ -1,207 +0,0 @@
|
|||
# =============================================================================
|
||||
# nomad/jobs/agents.hcl — All-role agent polling loop (Nomad service job)
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S4.1, issue #955). Runs the main bot
|
||||
# polling loop with all 7 agent roles (review, dev, gardener, architect,
|
||||
# planner, predictor, supervisor) against the local llama server.
|
||||
#
|
||||
# Host_volume contract:
|
||||
# This job mounts agent-data, project-repos, and ops-repo from
|
||||
# nomad/client.hcl. Paths under /srv/disinto/* are created by
|
||||
# lib/init/nomad/cluster-up.sh before any job references them.
|
||||
#
|
||||
# Vault integration (S4.1):
|
||||
# - vault { role = "service-agents" } at group scope — workload-identity
|
||||
# JWT exchanged for a Vault token carrying the composite service-agents
|
||||
# policy (vault/policies/service-agents.hcl), which grants read access
|
||||
# to all 7 bot KV namespaces + vault bot + shared forge config.
|
||||
# - template stanza renders per-bot FORGE_*_TOKEN + FORGE_PASS from Vault
|
||||
# KV v2 at kv/disinto/bots/<role>.
|
||||
# - Seeded on fresh boxes by tools/vault-seed-agents.sh.
|
||||
#
|
||||
# Not the runtime yet: docker-compose.yml is still the factory's live stack
|
||||
# until cutover. This file exists so CI can validate it and S4.2 can wire
|
||||
# `disinto init --backend=nomad --with agents` to `nomad job run` it.
|
||||
# =============================================================================
|
||||
|
||||
job "agents" {
|
||||
type = "service"
|
||||
datacenters = ["dc1"]
|
||||
|
||||
group "agents" {
|
||||
count = 1
|
||||
|
||||
# ── Vault workload identity (S4.1, issue #955) ───────────────────────────
|
||||
# Composite role covering all 7 bot identities + vault bot. Role defined
|
||||
# in vault/roles.yaml, policy in vault/policies/service-agents.hcl.
|
||||
# Bound claim pins nomad_job_id = "agents".
|
||||
vault {
|
||||
role = "service-agents"
|
||||
}
|
||||
|
||||
# No network port — agents are outbound-only (poll forgejo, call llama).
|
||||
# No service discovery block — nothing health-checks agents over HTTP.
|
||||
|
||||
volume "agent-data" {
|
||||
type = "host"
|
||||
source = "agent-data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
volume "project-repos" {
|
||||
type = "host"
|
||||
source = "project-repos"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
volume "ops-repo" {
|
||||
type = "host"
|
||||
source = "ops-repo"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
# Conservative restart — fail fast to the scheduler.
|
||||
restart {
|
||||
attempts = 3
|
||||
interval = "5m"
|
||||
delay = "15s"
|
||||
mode = "delay"
|
||||
}
|
||||
|
||||
# ── Service registration ────────────────────────────────────────────────
|
||||
# Agents are outbound-only (poll forgejo, call llama) — no HTTP/TCP
|
||||
# endpoint to probe. The Nomad native provider only supports tcp/http
|
||||
# checks, not script checks. Registering without a check block means
|
||||
# Nomad tracks health via task lifecycle: task running = healthy,
|
||||
# task dead = service deregistered. This matches the docker-compose
|
||||
# pgrep healthcheck semantics (process alive = healthy).
|
||||
service {
|
||||
name = "agents"
|
||||
provider = "nomad"
|
||||
}
|
||||
|
||||
task "agents" {
|
||||
driver = "docker"
|
||||
|
||||
config {
|
||||
image = "disinto/agents:local"
|
||||
force_pull = false
|
||||
|
||||
# apparmor=unconfined matches docker-compose — Claude Code needs
|
||||
# ptrace for node.js inspector and /proc access.
|
||||
security_opt = ["apparmor=unconfined"]
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
volume = "agent-data"
|
||||
destination = "/home/agent/data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
volume = "project-repos"
|
||||
destination = "/home/agent/repos"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
volume = "ops-repo"
|
||||
destination = "/home/agent/repos/_factory/disinto-ops"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
# ── Non-secret env ─────────────────────────────────────────────────────
|
||||
env {
|
||||
FORGE_URL = "http://forgejo:3000"
|
||||
FORGE_REPO = "disinto-admin/disinto"
|
||||
ANTHROPIC_BASE_URL = "http://10.10.10.1:8081"
|
||||
ANTHROPIC_API_KEY = "sk-no-key-required"
|
||||
CLAUDE_MODEL = "unsloth/Qwen3.5-35B-A3B"
|
||||
AGENT_ROLES = "review,dev,gardener,architect,planner,predictor,supervisor"
|
||||
POLL_INTERVAL = "300"
|
||||
DISINTO_CONTAINER = "1"
|
||||
PROJECT_NAME = "project"
|
||||
PROJECT_REPO_ROOT = "/home/agent/repos/project"
|
||||
CLAUDE_TIMEOUT = "7200"
|
||||
|
||||
# llama-specific Claude Code tuning
|
||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC = "1"
|
||||
CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS = "1"
|
||||
CLAUDE_AUTOCOMPACT_PCT_OVERRIDE = "60"
|
||||
}
|
||||
|
||||
# ── Vault-templated bot tokens (S4.1, issue #955) ─────────────────────
|
||||
# Renders per-bot FORGE_*_TOKEN + FORGE_PASS from Vault KV v2.
|
||||
# Each `with secret ...` block reads one bot's KV path; the `else`
|
||||
# branch emits short placeholders on fresh installs where the path
|
||||
# is absent. Seed with tools/vault-seed-agents.sh.
|
||||
#
|
||||
# Placeholder values kept < 16 chars to avoid secret-scan CI failures.
|
||||
# error_on_missing_key = false prevents template-pending hangs.
|
||||
template {
|
||||
destination = "secrets/bots.env"
|
||||
env = true
|
||||
change_mode = "restart"
|
||||
error_on_missing_key = false
|
||||
data = <<EOT
|
||||
{{- with secret "kv/data/disinto/bots/dev" -}}
|
||||
FORGE_TOKEN={{ .Data.data.token }}
|
||||
FORGE_PASS={{ .Data.data.pass }}
|
||||
{{- else -}}
|
||||
# WARNING: run tools/vault-seed-agents.sh
|
||||
FORGE_TOKEN=seed-me
|
||||
FORGE_PASS=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/review" -}}
|
||||
FORGE_REVIEW_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_REVIEW_TOKEN=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/gardener" -}}
|
||||
FORGE_GARDENER_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_GARDENER_TOKEN=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/architect" -}}
|
||||
FORGE_ARCHITECT_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_ARCHITECT_TOKEN=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/planner" -}}
|
||||
FORGE_PLANNER_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_PLANNER_TOKEN=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/predictor" -}}
|
||||
FORGE_PREDICTOR_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_PREDICTOR_TOKEN=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/supervisor" -}}
|
||||
FORGE_SUPERVISOR_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_SUPERVISOR_TOKEN=seed-me
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/bots/vault" -}}
|
||||
FORGE_VAULT_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
FORGE_VAULT_TOKEN=seed-me
|
||||
{{- end }}
|
||||
EOT
|
||||
}
|
||||
|
||||
# Agents run Claude/llama sessions — need CPU + memory headroom.
|
||||
resources {
|
||||
cpu = 500
|
||||
memory = 1024
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
# =============================================================================
|
||||
# nomad/jobs/chat.hcl — Claude chat UI (Nomad service job)
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S5.2, issue #989). Lightweight service
|
||||
# job for the Claude chat UI with sandbox hardening (#706).
|
||||
#
|
||||
# Build:
|
||||
# Custom image built from docker/chat/Dockerfile as disinto/chat:local
|
||||
# (same :local pattern as disinto/agents:local).
|
||||
#
|
||||
# Sandbox hardening (#706):
|
||||
# - Read-only root filesystem (enforced via entrypoint)
|
||||
# - tmpfs /tmp:size=64m for runtime temp files
|
||||
# - cap_drop ALL (no Linux capabilities)
|
||||
# - pids_limit 128 (prevent fork bombs)
|
||||
# - mem_limit 512m (matches compose sandbox hardening)
|
||||
#
|
||||
# Vault integration:
|
||||
# - vault { role = "service-chat" } at group scope
|
||||
# - Template stanza renders CHAT_OAUTH_CLIENT_ID, CHAT_OAUTH_CLIENT_SECRET,
|
||||
# FORWARD_AUTH_SECRET from kv/disinto/shared/chat
|
||||
# - Seeded on fresh boxes by tools/vault-seed-chat.sh
|
||||
#
|
||||
# Host volume:
|
||||
# - chat-history → /var/lib/chat/history (persists conversation history)
|
||||
#
|
||||
# Not the runtime yet: docker-compose.yml is still the factory's live stack
|
||||
# until cutover. This file exists so CI can validate it and S5.2 can wire
|
||||
# `disinto init --backend=nomad --with chat` to `nomad job run` it.
|
||||
# =============================================================================
|
||||
|
||||
job "chat" {
|
||||
type = "service"
|
||||
datacenters = ["dc1"]
|
||||
|
||||
group "chat" {
|
||||
count = 1
|
||||
|
||||
# ── Vault workload identity (S5.2, issue #989) ───────────────────────────
|
||||
# Role `service-chat` defined in vault/roles.yaml, policy in
|
||||
# vault/policies/service-chat.hcl. Bound claim pins nomad_job_id = "chat".
|
||||
vault {
|
||||
role = "service-chat"
|
||||
}
|
||||
|
||||
# ── Network ──────────────────────────────────────────────────────────────
|
||||
# External port 8080 for chat UI access (via edge proxy or direct).
|
||||
network {
|
||||
port "http" {
|
||||
static = 8080
|
||||
to = 8080
|
||||
}
|
||||
}
|
||||
|
||||
# ── Host volumes ─────────────────────────────────────────────────────────
|
||||
# chat-history volume: declared in nomad/client.hcl, path
|
||||
# /srv/disinto/chat-history on the factory box.
|
||||
volume "chat-history" {
|
||||
type = "host"
|
||||
source = "chat-history"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ── Restart policy ───────────────────────────────────────────────────────
|
||||
restart {
|
||||
attempts = 3
|
||||
interval = "5m"
|
||||
delay = "15s"
|
||||
mode = "delay"
|
||||
}
|
||||
|
||||
# ── Service registration ─────────────────────────────────────────────────
|
||||
service {
|
||||
name = "chat"
|
||||
port = "http"
|
||||
provider = "nomad"
|
||||
|
||||
check {
|
||||
type = "http"
|
||||
path = "/health"
|
||||
interval = "10s"
|
||||
timeout = "3s"
|
||||
}
|
||||
}
|
||||
|
||||
task "chat" {
|
||||
driver = "docker"
|
||||
|
||||
config {
|
||||
image = "disinto/chat:local"
|
||||
force_pull = false
|
||||
# Sandbox hardening (#706): cap_drop ALL, pids_limit 128, tmpfs /tmp
|
||||
# ReadonlyRootfs enforced via entrypoint script (fails if running as root)
|
||||
cap_drop = ["ALL"]
|
||||
pids_limit = 128
|
||||
mount {
|
||||
type = "tmpfs"
|
||||
target = "/tmp"
|
||||
readonly = false
|
||||
tmpfs_options {
|
||||
size = 67108864 # 64MB in bytes
|
||||
}
|
||||
}
|
||||
# Security options for sandbox hardening
|
||||
# apparmor=unconfined needed for Claude CLI ptrace access
|
||||
# no-new-privileges prevents privilege escalation
|
||||
security_opt = ["apparmor=unconfined", "no-new-privileges"]
|
||||
}
|
||||
|
||||
# ── Volume mounts ──────────────────────────────────────────────────────
|
||||
# Mount chat-history for conversation persistence
|
||||
volume_mount {
|
||||
volume = "chat-history"
|
||||
destination = "/var/lib/chat/history"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ── Environment: secrets from Vault (S5.2) ──────────────────────────────
|
||||
# CHAT_OAUTH_CLIENT_ID, CHAT_OAUTH_CLIENT_SECRET, FORWARD_AUTH_SECRET
|
||||
# rendered from kv/disinto/shared/chat via template stanza.
|
||||
env {
|
||||
FORGE_URL = "http://forgejo:3000"
|
||||
CHAT_MAX_REQUESTS_PER_HOUR = "60"
|
||||
CHAT_MAX_REQUESTS_PER_DAY = "1000"
|
||||
}
|
||||
|
||||
# ── Vault-templated secrets (S5.2, issue #989) ─────────────────────────
|
||||
# Renders chat-secrets.env from Vault KV v2 at kv/disinto/shared/chat.
|
||||
# Placeholder values kept < 16 chars to avoid secret-scan CI failures.
|
||||
template {
|
||||
destination = "secrets/chat-secrets.env"
|
||||
env = true
|
||||
change_mode = "restart"
|
||||
error_on_missing_key = false
|
||||
data = <<EOT
|
||||
{{- with secret "kv/data/disinto/shared/chat" -}}
|
||||
CHAT_OAUTH_CLIENT_ID={{ .Data.data.chat_oauth_client_id }}
|
||||
CHAT_OAUTH_CLIENT_SECRET={{ .Data.data.chat_oauth_client_secret }}
|
||||
FORWARD_AUTH_SECRET={{ .Data.data.forward_auth_secret }}
|
||||
{{- else -}}
|
||||
# WARNING: run tools/vault-seed-chat.sh
|
||||
CHAT_OAUTH_CLIENT_ID=seed-me
|
||||
CHAT_OAUTH_CLIENT_SECRET=seed-me
|
||||
FORWARD_AUTH_SECRET=seed-me
|
||||
{{- end -}}
|
||||
EOT
|
||||
}
|
||||
|
||||
# ── Sandbox hardening (S5.2, #706) ────────────────────────────────────
|
||||
# Memory = 512MB (matches docker-compose sandbox hardening)
|
||||
resources {
|
||||
cpu = 200
|
||||
memory = 512
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,278 +0,0 @@
|
|||
# =============================================================================
|
||||
# nomad/jobs/edge.hcl — Edge proxy (Caddy + dispatcher sidecar) (Nomad service job)
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S5.1, issue #988). Caddy reverse proxy
|
||||
# routes traffic to Forgejo, Woodpecker, staging, and chat services. The
|
||||
# dispatcher sidecar polls disinto-ops for vault actions and dispatches them
|
||||
# via Nomad batch jobs.
|
||||
#
|
||||
# Host networking (issue #1031):
|
||||
# Caddy uses network_mode = "host" so upstreams are reached at
|
||||
# 127.0.0.1:<port> (forgejo :3000, woodpecker :8000, chat :8080).
|
||||
# Staging uses Nomad service discovery (S5-fix-7, issue #1018).
|
||||
#
|
||||
# Host_volume contract:
|
||||
# This job mounts caddy-data from nomad/client.hcl. Path
|
||||
# /srv/disinto/caddy-data is created by lib/init/nomad/cluster-up.sh before
|
||||
# any job references it. Keep the `source = "caddy-data"` below in sync
|
||||
# with the host_volume stanza in client.hcl.
|
||||
#
|
||||
# Build step (S5.1):
|
||||
# docker/edge/Dockerfile is custom (adds bash, jq, curl, git, docker-cli,
|
||||
# python3, openssh-client, autossh to caddy:latest). Build as
|
||||
# disinto/edge:local using the same pattern as disinto/agents:local.
|
||||
# Command: docker build -t disinto/edge:local -f docker/edge/Dockerfile docker/edge
|
||||
#
|
||||
# Not the runtime yet: docker-compose.yml is still the factory's live stack
|
||||
# until cutover. This file exists so CI can validate it and S5.2 can wire
|
||||
# `disinto init --backend=nomad --with edge` to `nomad job run` it.
|
||||
# =============================================================================
|
||||
|
||||
job "edge" {
|
||||
type = "service"
|
||||
datacenters = ["dc1"]
|
||||
|
||||
group "edge" {
|
||||
count = 1
|
||||
|
||||
# ── Vault workload identity for dispatcher (S5.1, issue #988) ──────────
|
||||
# Service role for dispatcher task to fetch vault actions from KV v2.
|
||||
# Role defined in vault/roles.yaml, policy in vault/policies/dispatcher.hcl.
|
||||
vault {
|
||||
role = "service-dispatcher"
|
||||
}
|
||||
|
||||
# ── Network ports (S5.1, issue #988) ──────────────────────────────────
|
||||
# Caddy listens on :80 and :443. Expose both on the host.
|
||||
network {
|
||||
port "http" {
|
||||
static = 80
|
||||
to = 80
|
||||
}
|
||||
|
||||
port "https" {
|
||||
static = 443
|
||||
to = 443
|
||||
}
|
||||
}
|
||||
|
||||
# ── Host-volume mounts (S5.1, issue #988) ─────────────────────────────
|
||||
# caddy-data: ACME certificates, Caddy config state.
|
||||
volume "caddy-data" {
|
||||
type = "host"
|
||||
source = "caddy-data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ops-repo: disinto-ops clone for vault actions polling.
|
||||
volume "ops-repo" {
|
||||
type = "host"
|
||||
source = "ops-repo"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ── Conservative restart policy ───────────────────────────────────────
|
||||
# Caddy should be stable; dispatcher may restart on errors.
|
||||
restart {
|
||||
attempts = 3
|
||||
interval = "5m"
|
||||
delay = "15s"
|
||||
mode = "delay"
|
||||
}
|
||||
|
||||
# ── Service registration ───────────────────────────────────────────────
|
||||
# Caddy is an HTTP reverse proxy — health check on port 80.
|
||||
service {
|
||||
name = "edge"
|
||||
port = "http"
|
||||
provider = "nomad"
|
||||
|
||||
check {
|
||||
type = "http"
|
||||
path = "/"
|
||||
interval = "10s"
|
||||
timeout = "3s"
|
||||
}
|
||||
}
|
||||
|
||||
# ── Caddy task (S5.1, issue #988) ─────────────────────────────────────
|
||||
task "caddy" {
|
||||
driver = "docker"
|
||||
|
||||
config {
|
||||
# Use pre-built disinto/edge:local image (custom Dockerfile adds
|
||||
# bash, jq, curl, git, docker-cli, python3, openssh-client, autossh).
|
||||
image = "disinto/edge:local"
|
||||
force_pull = false
|
||||
network_mode = "host"
|
||||
ports = ["http", "https"]
|
||||
|
||||
# apparmor=unconfined matches docker-compose — needed for autossh
|
||||
# in the entrypoint script.
|
||||
security_opt = ["apparmor=unconfined"]
|
||||
}
|
||||
|
||||
# Mount caddy-data volume for ACME state and config directory.
|
||||
# Caddyfile is mounted at /etc/caddy/Caddyfile by entrypoint-edge.sh.
|
||||
volume_mount {
|
||||
volume = "caddy-data"
|
||||
destination = "/data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ── Caddyfile via Nomad service discovery (S5-fix-7, issue #1018) ────
|
||||
# Renders staging upstream from Nomad service registration instead of
|
||||
# hardcoded staging:80. Caddy picks up /local/Caddyfile via entrypoint.
|
||||
# Forge URL via Nomad service discovery (issue #1034) — resolves forgejo
|
||||
# service address/port dynamically for bridge network compatibility.
|
||||
template {
|
||||
destination = "local/forge.env"
|
||||
env = true
|
||||
change_mode = "restart"
|
||||
data = <<EOT
|
||||
{{ range service "forgejo" -}}
|
||||
FORGE_URL=http://{{ .Address }}:{{ .Port }}
|
||||
{{- end }}
|
||||
EOT
|
||||
}
|
||||
|
||||
template {
|
||||
destination = "local/Caddyfile"
|
||||
change_mode = "restart"
|
||||
data = <<EOT
|
||||
# Caddyfile — edge proxy configuration (Nomad-rendered)
|
||||
# Staging upstream discovered via Nomad service registration.
|
||||
|
||||
:80 {
|
||||
# Redirect root to Forgejo
|
||||
handle / {
|
||||
redir /forge/ 302
|
||||
}
|
||||
|
||||
# Reverse proxy to Forgejo
|
||||
handle /forge/* {
|
||||
reverse_proxy 127.0.0.1:3000
|
||||
}
|
||||
|
||||
# Reverse proxy to Woodpecker CI
|
||||
handle /ci/* {
|
||||
reverse_proxy 127.0.0.1:8000
|
||||
}
|
||||
|
||||
# Reverse proxy to staging — dynamic port via Nomad service discovery
|
||||
handle /staging/* {
|
||||
{{ range nomadService "staging" }} reverse_proxy {{ .Address }}:{{ .Port }}
|
||||
{{ end }} }
|
||||
|
||||
# Chat service — reverse proxy to disinto-chat backend (#705)
|
||||
# OAuth routes bypass forward_auth — unauthenticated users need these (#709)
|
||||
handle /chat/login {
|
||||
reverse_proxy 127.0.0.1:8080
|
||||
}
|
||||
handle /chat/oauth/callback {
|
||||
reverse_proxy 127.0.0.1:8080
|
||||
}
|
||||
# Defense-in-depth: forward_auth stamps X-Forwarded-User from session (#709)
|
||||
handle /chat/* {
|
||||
forward_auth 127.0.0.1:8080 {
|
||||
uri /chat/auth/verify
|
||||
copy_headers X-Forwarded-User
|
||||
header_up X-Forward-Auth-Secret {$FORWARD_AUTH_SECRET}
|
||||
}
|
||||
reverse_proxy 127.0.0.1:8080
|
||||
}
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
# ── Non-secret env ───────────────────────────────────────────────────
|
||||
env {
|
||||
FORGE_REPO = "disinto-admin/disinto"
|
||||
DISINTO_CONTAINER = "1"
|
||||
PROJECT_NAME = "disinto"
|
||||
}
|
||||
|
||||
# Caddy needs CPU + memory headroom for reverse proxy work.
|
||||
resources {
|
||||
cpu = 200
|
||||
memory = 256
|
||||
}
|
||||
}
|
||||
|
||||
# ── Dispatcher task (S5.1, issue #988) ────────────────────────────────
|
||||
task "dispatcher" {
|
||||
driver = "docker"
|
||||
|
||||
config {
|
||||
# Use same disinto/agents:local image as other agents.
|
||||
image = "disinto/agents:local"
|
||||
force_pull = false
|
||||
network_mode = "host"
|
||||
|
||||
# apparmor=unconfined matches docker-compose.
|
||||
security_opt = ["apparmor=unconfined"]
|
||||
|
||||
# Mount docker.sock via bind-volume (not host volume) for legacy
|
||||
# docker backend compat. Nomad host volumes require named volumes
|
||||
# from client.hcl; socket files cannot be host volumes.
|
||||
volumes = ["/var/run/docker.sock:/var/run/docker.sock:ro"]
|
||||
}
|
||||
|
||||
# Mount ops-repo for vault actions polling.
|
||||
volume_mount {
|
||||
volume = "ops-repo"
|
||||
destination = "/home/agent/repos/disinto-ops"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ── Forge URL via Nomad service discovery (issue #1034) ──────────
|
||||
# Resolves forgejo service address/port dynamically for bridge network
|
||||
# compatibility. Template-scoped to dispatcher task (Nomad doesn't
|
||||
# propagate templates across tasks).
|
||||
template {
|
||||
destination = "local/forge.env"
|
||||
env = true
|
||||
change_mode = "restart"
|
||||
data = <<EOT
|
||||
{{ range service "forgejo" -}}
|
||||
FORGE_URL=http://{{ .Address }}:{{ .Port }}
|
||||
{{- end }}
|
||||
EOT
|
||||
}
|
||||
|
||||
# ── Vault-templated secrets (S5.1, issue #988) ──────────────────────
|
||||
# Renders FORGE_TOKEN from Vault KV v2 for ops repo access.
|
||||
template {
|
||||
destination = "secrets/dispatcher.env"
|
||||
env = true
|
||||
change_mode = "restart"
|
||||
error_on_missing_key = false
|
||||
data = <<EOT
|
||||
{{- with secret "kv/data/disinto/shared/ops-repo" -}}
|
||||
FORGE_TOKEN={{ .Data.data.token }}
|
||||
{{- else -}}
|
||||
# WARNING: kv/disinto/shared/ops-repo is empty — run tools/vault-seed-ops-repo.sh
|
||||
FORGE_TOKEN=seed-me
|
||||
{{- end }}
|
||||
EOT
|
||||
}
|
||||
|
||||
# ── Non-secret env ───────────────────────────────────────────────────
|
||||
env {
|
||||
DISPATCHER_BACKEND = "nomad"
|
||||
FORGE_REPO = "disinto-admin/disinto"
|
||||
FORGE_OPS_REPO = "disinto-admin/disinto-ops"
|
||||
PRIMARY_BRANCH = "main"
|
||||
DISINTO_CONTAINER = "1"
|
||||
OPS_REPO_ROOT = "/home/agent/repos/disinto-ops"
|
||||
FORGE_ADMIN_USERS = "vault-bot,admin"
|
||||
}
|
||||
|
||||
# Dispatcher is lightweight — minimal CPU + memory.
|
||||
resources {
|
||||
cpu = 100
|
||||
memory = 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
# =============================================================================
|
||||
# nomad/jobs/staging.hcl — Staging file server (Nomad service job)
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S5.2, issue #989). Lightweight service job
|
||||
# for the staging file server using Caddy as a static file server.
|
||||
#
|
||||
# Mount contract:
|
||||
# This job mounts the `docker/` directory as `/srv/site` (read-only).
|
||||
# The docker/ directory contains static content (images, HTML, etc.)
|
||||
# served to staging environment users.
|
||||
#
|
||||
# Network:
|
||||
# Dynamic host port — edge discovers via Nomad service registration.
|
||||
# No static port to avoid collisions with edge (which owns 80/443).
|
||||
#
|
||||
# Not the runtime yet: docker-compose.yml is still the factory's live stack
|
||||
# until cutover. This file exists so CI can validate it and S5.2 can wire
|
||||
# `disinto init --backend=nomad --with staging` to `nomad job run` it.
|
||||
# =============================================================================
|
||||
|
||||
job "staging" {
|
||||
type = "service"
|
||||
datacenters = ["dc1"]
|
||||
|
||||
group "staging" {
|
||||
count = 1
|
||||
|
||||
# No Vault integration needed — no secrets required (static file server)
|
||||
|
||||
# Internal service — dynamic host port. Edge discovers via Nomad service.
|
||||
network {
|
||||
port "http" {
|
||||
to = 80
|
||||
}
|
||||
}
|
||||
|
||||
volume "site-content" {
|
||||
type = "host"
|
||||
source = "site-content"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
restart {
|
||||
attempts = 3
|
||||
interval = "5m"
|
||||
delay = "15s"
|
||||
mode = "delay"
|
||||
}
|
||||
|
||||
service {
|
||||
name = "staging"
|
||||
port = "http"
|
||||
provider = "nomad"
|
||||
|
||||
check {
|
||||
type = "http"
|
||||
path = "/"
|
||||
interval = "10s"
|
||||
timeout = "3s"
|
||||
}
|
||||
}
|
||||
|
||||
task "staging" {
|
||||
driver = "docker"
|
||||
|
||||
config {
|
||||
image = "caddy:alpine"
|
||||
ports = ["http"]
|
||||
command = "caddy"
|
||||
args = ["file-server", "--root", "/srv/site"]
|
||||
}
|
||||
|
||||
# Mount docker/ directory as /srv/site:ro (static content)
|
||||
volume_mount {
|
||||
volume = "site-content"
|
||||
destination = "/srv/site"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
resources {
|
||||
cpu = 100
|
||||
memory = 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
# =============================================================================
|
||||
# nomad/jobs/vault-runner.hcl — Parameterized batch job for vault action dispatch
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S5.3, issue #990). Replaces the
|
||||
# `docker run --rm vault-runner-${action_id}` pattern in dispatcher.sh with
|
||||
# a Nomad-native parameterized batch job. Dispatched by the edge dispatcher
|
||||
# (S5.4) via `nomad job dispatch`.
|
||||
#
|
||||
# Parameterized meta:
|
||||
# action_id — vault action identifier (used by entrypoint-runner.sh)
|
||||
# secrets_csv — comma-separated secret names (e.g. "GITHUB_TOKEN,DEPLOY_KEY")
|
||||
#
|
||||
# Vault integration (approach A — pre-defined templates):
|
||||
# All 6 known runner secrets are rendered via template stanzas with
|
||||
# error_on_missing_key = false. Secrets not granted by the dispatch's
|
||||
# Vault policies render as empty strings. The dispatcher (S5.4) sets
|
||||
# vault { policies = [...] } per-dispatch based on the action TOML's
|
||||
# secrets=[...] list, scoping access to only the declared secrets.
|
||||
#
|
||||
# Cleanup: Nomad garbage-collects completed batch dispatches automatically.
|
||||
# =============================================================================
|
||||
|
||||
job "vault-runner" {
|
||||
type = "batch"
|
||||
datacenters = ["dc1"]
|
||||
|
||||
parameterized {
|
||||
meta_required = ["action_id", "secrets_csv"]
|
||||
}
|
||||
|
||||
group "runner" {
|
||||
count = 1
|
||||
|
||||
# ── Vault workload identity ──────────────────────────────────────────────
|
||||
# Per-dispatch policies are composed by the dispatcher (S5.4) based on the
|
||||
# action TOML's secrets=[...] list. Each policy grants read access to
|
||||
# exactly one kv/data/disinto/runner/<NAME> path. Roles defined in
|
||||
# vault/roles.yaml (runner-<NAME>), policies in vault/policies/.
|
||||
vault {}
|
||||
|
||||
volume "ops-repo" {
|
||||
type = "host"
|
||||
source = "ops-repo"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
# No restart for batch — fail fast, let the dispatcher handle retries.
|
||||
restart {
|
||||
attempts = 0
|
||||
mode = "fail"
|
||||
}
|
||||
|
||||
task "runner" {
|
||||
driver = "docker"
|
||||
|
||||
config {
|
||||
image = "disinto/agents:local"
|
||||
force_pull = false
|
||||
entrypoint = ["bash"]
|
||||
args = [
|
||||
"/home/agent/disinto/docker/runner/entrypoint-runner.sh",
|
||||
"${NOMAD_META_action_id}",
|
||||
]
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
volume = "ops-repo"
|
||||
destination = "/home/agent/ops"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
# ── Non-secret env ───────────────────────────────────────────────────────
|
||||
env {
|
||||
DISINTO_CONTAINER = "1"
|
||||
FACTORY_ROOT = "/home/agent/disinto"
|
||||
OPS_REPO_ROOT = "/home/agent/ops"
|
||||
}
|
||||
|
||||
# ── Vault-templated runner secrets (approach A) ────────────────────────
|
||||
# Pre-defined templates for all 6 known runner secrets. Each renders
|
||||
# from kv/data/disinto/runner/<NAME>. Secrets not granted by the
|
||||
# dispatch's Vault policies produce empty env vars (harmless).
|
||||
# error_on_missing_key = false prevents template-pending hangs when
|
||||
# a secret path is absent or the policy doesn't grant access.
|
||||
#
|
||||
# Placeholder values kept < 16 chars to avoid secret-scan CI failures.
|
||||
template {
|
||||
destination = "secrets/runner.env"
|
||||
env = true
|
||||
error_on_missing_key = false
|
||||
data = <<EOT
|
||||
{{- with secret "kv/data/disinto/runner/GITHUB_TOKEN" -}}
|
||||
GITHUB_TOKEN={{ .Data.data.value }}
|
||||
{{- else -}}
|
||||
GITHUB_TOKEN=
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/runner/CODEBERG_TOKEN" -}}
|
||||
CODEBERG_TOKEN={{ .Data.data.value }}
|
||||
{{- else -}}
|
||||
CODEBERG_TOKEN=
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/runner/CLAWHUB_TOKEN" -}}
|
||||
CLAWHUB_TOKEN={{ .Data.data.value }}
|
||||
{{- else -}}
|
||||
CLAWHUB_TOKEN=
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/runner/DEPLOY_KEY" -}}
|
||||
DEPLOY_KEY={{ .Data.data.value }}
|
||||
{{- else -}}
|
||||
DEPLOY_KEY=
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/runner/NPM_TOKEN" -}}
|
||||
NPM_TOKEN={{ .Data.data.value }}
|
||||
{{- else -}}
|
||||
NPM_TOKEN=
|
||||
{{- end }}
|
||||
|
||||
{{ with secret "kv/data/disinto/runner/DOCKER_HUB_TOKEN" -}}
|
||||
DOCKER_HUB_TOKEN={{ .Data.data.value }}
|
||||
{{- else -}}
|
||||
DOCKER_HUB_TOKEN=
|
||||
{{- end }}
|
||||
EOT
|
||||
}
|
||||
|
||||
# Formula execution headroom — matches agents.hcl baseline.
|
||||
resources {
|
||||
cpu = 500
|
||||
memory = 1024
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,9 +8,8 @@
|
|||
#
|
||||
# Host networking:
|
||||
# Uses network_mode = "host" to match the compose setup. The Woodpecker
|
||||
# server gRPC endpoint is addressed via Nomad service discovery using
|
||||
# the host's IP address (10.10.10.x:9000), since the server's port
|
||||
# binding in Nomad binds to the allocation's IP, not localhost.
|
||||
# server gRPC endpoint is addressed as "localhost:9000" since both
|
||||
# server and agent run on the same host.
|
||||
#
|
||||
# Vault integration:
|
||||
# - vault { role = "service-woodpecker-agent" } at the group scope — the
|
||||
|
|
@ -83,13 +82,8 @@ job "woodpecker-agent" {
|
|||
|
||||
# Non-secret env — server address, gRPC security, concurrency limit,
|
||||
# and health check endpoint. Nothing sensitive here.
|
||||
#
|
||||
# WOODPECKER_SERVER uses Nomad's attribute template to get the host's
|
||||
# IP address (10.10.10.x). The server's gRPC port 9000 is bound via
|
||||
# Nomad's port stanza to the allocation's IP (not localhost), so the
|
||||
# agent must use the LXC's eth0 IP, not 127.0.0.1.
|
||||
env {
|
||||
WOODPECKER_SERVER = "${attr.unique.network.ip-address}:9000"
|
||||
WOODPECKER_SERVER = "localhost:9000"
|
||||
WOODPECKER_GRPC_SECURE = "false"
|
||||
WOODPECKER_MAX_WORKFLOWS = "1"
|
||||
WOODPECKER_HEALTHCHECK_ADDR = ":3333"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Planner Agent
|
||||
|
||||
**Role**: Strategic planning using a Prerequisite Tree (Theory of Constraints),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Predictor Agent
|
||||
|
||||
**Role**: Abstract adversary (the "goblin"). Runs a 2-step formula
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Review Agent
|
||||
|
||||
**Role**: AI-powered PR review — post structured findings and formal
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# Supervisor Agent
|
||||
|
||||
**Role**: Health monitoring and auto-remediation, executed as a formula-driven
|
||||
|
|
@ -24,18 +24,12 @@ Both invoke the same `supervisor-run.sh`. Sources `lib/guard.sh` and calls `chec
|
|||
files for `PHASE:escalate` entries and auto-removes any whose linked issue
|
||||
is confirmed closed (24h grace period after closure to avoid races). Reports
|
||||
**stale crashed worktrees** (worktrees preserved after crash) — supervisor
|
||||
housekeeping removes them after 24h. Collects **Woodpecker agent health**
|
||||
(added #933): container `disinto-woodpecker-agent` health/running status,
|
||||
gRPC error count in last 20 min, fast-failure pipeline count (<60s, last 15 min),
|
||||
and overall health verdict (healthy/unhealthy). Unhealthy verdict triggers
|
||||
automatic container restart + `blocked:ci_exhausted` issue recovery in
|
||||
`supervisor-run.sh` before the Claude session starts.
|
||||
housekeeping removes them after 24h. Also collects **Woodpecker agent health**:
|
||||
container status, gRPC error count (last 20m), fast-failure pipelines (<60s,
|
||||
last 15m), and overall health determination.
|
||||
- `formulas/run-supervisor.toml` — Execution spec: five steps (preflight review,
|
||||
health-assessment, decide-actions, report, journal) with `needs` dependencies.
|
||||
Claude evaluates all metrics and takes actions in a single interactive session.
|
||||
Health-assessment now includes P2 **Woodpecker agent unhealthy** classification
|
||||
(container not running, ≥3 gRPC errors/20m, or ≥3 fast-failure pipelines/15m);
|
||||
decide-actions documents the pre-session auto-recovery path
|
||||
Claude evaluates all metrics and takes actions in a single interactive session
|
||||
- `$OPS_REPO_ROOT/knowledge/*.md` — Domain-specific remediation guides (memory,
|
||||
disk, CI, git, dev-agent, review-agent, forge)
|
||||
|
||||
|
|
|
|||
|
|
@ -215,44 +215,7 @@ setup_file() {
|
|||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with unknown-service --dry-run
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"unknown service"* ]]
|
||||
[[ "$output" == *"known: forgejo, woodpecker-server, woodpecker-agent, agents, staging, chat, edge"* ]]
|
||||
}
|
||||
|
||||
# S3.4: woodpecker auto-expansion and forgejo auto-inclusion
|
||||
@test "disinto init --backend=nomad --with woodpecker auto-expands to server+agent" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with woodpecker --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"services to deploy: forgejo,woodpecker-server,woodpecker-agent"* ]]
|
||||
[[ "$output" == *"deployment order: forgejo woodpecker-server woodpecker-agent"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with woodpecker auto-includes forgejo with note" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with woodpecker --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"Note: --with woodpecker implies --with forgejo"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with forgejo,woodpecker expands woodpecker" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with forgejo,woodpecker --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
# Order follows input: forgejo first, then woodpecker expanded
|
||||
[[ "$output" == *"services to deploy: forgejo,woodpecker-server,woodpecker-agent"* ]]
|
||||
[[ "$output" == *"deployment order: forgejo woodpecker-server woodpecker-agent"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with woodpecker seeds both forgejo and woodpecker" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with woodpecker --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"tools/vault-seed-forgejo.sh --dry-run"* ]]
|
||||
[[ "$output" == *"tools/vault-seed-woodpecker.sh --dry-run"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with forgejo,woodpecker deploys all three services" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with forgejo,woodpecker --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"forgejo.hcl"* ]]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"woodpecker-server.hcl"* ]]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"woodpecker-agent.hcl"* ]]
|
||||
[[ "$output" == *"known: forgejo"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with forgejo (flag=value syntax) works" {
|
||||
|
|
@ -385,60 +348,3 @@ setup_file() {
|
|||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"--empty and --import-env/--import-sops/--age-key are mutually exclusive"* ]]
|
||||
}
|
||||
|
||||
# S4.2: agents service auto-expansion and dependencies
|
||||
@test "disinto init --backend=nomad --with agents auto-includes forgejo and woodpecker" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with agents --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"services to deploy: forgejo,agents,woodpecker-server,woodpecker-agent"* ]]
|
||||
[[ "$output" == *"Note: --with agents implies --with forgejo"* ]]
|
||||
[[ "$output" == *"Note: --with agents implies --with woodpecker"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with agents deploys in correct order" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with agents --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"deployment order: forgejo woodpecker-server woodpecker-agent agents"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with agents seeds agents service" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with agents --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"tools/vault-seed-forgejo.sh --dry-run"* ]]
|
||||
[[ "$output" == *"tools/vault-seed-woodpecker.sh --dry-run"* ]]
|
||||
[[ "$output" == *"tools/vault-seed-agents.sh --dry-run"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with agents deploys all four services" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with agents --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"forgejo.hcl"* ]]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"woodpecker-server.hcl"* ]]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"woodpecker-agent.hcl"* ]]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"agents.hcl"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with woodpecker,agents expands correctly" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with woodpecker,agents --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
# woodpecker expands to server+agent, agents is already explicit
|
||||
# forgejo is auto-included by agents
|
||||
[[ "$output" == *"services to deploy: forgejo,woodpecker-server,woodpecker-agent,agents"* ]]
|
||||
[[ "$output" == *"deployment order: forgejo woodpecker-server woodpecker-agent agents"* ]]
|
||||
}
|
||||
|
||||
# S5.1 / #1035 — edge service seeds ops-repo (dispatcher FORGE_TOKEN)
|
||||
@test "disinto init --backend=nomad --with edge deploys edge" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with edge --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
# edge depends on all backend services, so all are included
|
||||
[[ "$output" == *"services to deploy: edge,forgejo"* ]]
|
||||
[[ "$output" == *"deployment order: forgejo woodpecker-server woodpecker-agent agents staging chat edge"* ]]
|
||||
[[ "$output" == *"[deploy] [dry-run] nomad job validate"*"edge.hcl"* ]]
|
||||
}
|
||||
|
||||
@test "disinto init --backend=nomad --with edge seeds ops-repo" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with edge --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"tools/vault-seed-ops-repo.sh --dry-run"* ]]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,7 +137,6 @@ setup() {
|
|||
"${VAULT_ADDR}/v1/kv/data/disinto/shared/woodpecker"
|
||||
[ "$status" -eq 0 ]
|
||||
echo "$output" | grep -q "wp-agent-secret"
|
||||
# Forgejo keys are normalized: WP_FORGEJO_* → forgejo_* (no wp_ prefix in key name)
|
||||
echo "$output" | grep -q "wp-forgejo-client"
|
||||
echo "$output" | grep -q "wp-forgejo-secret"
|
||||
echo "$output" | grep -q "wp-token"
|
||||
|
|
@ -295,8 +294,6 @@ setup() {
|
|||
"deploy-key-test"
|
||||
"npm-test-token"
|
||||
"dockerhub-test-token"
|
||||
# Note: forgejo-client and forgejo-secret are NOT in the output
|
||||
# because they are read from Vault, not logged
|
||||
)
|
||||
|
||||
for pattern in "${secret_patterns[@]}"; do
|
||||
|
|
|
|||
|
|
@ -391,13 +391,7 @@ EOF
|
|||
local val="${!key}"
|
||||
if [ -n "$val" ]; then
|
||||
local lowercase_key="${key,,}"
|
||||
# Normalize WP_FORGEJO_* → forgejo_* (strip wp_ prefix to match template)
|
||||
if [[ "$lowercase_key" =~ ^wp_(.+)$ ]]; then
|
||||
vault_key="${BASH_REMATCH[1]}"
|
||||
else
|
||||
vault_key="$lowercase_key"
|
||||
fi
|
||||
operations+=("woodpecker|$vault_key|$env_file|$key")
|
||||
operations+=("woodpecker|$lowercase_key|$env_file|$key")
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -1,176 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# tools/vault-seed-agents.sh — Idempotent seed for all bot KV paths
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S4.1, issue #955). Populates
|
||||
# kv/disinto/bots/<role> with token + pass for each of the 7 agent roles
|
||||
# plus the vault bot. Handles the "fresh factory, no .env import" case.
|
||||
#
|
||||
# Companion to tools/vault-import.sh — when that runs against a box with
|
||||
# an existing stack, it overwrites seeded values with real ones.
|
||||
#
|
||||
# Idempotency contract (per bot):
|
||||
# - Both token and pass present → skip, log "<role> unchanged".
|
||||
# - Either missing → generate random values for missing keys, preserve
|
||||
# existing keys, write back atomically.
|
||||
#
|
||||
# Preconditions:
|
||||
# - Vault reachable + unsealed at $VAULT_ADDR.
|
||||
# - VAULT_TOKEN set (env) or /etc/vault.d/root.token readable.
|
||||
# - curl, jq, openssl
|
||||
#
|
||||
# Usage:
|
||||
# tools/vault-seed-agents.sh
|
||||
# tools/vault-seed-agents.sh --dry-run
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (seed applied, or already applied)
|
||||
# 1 precondition / API / mount-mismatch failure
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# shellcheck source=../lib/hvault.sh
|
||||
source "${REPO_ROOT}/lib/hvault.sh"
|
||||
|
||||
KV_MOUNT="kv"
|
||||
TOKEN_BYTES=32 # 32 bytes → 64 hex chars
|
||||
PASS_BYTES=16 # 16 bytes → 32 hex chars
|
||||
|
||||
# All bot roles seeded by this script.
|
||||
BOT_ROLES=(dev review gardener architect planner predictor supervisor vault)
|
||||
|
||||
LOG_TAG="[vault-seed-agents]"
|
||||
log() { printf '%s %s\n' "$LOG_TAG" "$*"; }
|
||||
die() { printf '%s ERROR: %s\n' "$LOG_TAG" "$*" >&2; exit 1; }
|
||||
|
||||
# ── Flag parsing ─────────────────────────────────────────────────────────────
|
||||
# while/shift shape — distinct from forgejo (arity:value case) and
|
||||
# woodpecker (for-loop).
|
||||
DRY_RUN=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
-h|--help)
|
||||
printf 'Usage: %s [--dry-run]\n\n' "$(basename "$0")"
|
||||
printf 'Seed kv/disinto/bots/<role> with token + pass for all agent\n'
|
||||
printf 'roles. Idempotent: existing non-empty values are preserved.\n\n'
|
||||
printf ' --dry-run Print planned actions without writing.\n'
|
||||
exit 0
|
||||
;;
|
||||
*) die "invalid argument: ${1} (try --help)" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
for bin in curl jq openssl; do
|
||||
command -v "$bin" >/dev/null 2>&1 \
|
||||
|| die "required binary not found: ${bin}"
|
||||
done
|
||||
[ -n "${VAULT_ADDR:-}" ] \
|
||||
|| die "VAULT_ADDR unset — e.g. export VAULT_ADDR=http://127.0.0.1:8200"
|
||||
hvault_token_lookup >/dev/null \
|
||||
|| die "Vault auth probe failed — check VAULT_ADDR + VAULT_TOKEN"
|
||||
|
||||
# ── Step 1: ensure kv/ mount exists and is KV v2 ────────────────────────────
|
||||
log "── Step 1: ensure ${KV_MOUNT}/ is KV v2 ──"
|
||||
export DRY_RUN
|
||||
hvault_ensure_kv_v2 "$KV_MOUNT" "${LOG_TAG}" \
|
||||
|| die "KV mount check failed"
|
||||
|
||||
# ── Step 2: seed each bot role ───────────────────────────────────────────────
|
||||
total_generated=0
|
||||
|
||||
# Check if shared forge credentials exist for dev role fallback
|
||||
shared_forge_exists=0
|
||||
shared_forge_raw="$(hvault_get_or_empty "${KV_MOUNT}/data/disinto/shared/forge")" \
|
||||
|| true
|
||||
if [ -n "$shared_forge_raw" ]; then
|
||||
shared_forge_token="$(printf '%s' "$shared_forge_raw" | jq -r '.data.data.token // ""')"
|
||||
shared_forge_pass="$(printf '%s' "$shared_forge_raw" | jq -r '.data.data.pass // ""')"
|
||||
if [ -n "$shared_forge_token" ] && [ -n "$shared_forge_pass" ]; then
|
||||
shared_forge_exists=1
|
||||
fi
|
||||
fi
|
||||
|
||||
for role in "${BOT_ROLES[@]}"; do
|
||||
kv_logical="disinto/bots/${role}"
|
||||
kv_api="${KV_MOUNT}/data/${kv_logical}"
|
||||
|
||||
log "── seed ${kv_logical} ──"
|
||||
|
||||
existing_raw="$(hvault_get_or_empty "${kv_api}")" \
|
||||
|| die "failed to read ${kv_api}"
|
||||
|
||||
existing_token=""
|
||||
existing_pass=""
|
||||
existing_data="{}"
|
||||
if [ -n "$existing_raw" ]; then
|
||||
existing_data="$(printf '%s' "$existing_raw" | jq '.data.data // {}')"
|
||||
existing_token="$(printf '%s' "$existing_raw" | jq -r '.data.data.token // ""')"
|
||||
existing_pass="$(printf '%s' "$existing_raw" | jq -r '.data.data.pass // ""')"
|
||||
fi
|
||||
|
||||
generated=()
|
||||
desired_token="$existing_token"
|
||||
desired_pass="$existing_pass"
|
||||
|
||||
# Special case: dev role uses shared forge credentials if available
|
||||
if [ "$role" = "dev" ] && [ "$shared_forge_exists" -eq 1 ]; then
|
||||
# Use shared FORGE_TOKEN + FORGE_PASS for dev role
|
||||
if [ -z "$existing_token" ]; then
|
||||
desired_token="$shared_forge_token"
|
||||
generated+=("token")
|
||||
fi
|
||||
if [ -z "$existing_pass" ]; then
|
||||
desired_pass="$shared_forge_pass"
|
||||
generated+=("pass")
|
||||
fi
|
||||
else
|
||||
# Generate random values for missing keys
|
||||
if [ -z "$existing_token" ]; then
|
||||
generated+=("token")
|
||||
fi
|
||||
if [ -z "$existing_pass" ]; then
|
||||
generated+=("pass")
|
||||
fi
|
||||
|
||||
for key in "${generated[@]}"; do
|
||||
case "$key" in
|
||||
token) desired_token="$(openssl rand -hex "$TOKEN_BYTES")" ;;
|
||||
pass) desired_pass="$(openssl rand -hex "$PASS_BYTES")" ;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${#generated[@]}" -eq 0 ]; then
|
||||
log "${role}: unchanged"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
log "[dry-run] ${role}: would generate ${generated[*]}"
|
||||
total_generated=$(( total_generated + ${#generated[@]} ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Merge new keys into existing data to preserve any keys we don't own.
|
||||
payload="$(printf '%s' "$existing_data" \
|
||||
| jq --arg t "$desired_token" --arg p "$desired_pass" \
|
||||
'{data: (. + {token: $t, pass: $p})}')"
|
||||
|
||||
_hvault_request POST "${kv_api}" "$payload" >/dev/null \
|
||||
|| die "failed to write ${kv_api}"
|
||||
|
||||
log "${role}: generated ${generated[*]}"
|
||||
total_generated=$(( total_generated + ${#generated[@]} ))
|
||||
done
|
||||
|
||||
if [ "$total_generated" -eq 0 ]; then
|
||||
log "all bot paths already seeded — no-op"
|
||||
else
|
||||
log "done — ${total_generated} key(s) seeded across ${#BOT_ROLES[@]} bot paths"
|
||||
fi
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# tools/vault-seed-chat.sh — Idempotent seed for kv/disinto/shared/chat
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S5.2, issue #989). Populates the KV v2
|
||||
# path that nomad/jobs/chat.hcl reads from, so a clean-install factory
|
||||
# (no old-stack secrets to import) still has per-key values for
|
||||
# CHAT_OAUTH_CLIENT_ID, CHAT_OAUTH_CLIENT_SECRET, and FORWARD_AUTH_SECRET.
|
||||
#
|
||||
# Companion to tools/vault-import.sh (S2.2) — when that import runs against
|
||||
# a box with an existing stack, it overwrites these seeded values with the
|
||||
# real ones. Order doesn't matter: whichever runs last wins, and both
|
||||
# scripts are idempotent in the sense that re-running never rotates an
|
||||
# existing non-empty key.
|
||||
#
|
||||
# Uses _hvault_seed_key (lib/hvault.sh) for each key — the helper reads
|
||||
# existing data and merges to preserve sibling keys (KV v2 replaces .data
|
||||
# atomically).
|
||||
#
|
||||
# Preconditions:
|
||||
# - Vault reachable + unsealed at $VAULT_ADDR.
|
||||
# - VAULT_TOKEN set (env) or /etc/vault.d/root.token readable.
|
||||
# - The `kv/` mount is enabled as KV v2.
|
||||
#
|
||||
# Requires: VAULT_ADDR, VAULT_TOKEN, curl, jq, openssl
|
||||
#
|
||||
# Usage:
|
||||
# tools/vault-seed-chat.sh
|
||||
# tools/vault-seed-chat.sh --dry-run
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (seed applied, or already applied)
|
||||
# 1 precondition / API / mount-mismatch failure
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# shellcheck source=../lib/hvault.sh
|
||||
source "${REPO_ROOT}/lib/hvault.sh"
|
||||
|
||||
KV_MOUNT="kv"
|
||||
KV_LOGICAL_PATH="disinto/shared/chat"
|
||||
|
||||
# Keys to seed — array-driven loop (structurally distinct from forgejo's
|
||||
# sequential if-blocks and agents' role loop).
|
||||
SEED_KEYS=(chat_oauth_client_id chat_oauth_client_secret forward_auth_secret)
|
||||
|
||||
LOG_TAG="[vault-seed-chat]"
|
||||
log() { printf '%s %s\n' "$LOG_TAG" "$*"; }
|
||||
die() { printf '%s ERROR: %s\n' "$LOG_TAG" "$*" >&2; exit 1; }
|
||||
|
||||
# ── Flag parsing — [[ ]] guard + case: shape distinct from forgejo
|
||||
# (arity:value case), woodpecker (for-loop), agents (while/shift).
|
||||
DRY_RUN=0
|
||||
if [[ $# -gt 0 ]]; then
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
-h|--help)
|
||||
printf 'Usage: %s [--dry-run]\n\n' "$(basename "$0")"
|
||||
printf 'Seed kv/disinto/shared/chat with random OAuth client\n'
|
||||
printf 'credentials and forward auth secret if missing.\n'
|
||||
printf 'Idempotent: existing non-empty values are preserved.\n\n'
|
||||
printf ' --dry-run Show what would be seeded without writing.\n'
|
||||
exit 0
|
||||
;;
|
||||
*) die "invalid argument: ${1} (try --help)" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ── Preconditions — inline check-or-die (shape distinct from agents' array
|
||||
# loop and forgejo's continuation-line style) ─────────────────────────────
|
||||
command -v curl >/dev/null 2>&1 || die "curl not found"
|
||||
command -v jq >/dev/null 2>&1 || die "jq not found"
|
||||
command -v openssl >/dev/null 2>&1 || die "openssl not found"
|
||||
[ -n "${VAULT_ADDR:-}" ] || die "VAULT_ADDR unset — export VAULT_ADDR=http://127.0.0.1:8200"
|
||||
hvault_token_lookup >/dev/null || die "Vault auth probe failed — check VAULT_ADDR + VAULT_TOKEN"
|
||||
|
||||
# ── Step 1/2: ensure kv/ mount exists and is KV v2 ───────────────────────────
|
||||
log "── Step 1/2: ensure ${KV_MOUNT}/ is KV v2 ──"
|
||||
export DRY_RUN
|
||||
hvault_ensure_kv_v2 "$KV_MOUNT" "${LOG_TAG}" \
|
||||
|| die "KV mount check failed"
|
||||
|
||||
# ── Step 2/2: seed missing keys via _hvault_seed_key helper ──────────────────
|
||||
log "── Step 2/2: seed ${KV_LOGICAL_PATH} ──"
|
||||
|
||||
generated=()
|
||||
for key in "${SEED_KEYS[@]}"; do
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
# Check existence without writing
|
||||
existing=$(hvault_kv_get "$KV_LOGICAL_PATH" "$key" 2>/dev/null) || true
|
||||
if [ -z "$existing" ]; then
|
||||
generated+=("$key")
|
||||
log "[dry-run] ${key} would be generated"
|
||||
else
|
||||
log "[dry-run] ${key} unchanged"
|
||||
fi
|
||||
else
|
||||
rc=0
|
||||
_hvault_seed_key "$KV_LOGICAL_PATH" "$key" || rc=$?
|
||||
case "$rc" in
|
||||
0) generated+=("$key"); log "${key} generated" ;;
|
||||
1) log "${key} unchanged" ;;
|
||||
*) die "API error seeding ${key} (rc=${rc})" ;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${#generated[@]}" -eq 0 ]; then
|
||||
log "all keys present — no-op"
|
||||
else
|
||||
log "done — ${#generated[@]} key(s) seeded at kv/${KV_LOGICAL_PATH}"
|
||||
fi
|
||||
|
|
@ -1,149 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# tools/vault-seed-ops-repo.sh — Idempotent seed for kv/disinto/shared/ops-repo
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S5.1, issue #1035). Populates the KV v2
|
||||
# path that nomad/jobs/edge.hcl dispatcher task reads from, so the edge
|
||||
# proxy has FORGE_TOKEN for ops repo access.
|
||||
#
|
||||
# Seeds from kv/disinto/bots/vault (the vault bot credentials) — copies the
|
||||
# token field to kv/disinto/shared/ops-repo. This is the "service" path that
|
||||
# dispatcher uses, distinct from the "agent" path (bots/vault) used by
|
||||
# agent tasks under the service-agents policy.
|
||||
#
|
||||
# Idempotency contract:
|
||||
# - Key present with non-empty value → leave untouched, log "token unchanged".
|
||||
# - Key missing or empty → copy from bots/vault, log "token copied".
|
||||
# - If bots/vault is also empty → generate a random value, log "token generated".
|
||||
#
|
||||
# Preconditions:
|
||||
# - Vault reachable + unsealed at $VAULT_ADDR.
|
||||
# - VAULT_TOKEN set (env) or /etc/vault.d/root.token readable.
|
||||
# - The `kv/` mount is enabled as KV v2.
|
||||
#
|
||||
# Requires:
|
||||
# - VAULT_ADDR (e.g. http://127.0.0.1:8200)
|
||||
# - VAULT_TOKEN (env OR /etc/vault.d/root.token, resolved by lib/hvault.sh)
|
||||
# - curl, jq, openssl
|
||||
#
|
||||
# Usage:
|
||||
# tools/vault-seed-ops-repo.sh
|
||||
# tools/vault-seed-ops-repo.sh --dry-run
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (seed applied, or already applied)
|
||||
# 1 precondition / API / mount-mismatch failure
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# shellcheck source=../lib/hvault.sh
|
||||
source "${REPO_ROOT}/lib/hvault.sh"
|
||||
|
||||
# KV v2 mount + logical paths
|
||||
KV_MOUNT="kv"
|
||||
OPS_REPO_PATH="disinto/shared/ops-repo"
|
||||
VAULT_BOT_PATH="disinto/bots/vault"
|
||||
|
||||
OPS_REPO_API="${KV_MOUNT}/data/${OPS_REPO_PATH}"
|
||||
VAULT_BOT_API="${KV_MOUNT}/data/${VAULT_BOT_PATH}"
|
||||
|
||||
log() { printf '[vault-seed-ops-repo] %s\n' "$*"; }
|
||||
die() { printf '[vault-seed-ops-repo] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# ── Flag parsing ─────────────────────────────────────────────────────────────
|
||||
DRY_RUN=0
|
||||
case "$#:${1-}" in
|
||||
0:)
|
||||
;;
|
||||
1:--dry-run)
|
||||
DRY_RUN=1
|
||||
;;
|
||||
1:-h|1:--help)
|
||||
printf 'Usage: %s [--dry-run]\n\n' "$(basename "$0")"
|
||||
printf 'Seed kv/disinto/shared/ops-repo with FORGE_TOKEN.\n\n'
|
||||
printf 'Copies token from kv/disinto/bots/vault if present;\n'
|
||||
printf 'otherwise generates a random value. Idempotent:\n'
|
||||
printf 'existing non-empty values are left untouched.\n\n'
|
||||
printf ' --dry-run Print planned actions without writing.\n'
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "invalid arguments: $* (try --help)"
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
for bin in curl jq openssl; do
|
||||
command -v "$bin" >/dev/null 2>&1 \
|
||||
|| die "required binary not found: ${bin}"
|
||||
done
|
||||
|
||||
[ -n "${VAULT_ADDR:-}" ] \
|
||||
|| die "VAULT_ADDR unset — e.g. export VAULT_ADDR=http://127.0.0.1:8200"
|
||||
hvault_token_lookup >/dev/null \
|
||||
|| die "Vault auth probe failed — check VAULT_ADDR + VAULT_TOKEN"
|
||||
|
||||
# ── Step 1/2: ensure kv/ mount exists and is KV v2 ───────────────────────────
|
||||
log "── Step 1/2: ensure ${KV_MOUNT}/ is KV v2 ──"
|
||||
export DRY_RUN
|
||||
hvault_ensure_kv_v2 "$KV_MOUNT" "[vault-seed-ops-repo]" \
|
||||
|| die "KV mount check failed"
|
||||
|
||||
# ── Step 2/2: seed ops-repo from vault bot ───────────────────────────────────
|
||||
log "── Step 2/2: seed ${OPS_REPO_API} ──"
|
||||
|
||||
# Read existing ops-repo value
|
||||
existing_raw="$(hvault_get_or_empty "${OPS_REPO_API}")" \
|
||||
|| die "failed to read ${OPS_REPO_API}"
|
||||
|
||||
existing_token=""
|
||||
if [ -n "$existing_raw" ]; then
|
||||
existing_token="$(printf '%s' "$existing_raw" | jq -r '.data.data.token // ""')"
|
||||
fi
|
||||
|
||||
desired_token="$existing_token"
|
||||
action=""
|
||||
|
||||
if [ -z "$existing_token" ]; then
|
||||
# Token missing — try to copy from vault bot
|
||||
bot_raw="$(hvault_get_or_empty "${VAULT_BOT_API}")" || true
|
||||
if [ -n "$bot_raw" ]; then
|
||||
bot_token="$(printf '%s' "$bot_raw" | jq -r '.data.data.token // ""')"
|
||||
if [ -n "$bot_token" ]; then
|
||||
desired_token="$bot_token"
|
||||
action="copied"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If still no token, generate one
|
||||
if [ -z "$desired_token" ]; then
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
action="generated (dry-run)"
|
||||
else
|
||||
desired_token="$(openssl rand -hex 32)"
|
||||
action="generated"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$action" ]; then
|
||||
log "all keys present at ${OPS_REPO_API} — no-op"
|
||||
log "token unchanged"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
log "[dry-run] ${OPS_REPO_PATH}: would ${action} token"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Write the token
|
||||
payload="$(jq -n --arg t "$desired_token" '{data: {token: $t}}')"
|
||||
_hvault_request POST "${OPS_REPO_API}" "$payload" >/dev/null \
|
||||
|| die "failed to write ${OPS_REPO_API}"
|
||||
|
||||
log "${OPS_REPO_PATH}: ${action} token"
|
||||
log "done — ${OPS_REPO_API} seeded"
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: a467d613a44b9b475a60c14c4162621e846969ea -->
|
||||
<!-- last-reviewed: 8ad5aca6bbee77634b3c63523042b1d39cefa96a -->
|
||||
# vault/policies/ — Agent Instructions
|
||||
|
||||
HashiCorp Vault ACL policies for the disinto factory. One `.hcl` file per
|
||||
|
|
@ -30,9 +30,6 @@ KV v2). Vault addresses KV v2 data at `kv/data/<path>` and metadata at
|
|||
|---|---|
|
||||
| `service-forgejo` | `kv/data/disinto/shared/forgejo/*` |
|
||||
| `service-woodpecker` | `kv/data/disinto/shared/woodpecker/*` |
|
||||
| `service-agents` | All 7 `kv/data/disinto/bots/<role>/*` namespaces + `kv/data/disinto/shared/forge/*`; composite policy for the `agents` Nomad job (S4.1) |
|
||||
| `service-chat` | `kv/data/disinto/shared/chat/*`; read-only OAuth client config + forward-auth secret for the chat Nomad job (S5.2, #989) |
|
||||
| `service-dispatcher` | `kv/data/disinto/runner/*` (list+read) + `kv/data/disinto/shared/ops-repo/*` (read); used by edge dispatcher sidecar (S5.1, #988) |
|
||||
| `bot-<role>` (dev, review, gardener, architect, planner, predictor, supervisor, vault, dev-qwen) | `kv/data/disinto/bots/<role>/*` + `kv/data/disinto/shared/forge/*` |
|
||||
| `runner-<TOKEN>` (GITHUB\_TOKEN, CODEBERG\_TOKEN, CLAWHUB\_TOKEN, DEPLOY\_KEY, NPM\_TOKEN, DOCKER\_HUB\_TOKEN) | `kv/data/disinto/runner/<TOKEN>` (exactly one) |
|
||||
| `dispatcher` | `kv/data/disinto/runner/*` + `kv/data/disinto/shared/ops-repo/*` |
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Architect agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the architect-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/architect" {
|
||||
path "kv/data/disinto/bots/architect/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/architect" {
|
||||
path "kv/metadata/disinto/bots/architect/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,14 +5,14 @@
|
|||
# via workload identity (S2.4). KV path mirrors the bot basename:
|
||||
# kv/disinto/bots/dev-qwen/*.
|
||||
|
||||
path "kv/data/disinto/bots/dev-qwen" {
|
||||
path "kv/data/disinto/bots/dev-qwen/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/dev-qwen" {
|
||||
path "kv/metadata/disinto/bots/dev-qwen/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Dev agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the dev-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/dev" {
|
||||
path "kv/data/disinto/bots/dev/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/dev" {
|
||||
path "kv/metadata/disinto/bots/dev/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Gardener agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the gardener-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/gardener" {
|
||||
path "kv/data/disinto/bots/gardener/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/gardener" {
|
||||
path "kv/metadata/disinto/bots/gardener/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Planner agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the planner-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/planner" {
|
||||
path "kv/data/disinto/bots/planner/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/planner" {
|
||||
path "kv/metadata/disinto/bots/planner/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Predictor agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the predictor-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/predictor" {
|
||||
path "kv/data/disinto/bots/predictor/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/predictor" {
|
||||
path "kv/metadata/disinto/bots/predictor/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Review agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the review-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/review" {
|
||||
path "kv/data/disinto/bots/review/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/review" {
|
||||
path "kv/metadata/disinto/bots/review/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
# Supervisor agent: reads its own bot KV namespace + the shared forge URL.
|
||||
# Attached to the supervisor-agent Nomad job via workload identity (S2.4).
|
||||
|
||||
path "kv/data/disinto/bots/supervisor" {
|
||||
path "kv/data/disinto/bots/supervisor/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/supervisor" {
|
||||
path "kv/metadata/disinto/bots/supervisor/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@
|
|||
# NOTE: distinct from the runner-* policies, which gate per-secret access
|
||||
# for vault-runner ephemeral dispatches (Step 5).
|
||||
|
||||
path "kv/data/disinto/bots/vault" {
|
||||
path "kv/data/disinto/bots/vault/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/vault" {
|
||||
path "kv/metadata/disinto/bots/vault/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
path "kv/data/disinto/shared/forge/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ path "kv/metadata/disinto/runner/*" {
|
|||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/ops-repo" {
|
||||
path "kv/data/disinto/shared/ops-repo/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/shared/ops-repo" {
|
||||
path "kv/metadata/disinto/shared/ops-repo/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,76 +0,0 @@
|
|||
# vault/policies/service-agents.hcl
|
||||
#
|
||||
# Composite policy for the `agents` Nomad job (S4.1, issue #955).
|
||||
# Grants read access to all 7 bot KV namespaces + shared forge config,
|
||||
# so a single job running all agent roles can pull per-bot tokens from
|
||||
# Vault via workload identity.
|
||||
|
||||
# ── Per-bot KV paths (token + pass per role) ─────────────────────────────────
|
||||
path "kv/data/disinto/bots/dev" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/dev" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/review" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/review" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/gardener" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/gardener" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/architect" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/architect" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/planner" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/planner" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/predictor" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/predictor" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/supervisor" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/supervisor" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/bots/vault" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/bots/vault" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
# ── Shared forge config (URL, bot usernames) ─────────────────────────────────
|
||||
path "kv/data/disinto/shared/forge" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
# vault/policies/service-chat.hcl
|
||||
#
|
||||
# Read-only access to shared Chat secrets (OAuth client config, forward auth
|
||||
# secret). Attached to the Chat Nomad job via workload identity (S5.2).
|
||||
#
|
||||
# Scope: kv/disinto/shared/chat — entries owned by the operator and
|
||||
# shared between the chat service and edge proxy.
|
||||
|
||||
path "kv/data/disinto/shared/chat" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/shared/chat" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
# vault/policies/service-dispatcher.hcl
|
||||
#
|
||||
# Edge dispatcher policy: needs to enumerate the runner secret namespace
|
||||
# (to check secret presence before dispatching) and read the shared
|
||||
# ops-repo credentials (token + clone URL) it uses to fetch action TOMLs.
|
||||
#
|
||||
# Scope:
|
||||
# - kv/disinto/runner/* — read all per-secret values + list keys
|
||||
# - kv/disinto/shared/ops-repo/* — read the ops-repo creds bundle
|
||||
#
|
||||
# The actual ephemeral runner container created per dispatch gets the
|
||||
# narrow runner-<NAME> policies, NOT this one. This policy stays bound
|
||||
# to the long-running dispatcher only.
|
||||
|
||||
path "kv/data/disinto/runner/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/runner/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
||||
path "kv/data/disinto/shared/ops-repo" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/shared/ops-repo" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
|
@ -6,10 +6,10 @@
|
|||
# Scope: kv/disinto/shared/woodpecker/* — entries owned by the operator
|
||||
# and consumed by woodpecker-server + woodpecker-agent.
|
||||
|
||||
path "kv/data/disinto/shared/woodpecker" {
|
||||
path "kv/data/disinto/shared/woodpecker/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/disinto/shared/woodpecker" {
|
||||
path "kv/metadata/disinto/shared/woodpecker/*" {
|
||||
capabilities = ["list", "read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,21 +62,6 @@ roles:
|
|||
namespace: default
|
||||
job_id: woodpecker-agent
|
||||
|
||||
# ── Agents composite (nomad/jobs/agents.hcl — S4.1) ──────────────────────
|
||||
# Single job running all 7 agent roles. Uses a composite policy
|
||||
# (vault/policies/service-agents.hcl) that unions all bot KV paths.
|
||||
- name: service-agents
|
||||
policy: service-agents
|
||||
namespace: default
|
||||
job_id: agents
|
||||
|
||||
# ── Chat UI (nomad/jobs/chat.hcl — S5.2) ─────────────────────────────────
|
||||
# Claude chat UI service with OAuth secrets. Uses vault/policies/service-chat.hcl.
|
||||
- name: service-chat
|
||||
policy: service-chat
|
||||
namespace: default
|
||||
job_id: chat
|
||||
|
||||
# ── Per-agent bots (nomad/jobs/bot-<role>.hcl — land in later steps) ───────
|
||||
# job_id placeholders match the policy name 1:1 until each bot's jobspec
|
||||
# lands. When a bot's jobspec is added under nomad/jobs/, update the
|
||||
|
|
@ -128,10 +113,10 @@ roles:
|
|||
job_id: bot-vault
|
||||
|
||||
# ── Edge dispatcher ────────────────────────────────────────────────────────
|
||||
- name: service-dispatcher
|
||||
policy: service-dispatcher
|
||||
- name: dispatcher
|
||||
policy: dispatcher
|
||||
namespace: default
|
||||
job_id: edge
|
||||
job_id: dispatcher
|
||||
|
||||
# ── Per-secret runner roles ────────────────────────────────────────────────
|
||||
# vault-runner (Step 5) composes runner-<NAME> policies onto each
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue