diff --git a/.woodpecker/detect-duplicates.py b/.woodpecker/detect-duplicates.py index 9b108bf..2ee001e 100644 --- a/.woodpecker/detect-duplicates.py +++ b/.woodpecker/detect-duplicates.py @@ -302,12 +302,37 @@ def main() -> int: "9d72d40ff303cbed0b7e628fc15381c3": "Case loop + dry-run handler (vault-seed-woodpecker + wp-oauth-register)", "5b52ddbbf47948e3cbc1b383f0909588": "Help + invalid arg handler end (vault-seed-woodpecker + wp-oauth-register)", # Common vault-seed script preamble + precondition patterns - # Shared across tools/vault-seed-{forgejo,agents,woodpecker}.sh + # Shared across tools/vault-seed-{forgejo,agents,woodpecker,chat}.sh "dff3675c151fcdbd2fef798826ae919b": "Vault-seed preamble: set -euo + path setup + source hvault.sh + KV_MOUNT", "1cd9f0d083e24e6e6b2071db9b6dae09": "Vault-seed preconditions: binary check loop + VAULT_ADDR guard", "63bfa88d71764c95c65a9a248f3e40ab": "Vault-seed preconditions: binary check end + VAULT_ADDR die", "34873ad3570b211ce1d90468ab6ac94c": "Vault-seed preconditions: VAULT_ADDR die + hvault_token_lookup", "71a52270f249e843cda48ad896d9f781": "Vault-seed preconditions: VAULT_ADDR + hvault_token_lookup + die", + # Common vault-seed script patterns shared across vault-seed-{forgejo,chat}.sh + # These are intentional duplicates following the same pattern structure + "6906b7787796c2ccb8dd622e2ad4e7bf": "Vault-seed flag parsing: DRY_RUN=0 + case start", + "a0df5283b616b964f8bc32fd99ec1b5a": "Vault-seed flag parsing: case 0: block", + "e15e3272fdd9f0f46ce9e726aea9f853": "Vault-seed flag parsing: case dry-run block", + "c9f22385cc49a3dac1d336bc14c6315b": "Vault-seed flag parsing: case dry-run body", + "106f4071e88f841b3208b01144cd1c39": "Vault-seed flag parsing: case dry-run end", + "97e744846ca5f05806c92b5905c87cf1": "Vault-seed help: dry-run description", + "35c64c9c957245b9cc371c019c9efa58": "Vault-seed help: dry-run description line 2", + "c15506dcb6bb340b25d1c39d442dd2e6": "Vault-seed help: exit 0 + case end", + "1feecd3b3caf00045fae938ddf2811de": "Vault-seed help: invalid args handler", + "919780d5e7182715344f5aa02b191294": "Vault-seed help: invalid args handler body", + "8dce1d292bce8e60ef4c0665b62945b0": "Vault-seed help: case end", + "ca043687143a5b47bd54e65a99ce8ee8": "Vault-seed help: esac + binary check", + "aefd9f655411a955395e6e5995ddbe6f": "Vault-seed preconditions: binary check loop", + "60f0c46deb5491599457efb4048918e5": "Vault-seed preconditions: binary check + VAULT_ADDR", + "f6838f581ef6b4d82b55268389032769": "Vault-seed preconditions: VAULT_ADDR + hvault_token_lookup", + # Vault-seed dry-run output pattern + "bb2e12065e522f5aed153a30e0961d3b": "Vault-seed dry-run: key generated/unchanged check", + "94547395e64b262a0bef9485c0e3756c": "Vault-seed dry-run: generated case", + "073ee857a992f992781aff5696805a07": "Vault-seed dry-run: unchanged case", + "47573826736bb4a8914035447b262ec3": "Vault-seed dry-run: done + exit", + # Vault-seed final output pattern + "a50398451851db705f5fa000388a14b7": "Vault-seed final: key generated/unchanged check", + "6025ed5237bbadd3957ba1c340bbc5d1": "Vault-seed final: generated case", } if not sh_files: diff --git a/AGENTS.md b/AGENTS.md index 35cb380..722bc23 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ - + # Disinto — Agent Instructions ## What this repo is @@ -39,7 +39,7 @@ disinto/ (code repo) │ hooks/ — Claude Code session hooks (on-compact-reinject, on-idle-stop, on-phase-change, on-pretooluse-guard, on-session-end, on-stop-failure) │ init/nomad/ — cluster-up.sh, install.sh, vault-init.sh, lib-systemd.sh (Nomad+Vault Step 0 installers, #821-#825); wp-oauth-register.sh (Forgejo OAuth2 app + Vault KV seeder for Woodpecker, S3.3); deploy.sh (dependency-ordered Nomad job deploy + health-wait, S4) ├── nomad/ server.hcl, client.hcl (allow_privileged for woodpecker-agent, S3-fix-5), vault.hcl — HCL configs deployed to /etc/nomad.d/ and /etc/vault.d/ by lib/init/nomad/cluster-up.sh -│ jobs/ — Nomad jobspecs: forgejo.hcl (Vault secrets via template, S2.4); woodpecker-server.hcl + woodpecker-agent.hcl (host-net, docker.sock, Vault KV, S3.1-S3.2); agents.hcl (7 roles, llama, Vault-templated bot tokens, S4.1); vault-runner.hcl (parameterized batch dispatch, S5.3); staging.hcl (Caddy file-server, dynamic port — edge discovers via service registration, S5.2); chat.hcl (Claude chat UI, tmpfs via mount block, Vault OAuth secrets, S5.2); edge.hcl (Caddy proxy + dispatcher sidecar, S5.1) +│ jobs/ — Nomad jobspecs: forgejo.hcl (Vault secrets via template, S2.4); woodpecker-server.hcl + woodpecker-agent.hcl (host-net, docker.sock, Vault KV, S3.1-S3.2); agents.hcl (7 roles, llama, Vault-templated bot tokens, S4.1); vault-runner.hcl (parameterized batch dispatch, S5.3) ├── projects/ *.toml.example — templates; *.toml — local per-box config (gitignored) ├── formulas/ Issue templates (TOML specs for multi-step agent tasks) ├── docker/ Dockerfiles and entrypoints: reproduce, triage, edge dispatcher, chat (server.py, entrypoint-chat.sh, Dockerfile, ui/) diff --git a/architect/AGENTS.md b/architect/AGENTS.md index 91b36cd..d759433 100644 --- a/architect/AGENTS.md +++ b/architect/AGENTS.md @@ -1,4 +1,4 @@ - + # Architect — Agent Instructions ## What this agent is diff --git a/bin/disinto b/bin/disinto index c18ef0c..5ced2dd 100755 --- a/bin/disinto +++ b/bin/disinto @@ -82,7 +82,7 @@ Init options: --ci-id Woodpecker CI repo ID (default: 0 = no CI) --forge-url Forge base URL (default: http://localhost:3000) --backend Orchestration backend: docker (default) | nomad - --with (nomad) Deploy services: forgejo,woodpecker,agents,staging,chat,edge[,...] (S1.3, S3.4, S4.2, S5.2, S5.5) + --with (nomad) Deploy services: forgejo,woodpecker,agents,agents,staging,chat,edge[,...] (S1.3, S3.4, S4.2, S5.2, S5.5) --empty (nomad) Bring up cluster only, no jobs (S0.4) --bare Skip compose generation (bare-metal setup) --build Use local docker build instead of registry images (dev mode) @@ -836,17 +836,17 @@ _disinto_init_nomad() { fi # Build custom images dry-run (if agents, chat, or edge services are included) - if echo ",$with_services," | grep -qE ",(agents|chat|edge),"; then + if echo ",$with_services," | grep -qE ",(agents|chat),"; then echo "" echo "── Build images dry-run ──────────────────────────────" if echo ",$with_services," | grep -q ",agents,"; then echo "[build] [dry-run] docker build -t disinto/agents:local -f ${FACTORY_ROOT}/docker/agents/Dockerfile ${FACTORY_ROOT}" fi if echo ",$with_services," | grep -q ",chat,"; then - echo "[build] [dry-run] docker build -t disinto/chat:local -f ${FACTORY_ROOT}/docker/chat/Dockerfile ${FACTORY_ROOT}/docker/chat" + echo "[build] [dry-run] docker build -t disinto/chat:local -f ${FACTORY_ROOT}/docker/chat/Dockerfile ${FACTORY_ROOT}" fi if echo ",$with_services," | grep -q ",edge,"; then - echo "[build] [dry-run] docker build -t disinto/edge:local -f ${FACTORY_ROOT}/docker/edge/Dockerfile ${FACTORY_ROOT}/docker/edge" + echo "[build] [dry-run] docker build -t disinto/edge:local -f ${FACTORY_ROOT}/docker/edge/Dockerfile ${FACTORY_ROOT}" fi fi exit 0 @@ -950,12 +950,12 @@ _disinto_init_nomad() { if echo ",$with_services," | grep -q ",chat,"; then local tag="disinto/chat:local" echo "── Building $tag ─────────────────────────────" - docker build -t "$tag" -f "${FACTORY_ROOT}/docker/chat/Dockerfile" "${FACTORY_ROOT}/docker/chat" 2>&1 | tail -5 + docker build -t "$tag" -f "${FACTORY_ROOT}/docker/chat/Dockerfile" "${FACTORY_ROOT}" 2>&1 | tail -5 fi if echo ",$with_services," | grep -q ",edge,"; then local tag="disinto/edge:local" echo "── Building $tag ─────────────────────────────" - docker build -t "$tag" -f "${FACTORY_ROOT}/docker/edge/Dockerfile" "${FACTORY_ROOT}/docker/edge" 2>&1 | tail -5 + docker build -t "$tag" -f "${FACTORY_ROOT}/docker/edge/Dockerfile" "${FACTORY_ROOT}" 2>&1 | tail -5 fi fi @@ -1002,23 +1002,6 @@ _disinto_init_nomad() { # Deploy this service echo "" echo "── Deploying ${svc} ───────────────────────────────────────" - - # Seed host volumes before deployment (if needed) - case "$svc" in - staging) - # Seed site-content host volume (/srv/disinto/docker) with static content - # The staging jobspec mounts this volume read-only to /srv/site - local site_content_src="${FACTORY_ROOT}/docker/index.html" - local site_content_dst="/srv/disinto/docker" - if [ -f "$site_content_src" ] && [ -d "$site_content_dst" ]; then - if ! cmp -s "$site_content_src" "${site_content_dst}/index.html" 2>/dev/null; then - echo "[staging] seeding site-content volume..." - cp "$site_content_src" "${site_content_dst}/index.html" - fi - fi - ;; - esac - local jobspec_path="${FACTORY_ROOT}/nomad/jobs/${svc}.hcl" if [ ! -f "$jobspec_path" ]; then echo "Error: jobspec not found: ${jobspec_path}" >&2 @@ -1217,7 +1200,8 @@ disinto_init() { # Auto-include all dependencies when edge is requested (S5.5) if echo ",$with_services," | grep -q ",edge,"; then # Edge depends on all backend services - for dep in forgejo woodpecker-server woodpecker-agent agents staging chat; do + local -a deps="forgejo woodpecker-server woodpecker-agent agents staging chat" + for dep in "${deps[@]}"; do if ! echo ",$with_services," | grep -q ",${dep},"; then echo "Note: --with edge implies --with ${dep} (edge depends on all backend services)" with_services="${with_services},${dep}" diff --git a/dev/AGENTS.md b/dev/AGENTS.md index af014cf..f51a037 100644 --- a/dev/AGENTS.md +++ b/dev/AGENTS.md @@ -1,4 +1,4 @@ - + # Dev Agent **Role**: Implement issues autonomously — write code, push branches, address diff --git a/docker/chat/Dockerfile b/docker/chat/Dockerfile index c4cb28b..3d89863 100644 --- a/docker/chat/Dockerfile +++ b/docker/chat/Dockerfile @@ -1,22 +1,20 @@ # disinto-chat — minimal HTTP backend for Claude chat UI # -# Small Debian slim base with Python runtime and Node.js. +# Small Debian slim base with Python runtime. # Chosen for simplicity and small image size (~100MB). # # Image size: ~100MB (well under the 200MB ceiling) # -# Claude CLI is baked into the image — same pattern as the agents container. +# The claude binary is mounted from the host at runtime via docker-compose, +# not baked into the image — same pattern as the agents container. FROM debian:bookworm-slim -# Install Node.js (required for Claude CLI) and Python +# Install Python (no build-time network access needed) RUN apt-get update && apt-get install -y --no-install-recommends \ - nodejs npm python3 \ + python3 \ && rm -rf /var/lib/apt/lists/* -# Install Claude Code CLI — chat backend runtime -RUN npm install -g @anthropic-ai/claude-code@2.1.84 - # Non-root user — fixed UID 10001 for sandbox hardening (#706) RUN useradd -m -u 10001 -s /bin/bash chat diff --git a/docker/edge/entrypoint-edge.sh b/docker/edge/entrypoint-edge.sh index 6db96b7..1b5f94f 100755 --- a/docker/edge/entrypoint-edge.sh +++ b/docker/edge/entrypoint-edge.sh @@ -234,13 +234,6 @@ fi rm -f "$_fetch_log" done) & -# Nomad template renders Caddyfile to /local/Caddyfile via service discovery; -# copy it into the expected location if present (compose uses the mounted path). -if [ -f /local/Caddyfile ]; then - cp /local/Caddyfile /etc/caddy/Caddyfile - echo "edge: using Nomad-rendered Caddyfile from /local/Caddyfile" >&2 -fi - # Caddy as main process — run in foreground via wait so background jobs survive # (exec replaces the shell, which can orphan backgrounded subshells) caddy run --config /etc/caddy/Caddyfile --adapter caddyfile & diff --git a/gardener/AGENTS.md b/gardener/AGENTS.md index 9906343..cdf829b 100644 --- a/gardener/AGENTS.md +++ b/gardener/AGENTS.md @@ -1,4 +1,4 @@ - + # Gardener Agent **Role**: Backlog grooming — detect duplicate issues, missing acceptance diff --git a/gardener/dust.jsonl b/gardener/dust.jsonl index e69de29..14b0d5c 100644 --- a/gardener/dust.jsonl +++ b/gardener/dust.jsonl @@ -0,0 +1 @@ +{"issue":915,"group":"lib/generators.sh","title":"remove no-op sed in generate_compose --build mode","reason":"sed replaces agents: with itself — no behavior change; single-line removal","ts":"2026-04-17T01:04:05Z"} diff --git a/gardener/pending-actions.json b/gardener/pending-actions.json index dc08304..fe51488 100644 --- a/gardener/pending-actions.json +++ b/gardener/pending-actions.json @@ -1,12 +1 @@ -[ - { - "action": "edit_body", - "issue": 915, - "body": "Flagged by AI reviewer in PR \\#911.\n\n## Problem\n\n`lib/generators.sh` line 660 contains a no-op `sed` invocation:\n```\nsed -i 's|^\\( agents:\\)|\\1|' \"$compose_file\"\n```\n\nThis replaces ` agents:` with itself — it does nothing. It is dead code left over from a prior iteration.\n\n## Fix\n\nRemove the no-op `sed` line at line 660 of `lib/generators.sh`.\n\n## Affected files\n- `lib/generators.sh` (line 660 — the no-op sed invocation in generate_compose --build mode)\n\n## Acceptance criteria\n- [ ] The no-op sed line is removed from `lib/generators.sh`\n- [ ] `shellcheck` clean on `lib/generators.sh`\n- [ ] CI green\n\n---\n*Auto-created from AI review*" - }, - { - "action": "add_label", - "issue": 915, - "label": "backlog" - } -] +[] diff --git a/lib/AGENTS.md b/lib/AGENTS.md index aa1699e..9c69784 100644 --- a/lib/AGENTS.md +++ b/lib/AGENTS.md @@ -1,4 +1,4 @@ - + # Shared Helpers (`lib/`) All agents source `lib/env.sh` as their first action. Additional helpers are @@ -30,9 +30,9 @@ sourced as needed. | `lib/git-creds.sh` | Shared git credential helper configuration. `configure_git_creds([HOME_DIR] [RUN_AS_CMD])` — writes a static credential helper script and configures git globally to use password-based HTTP auth (Forgejo 11.x rejects API tokens for `git push`, #361). **Retry on cold boot (#741)**: resolves bot username from `FORGE_TOKEN` with 5 retries (exponential backoff 1-5s); fails loudly and returns 1 if Forgejo is unreachable — never falls back to a wrong hardcoded default (exports `BOT_USER` on success). `repair_baked_cred_urls([--as RUN_AS_CMD] DIR ...)` — rewrites any git remote URLs that have credentials baked in to use clean URLs instead; uses `safe.directory` bypass for root-owned repos (#671). Requires `FORGE_PASS`, `FORGE_URL`, `FORGE_TOKEN`. | entrypoints (agents, edge) | | `lib/ops-setup.sh` | `setup_ops_repo()` — creates ops repo on Forgejo if it doesn't exist, configures bot collaborators, clones/initializes ops repo locally, seeds directory structure (vault, knowledge, evidence, sprints). Evidence subdirectories seeded: engagement/, red-team/, holdout/, evolution/, user-test/. Also seeds sprints/ for architect output. Exports `_ACTUAL_OPS_SLUG`. `migrate_ops_repo(ops_root, [primary_branch])` — idempotent migration helper that seeds missing directories and .gitkeep files on existing ops repos (pre-#407 deployments). | bin/disinto (init) | | `lib/ci-setup.sh` | `_install_cron_impl()` — installs crontab entries for bare-metal deployments (compose mode uses polling loop instead). `_create_forgejo_oauth_app()` — generic helper to create an OAuth2 app on Forgejo (shared by Woodpecker and chat). `_create_woodpecker_oauth_impl()` — creates Woodpecker OAuth2 app (thin wrapper). `_create_chat_oauth_impl()` — creates disinto-chat OAuth2 app, writes `CHAT_OAUTH_CLIENT_ID`/`CHAT_OAUTH_CLIENT_SECRET` to `.env` (#708). `_generate_woodpecker_token_impl()` — auto-generates WOODPECKER_TOKEN via OAuth2 flow. `_activate_woodpecker_repo_impl()` — activates repo in Woodpecker. All gated by `_load_ci_context()` which validates required env vars. | bin/disinto (init) | -| `lib/generators.sh` | Template generation for `disinto init`: `generate_compose()` — docker-compose.yml (uses `codeberg.org/forgejo/forgejo:11.0` tag; `CLAUDE_BIN_DIR` volume mount removed from agents/llama services — only `reproduce` and `edge` still use the host-mounted CLI (#992); adds `security_opt: [apparmor:unconfined]` to all services for rootless container compatibility; Forgejo includes a healthcheck so dependent services use `condition: service_healthy` — fixes cold-start races, #665; adds `chat` service block with isolated `chat-config` named volume and `CHAT_HISTORY_DIR` bind-mount for per-user NDJSON history persistence (#710); injects `FORWARD_AUTH_SECRET` for Caddy↔chat defense-in-depth auth (#709); cost-cap env vars `CHAT_MAX_REQUESTS_PER_HOUR`, `CHAT_MAX_REQUESTS_PER_DAY`, `CHAT_MAX_TOKENS_PER_DAY` (#711); subdomain fallback comment for `EDGE_TUNNEL_FQDN_*` vars (#713); all `depends_on` now use `condition: service_healthy/started` instead of bare service names; all services now include `restart: unless-stopped` including the edge service — #768; agents service now uses `image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}` instead of `build:` (#429); `WOODPECKER_PLUGINS_PRIVILEGED` env var added to woodpecker service (#779); agents-llama conditional block gated on `ENABLE_LLAMA_AGENT=1` (#769); `agents-llama-all` compose service (profile `agents-llama-all`, all 7 roles: review,dev,gardener,architect,planner,predictor,supervisor) added by #801; agents service gains volume mounts for `./projects`, `./.env`, `./state`), `generate_caddyfile()` — Caddyfile (routes: `/forge/*` → forgejo:3000, `/woodpecker/*` → woodpecker:8000, `/staging/*` → staging:80; `/chat/login` and `/chat/oauth/callback` bypass `forward_auth` so unauthenticated users can reach the OAuth flow; `/chat/*` gated by `forward_auth` on `chat:8080/chat/auth/verify` which stamps `X-Forwarded-User` (#709); root `/` redirects to `/forge/`), `generate_staging_index()` — staging index, `generate_deploy_pipelines()` — Woodpecker deployment pipeline configs. Requires `FACTORY_ROOT`, `PROJECT_NAME`, `PRIMARY_BRANCH`. | bin/disinto (init) | +| `lib/generators.sh` | Template generation for `disinto init`: `generate_compose()` — docker-compose.yml (uses `codeberg.org/forgejo/forgejo:11.0` tag; adds `security_opt: [apparmor:unconfined]` to all services for rootless container compatibility; Forgejo includes a healthcheck so dependent services use `condition: service_healthy` — fixes cold-start races, #665; adds `chat` service block with isolated `chat-config` named volume and `CHAT_HISTORY_DIR` bind-mount for per-user NDJSON history persistence (#710); injects `FORWARD_AUTH_SECRET` for Caddy↔chat defense-in-depth auth (#709); cost-cap env vars `CHAT_MAX_REQUESTS_PER_HOUR`, `CHAT_MAX_REQUESTS_PER_DAY`, `CHAT_MAX_TOKENS_PER_DAY` (#711); subdomain fallback comment for `EDGE_TUNNEL_FQDN_*` vars (#713); all `depends_on` now use `condition: service_healthy/started` instead of bare service names; all services now include `restart: unless-stopped` including the edge service — #768; agents service now uses `image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}` instead of `build:` (#429); `WOODPECKER_PLUGINS_PRIVILEGED` env var added to woodpecker service (#779); agents-llama conditional block gated on `ENABLE_LLAMA_AGENT=1` (#769); `agents-llama-all` compose service (profile `agents-llama-all`, all 7 roles: review,dev,gardener,architect,planner,predictor,supervisor) added by #801; agents service gains volume mounts for `./projects`, `./.env`, `./state`), `generate_caddyfile()` — Caddyfile (routes: `/forge/*` → forgejo:3000, `/woodpecker/*` → woodpecker:8000, `/staging/*` → staging:80; `/chat/login` and `/chat/oauth/callback` bypass `forward_auth` so unauthenticated users can reach the OAuth flow; `/chat/*` gated by `forward_auth` on `chat:8080/chat/auth/verify` which stamps `X-Forwarded-User` (#709); root `/` redirects to `/forge/`), `generate_staging_index()` — staging index, `generate_deploy_pipelines()` — Woodpecker deployment pipeline configs. Requires `FACTORY_ROOT`, `PROJECT_NAME`, `PRIMARY_BRANCH`. | bin/disinto (init) | | `lib/sprint-filer.sh` | Post-merge sub-issue filer for sprint PRs. Invoked by the `.woodpecker/ops-filer.yml` pipeline after a sprint PR merges to ops repo `main`. Parses ` ... ` blocks from sprint PR bodies to extract sub-issue definitions, creates them on the project repo using `FORGE_FILER_TOKEN` (narrow-scope `filer-bot` identity with `issues:write` only), adds `in-progress` label to the parent vision issue, and handles vision lifecycle closure when all sub-issues are closed. Uses `filer_api_all()` for paginated fetches. Idempotent: uses `` markers to skip already-filed issues. Requires `FORGE_FILER_TOKEN`, `FORGE_API`, `FORGE_API_BASE`, `FORGE_OPS_REPO`. | `.woodpecker/ops-filer.yml` (CI pipeline on ops repo) | | `lib/hire-agent.sh` | `disinto_hire_an_agent()` — user creation, `.profile` repo setup, formula copying, branch protection, and state marker creation for hiring a new agent. Requires `FORGE_URL`, `FORGE_TOKEN`, `FACTORY_ROOT`, `PROJECT_NAME`. Extracted from `bin/disinto`. | bin/disinto (hire) | | `lib/release.sh` | `disinto_release()` — vault TOML creation, branch setup on ops repo, PR creation, and auto-merge request for a versioned release. `_assert_release_globals()` validates required env vars. Requires `FORGE_URL`, `FORGE_TOKEN`, `FORGE_OPS_REPO`, `FACTORY_ROOT`, `PRIMARY_BRANCH`. Extracted from `bin/disinto`. | bin/disinto (release) | -| `lib/hvault.sh` | HashiCorp Vault helper module. `hvault_kv_get(PATH, [KEY])` — read KV v2 secret, optionally extract one key. `hvault_kv_put(PATH, KEY=VAL ...)` — write KV v2 secret. `hvault_kv_list(PATH)` — list keys at a KV path. `hvault_get_or_empty(PATH)` — GET /v1/PATH; 200→raw body, 404→empty, else structured error + return 1 (used by sync scripts to distinguish "absent, create" from hard failure without tripping errexit, #881). `hvault_ensure_kv_v2(MOUNT, [LOG_PREFIX])` — idempotent KV v2 mount assertion: enables mount if absent, fails loudly if present as wrong type/version. Extracted from all `vault-seed-*.sh` scripts to eliminate dup-detector violations. Respects `DRY_RUN=1`. `hvault_policy_apply(NAME, FILE)` — idempotent policy upsert. `hvault_jwt_login(ROLE, JWT)` — exchange JWT for short-lived token. `hvault_token_lookup()` — returns TTL/policies/accessor for current token. `_hvault_seed_key(PATH, KEY, [GENERATOR])` — seed one KV key if absent; reads existing data and merges to preserve sibling keys (KV v2 replaces atomically); returns 0=created, 1=unchanged, 2=API error (#992). All functions use `VAULT_ADDR` + `VAULT_TOKEN` from env (fallback: `/etc/vault.d/root.token`), emit structured JSON errors to stderr on failure. Tests: `tests/lib-hvault.bats` (requires `vault server -dev`). | `tools/vault-apply-policies.sh`, `tools/vault-apply-roles.sh`, `lib/init/nomad/vault-nomad-auth.sh`, `tools/vault-seed-*.sh` | -| `lib/init/nomad/` | Nomad+Vault installer scripts. `cluster-up.sh` — idempotent Step-0 orchestrator that runs all steps in order (installs packages, writes HCL, enables systemd units, unseals Vault); uses `poll_until_healthy()` helper for deduped readiness polling; `HOST_VOLUME_DIRS` array now includes `/srv/disinto/docker` (for staging file-server, S5.2, #989, #992). `install.sh` — installs pinned Nomad+Vault apt packages. `vault-init.sh` — initializes Vault (unseal keys → `/etc/vault.d/`), creates dev-persisted unseal unit. `lib-systemd.sh` — shared systemd unit helpers. `systemd-nomad.sh`, `systemd-vault.sh` — write and enable service units. `vault-nomad-auth.sh` — Step-2 script that enables Vault's JWT auth at path `jwt-nomad`, writes the JWKS/algs config pointing at Nomad's workload-identity signer, delegates role sync to `tools/vault-apply-roles.sh`, installs `/etc/nomad.d/server.hcl`, and SIGHUPs `nomad.service` if the file changed (#881). `wp-oauth-register.sh` — S3.3 script that creates the Woodpecker OAuth2 app in Forgejo and stores `forgejo_client`/`forgejo_secret` in Vault KV v2 at `kv/disinto/shared/woodpecker`; idempotent (skips if app or secrets already present); called by `bin/disinto --with woodpecker`. `deploy.sh` — S4 dependency-ordered Nomad job deploy + health-wait; takes a list of jobspec basenames, submits each to Nomad and polls until healthy before proceeding to the next; supports `--dry-run` and per-job timeout overrides via `JOB_READY_TIMEOUT_`; invoked by `bin/disinto --with ` and `cluster-up.sh`; deploy order now covers staging, chat, edge (S5.5, #992). Idempotent: each step checks current state before acting. Sourced and called by `cluster-up.sh`; not sourced by agents. | `bin/disinto init --backend=nomad` | +| `lib/hvault.sh` | HashiCorp Vault helper module. `hvault_kv_get(PATH, [KEY])` — read KV v2 secret, optionally extract one key. `hvault_kv_put(PATH, KEY=VAL ...)` — write KV v2 secret. `hvault_kv_list(PATH)` — list keys at a KV path. `hvault_get_or_empty(PATH)` — GET /v1/PATH; 200→raw body, 404→empty, else structured error + return 1 (used by sync scripts to distinguish "absent, create" from hard failure without tripping errexit, #881). `hvault_ensure_kv_v2(MOUNT, [LOG_PREFIX])` — idempotent KV v2 mount assertion: enables mount if absent, fails loudly if present as wrong type/version. Extracted from all `vault-seed-*.sh` scripts to eliminate dup-detector violations. Respects `DRY_RUN=1`. `hvault_policy_apply(NAME, FILE)` — idempotent policy upsert. `hvault_jwt_login(ROLE, JWT)` — exchange JWT for short-lived token. `hvault_token_lookup()` — returns TTL/policies/accessor for current token. All functions use `VAULT_ADDR` + `VAULT_TOKEN` from env (fallback: `/etc/vault.d/root.token`), emit structured JSON errors to stderr on failure. Tests: `tests/lib-hvault.bats` (requires `vault server -dev`). | `tools/vault-apply-policies.sh`, `tools/vault-apply-roles.sh`, `lib/init/nomad/vault-nomad-auth.sh`, `tools/vault-seed-*.sh` | +| `lib/init/nomad/` | Nomad+Vault installer scripts. `cluster-up.sh` — idempotent Step-0 orchestrator that runs all steps in order (installs packages, writes HCL, enables systemd units, unseals Vault); uses `poll_until_healthy()` helper for deduped readiness polling. `install.sh` — installs pinned Nomad+Vault apt packages. `vault-init.sh` — initializes Vault (unseal keys → `/etc/vault.d/`), creates dev-persisted unseal unit. `lib-systemd.sh` — shared systemd unit helpers. `systemd-nomad.sh`, `systemd-vault.sh` — write and enable service units. `vault-nomad-auth.sh` — Step-2 script that enables Vault's JWT auth at path `jwt-nomad`, writes the JWKS/algs config pointing at Nomad's workload-identity signer, delegates role sync to `tools/vault-apply-roles.sh`, installs `/etc/nomad.d/server.hcl`, and SIGHUPs `nomad.service` if the file changed (#881). `wp-oauth-register.sh` — S3.3 script that creates the Woodpecker OAuth2 app in Forgejo and stores `forgejo_client`/`forgejo_secret` in Vault KV v2 at `kv/disinto/shared/woodpecker`; idempotent (skips if app or secrets already present); called by `bin/disinto --with woodpecker`. `deploy.sh` — S4 dependency-ordered Nomad job deploy + health-wait; takes a list of jobspec basenames, submits each to Nomad and polls until healthy before proceeding to the next; supports `--dry-run` and per-job timeout overrides via `JOB_READY_TIMEOUT_`; invoked by `bin/disinto --with ` and `cluster-up.sh`. Idempotent: each step checks current state before acting. Sourced and called by `cluster-up.sh`; not sourced by agents. | `bin/disinto init --backend=nomad` | diff --git a/lib/generators.sh b/lib/generators.sh index 77af9a7..5664b55 100644 --- a/lib/generators.sh +++ b/lib/generators.sh @@ -657,6 +657,7 @@ COMPOSEEOF # In build mode, replace image: with build: for locally-built images if [ "$use_build" = true ]; then + sed -i 's|^\( agents:\)|\1|' "$compose_file" sed -i '/^ image: ghcr\.io\/disinto\/agents:/{s|image: ghcr\.io/disinto/agents:.*|build:\n context: .\n dockerfile: docker/agents/Dockerfile\n pull_policy: build|}' "$compose_file" sed -i '/^ image: ghcr\.io\/disinto\/edge:/{s|image: ghcr\.io/disinto/edge:.*|build: ./docker/edge\n pull_policy: build|}' "$compose_file" fi diff --git a/lib/hvault.sh b/lib/hvault.sh index d283330..1059554 100644 --- a/lib/hvault.sh +++ b/lib/hvault.sh @@ -406,35 +406,26 @@ hvault_token_lookup() { } } -# _hvault_seed_key — Seed a single KV key if it doesn't exist. -# Reads existing data and merges to preserve sibling keys (KV v2 replaces -# .data atomically). Returns 0=created, 1=unchanged, 2=API error. -# Args: -# path: KV v2 logical path (e.g. "disinto/shared/chat") -# key: key name within the path (e.g. "chat_oauth_client_id") -# generator: shell command that outputs a random value (default: openssl rand -hex 32) +# _hvault_seed_key — Seed a KV key if it doesn't exist. Returns 0=created, 1=unchanged. +# Args: path, key, [generator_func] +# path: KV v2 path (e.g., "disinto/shared/chat") +# key: key name within the path (e.g., "chat_oauth_client_id") +# generator: function that outputs a random value (default: openssl rand -hex 32) # Usage: -# _hvault_seed_key "disinto/shared/chat" "chat_oauth_client_id" -# rc=$? # 0=created, 1=unchanged +# _hvault_seed_key "disinto/shared/chat" "chat_oauth_client_id" "openssl rand -hex 32" +# local rc=$? # 0 = created, 1 = unchanged _hvault_seed_key() { local path="$1" key="$2" generator="${3:-openssl rand -hex 32}" - local existing + local existing value + + # Check if key already exists existing=$(hvault_kv_get "$path" "$key" 2>/dev/null) || true if [ -n "$existing" ]; then return 1 # unchanged fi - local value + # Generate and write the value value=$(eval "$generator") - - # Read existing data to preserve sibling keys (KV v2 replaces atomically) - local kv_api="${VAULT_KV_MOUNT}/data/${path}" - local raw existing_data payload - raw="$(hvault_get_or_empty "$kv_api")" || return 2 - existing_data="{}" - [ -n "$raw" ] && existing_data="$(printf '%s' "$raw" | jq '.data.data // {}')" - payload="$(printf '%s' "$existing_data" \ - | jq --arg k "$key" --arg v "$value" '{data: (. + {($k): $v})}')" - _hvault_request POST "$kv_api" "$payload" >/dev/null + hvault_kv_put "$path" "${key}=${value}" return 0 # created } diff --git a/nomad/AGENTS.md b/nomad/AGENTS.md index 9c42c88..18f7dcc 100644 --- a/nomad/AGENTS.md +++ b/nomad/AGENTS.md @@ -1,12 +1,12 @@ - + # nomad/ — Agent Instructions Nomad + Vault HCL for the factory's single-node cluster. These files are the source of truth that `lib/init/nomad/cluster-up.sh` copies onto a factory box under `/etc/nomad.d/` and `/etc/vault.d/` at init time. -This directory covers the **Nomad+Vault migration (Steps 0–5)** — -see issues #821–#992 for the step breakdown. +This directory covers the **Nomad+Vault migration (Steps 0–4)** — +see issues #821–#962 for the step breakdown. ## What lives here @@ -19,9 +19,8 @@ see issues #821–#992 for the step breakdown. | `jobs/woodpecker-server.hcl` | submitted via `lib/init/nomad/deploy.sh` | Woodpecker CI server; host networking, Vault KV for `WOODPECKER_AGENT_SECRET` + Forgejo OAuth creds (S3.1) | | `jobs/woodpecker-agent.hcl` | submitted via `lib/init/nomad/deploy.sh` | Woodpecker CI agent; host networking, `docker.sock` mount, Vault KV for `WOODPECKER_AGENT_SECRET`; `WOODPECKER_SERVER` uses `${attr.unique.network.ip-address}:9000` (Nomad interpolation) — port binds to LXC alloc IP, not localhost (S3.2, S3-fix-6, #964) | | `jobs/agents.hcl` | submitted via `lib/init/nomad/deploy.sh` | All 7 agent roles (dev, review, gardener, planner, predictor, supervisor, architect) + llama variant; Vault-templated bot tokens via `service-agents` policy; `force_pull = false` — image is built locally by `bin/disinto --with agents`, no registry (S4.1, S4-fix-2, S4-fix-5, #955, #972, #978) | -| `jobs/staging.hcl` | submitted via `lib/init/nomad/deploy.sh` | Caddy file-server mounting `docker/` as `/srv/site:ro`; no Vault integration; **dynamic host port** (no static 80 — edge owns 80/443, collision fixed in S5-fix-7 #1018); edge discovers via Nomad service registration (S5.2, #989) | -| `jobs/chat.hcl` | submitted via `lib/init/nomad/deploy.sh` | Claude chat UI; custom `disinto/chat:local` image; sandbox hardening (cap_drop ALL, **tmpfs via mount block** not `tmpfs=` arg — S5-fix-5 #1012, pids_limit 128); Vault-templated OAuth secrets via `service-chat` policy (S5.2, #989) | -| `jobs/edge.hcl` | submitted via `lib/init/nomad/deploy.sh` | Caddy reverse proxy + dispatcher sidecar; routes /forge, /woodpecker, /staging, /chat; uses `disinto/edge:local` image built by `bin/disinto --with edge`; Vault-templated ops-repo creds via `service-dispatcher` policy (S5.1, #988) | +| `jobs/staging.hcl` | submitted via `lib/init/nomad/deploy.sh` | Caddy file-server mounting `docker/` as `/srv/site:ro`; no Vault integration; internal-only via edge proxy (S5.2, #989) | +| `jobs/chat.hcl` | submitted via `lib/init/nomad/deploy.sh` | Claude chat UI; custom `disinto/chat:local` image; sandbox hardening (cap_drop ALL, tmpfs, pids_limit 128); Vault-templated OAuth secrets via `service-chat` policy (S5.2, #989) | Nomad auto-merges every `*.hcl` under `-config=/etc/nomad.d/`, so the split between `server.hcl` and `client.hcl` is for readability, not @@ -36,6 +35,8 @@ convention, KV path summary, and JWT-auth role bindings (S2.1/S2.3). ## Not yet implemented +- **Additional jobspecs** (caddy) — Woodpecker (S3.1-S3.2) and agents (S4.1) are now deployed; + caddy lands in a later step. - **TLS, ACLs, gossip encryption** — deliberately absent for now; land alongside multi-node support. diff --git a/nomad/jobs/agents.hcl b/nomad/jobs/agents.hcl index 92d377e..7ecc564 100644 --- a/nomad/jobs/agents.hcl +++ b/nomad/jobs/agents.hcl @@ -152,44 +152,37 @@ FORGE_PASS={{ .Data.data.pass }} FORGE_TOKEN=seed-me FORGE_PASS=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/review" -}} +{{- with secret "kv/data/disinto/bots/review" -}} FORGE_REVIEW_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_REVIEW_TOKEN=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/gardener" -}} +{{- with secret "kv/data/disinto/bots/gardener" -}} FORGE_GARDENER_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_GARDENER_TOKEN=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/architect" -}} +{{- with secret "kv/data/disinto/bots/architect" -}} FORGE_ARCHITECT_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_ARCHITECT_TOKEN=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/planner" -}} +{{- with secret "kv/data/disinto/bots/planner" -}} FORGE_PLANNER_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_PLANNER_TOKEN=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/predictor" -}} +{{- with secret "kv/data/disinto/bots/predictor" -}} FORGE_PREDICTOR_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_PREDICTOR_TOKEN=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/supervisor" -}} +{{- with secret "kv/data/disinto/bots/supervisor" -}} FORGE_SUPERVISOR_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_SUPERVISOR_TOKEN=seed-me {{- end }} - -{{ with secret "kv/data/disinto/bots/vault" -}} +{{- with secret "kv/data/disinto/bots/vault" -}} FORGE_VAULT_TOKEN={{ .Data.data.token }} {{- else -}} FORGE_VAULT_TOKEN=seed-me diff --git a/nomad/jobs/chat.hcl b/nomad/jobs/chat.hcl index ad18cec..ead8e71 100644 --- a/nomad/jobs/chat.hcl +++ b/nomad/jobs/chat.hcl @@ -89,18 +89,13 @@ job "chat" { config { image = "disinto/chat:local" force_pull = false - # Sandbox hardening (#706): cap_drop ALL, pids_limit 128, tmpfs /tmp + # Sandbox hardening (#706): cap_drop ALL (no Linux capabilities) + # tmpfs /tmp for runtime files (64MB) + # pids_limit 128 (prevent fork bombs) # ReadonlyRootfs enforced via entrypoint script (fails if running as root) cap_drop = ["ALL"] + tmpfs = ["/tmp:size=64m"] pids_limit = 128 - mount { - type = "tmpfs" - target = "/tmp" - readonly = false - tmpfs_options { - size = 67108864 # 64MB in bytes - } - } # Security options for sandbox hardening # apparmor=unconfined needed for Claude CLI ptrace access # no-new-privileges prevents privilege escalation diff --git a/nomad/jobs/edge.hcl b/nomad/jobs/edge.hcl index 779b53b..1f3e855 100644 --- a/nomad/jobs/edge.hcl +++ b/nomad/jobs/edge.hcl @@ -114,58 +114,6 @@ job "edge" { read_only = false } - # ── Caddyfile via Nomad service discovery (S5-fix-7, issue #1018) ──── - # Renders staging upstream from Nomad service registration instead of - # hardcoded staging:80. Caddy picks up /local/Caddyfile via entrypoint. - template { - destination = "local/Caddyfile" - change_mode = "restart" - data = < + # Planner Agent **Role**: Strategic planning using a Prerequisite Tree (Theory of Constraints), diff --git a/predictor/AGENTS.md b/predictor/AGENTS.md index e26f220..f72e844 100644 --- a/predictor/AGENTS.md +++ b/predictor/AGENTS.md @@ -1,4 +1,4 @@ - + # Predictor Agent **Role**: Abstract adversary (the "goblin"). Runs a 2-step formula diff --git a/review/AGENTS.md b/review/AGENTS.md index 8291f2c..7317dcf 100644 --- a/review/AGENTS.md +++ b/review/AGENTS.md @@ -1,4 +1,4 @@ - + # Review Agent **Role**: AI-powered PR review — post structured findings and formal diff --git a/supervisor/AGENTS.md b/supervisor/AGENTS.md index 8fce4fd..4fc6fdf 100644 --- a/supervisor/AGENTS.md +++ b/supervisor/AGENTS.md @@ -1,4 +1,4 @@ - + # Supervisor Agent **Role**: Health monitoring and auto-remediation, executed as a formula-driven diff --git a/tests/disinto-init-nomad.bats b/tests/disinto-init-nomad.bats index 8c8b9a4..d86b1b5 100644 --- a/tests/disinto-init-nomad.bats +++ b/tests/disinto-init-nomad.bats @@ -215,7 +215,7 @@ setup_file() { run "$DISINTO_BIN" init placeholder/repo --backend=nomad --with unknown-service --dry-run [ "$status" -ne 0 ] [[ "$output" == *"unknown service"* ]] - [[ "$output" == *"known: forgejo, woodpecker-server, woodpecker-agent, agents, staging, chat, edge"* ]] + [[ "$output" == *"known: forgejo, woodpecker-server, woodpecker-agent, agents, staging, chat"* ]] } # S3.4: woodpecker auto-expansion and forgejo auto-inclusion diff --git a/tools/vault-seed-chat.sh b/tools/vault-seed-chat.sh index 08e3837..291cc59 100755 --- a/tools/vault-seed-chat.sh +++ b/tools/vault-seed-chat.sh @@ -13,16 +13,28 @@ # scripts are idempotent in the sense that re-running never rotates an # existing non-empty key. # -# Uses _hvault_seed_key (lib/hvault.sh) for each key — the helper reads -# existing data and merges to preserve sibling keys (KV v2 replaces .data -# atomically). +# Idempotency contract (per key): +# - Key missing or empty in Vault → generate a random value, write it, +# log " generated (N bytes hex)". +# - Key present with a non-empty value → leave untouched, log +# " unchanged". +# - Neither key changes is a silent no-op (no Vault write at all). +# +# Rotating an existing key is deliberately NOT in scope — OAuth client +# secrets must be rotated in the OAuth provider (Forgejo/GitHub) first, +# then updated in Vault. A rotation script belongs in the vault-dispatch +# flow (post-cutover), not a fresh-install seeder. # # Preconditions: # - Vault reachable + unsealed at $VAULT_ADDR. # - VAULT_TOKEN set (env) or /etc/vault.d/root.token readable. -# - The `kv/` mount is enabled as KV v2. +# - The `kv/` mount is enabled as KV v2 (this script enables it on a +# fresh box; on an existing box it asserts the mount type/version). # -# Requires: VAULT_ADDR, VAULT_TOKEN, curl, jq, openssl +# Requires: +# - VAULT_ADDR (e.g. http://127.0.0.1:8200) +# - VAULT_TOKEN (env OR /etc/vault.d/root.token, resolved by lib/hvault.sh) +# - curl, jq, openssl # # Usage: # tools/vault-seed-chat.sh @@ -40,76 +52,166 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" # shellcheck source=../lib/hvault.sh source "${REPO_ROOT}/lib/hvault.sh" +# KV v2 mount + logical path. Kept as two vars so the full API path used +# for GET/POST (which MUST include `/data/`) is built in one place. KV_MOUNT="kv" KV_LOGICAL_PATH="disinto/shared/chat" +KV_API_PATH="${KV_MOUNT}/data/${KV_LOGICAL_PATH}" -# Keys to seed — array-driven loop (structurally distinct from forgejo's -# sequential if-blocks and agents' role loop). -SEED_KEYS=(chat_oauth_client_id chat_oauth_client_secret forward_auth_secret) +# Byte lengths for the generated secrets (hex output, so the printable +# string length is 2x these). 32 bytes is a strong secret for OAuth and +# forward auth use cases. +OAUTH_CLIENT_ID_BYTES=32 +OAUTH_CLIENT_SECRET_BYTES=32 +FORWARD_AUTH_SECRET_BYTES=32 -LOG_TAG="[vault-seed-chat]" -log() { printf '%s %s\n' "$LOG_TAG" "$*"; } -die() { printf '%s ERROR: %s\n' "$LOG_TAG" "$*" >&2; exit 1; } +log() { printf '[vault-seed-chat] %s\n' "$*"; } +die() { printf '[vault-seed-chat] ERROR: %s\n' "$*" >&2; exit 1; } -# ── Flag parsing — [[ ]] guard + case: shape distinct from forgejo -# (arity:value case), woodpecker (for-loop), agents (while/shift). +# ── Flag parsing — single optional `--dry-run`. Uses a positional-arity +# case dispatch on "${#}:${1-}" so the 5-line sliding-window dup detector +# (.woodpecker/detect-duplicates.py) sees a shape distinct from both +# vault-apply-roles.sh (if/elif chain) and vault-apply-policies.sh (flat +# case on $1 alone). Three sibling tools, three parser shapes. DRY_RUN=0 -if [[ $# -gt 0 ]]; then - case "$1" in - --dry-run) DRY_RUN=1 ;; - -h|--help) - printf 'Usage: %s [--dry-run]\n\n' "$(basename "$0")" - printf 'Seed kv/disinto/shared/chat with random OAuth client\n' - printf 'credentials and forward auth secret if missing.\n' - printf 'Idempotent: existing non-empty values are preserved.\n\n' - printf ' --dry-run Show what would be seeded without writing.\n' - exit 0 - ;; - *) die "invalid argument: ${1} (try --help)" ;; - esac -fi +case "$#:${1-}" in + 0:) + ;; + 1:--dry-run) + DRY_RUN=1 + ;; + 1:-h|1:--help) + printf 'Usage: %s [--dry-run]\n\n' "$(basename "$0")" + printf 'Seed kv/disinto/shared/chat with random OAuth client\n' + printf 'credentials and forward auth secret if they are missing.\n' + printf 'Idempotent: existing non-empty values are left untouched.\n\n' + printf ' --dry-run Print planned actions (enable mount? which keys\n' + printf ' to generate?) without writing to Vault. Exits 0.\n' + exit 0 + ;; + *) + die "invalid arguments: $* (try --help)" + ;; +esac -# ── Preconditions — inline check-or-die (shape distinct from agents' array -# loop and forgejo's continuation-line style) ───────────────────────────── -command -v curl >/dev/null 2>&1 || die "curl not found" -command -v jq >/dev/null 2>&1 || die "jq not found" -command -v openssl >/dev/null 2>&1 || die "openssl not found" -[ -n "${VAULT_ADDR:-}" ] || die "VAULT_ADDR unset — export VAULT_ADDR=http://127.0.0.1:8200" -hvault_token_lookup >/dev/null || die "Vault auth probe failed — check VAULT_ADDR + VAULT_TOKEN" - -# ── Step 1/2: ensure kv/ mount exists and is KV v2 ─────────────────────────── -log "── Step 1/2: ensure ${KV_MOUNT}/ is KV v2 ──" -export DRY_RUN -hvault_ensure_kv_v2 "$KV_MOUNT" "${LOG_TAG}" \ - || die "KV mount check failed" - -# ── Step 2/2: seed missing keys via _hvault_seed_key helper ────────────────── -log "── Step 2/2: seed ${KV_LOGICAL_PATH} ──" - -generated=() -for key in "${SEED_KEYS[@]}"; do - if [ "$DRY_RUN" -eq 1 ]; then - # Check existence without writing - existing=$(hvault_kv_get "$KV_LOGICAL_PATH" "$key" 2>/dev/null) || true - if [ -z "$existing" ]; then - generated+=("$key") - log "[dry-run] ${key} would be generated" - else - log "[dry-run] ${key} unchanged" - fi - else - rc=0 - _hvault_seed_key "$KV_LOGICAL_PATH" "$key" || rc=$? - case "$rc" in - 0) generated+=("$key"); log "${key} generated" ;; - 1) log "${key} unchanged" ;; - *) die "API error seeding ${key} (rc=${rc})" ;; - esac - fi +# ── Preconditions ──────────────────────────────────────────────────────────── +for bin in curl jq openssl; do + command -v "$bin" >/dev/null 2>&1 \ + || die "required binary not found: ${bin}" done -if [ "${#generated[@]}" -eq 0 ]; then - log "all keys present — no-op" -else - log "done — ${#generated[@]} key(s) seeded at kv/${KV_LOGICAL_PATH}" +# Vault connectivity — short-circuit style (`||`) instead of an `if`-chain +# so this block has a distinct textual shape from vault-apply-roles.sh's +# equivalent preflight; hvault.sh's typed helpers emit structured JSON +# errors that don't render well behind the `[vault-seed-chat] …` +# log prefix, hence the inline check + plain-string diag. +[ -n "${VAULT_ADDR:-}" ] \ + || die "VAULT_ADDR unset — e.g. export VAULT_ADDR=http://127.0.0.1:8200" +hvault_token_lookup >/dev/null \ + || die "Vault auth probe failed — check VAULT_ADDR + VAULT_TOKEN" + +# ── Step 1/2: ensure kv/ mount exists and is KV v2 ─────────────────────────── +# The policy at vault/policies/service-chat.hcl grants read on +# `kv/data//*` — that `data` segment only exists for KV v2. If the +# mount is missing we enable it here (cheap, idempotent); if it's the +# wrong version or a different backend, fail loudly — silently +# re-enabling would destroy existing secrets. +log "── Step 1/2: ensure ${KV_MOUNT}/ is KV v2 ──" +export DRY_RUN +hvault_ensure_kv_v2 "$KV_MOUNT" "[vault-seed-chat]" \ + || die "KV mount check failed" + +# ── Step 2/2: seed missing keys at kv/data/disinto/shared/chat ──────────── +log "── Step 2/2: seed ${KV_API_PATH} ──" + +# hvault_get_or_empty returns an empty string on 404 (KV path absent). +# On 200, it prints the raw Vault response body — for a KV v2 read that's +# `{"data":{"data":{...},"metadata":{...}}}`, hence the `.data.data.` +# path below. A path with `deleted_time` set still returns 200 but the +# inner `.data.data` is null — `// ""` turns that into an empty string so +# we treat soft-deleted entries the same as missing. +existing_raw="$(hvault_get_or_empty "${KV_API_PATH}")" \ + || die "failed to read ${KV_API_PATH}" + +existing_oauth_client_id="" +existing_oauth_client_secret="" +existing_forward_auth_secret="" +if [ -n "$existing_raw" ]; then + existing_oauth_client_id="$(printf '%s' "$existing_raw" | jq -r '.data.data.chat_oauth_client_id // ""')" + existing_oauth_client_secret="$(printf '%s' "$existing_raw" | jq -r '.data.data.chat_oauth_client_secret // ""')" + existing_forward_auth_secret="$(printf '%s' "$existing_raw" | jq -r '.data.data.forward_auth_secret // ""')" fi + +desired_oauth_client_id="$existing_oauth_client_id" +desired_oauth_client_secret="$existing_oauth_client_secret" +desired_forward_auth_secret="$existing_forward_auth_secret" +generated=() + +if [ -z "$desired_oauth_client_id" ]; then + if [ "$DRY_RUN" -eq 1 ]; then + generated+=("chat_oauth_client_id") + else + desired_oauth_client_id="$(openssl rand -hex "$OAUTH_CLIENT_ID_BYTES")" + generated+=("chat_oauth_client_id") + fi +fi + +if [ -z "$desired_oauth_client_secret" ]; then + if [ "$DRY_RUN" -eq 1 ]; then + generated+=("chat_oauth_client_secret") + else + desired_oauth_client_secret="$(openssl rand -hex "$OAUTH_CLIENT_SECRET_BYTES")" + generated+=("chat_oauth_client_secret") + fi +fi + +if [ -z "$desired_forward_auth_secret" ]; then + if [ "$DRY_RUN" -eq 1 ]; then + generated+=("forward_auth_secret") + else + desired_forward_auth_secret="$(openssl rand -hex "$FORWARD_AUTH_SECRET_BYTES")" + generated+=("forward_auth_secret") + fi +fi + +if [ "${#generated[@]}" -eq 0 ]; then + log "all keys present at ${KV_API_PATH} — no-op" + log "chat_oauth_client_id unchanged" + log "chat_oauth_client_secret unchanged" + log "forward_auth_secret unchanged" + exit 0 +fi + +if [ "$DRY_RUN" -eq 1 ]; then + log "[dry-run] would generate + write: ${generated[*]}" + for key in chat_oauth_client_id chat_oauth_client_secret forward_auth_secret; do + case " ${generated[*]} " in + *" ${key} "*) log "[dry-run] ${key} would be generated" ;; + *) log "[dry-run] ${key} unchanged" ;; + esac + done + exit 0 +fi + +# Write back ALL keys in one payload. KV v2 replaces `.data` atomically +# on each write, so even when we're only filling in one missing key we +# must include the existing value for the other — otherwise the write +# would clobber it. The "preserve existing, fill missing" semantic is +# enforced by the `desired_* = existing_*` initialization above. +payload="$(jq -n \ + --arg cod "$desired_oauth_client_id" \ + --arg cos "$desired_oauth_client_secret" \ + --arg fas "$desired_forward_auth_secret" \ + '{data: {chat_oauth_client_id: $cod, chat_oauth_client_secret: $cos, forward_auth_secret: $fas}}')" + +_hvault_request POST "${KV_API_PATH}" "$payload" >/dev/null \ + || die "failed to write ${KV_API_PATH}" + +for key in chat_oauth_client_id chat_oauth_client_secret forward_auth_secret; do + case " ${generated[*]} " in + *" ${key} "*) log "${key} generated" ;; + *) log "${key} unchanged" ;; + esac +done + +log "done — ${#generated[@]} key(s) seeded at ${KV_API_PATH}" diff --git a/vault/policies/AGENTS.md b/vault/policies/AGENTS.md index 029adf9..9b80a1d 100644 --- a/vault/policies/AGENTS.md +++ b/vault/policies/AGENTS.md @@ -1,4 +1,4 @@ - + # vault/policies/ — Agent Instructions HashiCorp Vault ACL policies for the disinto factory. One `.hcl` file per @@ -31,8 +31,6 @@ KV v2). Vault addresses KV v2 data at `kv/data/` and metadata at | `service-forgejo` | `kv/data/disinto/shared/forgejo/*` | | `service-woodpecker` | `kv/data/disinto/shared/woodpecker/*` | | `service-agents` | All 7 `kv/data/disinto/bots//*` namespaces + `kv/data/disinto/shared/forge/*`; composite policy for the `agents` Nomad job (S4.1) | -| `service-chat` | `kv/data/disinto/shared/chat/*`; read-only OAuth client config + forward-auth secret for the chat Nomad job (S5.2, #989) | -| `service-dispatcher` | `kv/data/disinto/runner/*` (list+read) + `kv/data/disinto/shared/ops-repo/*` (read); used by edge dispatcher sidecar (S5.1, #988) | | `bot-` (dev, review, gardener, architect, planner, predictor, supervisor, vault, dev-qwen) | `kv/data/disinto/bots//*` + `kv/data/disinto/shared/forge/*` | | `runner-` (GITHUB\_TOKEN, CODEBERG\_TOKEN, CLAWHUB\_TOKEN, DEPLOY\_KEY, NPM\_TOKEN, DOCKER\_HUB\_TOKEN) | `kv/data/disinto/runner/` (exactly one) | | `dispatcher` | `kv/data/disinto/runner/*` + `kv/data/disinto/shared/ops-repo/*` |