From 96870d9f3035697194cb123abdb75e10d430ed42 Mon Sep 17 00:00:00 2001 From: Agent Date: Thu, 16 Apr 2026 18:21:41 +0000 Subject: [PATCH 1/6] fix: fix: vault_request RETURN trap fires prematurely when vault-env.sh is sourced (#773) --- lib/action-vault.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/action-vault.sh b/lib/action-vault.sh index 6348cc6..7602a39 100644 --- a/lib/action-vault.sh +++ b/lib/action-vault.sh @@ -128,7 +128,6 @@ vault_request() { # Validate TOML content local tmp_toml tmp_toml=$(mktemp /tmp/vault-XXXXXX.toml) - trap 'rm -f "$tmp_toml"' RETURN printf '%s' "$toml_content" > "$tmp_toml" @@ -136,6 +135,7 @@ vault_request() { local vault_env="${FACTORY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}/action-vault/vault-env.sh" if [ ! -f "$vault_env" ]; then echo "ERROR: vault-env.sh not found at $vault_env" >&2 + rm -f "$tmp_toml" return 1 fi @@ -145,11 +145,15 @@ vault_request() { if ! source "$vault_env"; then FORGE_TOKEN="${_saved_forge_token:-}" echo "ERROR: failed to source vault-env.sh" >&2 + rm -f "$tmp_toml" return 1 fi # Restore caller's FORGE_TOKEN after validation FORGE_TOKEN="${_saved_forge_token:-}" + # Set trap AFTER sourcing vault-env.sh to avoid RETURN trap firing during source + trap 'rm -f "$tmp_toml"' RETURN + # Run validation if ! validate_vault_action "$tmp_toml"; then echo "ERROR: TOML validation failed" >&2 From 7081c98a79e5e0612ba286b565425b110d7d524a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Apr 2026 19:04:04 +0000 Subject: [PATCH 2/6] =?UTF-8?q?fix:=20[nomad-step-2]=20S2.5=20=E2=80=94=20?= =?UTF-8?q?bin/disinto=20init=20--import-env=20/=20--import-sops=20/=20--a?= =?UTF-8?q?ge-key=20wire-up=20(#883)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the Step-2 building blocks (import, auth, policies) into `disinto init --backend=nomad` so a single command on a fresh LXC provisions cluster + policies + auth + imports secrets + deploys services. Adds three flags to `disinto init --backend=nomad`: --import-env PATH plaintext .env from old stack --import-sops PATH sops-encrypted .env.vault.enc (requires --age-key) --age-key PATH age keyfile to decrypt --import-sops Flow: cluster-up.sh → vault-apply-policies.sh → vault-nomad-auth.sh → (optional) vault-import.sh → deploy.sh. Policies + auth run on every nomad real-run path (idempotent); import runs only when --import-* is set; all layers safe to re-run. Flag validation: --import-sops without --age-key → error --age-key without --import-sops → error --import-env alone (no sops) → OK --backend=docker + any --import-* → error Dry-run prints a five-section plan (cluster-up + policies + auth + import + deploy) with every argv that would be executed; touches nothing, logs no secret values. Dry-run output prints one line per --import-* flag that is actually set — not in an if/elif chain — so all three paths appear when all three flags are passed. Prior attempts regressed this invariant. Tests: tests/disinto-init-nomad.bats +10 cases covering flag validation, dry-run plan shape (each flag prints its own path), policies+auth always-on (without --import-*), and --flag=value form. Docs: docs/nomad-migration.md new file — cutover-day runbook with invocation shape, flag summary, idempotency contract, dry-run, and secret-hygiene notes. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/disinto | 153 +++++++++++++++++++++++++++++++++- docs/nomad-migration.md | 121 +++++++++++++++++++++++++++ tests/disinto-init-nomad.bats | 89 ++++++++++++++++++++ 3 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 docs/nomad-migration.md diff --git a/bin/disinto b/bin/disinto index 6128b7c..a752bac 100755 --- a/bin/disinto +++ b/bin/disinto @@ -89,6 +89,9 @@ Init options: --yes Skip confirmation prompts --rotate-tokens Force regeneration of all bot tokens/passwords (idempotent by default) --dry-run Print every intended action without executing + --import-env (nomad) Path to .env file for import into Vault KV (S2.5) + --import-sops (nomad) Path to sops-encrypted .env.vault.enc for import (S2.5) + --age-key (nomad) Path to age keyfile (required with --import-sops) (S2.5) Hire an agent options: --formula Path to role formula TOML (default: formulas/.toml) @@ -664,8 +667,12 @@ prompt_admin_password() { # `sudo disinto init ...` directly. _disinto_init_nomad() { local dry_run="${1:-false}" empty="${2:-false}" with_services="${3:-}" + local import_env="${4:-}" import_sops="${5:-}" age_key="${6:-}" local cluster_up="${FACTORY_ROOT}/lib/init/nomad/cluster-up.sh" local deploy_sh="${FACTORY_ROOT}/lib/init/nomad/deploy.sh" + local vault_policies_sh="${FACTORY_ROOT}/tools/vault-apply-policies.sh" + local vault_auth_sh="${FACTORY_ROOT}/lib/init/nomad/vault-nomad-auth.sh" + local vault_import_sh="${FACTORY_ROOT}/tools/vault-import.sh" if [ ! -x "$cluster_up" ]; then echo "Error: ${cluster_up} not found or not executable" >&2 @@ -677,6 +684,27 @@ _disinto_init_nomad() { exit 1 fi + # Step 2/3/4 scripts must exist as soon as any --import-* flag is set, + # since we unconditionally invoke policies+auth and optionally import. + local import_any=false + if [ -n "$import_env" ] || [ -n "$import_sops" ]; then + import_any=true + fi + if [ "$import_any" = true ]; then + if [ ! -x "$vault_policies_sh" ]; then + echo "Error: ${vault_policies_sh} not found or not executable" >&2 + exit 1 + fi + if [ ! -x "$vault_auth_sh" ]; then + echo "Error: ${vault_auth_sh} not found or not executable" >&2 + exit 1 + fi + if [ ! -x "$vault_import_sh" ]; then + echo "Error: ${vault_import_sh} not found or not executable" >&2 + exit 1 + fi + fi + # --empty and default both invoke cluster-up today. Log the requested # mode so the dispatch is visible in factory bootstrap logs — Step 1 # will branch on $empty to gate the job-deployment path. @@ -686,7 +714,7 @@ _disinto_init_nomad() { echo "nomad backend: default (cluster-up; jobs deferred to Step 1)" fi - # Dry-run: print cluster-up plan + deploy.sh plan + # Dry-run: print cluster-up plan + policies/auth/import plan + deploy.sh plan if [ "$dry_run" = "true" ]; then echo "" echo "── Cluster-up dry-run ─────────────────────────────────" @@ -694,6 +722,38 @@ _disinto_init_nomad() { "${cmd[@]}" || true echo "" + # Vault policies + auth are invoked on every nomad real-run path + # regardless of --import-* flags (they're idempotent; S2.1 + S2.3). + # Mirror that ordering in the dry-run plan so the operator sees the + # full sequence Step 2 will execute. + echo "── Vault policies dry-run ─────────────────────────────" + echo "[policies] [dry-run] ${vault_policies_sh} --dry-run" + echo "" + echo "── Vault auth dry-run ─────────────────────────────────" + echo "[auth] [dry-run] ${vault_auth_sh}" + echo "" + + # Import plan: one line per --import-* flag that is actually set. + # Printing independently (not in an if/elif chain) means that all + # three flags appearing together each echo their own path — the + # regression that bit prior implementations of this issue (#883). + if [ "$import_any" = true ]; then + echo "── Vault import dry-run ───────────────────────────────" + [ -n "$import_env" ] && echo "[import] --import-env env file: ${import_env}" + [ -n "$import_sops" ] && echo "[import] --import-sops sops file: ${import_sops}" + [ -n "$age_key" ] && echo "[import] --age-key age key: ${age_key}" + local -a import_dry_cmd=("$vault_import_sh") + [ -n "$import_env" ] && import_dry_cmd+=("--env" "$import_env") + [ -n "$import_sops" ] && import_dry_cmd+=("--sops" "$import_sops") + [ -n "$age_key" ] && import_dry_cmd+=("--age-key" "$age_key") + import_dry_cmd+=("--dry-run") + echo "[import] [dry-run] ${import_dry_cmd[*]}" + echo "" + else + echo "[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services" + echo "" + fi + if [ -n "$with_services" ]; then echo "── Deploy services dry-run ────────────────────────────" echo "[deploy] services to deploy: ${with_services}" @@ -721,7 +781,7 @@ _disinto_init_nomad() { exit 0 fi - # Real run: cluster-up + deploy services + # Real run: cluster-up + policies + auth + (optional) import + deploy local -a cluster_cmd=("$cluster_up") if [ "$(id -u)" -eq 0 ]; then "${cluster_cmd[@]}" || exit $? @@ -733,6 +793,56 @@ _disinto_init_nomad() { sudo -n -- "${cluster_cmd[@]}" || exit $? fi + # Apply Vault policies (S2.1) — idempotent, safe to re-run. + echo "" + echo "── Applying Vault policies ────────────────────────────" + local -a policies_cmd=("$vault_policies_sh") + if [ "$(id -u)" -eq 0 ]; then + "${policies_cmd[@]}" || exit $? + else + if ! command -v sudo >/dev/null 2>&1; then + echo "Error: vault-apply-policies.sh must run as root and sudo is not installed" >&2 + exit 1 + fi + sudo -n -- "${policies_cmd[@]}" || exit $? + fi + + # Configure Vault JWT auth + Nomad workload identity (S2.3) — idempotent. + echo "" + echo "── Configuring Vault JWT auth ─────────────────────────" + local -a auth_cmd=("$vault_auth_sh") + if [ "$(id -u)" -eq 0 ]; then + "${auth_cmd[@]}" || exit $? + else + if ! command -v sudo >/dev/null 2>&1; then + echo "Error: vault-nomad-auth.sh must run as root and sudo is not installed" >&2 + exit 1 + fi + sudo -n -- "${auth_cmd[@]}" || exit $? + fi + + # Import secrets if any --import-* flag is set (S2.2). + if [ "$import_any" = true ]; then + echo "" + echo "── Importing secrets into Vault ───────────────────────" + local -a import_cmd=("$vault_import_sh") + [ -n "$import_env" ] && import_cmd+=("--env" "$import_env") + [ -n "$import_sops" ] && import_cmd+=("--sops" "$import_sops") + [ -n "$age_key" ] && import_cmd+=("--age-key" "$age_key") + if [ "$(id -u)" -eq 0 ]; then + "${import_cmd[@]}" || exit $? + else + if ! command -v sudo >/dev/null 2>&1; then + echo "Error: vault-import.sh must run as root and sudo is not installed" >&2 + exit 1 + fi + sudo -n -- "${import_cmd[@]}" || exit $? + fi + else + echo "" + echo "[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services" + fi + # Deploy services if requested if [ -n "$with_services" ]; then echo "" @@ -777,6 +887,16 @@ _disinto_init_nomad() { echo "" echo "── Summary ────────────────────────────────────────────" echo "Cluster: Nomad+Vault cluster is up" + echo "Policies: applied (Vault ACL)" + echo "Auth: Vault JWT auth + Nomad workload identity configured" + if [ "$import_any" = true ]; then + local import_desc="" + [ -n "$import_env" ] && import_desc+="${import_env} " + [ -n "$import_sops" ] && import_desc+="${import_sops} " + echo "Imported: ${import_desc% }" + else + echo "Imported: (none — seed kv/disinto/* manually before deploying secret-dependent services)" + fi echo "Deployed: ${with_services}" if echo "$with_services" | grep -q "forgejo"; then echo "Ports: forgejo: 3000" @@ -803,6 +923,7 @@ disinto_init() { # Parse flags local branch="" repo_root="" ci_id="0" auto_yes=false forge_url_flag="" bare=false rotate_tokens=false use_build=false dry_run=false backend="docker" empty=false with_services="" + local import_env="" import_sops="" age_key="" while [ $# -gt 0 ]; do case "$1" in --branch) branch="$2"; shift 2 ;; @@ -819,6 +940,12 @@ disinto_init() { --yes) auto_yes=true; shift ;; --rotate-tokens) rotate_tokens=true; shift ;; --dry-run) dry_run=true; shift ;; + --import-env) import_env="$2"; shift 2 ;; + --import-env=*) import_env="${1#--import-env=}"; shift ;; + --import-sops) import_sops="$2"; shift 2 ;; + --import-sops=*) import_sops="${1#--import-sops=}"; shift ;; + --age-key) age_key="$2"; shift 2 ;; + --age-key=*) age_key="${1#--age-key=}"; shift ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac done @@ -859,11 +986,31 @@ disinto_init() { exit 1 fi + # --import-* flag validation (S2.5). These three flags form an import + # triple and must be consistent before dispatch: sops encryption is + # useless without the age key to decrypt it, so either both --import-sops + # and --age-key are present or neither is. --import-env alone is fine + # (it just imports the plaintext dotenv). All three flags are nomad-only. + if [ -n "$import_sops" ] && [ -z "$age_key" ]; then + echo "Error: --import-sops requires --age-key" >&2 + exit 1 + fi + if [ -n "$age_key" ] && [ -z "$import_sops" ]; then + echo "Error: --age-key requires --import-sops" >&2 + exit 1 + fi + if { [ -n "$import_env" ] || [ -n "$import_sops" ] || [ -n "$age_key" ]; } \ + && [ "$backend" != "nomad" ]; then + echo "Error: --import-env, --import-sops, and --age-key require --backend=nomad" >&2 + exit 1 + fi + # Dispatch on backend — the nomad path runs lib/init/nomad/cluster-up.sh # (S0.4). The default and --empty variants are identical today; Step 1 # will branch on $empty to add job deployment to the default path. if [ "$backend" = "nomad" ]; then - _disinto_init_nomad "$dry_run" "$empty" "$with_services" + _disinto_init_nomad "$dry_run" "$empty" "$with_services" \ + "$import_env" "$import_sops" "$age_key" # shellcheck disable=SC2317 # _disinto_init_nomad always exits today; # `return` is defensive against future refactors. return diff --git a/docs/nomad-migration.md b/docs/nomad-migration.md new file mode 100644 index 0000000..8984b10 --- /dev/null +++ b/docs/nomad-migration.md @@ -0,0 +1,121 @@ + +# Nomad+Vault migration — cutover-day runbook + +`disinto init --backend=nomad` is the single entry-point that turns a fresh +LXC (with the disinto repo cloned) into a running Nomad+Vault cluster with +policies applied, JWT workload-identity auth configured, secrets imported +from the old docker stack, and services deployed. + +## Cutover-day invocation + +On the new LXC, as root (or an operator with NOPASSWD sudo): + +```bash +# Copy the plaintext .env + sops-encrypted .env.vault.enc + age keyfile +# from the old box first (out of band — SSH, USB, whatever your ops +# procedure allows). Then: + +sudo ./bin/disinto init \ + --backend=nomad \ + --import-env /tmp/.env \ + --import-sops /tmp/.env.vault.enc \ + --age-key /tmp/keys.txt \ + --with forgejo +``` + +This runs, in order: + +1. **`lib/init/nomad/cluster-up.sh`** (S0) — installs Nomad + Vault + binaries, writes `/etc/nomad.d/*`, initializes Vault, starts both + services, waits for the Nomad node to become ready. +2. **`tools/vault-apply-policies.sh`** (S2.1) — syncs every + `vault/policies/*.hcl` into Vault as an ACL policy. Idempotent. +3. **`lib/init/nomad/vault-nomad-auth.sh`** (S2.3) — enables Vault's + JWT auth method at `jwt-nomad`, points it at Nomad's JWKS, writes + one role per policy, reloads Nomad so jobs can exchange + workload-identity tokens for Vault tokens. Idempotent. +4. **`tools/vault-import.sh`** (S2.2) — reads `/tmp/.env` and the + sops-decrypted `/tmp/.env.vault.enc`, writes them to the KV paths + matching the S2.1 policy layout (`kv/disinto/bots/*`, `kv/disinto/shared/*`, + `kv/disinto/runner/*`). Idempotent (overwrites KV v2 data in place). +5. **`lib/init/nomad/deploy.sh forgejo`** (S1) — validates + runs the + `nomad/jobs/forgejo.hcl` jobspec. Forgejo reads its admin creds from + Vault via the `template` stanza (S2.4). + +## Flag summary + +| Flag | Meaning | +|---|---| +| `--backend=nomad` | Switch the init dispatcher to the Nomad+Vault path (instead of docker compose). | +| `--empty` | Bring the cluster up, skip policies/auth/import/deploy. Escape hatch for debugging. | +| `--with forgejo[,…]` | Deploy these services after the cluster is up. | +| `--import-env PATH` | Plaintext `.env` from the old stack. Optional. | +| `--import-sops PATH` | Sops-encrypted `.env.vault.enc` from the old stack. Requires `--age-key`. | +| `--age-key PATH` | Age keyfile used to decrypt `--import-sops`. Requires `--import-sops`. | +| `--dry-run` | Print the full plan (cluster-up + policies + auth + import + deploy) and exit. Touches nothing. | + +### Flag validation + +- `--import-sops` without `--age-key` → error. +- `--age-key` without `--import-sops` → error. +- `--import-env` alone (no sops) → OK (imports just the plaintext `.env`). +- `--backend=docker` with any `--import-*` flag → error. + +## Idempotency + +Every layer is idempotent by design. Re-running the same command on an +already-provisioned box is a no-op at every step: + +- **Cluster-up:** second run detects running `nomad`/`vault` systemd + units and state files, skips re-init. +- **Policies:** byte-for-byte compare against on-server policy text; + "unchanged" for every untouched file. +- **Auth:** skips auth-method create if `jwt-nomad/` already enabled, + skips config write if the JWKS + algs match, skips server.hcl write if + the file on disk is identical to the repo copy. +- **Import:** KV v2 writes overwrite in place (same path, same keys, + same values → no new version). +- **Deploy:** `nomad job run` is declarative; same jobspec → no new + allocation. + +## Dry-run + +```bash +./bin/disinto init --backend=nomad \ + --import-env /tmp/.env \ + --import-sops /tmp/.env.vault.enc \ + --age-key /tmp/keys.txt \ + --with forgejo \ + --dry-run +``` + +Prints the five-section plan — cluster-up, policies, auth, import, +deploy — with every path and every argv that would be executed. No +network, no sudo, no state mutation. See +`tests/disinto-init-nomad.bats` for the exact output shape. + +## No-import path + +If you already have `kv/disinto/*` seeded by other means (manual +`vault kv put`, a replica, etc.), omit all three `--import-*` flags. +`disinto init --backend=nomad --with forgejo` still applies policies, +configures auth, and deploys — but skips the import step with: + +``` +[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services +``` + +Forgejo's template stanza will fail to render (and thus the allocation +will stall) until those KV paths exist — so either import them or seed +them first. + +## Secret hygiene + +- Never log a secret value. The CLI only prints paths (`--import-env`, + `--age-key`) and KV *paths* (`kv/disinto/bots/review/token`), never + the values themselves. `tools/vault-import.sh` is the only thing that + reads the values, and it pipes them directly into Vault's HTTP API. +- The age keyfile must be mode 0400 — `vault-import.sh` refuses to + source a keyfile with looser permissions. +- `VAULT_ADDR` must be localhost during import — the import tool + refuses to run against a remote Vault, preventing accidental exposure. diff --git a/tests/disinto-init-nomad.bats b/tests/disinto-init-nomad.bats index 84cfa10..30c7f7c 100644 --- a/tests/disinto-init-nomad.bats +++ b/tests/disinto-init-nomad.bats @@ -191,3 +191,92 @@ setup_file() { [ "$status" -ne 0 ] [[ "$output" == *"--empty and --with are mutually exclusive"* ]] } + +# ── --import-env / --import-sops / --age-key (S2.5, #883) ──────────────────── +# +# Step 2.5 wires Vault policies + JWT auth + optional KV import into +# `disinto init --backend=nomad`. The tests below exercise the flag +# grammar (who-requires-whom + who-requires-backend=nomad) and the +# dry-run plan shape (each --import-* flag prints its own path line, +# independently). A prior attempt at this issue regressed the "print +# every set flag" invariant by using if/elif — covered by the +# "--import-env --import-sops --age-key" case. + +@test "disinto init --backend=nomad --import-env only is accepted" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env /tmp/.env --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"--import-env"* ]] + [[ "$output" == *"env file: /tmp/.env"* ]] +} + +@test "disinto init --backend=nomad --import-sops without --age-key errors" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-sops /tmp/.env.vault.enc --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"--import-sops requires --age-key"* ]] +} + +@test "disinto init --backend=nomad --age-key without --import-sops errors" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --age-key /tmp/keys.txt --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"--age-key requires --import-sops"* ]] +} + +@test "disinto init --backend=docker --import-env errors with backend requirement" { + run "$DISINTO_BIN" init placeholder/repo --backend=docker --import-env /tmp/.env + [ "$status" -ne 0 ] + [[ "$output" == *"--import-env, --import-sops, and --age-key require --backend=nomad"* ]] +} + +@test "disinto init --backend=nomad --import-sops --age-key --dry-run shows import plan" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Vault import dry-run"* ]] + [[ "$output" == *"--import-sops"* ]] + [[ "$output" == *"--age-key"* ]] + [[ "$output" == *"sops file: /tmp/.env.vault.enc"* ]] + [[ "$output" == *"age key: /tmp/keys.txt"* ]] +} + +# When all three flags are set, each one must print its own path line — +# if/elif regressed this to "only one printed" in a prior attempt (#883). +@test "disinto init --backend=nomad --import-env --import-sops --age-key --dry-run shows full import plan" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env /tmp/.env --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Vault import dry-run"* ]] + [[ "$output" == *"env file: /tmp/.env"* ]] + [[ "$output" == *"sops file: /tmp/.env.vault.enc"* ]] + [[ "$output" == *"age key: /tmp/keys.txt"* ]] +} + +@test "disinto init --backend=nomad without import flags shows skip message" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"no --import-env/--import-sops"* ]] + [[ "$output" == *"skipping"* ]] +} + +@test "disinto init --backend=nomad --import-env --import-sops --age-key --with forgejo --dry-run shows all plans" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env /tmp/.env --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --with forgejo --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Vault import dry-run"* ]] + [[ "$output" == *"Vault policies dry-run"* ]] + [[ "$output" == *"Vault auth dry-run"* ]] + [[ "$output" == *"Deploy services dry-run"* ]] +} + +@test "disinto init --backend=nomad --dry-run prints policies + auth plan even without --import-*" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --dry-run + [ "$status" -eq 0 ] + # Policies + auth run on every nomad path (idempotent), so the dry-run + # plan always lists them — regardless of whether --import-* is set. + [[ "$output" == *"Vault policies dry-run"* ]] + [[ "$output" == *"Vault auth dry-run"* ]] + [[ "$output" != *"Vault import dry-run"* ]] +} + +# --import-env=PATH (=-form) must work alongside --import-env PATH. +@test "disinto init --backend=nomad --import-env=PATH (equals form) works" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env=/tmp/.env --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"env file: /tmp/.env"* ]] +} From 28eb182487c3f9ad2fe4918f7c0390a090adb583 Mon Sep 17 00:00:00 2001 From: dev-qwen2 Date: Thu, 16 Apr 2026 18:40:35 +0000 Subject: [PATCH 3/6] fix: Two parallel activation paths for llama agents (ENABLE_LLAMA_AGENT vs [agents.X] TOML) (#846) --- .env.example | 14 +-- bin/disinto | 14 --- docker/agents/entrypoint.sh | 32 +++++++ docs/agents-llama.md | 5 +- lib/forge-setup.sh | 166 ------------------------------------ lib/generators.sh | 130 ---------------------------- 6 files changed, 38 insertions(+), 323 deletions(-) diff --git a/.env.example b/.env.example index c1c0b98..a1f24d5 100644 --- a/.env.example +++ b/.env.example @@ -32,13 +32,10 @@ FORGE_URL=http://localhost:3000 # [CONFIG] local Forgejo instance # - FORGE_PASS_DEV_QWEN2 # Name conversion: tr 'a-z-' 'A-Z_' (lowercase→UPPER, hyphens→underscores). # The compose generator looks these up via the agent's `forge_user` field in -# the project TOML. The pre-existing `dev-qwen` llama agent uses -# FORGE_TOKEN_LLAMA / FORGE_PASS_LLAMA (kept for backwards-compat with the -# legacy `ENABLE_LLAMA_AGENT=1` single-agent path). +# the project TOML. Configure local-model agents via [agents.X] sections in +# projects/*.toml — this is the canonical activation path. FORGE_TOKEN= # [SECRET] dev-bot API token (default for all agents) FORGE_PASS= # [SECRET] dev-bot password for git HTTP push (#361) -FORGE_TOKEN_LLAMA= # [SECRET] dev-qwen API token (for agents-llama) -FORGE_PASS_LLAMA= # [SECRET] dev-qwen password for git HTTP push FORGE_REVIEW_TOKEN= # [SECRET] review-bot API token FORGE_REVIEW_PASS= # [SECRET] review-bot password for git HTTP push FORGE_PLANNER_TOKEN= # [SECRET] planner-bot API token @@ -107,13 +104,6 @@ FORWARD_AUTH_SECRET= # [SECRET] Shared secret for Caddy ↔ # Store all project secrets here so formulas reference env vars, never hardcode. BASE_RPC_URL= # [SECRET] on-chain RPC endpoint -# ── Local Qwen dev agent (optional) ────────────────────────────────────── -# Set ENABLE_LLAMA_AGENT=1 to emit agents-llama in docker-compose.yml. -# Requires a running llama-server reachable at ANTHROPIC_BASE_URL. -# See docs/agents-llama.md for details. -ENABLE_LLAMA_AGENT=0 # [CONFIG] 1 = enable agents-llama service -ANTHROPIC_BASE_URL= # [CONFIG] e.g. http://host.docker.internal:8081 - # ── Tuning ──────────────────────────────────────────────────────────────── CLAUDE_TIMEOUT=7200 # [CONFIG] max seconds per Claude invocation diff --git a/bin/disinto b/bin/disinto index 6128b7c..c6c2421 100755 --- a/bin/disinto +++ b/bin/disinto @@ -977,7 +977,6 @@ p.write_text(text) echo "" echo "[ensure] Forgejo admin user 'disinto-admin'" echo "[ensure] 8 bot users: dev-bot, review-bot, planner-bot, gardener-bot, vault-bot, supervisor-bot, predictor-bot, architect-bot" - echo "[ensure] 2 llama bot users: dev-qwen, dev-qwen-nightly" echo "[ensure] .profile repos for all bots" echo "[ensure] repo ${forge_repo} on Forgejo with collaborators" echo "[run] preflight checks" @@ -1173,19 +1172,6 @@ p.write_text(text) echo "Config: CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 saved to .env" fi - # Write local-Qwen dev agent env keys with safe defaults (#769) - if ! grep -q '^ENABLE_LLAMA_AGENT=' "$env_file" 2>/dev/null; then - cat >> "$env_file" <<'LLAMAENVEOF' - -# Local Qwen dev agent (optional) — set to 1 to enable -ENABLE_LLAMA_AGENT=0 -FORGE_TOKEN_LLAMA= -FORGE_PASS_LLAMA= -ANTHROPIC_BASE_URL= -LLAMAENVEOF - echo "Config: ENABLE_LLAMA_AGENT keys written to .env (disabled by default)" - fi - # Create labels on remote create_labels "$forge_repo" "$forge_url" diff --git a/docker/agents/entrypoint.sh b/docker/agents/entrypoint.sh index f838c15..7c58674 100644 --- a/docker/agents/entrypoint.sh +++ b/docker/agents/entrypoint.sh @@ -17,6 +17,38 @@ set -euo pipefail # - predictor: every 24 hours (288 iterations * 5 min) # - supervisor: every SUPERVISOR_INTERVAL seconds (default: 1200 = 20 min) +# ── Migration check: reject ENABLE_LLAMA_AGENT ─────────────────────────────── +# #846: The legacy ENABLE_LLAMA_AGENT env flag is no longer supported. +# Activation is now done exclusively via [agents.X] sections in project TOML. +# If this legacy flag is detected, fail immediately with a migration message. +if [ "${ENABLE_LLAMA_AGENT:-}" = "1" ]; then + cat <<'MIGRATION_ERR' +FATAL: ENABLE_LLAMA_AGENT is no longer supported. + +The legacy ENABLE_LLAMA_AGENT=1 flag has been removed (#846). +Activation is now done exclusively via [agents.X] sections in projects/*.toml. + +To migrate: + 1. Remove ENABLE_LLAMA_AGENT from your .env or .env.enc file + 2. Add an [agents.] section to your project TOML: + + [agents.dev-qwen] + base_url = "http://your-llama-server:8081" + model = "unsloth/Qwen3.5-35B-A3B" + api_key = "sk-no-key-required" + roles = ["dev"] + forge_user = "dev-qwen" + compact_pct = 60 + poll_interval = 60 + + 3. Run: disinto init + 4. Start the agent: docker compose up -d agents-dev-qwen + +See docs/agents-llama.md for full details. +MIGRATION_ERR + exit 1 +fi + DISINTO_BAKED="/home/agent/disinto" DISINTO_LIVE="/home/agent/repos/_factory" DISINTO_DIR="$DISINTO_BAKED" # start with baked copy; switched to live checkout after bootstrap diff --git a/docs/agents-llama.md b/docs/agents-llama.md index bc973b7..b3a1334 100644 --- a/docs/agents-llama.md +++ b/docs/agents-llama.md @@ -2,9 +2,12 @@ Local-model agents run the same agent code as the Claude-backed agents, but connect to a local llama-server (or compatible OpenAI-API endpoint) instead of -the Anthropic API. This document describes the current activation flow using +the Anthropic API. This document describes the canonical activation flow using `disinto hire-an-agent` and `[agents.X]` TOML configuration. +> **Note:** The legacy `ENABLE_LLAMA_AGENT=1` env flag has been removed (#846). +> Activation is now done exclusively via `[agents.X]` sections in project TOML. + ## Overview Local-model agents are configured via `[agents.]` sections in diff --git a/lib/forge-setup.sh b/lib/forge-setup.sh index 2b7b697..2f8b117 100644 --- a/lib/forge-setup.sh +++ b/lib/forge-setup.sh @@ -356,16 +356,6 @@ setup_forge() { [predictor-bot]="FORGE_PREDICTOR_PASS" [architect-bot]="FORGE_ARCHITECT_PASS" ) - # Llama bot users (local-model agents) — separate from main agents - # Each llama agent gets its own Forgejo user, token, and password - local -A llama_token_vars=( - [dev-qwen]="FORGE_TOKEN_LLAMA" - [dev-qwen-nightly]="FORGE_TOKEN_LLAMA_NIGHTLY" - ) - local -A llama_pass_vars=( - [dev-qwen]="FORGE_PASS_LLAMA" - [dev-qwen-nightly]="FORGE_PASS_LLAMA_NIGHTLY" - ) local bot_user bot_pass token token_var pass_var @@ -515,159 +505,12 @@ setup_forge() { fi done - # Create llama bot users and tokens (local-model agents) - # These are separate from the main agents and get their own credentials - echo "" - echo "── Setting up llama bot users ────────────────────────────" - - local llama_user llama_pass llama_token llama_token_var llama_pass_var - for llama_user in "${!llama_token_vars[@]}"; do - llama_token_var="${llama_token_vars[$llama_user]}" - llama_pass_var="${llama_pass_vars[$llama_user]}" - - # Check if token already exists in .env - local token_exists=false - if _token_exists_in_env "$llama_token_var" "$env_file"; then - token_exists=true - fi - - # Check if password already exists in .env - local pass_exists=false - if _pass_exists_in_env "$llama_pass_var" "$env_file"; then - pass_exists=true - fi - - # Check if llama bot user exists on Forgejo - local llama_user_exists=false - if curl -sf --max-time 5 \ - -H "Authorization: token ${admin_token}" \ - "${forge_url}/api/v1/users/${llama_user}" >/dev/null 2>&1; then - llama_user_exists=true - fi - - # Skip token/password regeneration if both exist in .env and not forcing rotation - if [ "$token_exists" = true ] && [ "$pass_exists" = true ] && [ "$rotate_tokens" = false ]; then - echo " ${llama_user} token and password preserved (use --rotate-tokens to force)" - # Still export the existing token for use within this run - local existing_token existing_pass - existing_token=$(grep "^${llama_token_var}=" "$env_file" | head -1 | cut -d= -f2-) - existing_pass=$(grep "^${llama_pass_var}=" "$env_file" | head -1 | cut -d= -f2-) - export "${llama_token_var}=${existing_token}" - export "${llama_pass_var}=${existing_pass}" - continue - fi - - # Generate new credentials if: - # - Token doesn't exist (first run) - # - Password doesn't exist (first run) - # - --rotate-tokens flag is set (explicit rotation) - if [ "$llama_user_exists" = false ]; then - # User doesn't exist - create it - llama_pass="llama-$(head -c 16 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9' | head -c 20)" - echo "Creating llama bot user: ${llama_user}" - local create_output - if ! create_output=$(_forgejo_exec forgejo admin user create \ - --username "${llama_user}" \ - --password "${llama_pass}" \ - --email "${llama_user}@disinto.local" \ - --must-change-password=false 2>&1); then - echo "Error: failed to create llama bot user '${llama_user}':" >&2 - echo " ${create_output}" >&2 - exit 1 - fi - # Forgejo 11.x ignores --must-change-password=false on create; - # explicitly clear the flag so basic-auth token creation works. - _forgejo_exec forgejo admin user change-password \ - --username "${llama_user}" \ - --password "${llama_pass}" \ - --must-change-password=false - - # Verify llama bot user was actually created - if ! curl -sf --max-time 5 \ - -H "Authorization: token ${admin_token}" \ - "${forge_url}/api/v1/users/${llama_user}" >/dev/null 2>&1; then - echo "Error: llama bot user '${llama_user}' not found after creation" >&2 - exit 1 - fi - echo " ${llama_user} user created" - else - # User exists - reset password if needed - echo " ${llama_user} user exists" - if [ "$rotate_tokens" = true ] || [ "$pass_exists" = false ]; then - llama_pass="llama-$(head -c 16 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9' | head -c 20)" - _forgejo_exec forgejo admin user change-password \ - --username "${llama_user}" \ - --password "${llama_pass}" \ - --must-change-password=false || { - echo "Error: failed to reset password for existing llama bot user '${llama_user}'" >&2 - exit 1 - } - echo " ${llama_user} password reset for token generation" - else - # Password exists, get it from .env - llama_pass=$(grep "^${llama_pass_var}=" "$env_file" | head -1 | cut -d= -f2-) - fi - fi - - # Generate token via API (basic auth as the llama user) - # First, delete any existing tokens to avoid name collision - local existing_llama_token_ids - existing_llama_token_ids=$(curl -sf \ - -u "${llama_user}:${llama_pass}" \ - "${forge_url}/api/v1/users/${llama_user}/tokens" 2>/dev/null \ - | jq -r '.[].id // empty' 2>/dev/null) || existing_llama_token_ids="" - - # Delete any existing tokens for this user - if [ -n "$existing_llama_token_ids" ]; then - while IFS= read -r tid; do - [ -n "$tid" ] && curl -sf -X DELETE \ - -u "${llama_user}:${llama_pass}" \ - "${forge_url}/api/v1/users/${llama_user}/tokens/${tid}" >/dev/null 2>&1 || true - done <<< "$existing_llama_token_ids" - fi - - llama_token=$(curl -sf -X POST \ - -u "${llama_user}:${llama_pass}" \ - -H "Content-Type: application/json" \ - "${forge_url}/api/v1/users/${llama_user}/tokens" \ - -d "{\"name\":\"disinto-${llama_user}-token\",\"scopes\":[\"all\"]}" 2>/dev/null \ - | jq -r '.sha1 // empty') || llama_token="" - - if [ -z "$llama_token" ]; then - echo "Error: failed to create API token for '${llama_user}'" >&2 - exit 1 - fi - - # Store token in .env under the llama-specific variable name - if grep -q "^${llama_token_var}=" "$env_file" 2>/dev/null; then - sed -i "s|^${llama_token_var}=.*|${llama_token_var}=${llama_token}|" "$env_file" - else - printf '%s=%s\n' "$llama_token_var" "$llama_token" >> "$env_file" - fi - export "${llama_token_var}=${llama_token}" - echo " ${llama_user} token generated and saved (${llama_token_var})" - - # Store password in .env for git HTTP push (#361) - # Forgejo 11.x API tokens don't work for git push; password auth does. - if grep -q "^${llama_pass_var}=" "$env_file" 2>/dev/null; then - sed -i "s|^${llama_pass_var}=.*|${llama_pass_var}=${llama_pass}|" "$env_file" - else - printf '%s=%s\n' "$llama_pass_var" "$llama_pass" >> "$env_file" - fi - export "${llama_pass_var}=${llama_pass}" - echo " ${llama_user} password saved (${llama_pass_var})" - done - # Create .profile repos for all bot users (if they don't already exist) # This runs the same logic as hire-an-agent Step 2-3 for idempotent setup echo "" echo "── Setting up .profile repos ────────────────────────────" local -a bot_users=(dev-bot review-bot planner-bot gardener-bot vault-bot supervisor-bot predictor-bot architect-bot) - # Add llama bot users to .profile repo creation - for llama_user in "${!llama_token_vars[@]}"; do - bot_users+=("$llama_user") - done local bot_user for bot_user in "${bot_users[@]}"; do @@ -775,15 +618,6 @@ setup_forge() { -d "{\"permission\":\"${bot_perm}\"}" >/dev/null 2>&1 || true done - # Add llama bot users as write collaborators for local-model agents - for llama_user in "${!llama_token_vars[@]}"; do - curl -sf -X PUT \ - -H "Authorization: token ${admin_token:-${FORGE_TOKEN}}" \ - -H "Content-Type: application/json" \ - "${forge_url}/api/v1/repos/${repo_slug}/collaborators/${llama_user}" \ - -d '{"permission":"write"}' >/dev/null 2>&1 || true - done - # Add disinto-admin as admin collaborator curl -sf -X PUT \ -H "Authorization: token ${admin_token:-${FORGE_TOKEN}}" \ diff --git a/lib/generators.sh b/lib/generators.sh index 3f88e39..0df5725 100644 --- a/lib/generators.sh +++ b/lib/generators.sh @@ -438,136 +438,6 @@ services: COMPOSEEOF - # ── Conditional agents-llama block (ENABLE_LLAMA_AGENT=1) ────────────── - # Local-Qwen dev agent — gated on ENABLE_LLAMA_AGENT so factories without - # a local llama endpoint don't try to start it. See docs/agents-llama.md. - if [ "${ENABLE_LLAMA_AGENT:-0}" = "1" ]; then - cat >> "$compose_file" <<'LLAMAEOF' - - agents-llama: - build: - context: . - dockerfile: docker/agents/Dockerfile - # Rebuild on every up (#887): makes docker/agents/ source changes reach this - # container without a manual \`docker compose build\`. Cache-fast when clean. - pull_policy: build - container_name: disinto-agents-llama - restart: unless-stopped - security_opt: - - apparmor=unconfined - volumes: - - agent-data:/home/agent/data - - project-repos:/home/agent/repos - - ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared} - - ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro - - ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro - - ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro - - ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro - - woodpecker-data:/woodpecker-data:ro - environment: - FORGE_URL: http://forgejo:3000 - FORGE_REPO: ${FORGE_REPO:-disinto-admin/disinto} - FORGE_TOKEN: ${FORGE_TOKEN_LLAMA:-} - FORGE_PASS: ${FORGE_PASS_LLAMA:-} - FORGE_BOT_USERNAMES: ${FORGE_BOT_USERNAMES:-} - WOODPECKER_TOKEN: ${WOODPECKER_TOKEN:-} - CLAUDE_TIMEOUT: ${CLAUDE_TIMEOUT:-7200} - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: ${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1} - CLAUDE_AUTOCOMPACT_PCT_OVERRIDE: "60" - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} - ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-} - FORGE_ADMIN_PASS: ${FORGE_ADMIN_PASS:-} - DISINTO_CONTAINER: "1" - PROJECT_NAME: ${PROJECT_NAME:-project} - PROJECT_REPO_ROOT: /home/agent/repos/${PROJECT_NAME:-project} - WOODPECKER_DATA_DIR: /woodpecker-data - WOODPECKER_REPO_ID: "PLACEHOLDER_WP_REPO_ID" - CLAUDE_CONFIG_DIR: ${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config} - POLL_INTERVAL: ${POLL_INTERVAL:-300} - AGENT_ROLES: dev - healthcheck: - test: ["CMD", "pgrep", "-f", "entrypoint.sh"] - interval: 60s - timeout: 5s - retries: 3 - start_period: 30s - depends_on: - forgejo: - condition: service_healthy - networks: - - disinto-net - - agents-llama-all: - build: - context: . - dockerfile: docker/agents/Dockerfile - # Rebuild on every up (#887): makes docker/agents/ source changes reach this - # container without a manual \`docker compose build\`. Cache-fast when clean. - pull_policy: build - container_name: disinto-agents-llama-all - restart: unless-stopped - profiles: ["agents-llama-all"] - security_opt: - - apparmor=unconfined - volumes: - - agent-data:/home/agent/data - - project-repos:/home/agent/repos - - ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared} - - ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro - - ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro - - ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro - - ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro - - woodpecker-data:/woodpecker-data:ro - environment: - FORGE_URL: http://forgejo:3000 - FORGE_REPO: ${FORGE_REPO:-disinto-admin/disinto} - FORGE_TOKEN: ${FORGE_TOKEN_LLAMA:-} - FORGE_PASS: ${FORGE_PASS_LLAMA:-} - FORGE_REVIEW_TOKEN: ${FORGE_REVIEW_TOKEN:-} - FORGE_PLANNER_TOKEN: ${FORGE_PLANNER_TOKEN:-} - FORGE_GARDENER_TOKEN: ${FORGE_GARDENER_TOKEN:-} - FORGE_VAULT_TOKEN: ${FORGE_VAULT_TOKEN:-} - FORGE_SUPERVISOR_TOKEN: ${FORGE_SUPERVISOR_TOKEN:-} - FORGE_PREDICTOR_TOKEN: ${FORGE_PREDICTOR_TOKEN:-} - FORGE_ARCHITECT_TOKEN: ${FORGE_ARCHITECT_TOKEN:-} - FORGE_FILER_TOKEN: ${FORGE_FILER_TOKEN:-} - FORGE_BOT_USERNAMES: ${FORGE_BOT_USERNAMES:-} - WOODPECKER_TOKEN: ${WOODPECKER_TOKEN:-} - CLAUDE_TIMEOUT: ${CLAUDE_TIMEOUT:-7200} - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: ${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1} - CLAUDE_AUTOCOMPACT_PCT_OVERRIDE: "60" - CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS: "1" - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} - ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-} - FORGE_ADMIN_PASS: ${FORGE_ADMIN_PASS:-} - DISINTO_CONTAINER: "1" - PROJECT_NAME: ${PROJECT_NAME:-project} - PROJECT_REPO_ROOT: /home/agent/repos/${PROJECT_NAME:-project} - WOODPECKER_DATA_DIR: /woodpecker-data - WOODPECKER_REPO_ID: "PLACEHOLDER_WP_REPO_ID" - CLAUDE_CONFIG_DIR: ${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config} - POLL_INTERVAL: ${POLL_INTERVAL:-300} - GARDENER_INTERVAL: ${GARDENER_INTERVAL:-21600} - ARCHITECT_INTERVAL: ${ARCHITECT_INTERVAL:-21600} - PLANNER_INTERVAL: ${PLANNER_INTERVAL:-43200} - SUPERVISOR_INTERVAL: ${SUPERVISOR_INTERVAL:-1200} - AGENT_ROLES: review,dev,gardener,architect,planner,predictor,supervisor - healthcheck: - test: ["CMD", "pgrep", "-f", "entrypoint.sh"] - interval: 60s - timeout: 5s - retries: 3 - start_period: 30s - depends_on: - forgejo: - condition: service_healthy - woodpecker: - condition: service_started - networks: - - disinto-net -LLAMAEOF - fi - # Resume the rest of the compose file (runner onward) cat >> "$compose_file" <<'COMPOSEEOF' From e003829eaa444b2a5802a9f2a9ac8e88261fc863 Mon Sep 17 00:00:00 2001 From: dev-qwen2 Date: Thu, 16 Apr 2026 19:05:43 +0000 Subject: [PATCH 4/6] fix: Remove agents-llama service references from docs and formulas (#846) - AGENTS.md: Replace agents-llama and agents-llama-all rows with generic 'Local-model agents' entry pointing to docs/agents-llama.md - formulas/release.sh: Remove agents-llama from docker compose stop/up commands (line 181-182) - formulas/release.toml: Remove agents-llama references from restart-agents step description (lines 192, 195, 206) These changes complete the removal of the legacy ENABLE_LLAMA_AGENT activation path. The release formula now only references the 'agents' service, which is the only service that exists after disinto init regenerates docker-compose.yml based on [agents.X] TOML sections. --- AGENTS.md | 3 +-- formulas/release.sh | 4 ++-- formulas/release.toml | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ef5f00d..ad3867b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,8 +122,7 @@ bash dev/phase-test.sh | Reproduce | `docker/reproduce/` | Bug reproduction using Playwright MCP | `formulas/reproduce.toml` | | Triage | `docker/reproduce/` | Deep root cause analysis | `formulas/triage.toml` | | Edge dispatcher | `docker/edge/` | Polls ops repo for vault actions, executes via Claude sessions | `docker/edge/dispatcher.sh` | -| agents-llama | `docker/agents/` (same image) | Local-Qwen dev agent (`AGENT_ROLES=dev`), gated on `ENABLE_LLAMA_AGENT=1` | [docs/agents-llama.md](docs/agents-llama.md) | -| agents-llama-all | `docker/agents/` (same image) | Local-Qwen all-roles agent (all 7 roles), profile `agents-llama-all` | [docs/agents-llama.md](docs/agents-llama.md) | +| Local-model agents | `docker/agents/` (same image) | Local llama-server agents configured via `[agents.X]` sections in project TOML | [docs/agents-llama.md](docs/agents-llama.md) | > **Vault:** Being redesigned as a PR-based approval workflow (issues #73-#77). > See [docs/VAULT.md](docs/VAULT.md) for the vault PR workflow details. diff --git a/formulas/release.sh b/formulas/release.sh index b8c4eb6..6526d1a 100644 --- a/formulas/release.sh +++ b/formulas/release.sh @@ -178,8 +178,8 @@ log "Tagged disinto/agents:${RELEASE_VERSION}" log "Step 6/6: Restarting agent containers" -docker compose stop agents agents-llama 2>/dev/null || true -docker compose up -d agents agents-llama +docker compose stop agents 2>/dev/null || true +docker compose up -d agents log "Agent containers restarted" # ── Done ───────────────────────────────────────────────────────────────── diff --git a/formulas/release.toml b/formulas/release.toml index f702f42..ccd7f95 100644 --- a/formulas/release.toml +++ b/formulas/release.toml @@ -189,10 +189,10 @@ Restart agent containers to use the new image. - docker compose pull agents 2. Stop and remove existing agent containers: - - docker compose down agents agents-llama 2>/dev/null || true + - docker compose down agents 3. Start agents with new image: - - docker compose up -d agents agents-llama + - docker compose up -d agents 4. Wait for containers to be healthy: - for i in {1..30}; do @@ -203,7 +203,7 @@ Restart agent containers to use the new image. - done 5. Verify containers are running: - - docker compose ps agents agents-llama + - docker compose ps agents 6. Log restart: - echo "Restarted agents containers" From aa3782748d103a2118ba402d67ad3034bbb727cd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Apr 2026 19:04:04 +0000 Subject: [PATCH 5/6] =?UTF-8?q?fix:=20[nomad-step-2]=20S2.5=20=E2=80=94=20?= =?UTF-8?q?bin/disinto=20init=20--import-env=20/=20--import-sops=20/=20--a?= =?UTF-8?q?ge-key=20wire-up=20(#883)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the Step-2 building blocks (import, auth, policies) into `disinto init --backend=nomad` so a single command on a fresh LXC provisions cluster + policies + auth + imports secrets + deploys services. Adds three flags to `disinto init --backend=nomad`: --import-env PATH plaintext .env from old stack --import-sops PATH sops-encrypted .env.vault.enc (requires --age-key) --age-key PATH age keyfile to decrypt --import-sops Flow: cluster-up.sh → vault-apply-policies.sh → vault-nomad-auth.sh → (optional) vault-import.sh → deploy.sh. Policies + auth run on every nomad real-run path (idempotent); import runs only when --import-* is set; all layers safe to re-run. Flag validation: --import-sops without --age-key → error --age-key without --import-sops → error --import-env alone (no sops) → OK --backend=docker + any --import-* → error Dry-run prints a five-section plan (cluster-up + policies + auth + import + deploy) with every argv that would be executed; touches nothing, logs no secret values. Dry-run output prints one line per --import-* flag that is actually set — not in an if/elif chain — so all three paths appear when all three flags are passed. Prior attempts regressed this invariant. Tests: tests/disinto-init-nomad.bats +10 cases covering flag validation, dry-run plan shape (each flag prints its own path), policies+auth always-on (without --import-*), and --flag=value form. Docs: docs/nomad-migration.md new file — cutover-day runbook with invocation shape, flag summary, idempotency contract, dry-run, and secret-hygiene notes. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/disinto | 153 +++++++++++++++++++++++++++++++++- docs/nomad-migration.md | 121 +++++++++++++++++++++++++++ tests/disinto-init-nomad.bats | 89 ++++++++++++++++++++ 3 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 docs/nomad-migration.md diff --git a/bin/disinto b/bin/disinto index c6c2421..6591a5c 100755 --- a/bin/disinto +++ b/bin/disinto @@ -89,6 +89,9 @@ Init options: --yes Skip confirmation prompts --rotate-tokens Force regeneration of all bot tokens/passwords (idempotent by default) --dry-run Print every intended action without executing + --import-env (nomad) Path to .env file for import into Vault KV (S2.5) + --import-sops (nomad) Path to sops-encrypted .env.vault.enc for import (S2.5) + --age-key (nomad) Path to age keyfile (required with --import-sops) (S2.5) Hire an agent options: --formula Path to role formula TOML (default: formulas/.toml) @@ -664,8 +667,12 @@ prompt_admin_password() { # `sudo disinto init ...` directly. _disinto_init_nomad() { local dry_run="${1:-false}" empty="${2:-false}" with_services="${3:-}" + local import_env="${4:-}" import_sops="${5:-}" age_key="${6:-}" local cluster_up="${FACTORY_ROOT}/lib/init/nomad/cluster-up.sh" local deploy_sh="${FACTORY_ROOT}/lib/init/nomad/deploy.sh" + local vault_policies_sh="${FACTORY_ROOT}/tools/vault-apply-policies.sh" + local vault_auth_sh="${FACTORY_ROOT}/lib/init/nomad/vault-nomad-auth.sh" + local vault_import_sh="${FACTORY_ROOT}/tools/vault-import.sh" if [ ! -x "$cluster_up" ]; then echo "Error: ${cluster_up} not found or not executable" >&2 @@ -677,6 +684,27 @@ _disinto_init_nomad() { exit 1 fi + # Step 2/3/4 scripts must exist as soon as any --import-* flag is set, + # since we unconditionally invoke policies+auth and optionally import. + local import_any=false + if [ -n "$import_env" ] || [ -n "$import_sops" ]; then + import_any=true + fi + if [ "$import_any" = true ]; then + if [ ! -x "$vault_policies_sh" ]; then + echo "Error: ${vault_policies_sh} not found or not executable" >&2 + exit 1 + fi + if [ ! -x "$vault_auth_sh" ]; then + echo "Error: ${vault_auth_sh} not found or not executable" >&2 + exit 1 + fi + if [ ! -x "$vault_import_sh" ]; then + echo "Error: ${vault_import_sh} not found or not executable" >&2 + exit 1 + fi + fi + # --empty and default both invoke cluster-up today. Log the requested # mode so the dispatch is visible in factory bootstrap logs — Step 1 # will branch on $empty to gate the job-deployment path. @@ -686,7 +714,7 @@ _disinto_init_nomad() { echo "nomad backend: default (cluster-up; jobs deferred to Step 1)" fi - # Dry-run: print cluster-up plan + deploy.sh plan + # Dry-run: print cluster-up plan + policies/auth/import plan + deploy.sh plan if [ "$dry_run" = "true" ]; then echo "" echo "── Cluster-up dry-run ─────────────────────────────────" @@ -694,6 +722,38 @@ _disinto_init_nomad() { "${cmd[@]}" || true echo "" + # Vault policies + auth are invoked on every nomad real-run path + # regardless of --import-* flags (they're idempotent; S2.1 + S2.3). + # Mirror that ordering in the dry-run plan so the operator sees the + # full sequence Step 2 will execute. + echo "── Vault policies dry-run ─────────────────────────────" + echo "[policies] [dry-run] ${vault_policies_sh} --dry-run" + echo "" + echo "── Vault auth dry-run ─────────────────────────────────" + echo "[auth] [dry-run] ${vault_auth_sh}" + echo "" + + # Import plan: one line per --import-* flag that is actually set. + # Printing independently (not in an if/elif chain) means that all + # three flags appearing together each echo their own path — the + # regression that bit prior implementations of this issue (#883). + if [ "$import_any" = true ]; then + echo "── Vault import dry-run ───────────────────────────────" + [ -n "$import_env" ] && echo "[import] --import-env env file: ${import_env}" + [ -n "$import_sops" ] && echo "[import] --import-sops sops file: ${import_sops}" + [ -n "$age_key" ] && echo "[import] --age-key age key: ${age_key}" + local -a import_dry_cmd=("$vault_import_sh") + [ -n "$import_env" ] && import_dry_cmd+=("--env" "$import_env") + [ -n "$import_sops" ] && import_dry_cmd+=("--sops" "$import_sops") + [ -n "$age_key" ] && import_dry_cmd+=("--age-key" "$age_key") + import_dry_cmd+=("--dry-run") + echo "[import] [dry-run] ${import_dry_cmd[*]}" + echo "" + else + echo "[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services" + echo "" + fi + if [ -n "$with_services" ]; then echo "── Deploy services dry-run ────────────────────────────" echo "[deploy] services to deploy: ${with_services}" @@ -721,7 +781,7 @@ _disinto_init_nomad() { exit 0 fi - # Real run: cluster-up + deploy services + # Real run: cluster-up + policies + auth + (optional) import + deploy local -a cluster_cmd=("$cluster_up") if [ "$(id -u)" -eq 0 ]; then "${cluster_cmd[@]}" || exit $? @@ -733,6 +793,56 @@ _disinto_init_nomad() { sudo -n -- "${cluster_cmd[@]}" || exit $? fi + # Apply Vault policies (S2.1) — idempotent, safe to re-run. + echo "" + echo "── Applying Vault policies ────────────────────────────" + local -a policies_cmd=("$vault_policies_sh") + if [ "$(id -u)" -eq 0 ]; then + "${policies_cmd[@]}" || exit $? + else + if ! command -v sudo >/dev/null 2>&1; then + echo "Error: vault-apply-policies.sh must run as root and sudo is not installed" >&2 + exit 1 + fi + sudo -n -- "${policies_cmd[@]}" || exit $? + fi + + # Configure Vault JWT auth + Nomad workload identity (S2.3) — idempotent. + echo "" + echo "── Configuring Vault JWT auth ─────────────────────────" + local -a auth_cmd=("$vault_auth_sh") + if [ "$(id -u)" -eq 0 ]; then + "${auth_cmd[@]}" || exit $? + else + if ! command -v sudo >/dev/null 2>&1; then + echo "Error: vault-nomad-auth.sh must run as root and sudo is not installed" >&2 + exit 1 + fi + sudo -n -- "${auth_cmd[@]}" || exit $? + fi + + # Import secrets if any --import-* flag is set (S2.2). + if [ "$import_any" = true ]; then + echo "" + echo "── Importing secrets into Vault ───────────────────────" + local -a import_cmd=("$vault_import_sh") + [ -n "$import_env" ] && import_cmd+=("--env" "$import_env") + [ -n "$import_sops" ] && import_cmd+=("--sops" "$import_sops") + [ -n "$age_key" ] && import_cmd+=("--age-key" "$age_key") + if [ "$(id -u)" -eq 0 ]; then + "${import_cmd[@]}" || exit $? + else + if ! command -v sudo >/dev/null 2>&1; then + echo "Error: vault-import.sh must run as root and sudo is not installed" >&2 + exit 1 + fi + sudo -n -- "${import_cmd[@]}" || exit $? + fi + else + echo "" + echo "[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services" + fi + # Deploy services if requested if [ -n "$with_services" ]; then echo "" @@ -777,6 +887,16 @@ _disinto_init_nomad() { echo "" echo "── Summary ────────────────────────────────────────────" echo "Cluster: Nomad+Vault cluster is up" + echo "Policies: applied (Vault ACL)" + echo "Auth: Vault JWT auth + Nomad workload identity configured" + if [ "$import_any" = true ]; then + local import_desc="" + [ -n "$import_env" ] && import_desc+="${import_env} " + [ -n "$import_sops" ] && import_desc+="${import_sops} " + echo "Imported: ${import_desc% }" + else + echo "Imported: (none — seed kv/disinto/* manually before deploying secret-dependent services)" + fi echo "Deployed: ${with_services}" if echo "$with_services" | grep -q "forgejo"; then echo "Ports: forgejo: 3000" @@ -803,6 +923,7 @@ disinto_init() { # Parse flags local branch="" repo_root="" ci_id="0" auto_yes=false forge_url_flag="" bare=false rotate_tokens=false use_build=false dry_run=false backend="docker" empty=false with_services="" + local import_env="" import_sops="" age_key="" while [ $# -gt 0 ]; do case "$1" in --branch) branch="$2"; shift 2 ;; @@ -819,6 +940,12 @@ disinto_init() { --yes) auto_yes=true; shift ;; --rotate-tokens) rotate_tokens=true; shift ;; --dry-run) dry_run=true; shift ;; + --import-env) import_env="$2"; shift 2 ;; + --import-env=*) import_env="${1#--import-env=}"; shift ;; + --import-sops) import_sops="$2"; shift 2 ;; + --import-sops=*) import_sops="${1#--import-sops=}"; shift ;; + --age-key) age_key="$2"; shift 2 ;; + --age-key=*) age_key="${1#--age-key=}"; shift ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac done @@ -859,11 +986,31 @@ disinto_init() { exit 1 fi + # --import-* flag validation (S2.5). These three flags form an import + # triple and must be consistent before dispatch: sops encryption is + # useless without the age key to decrypt it, so either both --import-sops + # and --age-key are present or neither is. --import-env alone is fine + # (it just imports the plaintext dotenv). All three flags are nomad-only. + if [ -n "$import_sops" ] && [ -z "$age_key" ]; then + echo "Error: --import-sops requires --age-key" >&2 + exit 1 + fi + if [ -n "$age_key" ] && [ -z "$import_sops" ]; then + echo "Error: --age-key requires --import-sops" >&2 + exit 1 + fi + if { [ -n "$import_env" ] || [ -n "$import_sops" ] || [ -n "$age_key" ]; } \ + && [ "$backend" != "nomad" ]; then + echo "Error: --import-env, --import-sops, and --age-key require --backend=nomad" >&2 + exit 1 + fi + # Dispatch on backend — the nomad path runs lib/init/nomad/cluster-up.sh # (S0.4). The default and --empty variants are identical today; Step 1 # will branch on $empty to add job deployment to the default path. if [ "$backend" = "nomad" ]; then - _disinto_init_nomad "$dry_run" "$empty" "$with_services" + _disinto_init_nomad "$dry_run" "$empty" "$with_services" \ + "$import_env" "$import_sops" "$age_key" # shellcheck disable=SC2317 # _disinto_init_nomad always exits today; # `return` is defensive against future refactors. return diff --git a/docs/nomad-migration.md b/docs/nomad-migration.md new file mode 100644 index 0000000..8984b10 --- /dev/null +++ b/docs/nomad-migration.md @@ -0,0 +1,121 @@ + +# Nomad+Vault migration — cutover-day runbook + +`disinto init --backend=nomad` is the single entry-point that turns a fresh +LXC (with the disinto repo cloned) into a running Nomad+Vault cluster with +policies applied, JWT workload-identity auth configured, secrets imported +from the old docker stack, and services deployed. + +## Cutover-day invocation + +On the new LXC, as root (or an operator with NOPASSWD sudo): + +```bash +# Copy the plaintext .env + sops-encrypted .env.vault.enc + age keyfile +# from the old box first (out of band — SSH, USB, whatever your ops +# procedure allows). Then: + +sudo ./bin/disinto init \ + --backend=nomad \ + --import-env /tmp/.env \ + --import-sops /tmp/.env.vault.enc \ + --age-key /tmp/keys.txt \ + --with forgejo +``` + +This runs, in order: + +1. **`lib/init/nomad/cluster-up.sh`** (S0) — installs Nomad + Vault + binaries, writes `/etc/nomad.d/*`, initializes Vault, starts both + services, waits for the Nomad node to become ready. +2. **`tools/vault-apply-policies.sh`** (S2.1) — syncs every + `vault/policies/*.hcl` into Vault as an ACL policy. Idempotent. +3. **`lib/init/nomad/vault-nomad-auth.sh`** (S2.3) — enables Vault's + JWT auth method at `jwt-nomad`, points it at Nomad's JWKS, writes + one role per policy, reloads Nomad so jobs can exchange + workload-identity tokens for Vault tokens. Idempotent. +4. **`tools/vault-import.sh`** (S2.2) — reads `/tmp/.env` and the + sops-decrypted `/tmp/.env.vault.enc`, writes them to the KV paths + matching the S2.1 policy layout (`kv/disinto/bots/*`, `kv/disinto/shared/*`, + `kv/disinto/runner/*`). Idempotent (overwrites KV v2 data in place). +5. **`lib/init/nomad/deploy.sh forgejo`** (S1) — validates + runs the + `nomad/jobs/forgejo.hcl` jobspec. Forgejo reads its admin creds from + Vault via the `template` stanza (S2.4). + +## Flag summary + +| Flag | Meaning | +|---|---| +| `--backend=nomad` | Switch the init dispatcher to the Nomad+Vault path (instead of docker compose). | +| `--empty` | Bring the cluster up, skip policies/auth/import/deploy. Escape hatch for debugging. | +| `--with forgejo[,…]` | Deploy these services after the cluster is up. | +| `--import-env PATH` | Plaintext `.env` from the old stack. Optional. | +| `--import-sops PATH` | Sops-encrypted `.env.vault.enc` from the old stack. Requires `--age-key`. | +| `--age-key PATH` | Age keyfile used to decrypt `--import-sops`. Requires `--import-sops`. | +| `--dry-run` | Print the full plan (cluster-up + policies + auth + import + deploy) and exit. Touches nothing. | + +### Flag validation + +- `--import-sops` without `--age-key` → error. +- `--age-key` without `--import-sops` → error. +- `--import-env` alone (no sops) → OK (imports just the plaintext `.env`). +- `--backend=docker` with any `--import-*` flag → error. + +## Idempotency + +Every layer is idempotent by design. Re-running the same command on an +already-provisioned box is a no-op at every step: + +- **Cluster-up:** second run detects running `nomad`/`vault` systemd + units and state files, skips re-init. +- **Policies:** byte-for-byte compare against on-server policy text; + "unchanged" for every untouched file. +- **Auth:** skips auth-method create if `jwt-nomad/` already enabled, + skips config write if the JWKS + algs match, skips server.hcl write if + the file on disk is identical to the repo copy. +- **Import:** KV v2 writes overwrite in place (same path, same keys, + same values → no new version). +- **Deploy:** `nomad job run` is declarative; same jobspec → no new + allocation. + +## Dry-run + +```bash +./bin/disinto init --backend=nomad \ + --import-env /tmp/.env \ + --import-sops /tmp/.env.vault.enc \ + --age-key /tmp/keys.txt \ + --with forgejo \ + --dry-run +``` + +Prints the five-section plan — cluster-up, policies, auth, import, +deploy — with every path and every argv that would be executed. No +network, no sudo, no state mutation. See +`tests/disinto-init-nomad.bats` for the exact output shape. + +## No-import path + +If you already have `kv/disinto/*` seeded by other means (manual +`vault kv put`, a replica, etc.), omit all three `--import-*` flags. +`disinto init --backend=nomad --with forgejo` still applies policies, +configures auth, and deploys — but skips the import step with: + +``` +[import] no --import-env/--import-sops — skipping; set them or seed kv/disinto/* manually before deploying secret-dependent services +``` + +Forgejo's template stanza will fail to render (and thus the allocation +will stall) until those KV paths exist — so either import them or seed +them first. + +## Secret hygiene + +- Never log a secret value. The CLI only prints paths (`--import-env`, + `--age-key`) and KV *paths* (`kv/disinto/bots/review/token`), never + the values themselves. `tools/vault-import.sh` is the only thing that + reads the values, and it pipes them directly into Vault's HTTP API. +- The age keyfile must be mode 0400 — `vault-import.sh` refuses to + source a keyfile with looser permissions. +- `VAULT_ADDR` must be localhost during import — the import tool + refuses to run against a remote Vault, preventing accidental exposure. diff --git a/tests/disinto-init-nomad.bats b/tests/disinto-init-nomad.bats index 84cfa10..30c7f7c 100644 --- a/tests/disinto-init-nomad.bats +++ b/tests/disinto-init-nomad.bats @@ -191,3 +191,92 @@ setup_file() { [ "$status" -ne 0 ] [[ "$output" == *"--empty and --with are mutually exclusive"* ]] } + +# ── --import-env / --import-sops / --age-key (S2.5, #883) ──────────────────── +# +# Step 2.5 wires Vault policies + JWT auth + optional KV import into +# `disinto init --backend=nomad`. The tests below exercise the flag +# grammar (who-requires-whom + who-requires-backend=nomad) and the +# dry-run plan shape (each --import-* flag prints its own path line, +# independently). A prior attempt at this issue regressed the "print +# every set flag" invariant by using if/elif — covered by the +# "--import-env --import-sops --age-key" case. + +@test "disinto init --backend=nomad --import-env only is accepted" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env /tmp/.env --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"--import-env"* ]] + [[ "$output" == *"env file: /tmp/.env"* ]] +} + +@test "disinto init --backend=nomad --import-sops without --age-key errors" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-sops /tmp/.env.vault.enc --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"--import-sops requires --age-key"* ]] +} + +@test "disinto init --backend=nomad --age-key without --import-sops errors" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --age-key /tmp/keys.txt --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"--age-key requires --import-sops"* ]] +} + +@test "disinto init --backend=docker --import-env errors with backend requirement" { + run "$DISINTO_BIN" init placeholder/repo --backend=docker --import-env /tmp/.env + [ "$status" -ne 0 ] + [[ "$output" == *"--import-env, --import-sops, and --age-key require --backend=nomad"* ]] +} + +@test "disinto init --backend=nomad --import-sops --age-key --dry-run shows import plan" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Vault import dry-run"* ]] + [[ "$output" == *"--import-sops"* ]] + [[ "$output" == *"--age-key"* ]] + [[ "$output" == *"sops file: /tmp/.env.vault.enc"* ]] + [[ "$output" == *"age key: /tmp/keys.txt"* ]] +} + +# When all three flags are set, each one must print its own path line — +# if/elif regressed this to "only one printed" in a prior attempt (#883). +@test "disinto init --backend=nomad --import-env --import-sops --age-key --dry-run shows full import plan" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env /tmp/.env --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Vault import dry-run"* ]] + [[ "$output" == *"env file: /tmp/.env"* ]] + [[ "$output" == *"sops file: /tmp/.env.vault.enc"* ]] + [[ "$output" == *"age key: /tmp/keys.txt"* ]] +} + +@test "disinto init --backend=nomad without import flags shows skip message" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"no --import-env/--import-sops"* ]] + [[ "$output" == *"skipping"* ]] +} + +@test "disinto init --backend=nomad --import-env --import-sops --age-key --with forgejo --dry-run shows all plans" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env /tmp/.env --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --with forgejo --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"Vault import dry-run"* ]] + [[ "$output" == *"Vault policies dry-run"* ]] + [[ "$output" == *"Vault auth dry-run"* ]] + [[ "$output" == *"Deploy services dry-run"* ]] +} + +@test "disinto init --backend=nomad --dry-run prints policies + auth plan even without --import-*" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --dry-run + [ "$status" -eq 0 ] + # Policies + auth run on every nomad path (idempotent), so the dry-run + # plan always lists them — regardless of whether --import-* is set. + [[ "$output" == *"Vault policies dry-run"* ]] + [[ "$output" == *"Vault auth dry-run"* ]] + [[ "$output" != *"Vault import dry-run"* ]] +} + +# --import-env=PATH (=-form) must work alongside --import-env PATH. +@test "disinto init --backend=nomad --import-env=PATH (equals form) works" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --import-env=/tmp/.env --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"env file: /tmp/.env"* ]] +} From ece5d9b6cc640a3c67f8789f7f40a38902440707 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Apr 2026 19:25:27 +0000 Subject: [PATCH 6/6] =?UTF-8?q?fix:=20[nomad-step-2]=20S2.5=20review=20?= =?UTF-8?q?=E2=80=94=20gate=20policies/auth/import=20on=20--empty;=20rejec?= =?UTF-8?q?t=20--empty=20+=20--import-*=20(#883)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review #907 blocker: docs/nomad-migration.md claimed --empty "skips policies/auth/import/deploy" but _disinto_init_nomad had no $empty gate around those blocks — operators reaching the "cluster-only escape hatch" would still invoke vault-apply-policies.sh and vault-nomad-auth.sh, contradicting the runbook. Changes: - _disinto_init_nomad: exit 0 immediately after cluster-up when --empty is set, in both dry-run and real-run branches. Only the cluster-up plan appears; no policies, no auth, no import, no deploy. Matches the docs. - disinto_init: reject --empty combined with any --import-* flag. --empty discards the import step, so the combination silently does nothing (worse failure mode than a clear error up front). Symmetric to the existing --empty vs --with check. - Pre-flight existence check for policies/auth scripts now runs unconditionally on the non-empty path (previously gated on --import-*), matching the unconditional invocation. Import-script check stays gated on --import-*. Non-blocking observation also addressed: the pre-flight guard comment + actual predicate were inconsistent ("unconditionally invoke policies+auth" but only checked on import). Now the predicate matches: [ "$empty" != "true" ] gates policies/auth, and an inner --import-* guard gates the import script. Tests (+3): - --empty --dry-run shows no S2.x sections (negative assertions) - --empty --import-env rejected - --empty --import-sops --age-key rejected 30/30 nomad tests pass; shellcheck clean. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/disinto | 38 +++++++++++++++++++++++++++++++---- docs/nomad-migration.md | 3 +++ tests/disinto-init-nomad.bats | 30 +++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/bin/disinto b/bin/disinto index 6591a5c..2b676a3 100755 --- a/bin/disinto +++ b/bin/disinto @@ -684,13 +684,21 @@ _disinto_init_nomad() { exit 1 fi - # Step 2/3/4 scripts must exist as soon as any --import-* flag is set, - # since we unconditionally invoke policies+auth and optionally import. + # --empty short-circuits after cluster-up: no policies, no auth, no + # import, no deploy. It's the "cluster-only escape hatch" for debugging + # (docs/nomad-migration.md). Caller-side validation already rejects + # --empty combined with --with or any --import-* flag, so reaching + # this branch with those set is a bug in the caller. + # + # On the default (non-empty) path, vault-apply-policies.sh and + # vault-nomad-auth.sh are invoked unconditionally — they are idempotent + # and cheap to re-run, and subsequent --with deployments depend on + # them. vault-import.sh is invoked only when an --import-* flag is set. local import_any=false if [ -n "$import_env" ] || [ -n "$import_sops" ]; then import_any=true fi - if [ "$import_any" = true ]; then + if [ "$empty" != "true" ]; then if [ ! -x "$vault_policies_sh" ]; then echo "Error: ${vault_policies_sh} not found or not executable" >&2 exit 1 @@ -699,7 +707,7 @@ _disinto_init_nomad() { echo "Error: ${vault_auth_sh} not found or not executable" >&2 exit 1 fi - if [ ! -x "$vault_import_sh" ]; then + if [ "$import_any" = true ] && [ ! -x "$vault_import_sh" ]; then echo "Error: ${vault_import_sh} not found or not executable" >&2 exit 1 fi @@ -722,6 +730,13 @@ _disinto_init_nomad() { "${cmd[@]}" || true echo "" + # --empty skips policies/auth/import/deploy — cluster-up only, no + # workloads. The operator-visible dry-run plan must match the real + # run, so short-circuit here too. + if [ "$empty" = "true" ]; then + exit 0 + fi + # Vault policies + auth are invoked on every nomad real-run path # regardless of --import-* flags (they're idempotent; S2.1 + S2.3). # Mirror that ordering in the dry-run plan so the operator sees the @@ -793,6 +808,12 @@ _disinto_init_nomad() { sudo -n -- "${cluster_cmd[@]}" || exit $? fi + # --empty short-circuits here: cluster-up only, no policies/auth/import + # and no deploy. Matches the dry-run plan above and the docs/runbook. + if [ "$empty" = "true" ]; then + exit 0 + fi + # Apply Vault policies (S2.1) — idempotent, safe to re-run. echo "" echo "── Applying Vault policies ────────────────────────────" @@ -1005,6 +1026,15 @@ disinto_init() { exit 1 fi + # --empty is the cluster-only escape hatch — it skips policies, auth, + # import, and deploy. Pairing it with --import-* silently does nothing, + # which is a worse failure mode than a clear error. Reject explicitly. + if [ "$empty" = true ] \ + && { [ -n "$import_env" ] || [ -n "$import_sops" ] || [ -n "$age_key" ]; }; then + echo "Error: --empty and --import-env/--import-sops/--age-key are mutually exclusive" >&2 + exit 1 + fi + # Dispatch on backend — the nomad path runs lib/init/nomad/cluster-up.sh # (S0.4). The default and --empty variants are identical today; Step 1 # will branch on $empty to add job deployment to the default path. diff --git a/docs/nomad-migration.md b/docs/nomad-migration.md index 8984b10..02ff023 100644 --- a/docs/nomad-migration.md +++ b/docs/nomad-migration.md @@ -60,6 +60,9 @@ This runs, in order: - `--age-key` without `--import-sops` → error. - `--import-env` alone (no sops) → OK (imports just the plaintext `.env`). - `--backend=docker` with any `--import-*` flag → error. +- `--empty` with any `--import-*` flag → error (mutually exclusive: `--empty` + skips the import step, so pairing them silently discards the import + intent). ## Idempotency diff --git a/tests/disinto-init-nomad.bats b/tests/disinto-init-nomad.bats index 30c7f7c..f38805e 100644 --- a/tests/disinto-init-nomad.bats +++ b/tests/disinto-init-nomad.bats @@ -280,3 +280,33 @@ setup_file() { [ "$status" -eq 0 ] [[ "$output" == *"env file: /tmp/.env"* ]] } + +# --empty short-circuits after cluster-up: no policies, no auth, no +# import, no deploy. The dry-run plan must match that — cluster-up plan +# appears, but none of the S2.x section banners do. +@test "disinto init --backend=nomad --empty --dry-run skips policies/auth/import sections" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --empty --dry-run + [ "$status" -eq 0 ] + # Cluster-up still runs (it's what --empty brings up). + [[ "$output" == *"Cluster-up dry-run"* ]] + # Policies + auth + import must NOT appear under --empty. + [[ "$output" != *"Vault policies dry-run"* ]] + [[ "$output" != *"Vault auth dry-run"* ]] + [[ "$output" != *"Vault import dry-run"* ]] + [[ "$output" != *"no --import-env/--import-sops"* ]] +} + +# --empty + any --import-* flag silently does nothing (import is skipped), +# so the CLI rejects the combination up front rather than letting it +# look like the import "succeeded". +@test "disinto init --backend=nomad --empty --import-env errors" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --empty --import-env /tmp/.env --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"--empty and --import-env/--import-sops/--age-key are mutually exclusive"* ]] +} + +@test "disinto init --backend=nomad --empty --import-sops --age-key errors" { + run "$DISINTO_BIN" init placeholder/repo --backend=nomad --empty --import-sops /tmp/.env.vault.enc --age-key /tmp/keys.txt --dry-run + [ "$status" -ne 0 ] + [[ "$output" == *"--empty and --import-env/--import-sops/--age-key are mutually exclusive"* ]] +}