Compare commits
78 commits
fix/issue-
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 2a7ae0b7ea | |||
|
|
14c67f36e6 | ||
|
|
e5c41dd502 | ||
|
|
5150f8c486 | ||
| 271ec9d8f5 | |||
|
|
481175e043 | ||
|
|
d2c6b33271 | ||
| accd10ec67 | |||
|
|
24cb8f83a2 | ||
| 75bec43c4a | |||
|
|
06ead3a19d | ||
| 74f49e1c2f | |||
|
|
de00400bc4 | ||
| 32ab84a87c | |||
|
|
c236350e00 | ||
| c363ee0aea | |||
|
|
519742e5e7 | ||
| 131d0471f2 | |||
|
|
4487d1512c | ||
|
|
ef40433fff | ||
| 7513e93d6d | |||
|
|
0bfa31da49 | ||
|
|
8e885bed02 | ||
| 34447d31dc | |||
|
|
9d8f322005 | ||
| 55cce66468 | |||
|
|
14458f1f17 | ||
|
|
fbb246c626 | ||
| faf6490877 | |||
|
|
88b377ecfb | ||
|
|
d020847772 | ||
| 98ec610645 | |||
|
|
f8c3ada077 | ||
| 8315a4ecf5 | |||
|
|
b6f2d83a28 | ||
|
|
2465841b84 | ||
| 5c40b59359 | |||
|
|
19f10e33e6 | ||
| 6a4ca5c3a0 | |||
|
|
8799a8c676 | ||
| 3b366ad96e | |||
|
|
aa298eb2ad | ||
|
|
9dbc43ab23 | ||
| 1d4e28843e | |||
|
|
f90702f930 | ||
| defec3b255 | |||
|
|
88676e65ae | ||
| a87dcdf40b | |||
| b8cb8c5c32 | |||
|
|
0937707fe5 | ||
|
|
e9a018db5c | ||
| 18190874ca | |||
|
|
5a2a9e1c74 | ||
| 182c40b9fc | |||
|
|
241ce96046 | ||
|
|
987413ab3a | ||
| 02e86c3589 | |||
|
|
175716a847 | ||
| d6c8fd8127 | |||
|
|
5dda6dc8e9 | ||
| 49cc870f54 | |||
|
|
ec7bc8ff2c | ||
| f27c66a7e0 | |||
|
|
53ce7ad475 | ||
| c644660bda | |||
| 91f36b2692 | |||
|
|
a8d393f3bd | ||
| d0c0ef724a | |||
|
|
539862679d | ||
| 250788952f | |||
|
|
0104ac06a8 | ||
| c71b6d4f95 | |||
|
|
92f19cb2b3 | ||
| be463c5b43 | |||
|
|
0baac1a7d8 | ||
| 0db4c84818 | |||
| 378da77adf | |||
|
|
fd9ba028bc |
71 changed files with 4132 additions and 573 deletions
|
|
@ -1,8 +1,7 @@
|
|||
# Secrets — prevent .env files from being baked into the image
|
||||
# Secrets — prevent .env files and encrypted secrets from being baked into the image
|
||||
.env
|
||||
.env.enc
|
||||
.env.vault
|
||||
.env.vault.enc
|
||||
secrets/
|
||||
|
||||
# Version control — .git is huge and not needed in image
|
||||
.git
|
||||
|
|
|
|||
32
.env.example
32
.env.example
|
|
@ -63,6 +63,10 @@ FORGE_BOT_USERNAMES=dev-bot,review-bot,planner-bot,gardener-bot,vault-bot,superv
|
|||
WOODPECKER_TOKEN= # [SECRET] Woodpecker API token
|
||||
WOODPECKER_SERVER=http://localhost:8000 # [CONFIG] Woodpecker server URL
|
||||
WOODPECKER_AGENT_SECRET= # [SECRET] shared secret for server↔agent auth (auto-generated)
|
||||
# Woodpecker privileged-plugin allowlist — comma-separated image names
|
||||
# Add plugins/docker (and others) here to allow privileged execution
|
||||
WOODPECKER_PLUGINS_PRIVILEGED=plugins/docker
|
||||
|
||||
# WOODPECKER_REPO_ID — now per-project, set in projects/*.toml [ci] section
|
||||
|
||||
# Woodpecker Postgres (for direct DB queries)
|
||||
|
|
@ -79,24 +83,42 @@ FORWARD_AUTH_SECRET= # [SECRET] Shared secret for Caddy ↔
|
|||
|
||||
# ── Vault-only secrets (DO NOT put these in .env) ────────────────────────
|
||||
# These tokens grant access to external systems (GitHub, ClawHub, deploy targets).
|
||||
# They live ONLY in .env.vault.enc and are injected into the ephemeral runner
|
||||
# container at fire time (#745). lib/env.sh explicitly unsets them so agents
|
||||
# can never hold them directly — all external actions go through vault dispatch.
|
||||
# They live ONLY in secrets/<NAME>.enc (age-encrypted, one file per key) and are
|
||||
# decrypted into the ephemeral runner container at fire time (#745, #777).
|
||||
# lib/env.sh explicitly unsets them so agents can never hold them directly —
|
||||
# all external actions go through vault dispatch.
|
||||
#
|
||||
# GITHUB_TOKEN — GitHub API access (publish, deploy, post)
|
||||
# CLAWHUB_TOKEN — ClawHub registry credentials (publish)
|
||||
# CADDY_SSH_KEY — SSH key for Caddy log collection
|
||||
# (deploy keys) — SSH keys for deployment targets
|
||||
#
|
||||
# To manage vault secrets: disinto secrets edit-vault
|
||||
# (vault redesign in progress: PR-based approval, see #73-#77)
|
||||
# To manage secrets: disinto secrets add/show/remove/list
|
||||
|
||||
# ── Project-specific secrets ──────────────────────────────────────────────
|
||||
# Store all project secrets here so formulas reference env vars, never hardcode.
|
||||
BASE_RPC_URL= # [SECRET] on-chain RPC endpoint
|
||||
|
||||
# ── Local Qwen dev agent (optional) ──────────────────────────────────────
|
||||
# Set ENABLE_LLAMA_AGENT=1 to emit agents-llama in docker-compose.yml.
|
||||
# Requires a running llama-server reachable at ANTHROPIC_BASE_URL.
|
||||
# See docs/agents-llama.md for details.
|
||||
ENABLE_LLAMA_AGENT=0 # [CONFIG] 1 = enable agents-llama service
|
||||
ANTHROPIC_BASE_URL= # [CONFIG] e.g. http://host.docker.internal:8081
|
||||
|
||||
# ── Tuning ────────────────────────────────────────────────────────────────
|
||||
CLAUDE_TIMEOUT=7200 # [CONFIG] max seconds per Claude invocation
|
||||
|
||||
# ── Host paths (Nomad-portable) ────────────────────────────────────────────
|
||||
# These env vars externalize host-side bind-mount paths from docker-compose.yml.
|
||||
# At cutover, Nomad jobspecs reference the same vars — no path translation.
|
||||
# Defaults point at current paths so an empty .env override still works.
|
||||
CLAUDE_BIN_DIR=/usr/local/bin/claude # [CONFIG] host path to claude CLI binary (resolved by `disinto init`)
|
||||
CLAUDE_CONFIG_FILE=${HOME}/.claude.json # [CONFIG] host path to claude config JSON file
|
||||
CLAUDE_DIR=${HOME}/.claude # [CONFIG] host path to .claude directory (reproduce/edge)
|
||||
AGENT_SSH_DIR=${HOME}/.ssh # [CONFIG] host path to SSH keys directory
|
||||
SOPS_AGE_DIR=${HOME}/.config/sops/age # [CONFIG] host path to SOPS age key directory
|
||||
|
||||
# ── Claude Code shared OAuth state ─────────────────────────────────────────
|
||||
# Shared directory used by every factory container so Claude Code's internal
|
||||
# proper-lockfile-based OAuth refresh lock works across containers. Both
|
||||
|
|
|
|||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -3,7 +3,6 @@
|
|||
|
||||
# Encrypted secrets — safe to commit (SOPS-encrypted with age)
|
||||
!.env.enc
|
||||
!.env.vault.enc
|
||||
!.sops.yaml
|
||||
|
||||
# Per-box project config (generated by disinto init)
|
||||
|
|
@ -33,6 +32,9 @@ docker/agents/bin/
|
|||
# Note: This file is now committed to track volume mount configuration
|
||||
# docker-compose.yml
|
||||
|
||||
# Generated Caddyfile — single source of truth is generate_caddyfile in lib/generators.sh
|
||||
docker/Caddyfile
|
||||
|
||||
# Python bytecode
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
|
|
|||
102
.woodpecker/nomad-validate.yml
Normal file
102
.woodpecker/nomad-validate.yml
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
# =============================================================================
|
||||
# .woodpecker/nomad-validate.yml — Static validation for Nomad+Vault artifacts
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.5, issue #825). Locks in the
|
||||
# "no-ad-hoc-steps" principle: every HCL/shell artifact under nomad/ or
|
||||
# lib/init/nomad/, plus the `disinto init` dispatcher, gets checked
|
||||
# before it can land.
|
||||
#
|
||||
# Triggers on PRs (and pushes) that touch any of:
|
||||
# nomad/** — HCL configs (server, client, vault)
|
||||
# lib/init/nomad/** — cluster-up / install / systemd / vault-init
|
||||
# bin/disinto — `disinto init --backend=nomad` dispatcher
|
||||
# tests/disinto-init-nomad.bats — the bats suite itself
|
||||
# .woodpecker/nomad-validate.yml — the pipeline definition
|
||||
#
|
||||
# Steps (all fail-closed — any error blocks merge):
|
||||
# 1. nomad-config-validate — `nomad config validate` on server + client HCL
|
||||
# 2. vault-operator-diagnose — `vault operator diagnose` syntax check on vault.hcl
|
||||
# 3. shellcheck-nomad — shellcheck the cluster-up + install scripts + disinto
|
||||
# 4. bats-init-nomad — `disinto init --backend=nomad --dry-run` smoke tests
|
||||
#
|
||||
# Pinned image versions match lib/init/nomad/install.sh (nomad 1.9.5 /
|
||||
# vault 1.18.5). Bump there AND here together — drift = CI passing on
|
||||
# syntax the runtime would reject.
|
||||
# =============================================================================
|
||||
|
||||
when:
|
||||
- event: [push, pull_request]
|
||||
path:
|
||||
- "nomad/**"
|
||||
- "lib/init/nomad/**"
|
||||
- "bin/disinto"
|
||||
- "tests/disinto-init-nomad.bats"
|
||||
- ".woodpecker/nomad-validate.yml"
|
||||
|
||||
# Authenticated clone — same pattern as .woodpecker/ci.yml. Forgejo is
|
||||
# configured with REQUIRE_SIGN_IN, so anonymous git clones fail (exit 128).
|
||||
# FORGE_TOKEN is injected globally via WOODPECKER_ENVIRONMENT.
|
||||
clone:
|
||||
git:
|
||||
image: alpine/git
|
||||
commands:
|
||||
- AUTH_URL=$(printf '%s' "$CI_REPO_CLONE_URL" | sed "s|://|://token:$FORGE_TOKEN@|")
|
||||
- git clone --depth 1 "$AUTH_URL" .
|
||||
- git fetch --depth 1 origin "$CI_COMMIT_REF"
|
||||
- git checkout FETCH_HEAD
|
||||
|
||||
steps:
|
||||
# ── 1. Nomad HCL syntax check ────────────────────────────────────────────
|
||||
# `nomad config validate` parses server.hcl + client.hcl and fails on any
|
||||
# HCL/semantic error (unknown block, invalid port range, bad driver cfg).
|
||||
# vault.hcl is excluded — it's a Vault config, not Nomad, so it goes
|
||||
# through the vault-operator-diagnose step instead.
|
||||
- name: nomad-config-validate
|
||||
image: hashicorp/nomad:1.9.5
|
||||
commands:
|
||||
- nomad config validate nomad/server.hcl nomad/client.hcl
|
||||
|
||||
# ── 2. Vault HCL syntax check ────────────────────────────────────────────
|
||||
# `vault operator diagnose` loads the config and runs a suite of checks.
|
||||
# Exit codes:
|
||||
# 0 — all checks green
|
||||
# 1 — at least one hard failure (bad HCL, bad schema, unreachable storage)
|
||||
# 2 — advisory warnings only (no hard failure)
|
||||
# Our factory dev-box vault.hcl deliberately runs TLS-disabled on a
|
||||
# localhost-only listener (documented in nomad/vault.hcl), which triggers
|
||||
# an advisory "Check Listener TLS" warning → exit 2. The config still
|
||||
# parses, so we tolerate exit 2 and fail only on exit 1 or crashes.
|
||||
# -skip=storage/-skip=listener disables the runtime-only checks (vault's
|
||||
# container has /vault/file so storage is fine, but explicit skip is cheap
|
||||
# insurance against future container-image drift).
|
||||
- name: vault-operator-diagnose
|
||||
image: hashicorp/vault:1.18.5
|
||||
commands:
|
||||
- |
|
||||
rc=0
|
||||
vault operator diagnose -config=nomad/vault.hcl -skip=storage -skip=listener || rc=$?
|
||||
case "$rc" in
|
||||
0) echo "vault config: all checks green" ;;
|
||||
2) echo "vault config: parse OK (rc=2 — advisory warnings only; TLS-disabled on localhost listener is by design)" ;;
|
||||
*) echo "vault config: hard failure (rc=$rc)" >&2; exit "$rc" ;;
|
||||
esac
|
||||
|
||||
# ── 3. Shellcheck ────────────────────────────────────────────────────────
|
||||
# Covers the new lib/init/nomad/*.sh scripts plus bin/disinto (which owns
|
||||
# the backend dispatcher). bin/disinto has no .sh extension so the
|
||||
# repo-wide shellcheck in .woodpecker/ci.yml skips it — this step is the
|
||||
# one place it gets checked.
|
||||
- name: shellcheck-nomad
|
||||
image: koalaman/shellcheck-alpine:stable
|
||||
commands:
|
||||
- shellcheck --severity=warning lib/init/nomad/*.sh bin/disinto
|
||||
|
||||
# ── 4. bats: `disinto init --backend=nomad --dry-run` ────────────────────
|
||||
# Smoke-tests the CLI dispatcher: both --backend=nomad variants exit 0
|
||||
# with the expected step list, and --backend=docker stays on the docker
|
||||
# path (regression guard). Pure dry-run — no sudo, no network.
|
||||
- name: bats-init-nomad
|
||||
image: alpine:3.19
|
||||
commands:
|
||||
- apk add --no-cache bash bats
|
||||
- bats tests/disinto-init-nomad.bats
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
# .woodpecker/ops-filer.yml — Sub-issue filer pipeline (#764)
|
||||
#
|
||||
# Triggered on push to main of the ops repo after a sprint PR merges.
|
||||
# Parses sprints/*.md for ## Sub-issues blocks and files them on the
|
||||
# project repo via filer-bot (FORGE_FILER_TOKEN).
|
||||
#
|
||||
# NOTE: This pipeline runs on the ops repo. It must be registered in the
|
||||
# ops repo's Woodpecker project. The filer script (lib/sprint-filer.sh)
|
||||
# lives in the code repo and is cloned into the workspace.
|
||||
#
|
||||
# Idempotency: safe to re-run — each sub-issue carries a decomposed-from
|
||||
# marker that the filer checks before creating.
|
||||
|
||||
when:
|
||||
branch: main
|
||||
event: push
|
||||
|
||||
steps:
|
||||
- name: file-subissues
|
||||
image: alpine:3
|
||||
commands:
|
||||
- apk add --no-cache bash curl jq
|
||||
# Clone the code repo to get the filer script
|
||||
- AUTH_URL=$(printf '%s' "${FORGE_URL}/disinto-admin/disinto.git" | sed "s|://|://token:${FORGE_FILER_TOKEN}@|")
|
||||
- git clone --depth 1 "$AUTH_URL" /tmp/code-repo
|
||||
# Run filer against all sprint files in the ops repo workspace
|
||||
- bash /tmp/code-repo/lib/sprint-filer.sh --all sprints/
|
||||
environment:
|
||||
FORGE_FILER_TOKEN:
|
||||
from_secret: forge_filer_token
|
||||
FORGE_URL:
|
||||
from_secret: forge_url
|
||||
FORGE_API:
|
||||
from_secret: forge_api
|
||||
FORGE_API_BASE:
|
||||
from_secret: forge_api_base
|
||||
64
.woodpecker/publish-images.yml
Normal file
64
.woodpecker/publish-images.yml
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
# .woodpecker/publish-images.yml — Build and push versioned container images
|
||||
# Triggered on tag pushes (e.g. v1.2.3). Builds and pushes:
|
||||
# - ghcr.io/disinto/agents:<tag>
|
||||
# - ghcr.io/disinto/reproduce:<tag>
|
||||
# - ghcr.io/disinto/edge:<tag>
|
||||
#
|
||||
# Requires GHCR_TOKEN secret configured in Woodpecker with push access
|
||||
# to ghcr.io/disinto.
|
||||
|
||||
when:
|
||||
event: tag
|
||||
ref: refs/tags/v*
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: alpine/git
|
||||
commands:
|
||||
- AUTH_URL=$(printf '%s' "$CI_REPO_CLONE_URL" | sed "s|://|://token:$FORGE_TOKEN@|")
|
||||
- git clone --depth 1 "$AUTH_URL" .
|
||||
- git fetch --depth 1 origin "$CI_COMMIT_REF"
|
||||
- git checkout FETCH_HEAD
|
||||
|
||||
steps:
|
||||
- name: build-and-push-agents
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: ghcr.io/disinto/agents
|
||||
registry: ghcr.io
|
||||
dockerfile: docker/agents/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
username: disinto
|
||||
password:
|
||||
from_secret: GHCR_TOKEN
|
||||
|
||||
- name: build-and-push-reproduce
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: ghcr.io/disinto/reproduce
|
||||
registry: ghcr.io
|
||||
dockerfile: docker/reproduce/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
username: disinto
|
||||
password:
|
||||
from_secret: GHCR_TOKEN
|
||||
|
||||
- name: build-and-push-edge
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: ghcr.io/disinto/edge
|
||||
registry: ghcr.io
|
||||
dockerfile: docker/edge/Dockerfile
|
||||
context: docker/edge
|
||||
tags:
|
||||
- ${CI_COMMIT_TAG}
|
||||
- latest
|
||||
username: disinto
|
||||
password:
|
||||
from_secret: GHCR_TOKEN
|
||||
68
.woodpecker/run-secret-scan.sh
Normal file
68
.woodpecker/run-secret-scan.sh
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# run-secret-scan.sh — CI wrapper for lib/secret-scan.sh
|
||||
#
|
||||
# Scans files changed in this PR for plaintext secrets.
|
||||
# Exits non-zero if any secret is detected.
|
||||
|
||||
# shellcheck source=../lib/secret-scan.sh
|
||||
source lib/secret-scan.sh
|
||||
|
||||
# Path patterns considered secret-adjacent
|
||||
SECRET_PATH_PATTERNS=(
|
||||
'\.env'
|
||||
'tools/vault-.*\.sh'
|
||||
'nomad/'
|
||||
'vault/'
|
||||
'action-vault/'
|
||||
'lib/hvault\.sh'
|
||||
'lib/action-vault\.sh'
|
||||
)
|
||||
|
||||
# Build a single regex from patterns
|
||||
path_regex=$(printf '%s|' "${SECRET_PATH_PATTERNS[@]}")
|
||||
path_regex="${path_regex%|}"
|
||||
|
||||
# Get files changed in this PR vs target branch.
|
||||
# Note: shallow clone (depth 50) may lack the merge base for very large PRs,
|
||||
# causing git diff to fail — || true means the gate skips rather than blocks.
|
||||
changed_files=$(git diff --name-only --diff-filter=ACMR "origin/${CI_COMMIT_TARGET_BRANCH}...HEAD" || true)
|
||||
|
||||
if [ -z "$changed_files" ]; then
|
||||
echo "secret-scan: no changed files found, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Filter to secret-adjacent paths only
|
||||
target_files=$(printf '%s\n' "$changed_files" | grep -E "$path_regex" || true)
|
||||
|
||||
if [ -z "$target_files" ]; then
|
||||
echo "secret-scan: no secret-adjacent files changed, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "secret-scan: scanning $(printf '%s\n' "$target_files" | wc -l) file(s):"
|
||||
printf ' %s\n' "$target_files"
|
||||
|
||||
failures=0
|
||||
while IFS= read -r file; do
|
||||
# Skip deleted files / non-existent
|
||||
[ -f "$file" ] || continue
|
||||
# Skip binary files
|
||||
file -b --mime-encoding "$file" 2>/dev/null | grep -q binary && continue
|
||||
|
||||
content=$(cat "$file")
|
||||
if ! scan_for_secrets "$content"; then
|
||||
echo "FAIL: secret detected in $file"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done <<< "$target_files"
|
||||
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "secret-scan: $failures file(s) contain potential secrets — merge blocked"
|
||||
echo "If these are false positives, verify patterns in lib/secret-scan.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "secret-scan: all files clean"
|
||||
32
.woodpecker/secret-scan.yml
Normal file
32
.woodpecker/secret-scan.yml
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# .woodpecker/secret-scan.yml — Block PRs that leak plaintext secrets
|
||||
#
|
||||
# Triggers on pull requests touching secret-adjacent paths.
|
||||
# Sources lib/secret-scan.sh and scans each changed file's content.
|
||||
# Exits non-zero if any potential secret is detected.
|
||||
|
||||
when:
|
||||
- event: pull_request
|
||||
path:
|
||||
- ".env*"
|
||||
- "tools/vault-*.sh"
|
||||
- "nomad/**/*"
|
||||
- "vault/**/*"
|
||||
- "action-vault/**/*"
|
||||
- "lib/hvault.sh"
|
||||
- "lib/action-vault.sh"
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: alpine/git
|
||||
commands:
|
||||
- AUTH_URL=$(printf '%s' "$CI_REPO_CLONE_URL" | sed "s|://|://token:$FORGE_TOKEN@|")
|
||||
- git clone --depth 50 "$AUTH_URL" .
|
||||
- git fetch --depth 50 origin "$CI_COMMIT_REF" "$CI_COMMIT_TARGET_BRANCH"
|
||||
- git checkout FETCH_HEAD
|
||||
|
||||
steps:
|
||||
- name: secret-scan
|
||||
image: alpine:3
|
||||
commands:
|
||||
- apk add --no-cache bash git grep file
|
||||
- bash .woodpecker/run-secret-scan.sh
|
||||
21
AGENTS.md
21
AGENTS.md
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Disinto — Agent Instructions
|
||||
|
||||
## What this repo is
|
||||
|
|
@ -31,11 +31,11 @@ disinto/ (code repo)
|
|||
├── supervisor/ supervisor-run.sh — formula-driven health monitoring (polling-loop executor)
|
||||
│ preflight.sh — pre-flight data collection for supervisor formula
|
||||
├── architect/ architect-run.sh — strategic decomposition of vision into sprints
|
||||
├── vault/ vault-env.sh — shared env setup (vault redesign in progress, see #73-#77)
|
||||
├── action-vault/ vault-env.sh — shared env setup (vault redesign in progress, see #73-#77)
|
||||
│ SCHEMA.md — vault item schema documentation
|
||||
│ validate.sh — vault item validator
|
||||
│ examples/ — example vault action TOMLs (promote, publish, release, webhook-call)
|
||||
├── lib/ env.sh, agent-sdk.sh, ci-helpers.sh, ci-debug.sh, load-project.sh, parse-deps.sh, guard.sh, mirrors.sh, pr-lifecycle.sh, issue-lifecycle.sh, worktree.sh, formula-session.sh, stack-lock.sh, forge-setup.sh, forge-push.sh, ops-setup.sh, ci-setup.sh, generators.sh, hire-agent.sh, release.sh, build-graph.py, branch-protection.sh, secret-scan.sh, tea-helpers.sh, vault.sh, ci-log-reader.py, git-creds.sh, sprint-filer.sh
|
||||
├── lib/ env.sh, agent-sdk.sh, ci-helpers.sh, ci-debug.sh, load-project.sh, parse-deps.sh, guard.sh, mirrors.sh, pr-lifecycle.sh, issue-lifecycle.sh, worktree.sh, formula-session.sh, stack-lock.sh, forge-setup.sh, forge-push.sh, ops-setup.sh, ci-setup.sh, generators.sh, hire-agent.sh, release.sh, build-graph.py, branch-protection.sh, secret-scan.sh, tea-helpers.sh, action-vault.sh, ci-log-reader.py, git-creds.sh, sprint-filer.sh, hvault.sh
|
||||
│ hooks/ — Claude Code session hooks (on-compact-reinject, on-idle-stop, on-phase-change, on-pretooluse-guard, on-session-end, on-stop-failure)
|
||||
├── projects/ *.toml.example — templates; *.toml — local per-box config (gitignored)
|
||||
├── formulas/ Issue templates (TOML specs for multi-step agent tasks)
|
||||
|
|
@ -43,7 +43,7 @@ disinto/ (code repo)
|
|||
├── tools/ Operational tools: edge-control/ (register.sh, install.sh, verify-chat-sandbox.sh)
|
||||
├── docs/ Protocol docs (PHASE-PROTOCOL.md, EVIDENCE-ARCHITECTURE.md)
|
||||
├── site/ disinto.ai website content
|
||||
├── tests/ Test files (mock-forgejo.py, smoke-init.sh)
|
||||
├── tests/ Test files (mock-forgejo.py, smoke-init.sh, lib-hvault.bats)
|
||||
├── templates/ Issue templates
|
||||
├── bin/ The `disinto` CLI script
|
||||
├── disinto-factory/ Setup documentation and skill
|
||||
|
|
@ -86,7 +86,7 @@ Each agent has a `.profile` repository on Forgejo storing `knowledge/lessons-lea
|
|||
- All scripts start with `#!/usr/bin/env bash` and `set -euo pipefail`
|
||||
- Source shared environment: `source "$(dirname "$0")/../lib/env.sh"`
|
||||
- Log to `$LOGFILE` using the `log()` function from env.sh or defined locally
|
||||
- Never hardcode secrets — agent secrets come from `.env.enc`, vault secrets from `.env.vault.enc` (or `.env`/`.env.vault` fallback)
|
||||
- Never hardcode secrets — agent secrets come from `.env.enc`, vault secrets from `secrets/<NAME>.enc` (age-encrypted, one file per key)
|
||||
- Never embed secrets in issue bodies, PR descriptions, or comments — use env var references (e.g. `$BASE_RPC_URL`)
|
||||
- ShellCheck must pass (CI runs `shellcheck` on all `.sh` files)
|
||||
- Avoid duplicate code — shared helpers go in `lib/`
|
||||
|
|
@ -114,10 +114,12 @@ bash dev/phase-test.sh
|
|||
| Planner | `planner/` | Strategic planning | [planner/AGENTS.md](planner/AGENTS.md) |
|
||||
| Predictor | `predictor/` | Infrastructure pattern detection | [predictor/AGENTS.md](predictor/AGENTS.md) |
|
||||
| Architect | `architect/` | Strategic decomposition (read-only on project repo) | [architect/AGENTS.md](architect/AGENTS.md) |
|
||||
| Filer | `lib/sprint-filer.sh` | Sub-issue filing from merged sprint PRs | `.woodpecker/ops-filer.yml` |
|
||||
| Filer | `lib/sprint-filer.sh` | Sub-issue filing from merged sprint PRs | ops repo pipeline (deferred, see #779) |
|
||||
| Reproduce | `docker/reproduce/` | Bug reproduction using Playwright MCP | `formulas/reproduce.toml` |
|
||||
| Triage | `docker/reproduce/` | Deep root cause analysis | `formulas/triage.toml` |
|
||||
| Edge dispatcher | `docker/edge/` | Polls ops repo for vault actions, executes via Claude sessions | `docker/edge/dispatcher.sh` |
|
||||
| agents-llama | `docker/agents/` (same image) | Local-Qwen dev agent (`AGENT_ROLES=dev`), gated on `ENABLE_LLAMA_AGENT=1` | [docs/agents-llama.md](docs/agents-llama.md) |
|
||||
| agents-llama-all | `docker/agents/` (same image) | Local-Qwen all-roles agent (all 7 roles), profile `agents-llama-all` | [docs/agents-llama.md](docs/agents-llama.md) |
|
||||
|
||||
> **Vault:** Being redesigned as a PR-based approval workflow (issues #73-#77).
|
||||
> See [docs/VAULT.md](docs/VAULT.md) for the vault PR workflow details.
|
||||
|
|
@ -178,8 +180,8 @@ Humans write these. Agents read and enforce them.
|
|||
| AD-002 | **Concurrency is bounded per LLM backend, not per project.** One concurrent Claude session per OAuth credential pool; one concurrent session per llama-server instance. Containers with disjoint backends may run in parallel. | The single-thread invariant is about *backends*, not pipelines. **(a) Anthropic OAuth credentials race on token refresh** — each container uses a per-session `CLAUDE_CONFIG_DIR`, so Claude Code's native lockfile-based OAuth refresh handles contention automatically without external serialization. (Legacy: set `CLAUDE_EXTERNAL_LOCK=1` to re-enable the old `flock session.lock` wrapper for rollback.) **(b) llama-server has finite VRAM and one KV cache** — parallel inference thrashes the cache and risks OOM. All llama-backed agents serialize on the same lock. **(c) Disjoint backends are free to parallelize.** Today `disinto-agents` (Anthropic OAuth, runs `review,gardener`) runs concurrently with `disinto-agents-llama` (llama, runs `dev`) on the same project — they share neither OAuth state nor llama VRAM. **(d) Per-project work-conflict safety** (no duplicate dev work, no merge conflicts on the same branch) is enforced by `issue_claim` (assignee + `in-progress` label) and per-issue worktrees — that's a separate guard that does NOT depend on this AD. |
|
||||
| AD-003 | The runtime creates and destroys, the formula preserves. | Runtime manages worktrees/sessions/temp. Formulas commit knowledge to git before signaling done. |
|
||||
| AD-004 | Event-driven > polling > fixed delays. | Never `waitForTimeout` or hardcoded sleep. Use phase files, webhooks, or poll loops with backoff. |
|
||||
| AD-005 | Secrets via env var indirection, never in issue bodies. | Issue bodies become code. Agent secrets go in `.env.enc`, vault secrets in `.env.vault.enc` (SOPS-encrypted when available; plaintext `.env`/`.env.vault` fallback supported). Referenced as `$VAR_NAME`. Runner gets only vault secrets; agents get only agent secrets. |
|
||||
| AD-006 | External actions go through vault dispatch, never direct. | Agents build addressables; only the vault exercises them (publishes, deploys, posts). Tokens for external systems (`GITHUB_TOKEN`, `CLAWHUB_TOKEN`, deploy keys) live only in `.env.vault.enc` and are injected into the ephemeral runner container. `lib/env.sh` unsets them so agents never hold them. PRs with direct external actions without vault dispatch get REQUEST_CHANGES. (Vault redesign in progress: PR-based approval on ops repo, see #73-#77) |
|
||||
| AD-005 | Secrets via env var indirection, never in issue bodies. | Issue bodies become code. Agent secrets go in `.env.enc` (SOPS-encrypted), vault secrets in `secrets/<NAME>.enc` (age-encrypted, one file per key). Referenced as `$VAR_NAME`. Runner gets only vault secrets; agents get only agent secrets. |
|
||||
| AD-006 | External actions go through vault dispatch, never direct. | Agents build addressables; only the vault exercises them (publishes, deploys, posts). Tokens for external systems (`GITHUB_TOKEN`, `CLAWHUB_TOKEN`, deploy keys) live only in `secrets/<NAME>.enc` and are decrypted into the ephemeral runner container. `lib/env.sh` unsets them so agents never hold them. PRs with direct external actions without vault dispatch get REQUEST_CHANGES. (Vault redesign in progress: PR-based approval on ops repo, see #73-#77) |
|
||||
|
||||
**Who enforces what:**
|
||||
- **Gardener** checks open backlog issues against ADs during grooming; closes violations with a comment referencing the AD number.
|
||||
|
|
@ -187,8 +189,6 @@ Humans write these. Agents read and enforce them.
|
|||
- **Dev-agent** reads AGENTS.md before implementing; refuses work that violates ADs.
|
||||
- **AD-002 is a runtime invariant; nothing for the gardener to check at issue-groom time.** OAuth concurrency is handled by per-session `CLAUDE_CONFIG_DIR` isolation (with `CLAUDE_EXTERNAL_LOCK` as a rollback flag). Per-issue work is enforced by `issue_claim`. A violation manifests as a 401 or VRAM OOM in agent logs, not as a malformed issue.
|
||||
|
||||
---
|
||||
|
||||
## Phase-Signaling Protocol
|
||||
|
||||
When running as a persistent tmux session, Claude must signal the orchestrator
|
||||
|
|
@ -197,5 +197,4 @@ at each phase boundary by writing to a phase file (e.g.
|
|||
|
||||
Key phases: `PHASE:awaiting_ci` → `PHASE:awaiting_review` → `PHASE:done`.
|
||||
Also: `PHASE:escalate` (needs human input), `PHASE:failed`.
|
||||
|
||||
See [docs/PHASE-PROTOCOL.md](docs/PHASE-PROTOCOL.md) for the complete spec, orchestrator reaction matrix, sequence diagram, and crash recovery.
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ blast_radius = "low" # optional: overrides policy.toml tier ("low"|"medium
|
|||
|
||||
## Secret Names
|
||||
|
||||
Secret names must be defined in `.env.vault.enc` on the ops repo. The vault validates that requested secrets exist in the allowlist before execution.
|
||||
Secret names must have a corresponding `secrets/<NAME>.enc` file (age-encrypted). The vault validates that requested secrets exist in the allowlist before execution.
|
||||
|
||||
Common secret names:
|
||||
- `CLAWHUB_TOKEN` - Token for ClawHub skill publishing
|
||||
|
|
@ -28,7 +28,7 @@ fi
|
|||
# VAULT ACTION VALIDATION
|
||||
# =============================================================================
|
||||
|
||||
# Allowed secret names - must match keys in .env.vault.enc
|
||||
# Allowed secret names - must match files in secrets/<NAME>.enc
|
||||
VAULT_ALLOWED_SECRETS="CLAWHUB_TOKEN GITHUB_TOKEN CODEBERG_TOKEN DEPLOY_KEY NPM_TOKEN DOCKER_HUB_TOKEN"
|
||||
|
||||
# Allowed mount aliases — well-known file-based credential directories
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Architect — Agent Instructions
|
||||
|
||||
## What this agent is
|
||||
|
|
@ -96,6 +96,7 @@ Run via `architect/architect-run.sh`, which:
|
|||
- Acquires a poll-loop lock (via `acquire_lock`) and checks available memory
|
||||
- Cleans up per-issue scratch files from previous runs (`/tmp/architect-{project}-scratch-*.md`)
|
||||
- Sources shared libraries (env.sh, formula-session.sh)
|
||||
- Exports `FORGE_TOKEN_OVERRIDE="${FORGE_ARCHITECT_TOKEN}"` BEFORE sourcing env.sh, ensuring architect-bot identity survives re-sourcing (#762)
|
||||
- Uses FORGE_ARCHITECT_TOKEN for authentication
|
||||
- Processes existing architect PRs via bash-driven design phase
|
||||
- Loads the formula and builds context from VISION.md, AGENTS.md, and ops repo
|
||||
|
|
|
|||
515
bin/disinto
515
bin/disinto
|
|
@ -81,9 +81,13 @@ Init options:
|
|||
--repo-root <path> Local clone path (default: ~/name)
|
||||
--ci-id <n> Woodpecker CI repo ID (default: 0 = no CI)
|
||||
--forge-url <url> Forge base URL (default: http://localhost:3000)
|
||||
--backend <value> Orchestration backend: docker (default) | nomad
|
||||
--empty (nomad) Bring up cluster only, no jobs (S0.4)
|
||||
--bare Skip compose generation (bare-metal setup)
|
||||
--build Use local docker build instead of registry images (dev mode)
|
||||
--yes Skip confirmation prompts
|
||||
--rotate-tokens Force regeneration of all bot tokens/passwords (idempotent by default)
|
||||
--dry-run Print every intended action without executing
|
||||
|
||||
Hire an agent options:
|
||||
--formula <path> Path to role formula TOML (default: formulas/<role>.toml)
|
||||
|
|
@ -203,18 +207,21 @@ generate_compose() {
|
|||
|
||||
# Generate docker/agents/ files if they don't already exist.
|
||||
# (Implementation in lib/generators.sh)
|
||||
# shellcheck disable=SC2120 # passthrough wrapper; forwards any future args to impl
|
||||
generate_agent_docker() {
|
||||
_generate_agent_docker_impl "$@"
|
||||
}
|
||||
|
||||
# Generate docker/Caddyfile template for edge proxy.
|
||||
# (Implementation in lib/generators.sh)
|
||||
# shellcheck disable=SC2120 # passthrough wrapper; forwards any future args to impl
|
||||
generate_caddyfile() {
|
||||
_generate_caddyfile_impl "$@"
|
||||
}
|
||||
|
||||
# Generate docker/index.html default page.
|
||||
# (Implementation in lib/generators.sh)
|
||||
# shellcheck disable=SC2120 # passthrough wrapper; forwards any future args to impl
|
||||
generate_staging_index() {
|
||||
_generate_staging_index_impl "$@"
|
||||
}
|
||||
|
|
@ -642,6 +649,63 @@ prompt_admin_password() {
|
|||
|
||||
# ── init command ─────────────────────────────────────────────────────────────
|
||||
|
||||
# Nomad backend init — dispatcher (Nomad+Vault migration, S0.4, issue #824).
|
||||
#
|
||||
# Today `--empty` and the default (no flag) both bring up an empty
|
||||
# single-node Nomad+Vault cluster via lib/init/nomad/cluster-up.sh. Step 1
|
||||
# will extend the default path to also deploy jobs; `--empty` will remain
|
||||
# the "cluster only, no workloads" escape hatch.
|
||||
#
|
||||
# Uses `sudo -n` when not already root — cluster-up.sh mutates /etc/,
|
||||
# /srv/, and systemd state, so it has to run as root. The `-n` keeps the
|
||||
# failure mode legible (no hanging TTY-prompted sudo inside a factory
|
||||
# init run); operators running without sudo-NOPASSWD should invoke
|
||||
# `sudo disinto init ...` directly.
|
||||
_disinto_init_nomad() {
|
||||
local dry_run="${1:-false}" empty="${2:-false}"
|
||||
local cluster_up="${FACTORY_ROOT}/lib/init/nomad/cluster-up.sh"
|
||||
|
||||
if [ ! -x "$cluster_up" ]; then
|
||||
echo "Error: ${cluster_up} not found or not executable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --empty and default both invoke cluster-up today. Log the requested
|
||||
# mode so the dispatch is visible in factory bootstrap logs — Step 1
|
||||
# will branch on $empty to gate the job-deployment path.
|
||||
if [ "$empty" = "true" ]; then
|
||||
echo "nomad backend: --empty (cluster-up only, no jobs)"
|
||||
else
|
||||
echo "nomad backend: default (cluster-up; jobs deferred to Step 1)"
|
||||
fi
|
||||
|
||||
# Dry-run forwards straight through; cluster-up.sh prints its own step
|
||||
# list and exits 0 without touching the box.
|
||||
local -a cmd=("$cluster_up")
|
||||
if [ "$dry_run" = "true" ]; then
|
||||
cmd+=("--dry-run")
|
||||
"${cmd[@]}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Real run — needs root. Invoke via sudo if we're not already root so
|
||||
# the command's exit code propagates directly. We don't distinguish
|
||||
# "sudo denied" from "cluster-up.sh failed" here; both surface as a
|
||||
# non-zero exit, and cluster-up.sh's own error messages cover the
|
||||
# latter case.
|
||||
local rc=0
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
"${cmd[@]}" || rc=$?
|
||||
else
|
||||
if ! command -v sudo >/dev/null 2>&1; then
|
||||
echo "Error: cluster-up.sh must run as root and sudo is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n -- "${cmd[@]}" || rc=$?
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
disinto_init() {
|
||||
local repo_url="${1:-}"
|
||||
if [ -z "$repo_url" ]; then
|
||||
|
|
@ -652,20 +716,49 @@ disinto_init() {
|
|||
shift
|
||||
|
||||
# Parse flags
|
||||
local branch="" repo_root="" ci_id="0" auto_yes=false forge_url_flag="" bare=false rotate_tokens=false
|
||||
local branch="" repo_root="" ci_id="0" auto_yes=false forge_url_flag="" bare=false rotate_tokens=false use_build=false dry_run=false backend="docker" empty=false
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--branch) branch="$2"; shift 2 ;;
|
||||
--repo-root) repo_root="$2"; shift 2 ;;
|
||||
--ci-id) ci_id="$2"; shift 2 ;;
|
||||
--forge-url) forge_url_flag="$2"; shift 2 ;;
|
||||
--backend) backend="$2"; shift 2 ;;
|
||||
--backend=*) backend="${1#--backend=}"; shift ;;
|
||||
--bare) bare=true; shift ;;
|
||||
--build) use_build=true; shift ;;
|
||||
--empty) empty=true; shift ;;
|
||||
--yes) auto_yes=true; shift ;;
|
||||
--rotate-tokens) rotate_tokens=true; shift ;;
|
||||
--dry-run) dry_run=true; shift ;;
|
||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate backend
|
||||
case "$backend" in
|
||||
docker|nomad) ;;
|
||||
*) echo "Error: invalid --backend value '${backend}' (expected: docker|nomad)" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# --empty is nomad-only today (the docker path has no concept of an
|
||||
# "empty cluster"). Reject explicitly rather than letting it silently
|
||||
# do nothing on --backend=docker.
|
||||
if [ "$empty" = true ] && [ "$backend" != "nomad" ]; then
|
||||
echo "Error: --empty is only valid with --backend=nomad" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Dispatch on backend — the nomad path runs lib/init/nomad/cluster-up.sh
|
||||
# (S0.4). The default and --empty variants are identical today; Step 1
|
||||
# will branch on $empty to add job deployment to the default path.
|
||||
if [ "$backend" = "nomad" ]; then
|
||||
_disinto_init_nomad "$dry_run" "$empty"
|
||||
# shellcheck disable=SC2317 # _disinto_init_nomad always exits today;
|
||||
# `return` is defensive against future refactors.
|
||||
return
|
||||
fi
|
||||
|
||||
# Export bare-metal flag for setup_forge
|
||||
export DISINTO_BARE="$bare"
|
||||
|
||||
|
|
@ -738,12 +831,92 @@ p.write_text(text)
|
|||
fi
|
||||
fi
|
||||
|
||||
# ── Dry-run mode: report intended actions and exit ─────────────────────────
|
||||
if [ "$dry_run" = true ]; then
|
||||
echo ""
|
||||
echo "── Dry-run: intended actions ────────────────────────────"
|
||||
local env_file="${FACTORY_ROOT}/.env"
|
||||
local rr="${repo_root:-/home/${USER}/${project_name}}"
|
||||
|
||||
if [ "$bare" = false ]; then
|
||||
[ -f "${FACTORY_ROOT}/docker-compose.yml" ] \
|
||||
&& echo "[skip] docker-compose.yml (exists)" \
|
||||
|| echo "[create] docker-compose.yml"
|
||||
fi
|
||||
|
||||
[ -f "$env_file" ] \
|
||||
&& echo "[exists] .env" \
|
||||
|| echo "[create] .env"
|
||||
|
||||
# Report token state from .env
|
||||
if [ -f "$env_file" ]; then
|
||||
local _var
|
||||
for _var in FORGE_ADMIN_TOKEN HUMAN_TOKEN FORGE_TOKEN FORGE_REVIEW_TOKEN \
|
||||
FORGE_PLANNER_TOKEN FORGE_GARDENER_TOKEN FORGE_VAULT_TOKEN \
|
||||
FORGE_SUPERVISOR_TOKEN FORGE_PREDICTOR_TOKEN FORGE_ARCHITECT_TOKEN; do
|
||||
if grep -q "^${_var}=" "$env_file" 2>/dev/null; then
|
||||
echo "[keep] ${_var} (preserved)"
|
||||
else
|
||||
echo "[create] ${_var}"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "[create] all tokens and passwords"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "[ensure] Forgejo admin user 'disinto-admin'"
|
||||
echo "[ensure] 8 bot users: dev-bot, review-bot, planner-bot, gardener-bot, vault-bot, supervisor-bot, predictor-bot, architect-bot"
|
||||
echo "[ensure] 2 llama bot users: dev-qwen, dev-qwen-nightly"
|
||||
echo "[ensure] .profile repos for all bots"
|
||||
echo "[ensure] repo ${forge_repo} on Forgejo with collaborators"
|
||||
echo "[run] preflight checks"
|
||||
|
||||
[ -d "${rr}/.git" ] \
|
||||
&& echo "[skip] clone ${rr} (exists)" \
|
||||
|| echo "[clone] ${repo_url} -> ${rr}"
|
||||
|
||||
echo "[push] to local Forgejo"
|
||||
echo "[ensure] ops repo disinto-admin/${project_name}-ops"
|
||||
echo "[ensure] branch protection on ${forge_repo}"
|
||||
|
||||
[ "$toml_exists" = true ] \
|
||||
&& echo "[skip] ${toml_path} (exists)" \
|
||||
|| echo "[create] ${toml_path}"
|
||||
|
||||
if [ "$bare" = false ]; then
|
||||
echo "[ensure] Woodpecker OAuth2 app"
|
||||
echo "[ensure] Chat OAuth2 app"
|
||||
echo "[ensure] WOODPECKER_AGENT_SECRET in .env"
|
||||
fi
|
||||
|
||||
echo "[ensure] labels on ${forge_repo}"
|
||||
|
||||
[ -f "${rr}/VISION.md" ] \
|
||||
&& echo "[skip] VISION.md (exists)" \
|
||||
|| echo "[create] VISION.md"
|
||||
|
||||
echo "[copy] issue templates"
|
||||
echo "[ensure] scheduling (cron or compose polling)"
|
||||
|
||||
if [ "$bare" = false ]; then
|
||||
echo "[start] docker compose stack"
|
||||
echo "[ensure] Woodpecker token + repo activation"
|
||||
fi
|
||||
|
||||
echo "[ensure] CLAUDE_CONFIG_DIR"
|
||||
echo "[ensure] state files (.dev-active, .reviewer-active, .gardener-active)"
|
||||
echo ""
|
||||
echo "Dry run complete — no changes made."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Generate compose files (unless --bare)
|
||||
if [ "$bare" = false ]; then
|
||||
local forge_port
|
||||
forge_port=$(printf '%s' "$forge_url" | sed -E 's|.*:([0-9]+)/?$|\1|')
|
||||
forge_port="${forge_port:-3000}"
|
||||
generate_compose "$forge_port"
|
||||
generate_compose "$forge_port" "$use_build"
|
||||
generate_agent_docker
|
||||
generate_caddyfile
|
||||
generate_staging_index
|
||||
|
|
@ -890,6 +1063,19 @@ p.write_text(text)
|
|||
echo "Config: CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 saved to .env"
|
||||
fi
|
||||
|
||||
# Write local-Qwen dev agent env keys with safe defaults (#769)
|
||||
if ! grep -q '^ENABLE_LLAMA_AGENT=' "$env_file" 2>/dev/null; then
|
||||
cat >> "$env_file" <<'LLAMAENVEOF'
|
||||
|
||||
# Local Qwen dev agent (optional) — set to 1 to enable
|
||||
ENABLE_LLAMA_AGENT=0
|
||||
FORGE_TOKEN_LLAMA=
|
||||
FORGE_PASS_LLAMA=
|
||||
ANTHROPIC_BASE_URL=
|
||||
LLAMAENVEOF
|
||||
echo "Config: ENABLE_LLAMA_AGENT keys written to .env (disabled by default)"
|
||||
fi
|
||||
|
||||
# Create labels on remote
|
||||
create_labels "$forge_repo" "$forge_url"
|
||||
|
||||
|
|
@ -1118,8 +1304,6 @@ disinto_secrets() {
|
|||
local subcmd="${1:-}"
|
||||
local enc_file="${FACTORY_ROOT}/.env.enc"
|
||||
local env_file="${FACTORY_ROOT}/.env"
|
||||
local vault_enc_file="${FACTORY_ROOT}/.env.vault.enc"
|
||||
local vault_env_file="${FACTORY_ROOT}/.env.vault"
|
||||
|
||||
# Shared helper: ensure sops+age and .sops.yaml exist
|
||||
_secrets_ensure_sops() {
|
||||
|
|
@ -1165,30 +1349,51 @@ disinto_secrets() {
|
|||
|
||||
case "$subcmd" in
|
||||
add)
|
||||
local name="${2:-}"
|
||||
# Parse flags
|
||||
local force=false
|
||||
shift # consume 'add'
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-f|--force) force=true; shift ;;
|
||||
-*) echo "Unknown flag: $1" >&2; exit 1 ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
local name="${1:-}"
|
||||
if [ -z "$name" ]; then
|
||||
echo "Usage: disinto secrets add <NAME>" >&2
|
||||
echo "Usage: disinto secrets add [-f|--force] <NAME>" >&2
|
||||
exit 1
|
||||
fi
|
||||
_secrets_ensure_age_key
|
||||
mkdir -p "$secrets_dir"
|
||||
|
||||
printf 'Enter value for %s: ' "$name" >&2
|
||||
local value
|
||||
IFS= read -rs value
|
||||
echo >&2
|
||||
if [ -t 0 ]; then
|
||||
# Interactive TTY — prompt with hidden input (original behavior)
|
||||
printf 'Enter value for %s: ' "$name" >&2
|
||||
IFS= read -rs value
|
||||
echo >&2
|
||||
else
|
||||
# Piped/redirected stdin — read raw bytes verbatim
|
||||
IFS= read -r -d '' value || true
|
||||
fi
|
||||
if [ -z "$value" ]; then
|
||||
echo "Error: empty value" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local enc_path="${secrets_dir}/${name}.enc"
|
||||
if [ -f "$enc_path" ]; then
|
||||
printf 'Secret %s already exists. Overwrite? [y/N] ' "$name" >&2
|
||||
local confirm
|
||||
read -r confirm
|
||||
if [ "$confirm" != "y" ] && [ "$confirm" != "Y" ]; then
|
||||
echo "Aborted." >&2
|
||||
if [ -f "$enc_path" ] && [ "$force" = false ]; then
|
||||
if [ -t 0 ]; then
|
||||
printf 'Secret %s already exists. Overwrite? [y/N] ' "$name" >&2
|
||||
local confirm
|
||||
read -r confirm
|
||||
if [ "$confirm" != "y" ] && [ "$confirm" != "Y" ]; then
|
||||
echo "Aborted." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Error: secret ${name} already exists (use -f to overwrite)" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
|
@ -1221,6 +1426,37 @@ disinto_secrets() {
|
|||
sops -d "$enc_file"
|
||||
fi
|
||||
;;
|
||||
remove)
|
||||
local name="${2:-}"
|
||||
if [ -z "$name" ]; then
|
||||
echo "Usage: disinto secrets remove <NAME>" >&2
|
||||
exit 1
|
||||
fi
|
||||
local enc_path="${secrets_dir}/${name}.enc"
|
||||
if [ ! -f "$enc_path" ]; then
|
||||
echo "Error: ${enc_path} not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$enc_path"
|
||||
echo "Removed: ${enc_path}"
|
||||
;;
|
||||
list)
|
||||
if [ ! -d "$secrets_dir" ]; then
|
||||
echo "No secrets directory found." >&2
|
||||
exit 0
|
||||
fi
|
||||
local found=false
|
||||
for enc_file_path in "${secrets_dir}"/*.enc; do
|
||||
[ -f "$enc_file_path" ] || continue
|
||||
found=true
|
||||
local secret_name
|
||||
secret_name=$(basename "$enc_file_path" .enc)
|
||||
echo "$secret_name"
|
||||
done
|
||||
if [ "$found" = false ]; then
|
||||
echo "No secrets stored." >&2
|
||||
fi
|
||||
;;
|
||||
edit)
|
||||
if [ ! -f "$enc_file" ]; then
|
||||
echo "Error: ${enc_file} not found. Run 'disinto secrets migrate' first." >&2
|
||||
|
|
@ -1244,54 +1480,100 @@ disinto_secrets() {
|
|||
rm -f "$env_file"
|
||||
echo "Migrated: .env -> .env.enc (plaintext removed)"
|
||||
;;
|
||||
edit-vault)
|
||||
if [ ! -f "$vault_enc_file" ]; then
|
||||
echo "Error: ${vault_enc_file} not found. Run 'disinto secrets migrate-vault' first." >&2
|
||||
migrate-from-vault)
|
||||
# One-shot migration: split .env.vault.enc into secrets/<KEY>.enc files (#777)
|
||||
local vault_enc_file="${FACTORY_ROOT}/.env.vault.enc"
|
||||
local vault_env_file="${FACTORY_ROOT}/.env.vault"
|
||||
local source_file=""
|
||||
|
||||
if [ -f "$vault_enc_file" ] && command -v sops &>/dev/null; then
|
||||
source_file="$vault_enc_file"
|
||||
elif [ -f "$vault_env_file" ]; then
|
||||
source_file="$vault_env_file"
|
||||
else
|
||||
echo "Error: neither .env.vault.enc nor .env.vault found — nothing to migrate." >&2
|
||||
exit 1
|
||||
fi
|
||||
sops "$vault_enc_file"
|
||||
;;
|
||||
show-vault)
|
||||
if [ ! -f "$vault_enc_file" ]; then
|
||||
echo "Error: ${vault_enc_file} not found." >&2
|
||||
|
||||
_secrets_ensure_age_key
|
||||
mkdir -p "$secrets_dir"
|
||||
|
||||
# Decrypt vault to temp dotenv
|
||||
local tmp_dotenv
|
||||
tmp_dotenv=$(mktemp /tmp/disinto-vault-migrate-XXXXXX)
|
||||
trap 'rm -f "$tmp_dotenv"' RETURN
|
||||
|
||||
if [ "$source_file" = "$vault_enc_file" ]; then
|
||||
if ! sops -d --output-type dotenv "$vault_enc_file" > "$tmp_dotenv" 2>/dev/null; then
|
||||
rm -f "$tmp_dotenv"
|
||||
echo "Error: failed to decrypt .env.vault.enc" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
cp "$vault_env_file" "$tmp_dotenv"
|
||||
fi
|
||||
|
||||
# Parse each KEY=VALUE and encrypt into secrets/<KEY>.enc
|
||||
local count=0
|
||||
local failed=0
|
||||
while IFS='=' read -r key value; do
|
||||
# Skip empty lines and comments
|
||||
[[ -z "$key" || "$key" =~ ^[[:space:]]*# ]] && continue
|
||||
# Trim whitespace from key
|
||||
key=$(echo "$key" | xargs)
|
||||
[ -z "$key" ] && continue
|
||||
|
||||
local enc_path="${secrets_dir}/${key}.enc"
|
||||
if printf '%s' "$value" | age -r "$AGE_PUBLIC_KEY" -o "$enc_path" 2>/dev/null; then
|
||||
# Verify round-trip
|
||||
local check
|
||||
check=$(age -d -i "$age_key_file" "$enc_path" 2>/dev/null) || { failed=$((failed + 1)); echo " FAIL (verify): ${key}" >&2; continue; }
|
||||
if [ "$check" = "$value" ]; then
|
||||
echo " OK: ${key} -> secrets/${key}.enc"
|
||||
count=$((count + 1))
|
||||
else
|
||||
echo " FAIL (mismatch): ${key}" >&2
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
else
|
||||
echo " FAIL (encrypt): ${key}" >&2
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
done < "$tmp_dotenv"
|
||||
|
||||
rm -f "$tmp_dotenv"
|
||||
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
echo "Error: ${failed} secret(s) failed migration. Vault files NOT removed." >&2
|
||||
exit 1
|
||||
fi
|
||||
sops -d "$vault_enc_file"
|
||||
;;
|
||||
migrate-vault)
|
||||
if [ ! -f "$vault_env_file" ]; then
|
||||
echo "Error: ${vault_env_file} not found — nothing to migrate." >&2
|
||||
echo " Create .env.vault with vault secrets (GITHUB_TOKEN, deploy keys, etc.)" >&2
|
||||
exit 1
|
||||
|
||||
if [ "$count" -eq 0 ]; then
|
||||
echo "Warning: no secrets found in vault file." >&2
|
||||
else
|
||||
echo "Migrated ${count} secret(s) to secrets/*.enc"
|
||||
# Remove old vault files on success
|
||||
rm -f "$vault_enc_file" "$vault_env_file"
|
||||
echo "Removed: .env.vault.enc / .env.vault"
|
||||
fi
|
||||
_secrets_ensure_sops
|
||||
encrypt_env_file "$vault_env_file" "$vault_enc_file"
|
||||
# Verify decryption works before removing plaintext
|
||||
if ! sops -d "$vault_enc_file" >/dev/null 2>&1; then
|
||||
echo "Error: failed to verify .env.vault.enc decryption" >&2
|
||||
rm -f "$vault_enc_file"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$vault_env_file"
|
||||
echo "Migrated: .env.vault -> .env.vault.enc (plaintext removed)"
|
||||
;;
|
||||
*)
|
||||
cat <<EOF >&2
|
||||
Usage: disinto secrets <subcommand>
|
||||
|
||||
Individual secrets (secrets/<NAME>.enc):
|
||||
add <NAME> Prompt for value, encrypt, store in secrets/<NAME>.enc
|
||||
show <NAME> Decrypt and print an individual secret
|
||||
Secrets (secrets/<NAME>.enc — age-encrypted, one file per key):
|
||||
add <NAME> Prompt for value, encrypt, store in secrets/<NAME>.enc
|
||||
show <NAME> Decrypt and print a secret
|
||||
remove <NAME> Remove a secret
|
||||
list List all stored secrets
|
||||
|
||||
Agent secrets (.env.enc):
|
||||
edit Edit agent secrets (FORGE_TOKEN, CLAUDE_API_KEY, etc.)
|
||||
show Show decrypted agent secrets (no argument)
|
||||
migrate Encrypt .env -> .env.enc
|
||||
Agent secrets (.env.enc — sops-encrypted dotenv):
|
||||
edit Edit agent secrets (FORGE_TOKEN, CLAUDE_API_KEY, etc.)
|
||||
show Show decrypted agent secrets (no argument)
|
||||
migrate Encrypt .env -> .env.enc
|
||||
|
||||
Vault secrets (.env.vault.enc):
|
||||
edit-vault Edit vault secrets (GITHUB_TOKEN, deploy keys, etc.)
|
||||
show-vault Show decrypted vault secrets
|
||||
migrate-vault Encrypt .env.vault -> .env.vault.enc
|
||||
Migration:
|
||||
migrate-from-vault Split .env.vault.enc into secrets/<KEY>.enc (one-shot)
|
||||
EOF
|
||||
exit 1
|
||||
;;
|
||||
|
|
@ -1303,7 +1585,8 @@ EOF
|
|||
disinto_run() {
|
||||
local action_id="${1:?Usage: disinto run <action-id>}"
|
||||
local compose_file="${FACTORY_ROOT}/docker-compose.yml"
|
||||
local vault_enc="${FACTORY_ROOT}/.env.vault.enc"
|
||||
local secrets_dir="${FACTORY_ROOT}/secrets"
|
||||
local age_key_file="${HOME}/.config/sops/age/keys.txt"
|
||||
|
||||
if [ ! -f "$compose_file" ]; then
|
||||
echo "Error: docker-compose.yml not found" >&2
|
||||
|
|
@ -1311,29 +1594,42 @@ disinto_run() {
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$vault_enc" ]; then
|
||||
echo "Error: .env.vault.enc not found — create vault secrets first" >&2
|
||||
echo " Run 'disinto secrets migrate-vault' after creating .env.vault" >&2
|
||||
if [ ! -d "$secrets_dir" ]; then
|
||||
echo "Error: secrets/ directory not found — create secrets first" >&2
|
||||
echo " Run 'disinto secrets add <NAME>' to add secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v sops &>/dev/null; then
|
||||
echo "Error: sops not found — required to decrypt vault secrets" >&2
|
||||
if ! command -v age &>/dev/null; then
|
||||
echo "Error: age not found — required to decrypt secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Decrypt vault secrets to temp file
|
||||
if [ ! -f "$age_key_file" ]; then
|
||||
echo "Error: age key not found at ${age_key_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Decrypt all secrets/*.enc into a temp env file for the runner
|
||||
local tmp_env
|
||||
tmp_env=$(mktemp /tmp/disinto-vault-XXXXXX)
|
||||
tmp_env=$(mktemp /tmp/disinto-secrets-XXXXXX)
|
||||
trap 'rm -f "$tmp_env"' EXIT
|
||||
|
||||
if ! sops -d --output-type dotenv "$vault_enc" > "$tmp_env" 2>/dev/null; then
|
||||
rm -f "$tmp_env"
|
||||
echo "Error: failed to decrypt .env.vault.enc" >&2
|
||||
exit 1
|
||||
fi
|
||||
local count=0
|
||||
for enc_path in "${secrets_dir}"/*.enc; do
|
||||
[ -f "$enc_path" ] || continue
|
||||
local key
|
||||
key=$(basename "$enc_path" .enc)
|
||||
local val
|
||||
val=$(age -d -i "$age_key_file" "$enc_path" 2>/dev/null) || {
|
||||
echo "Warning: failed to decrypt ${enc_path}" >&2
|
||||
continue
|
||||
}
|
||||
printf '%s=%s\n' "$key" "$val" >> "$tmp_env"
|
||||
count=$((count + 1))
|
||||
done
|
||||
|
||||
echo "Vault secrets decrypted to tmpfile"
|
||||
echo "Decrypted ${count} secret(s) to tmpfile"
|
||||
|
||||
# Run action in ephemeral runner container
|
||||
local rc=0
|
||||
|
|
@ -1404,21 +1700,96 @@ download_agent_binaries() {
|
|||
|
||||
# ── up command ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Regenerate a file idempotently: run the generator, compare output, backup if changed.
|
||||
# Usage: _regen_file <target_file> <generator_fn> [args...]
|
||||
_regen_file() {
|
||||
local target="$1"; shift
|
||||
local generator="$1"; shift
|
||||
local basename
|
||||
basename=$(basename "$target")
|
||||
|
||||
# Move existing file aside so the generator (which skips if file exists)
|
||||
# produces a fresh copy.
|
||||
local stashed=""
|
||||
if [ -f "$target" ]; then
|
||||
stashed=$(mktemp "${target}.stash.XXXXXX")
|
||||
mv "$target" "$stashed"
|
||||
fi
|
||||
|
||||
# Run the generator — it writes $target from scratch.
|
||||
# If the generator fails, restore the stashed original so it is not stranded.
|
||||
if ! "$generator" "$@"; then
|
||||
if [ -n "$stashed" ]; then
|
||||
mv "$stashed" "$target"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -z "$stashed" ]; then
|
||||
# No previous file — first generation
|
||||
echo "regenerated: ${basename} (new)"
|
||||
return
|
||||
fi
|
||||
|
||||
if cmp -s "$stashed" "$target"; then
|
||||
# Content unchanged — restore original to preserve mtime
|
||||
mv "$stashed" "$target"
|
||||
echo "unchanged: ${basename}"
|
||||
else
|
||||
# Content changed — keep new, save old as .prev
|
||||
mv "$stashed" "${target}.prev"
|
||||
echo "regenerated: ${basename} (previous saved as ${basename}.prev)"
|
||||
fi
|
||||
}
|
||||
|
||||
disinto_up() {
|
||||
local compose_file="${FACTORY_ROOT}/docker-compose.yml"
|
||||
local caddyfile="${FACTORY_ROOT}/docker/Caddyfile"
|
||||
if [ ! -f "$compose_file" ]; then
|
||||
echo "Error: docker-compose.yml not found" >&2
|
||||
echo " Run 'disinto init <repo-url>' first (without --bare)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pre-build: download binaries to docker/agents/bin/ to avoid network calls during docker build
|
||||
echo "── Pre-build: downloading agent binaries ────────────────────────"
|
||||
if ! download_agent_binaries; then
|
||||
echo "Error: failed to download agent binaries" >&2
|
||||
exit 1
|
||||
# Parse --no-regen flag; remaining args pass through to docker compose
|
||||
local no_regen=false
|
||||
local -a compose_args=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-regen) no_regen=true ;;
|
||||
*) compose_args+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Regenerate compose & Caddyfile from generators ──────────────────────
|
||||
if [ "$no_regen" = true ]; then
|
||||
echo "Warning: running with unmanaged compose — hand-edits will drift" >&2
|
||||
else
|
||||
# Determine forge_port from FORGE_URL (same logic as init)
|
||||
local forge_url="${FORGE_URL:-http://localhost:3000}"
|
||||
local forge_port
|
||||
forge_port=$(printf '%s' "$forge_url" | sed -E 's|.*:([0-9]+)/?$|\1|')
|
||||
forge_port="${forge_port:-3000}"
|
||||
|
||||
# Detect build mode from existing compose
|
||||
local use_build=false
|
||||
if grep -q '^\s*build:' "$compose_file"; then
|
||||
use_build=true
|
||||
fi
|
||||
|
||||
_regen_file "$compose_file" generate_compose "$forge_port" "$use_build"
|
||||
_regen_file "$caddyfile" generate_caddyfile
|
||||
fi
|
||||
|
||||
# Pre-build: download binaries only when compose uses local build
|
||||
if grep -q '^\s*build:' "$compose_file"; then
|
||||
echo "── Pre-build: downloading agent binaries ────────────────────────"
|
||||
if ! download_agent_binaries; then
|
||||
echo "Error: failed to download agent binaries" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Decrypt secrets to temp .env if SOPS available and .env.enc exists
|
||||
local tmp_env=""
|
||||
|
|
@ -1431,7 +1802,7 @@ disinto_up() {
|
|||
echo "Decrypted secrets for compose"
|
||||
fi
|
||||
|
||||
docker compose -f "$compose_file" up -d "$@"
|
||||
docker compose -f "$compose_file" up -d --build --remove-orphans ${compose_args[@]+"${compose_args[@]}"}
|
||||
echo "Stack is up"
|
||||
|
||||
# Clean up temp .env (also handled by EXIT trap if compose fails)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Dev Agent
|
||||
|
||||
**Role**: Implement issues autonomously — write code, push branches, address
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ services:
|
|||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${HOME}/.claude.json:/home/agent/.claude.json:ro
|
||||
- CLAUDE_BIN_PLACEHOLDER:/usr/local/bin/claude:ro
|
||||
- ${HOME}/.ssh:/home/agent/.ssh:ro
|
||||
- ${HOME}/.config/sops/age:/home/agent/.config/sops/age:ro
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
environment:
|
||||
- FORGE_URL=http://forgejo:3000
|
||||
|
|
@ -49,6 +49,13 @@ services:
|
|||
- GARDENER_INTERVAL=${GARDENER_INTERVAL:-21600}
|
||||
- ARCHITECT_INTERVAL=${ARCHITECT_INTERVAL:-21600}
|
||||
- PLANNER_INTERVAL=${PLANNER_INTERVAL:-43200}
|
||||
- SUPERVISOR_INTERVAL=${SUPERVISOR_INTERVAL:-1200}
|
||||
healthcheck:
|
||||
test: ["CMD", "pgrep", "-f", "entrypoint.sh"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
|
|
@ -70,10 +77,10 @@ services:
|
|||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${HOME}/.claude.json:/home/agent/.claude.json:ro
|
||||
- CLAUDE_BIN_PLACEHOLDER:/usr/local/bin/claude:ro
|
||||
- ${HOME}/.ssh:/home/agent/.ssh:ro
|
||||
- ${HOME}/.config/sops/age:/home/agent/.config/sops/age:ro
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
environment:
|
||||
- FORGE_URL=http://forgejo:3000
|
||||
|
|
@ -103,6 +110,80 @@ services:
|
|||
- CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}
|
||||
- POLL_INTERVAL=${POLL_INTERVAL:-300}
|
||||
- AGENT_ROLES=dev
|
||||
healthcheck:
|
||||
test: ["CMD", "pgrep", "-f", "entrypoint.sh"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
woodpecker:
|
||||
condition: service_started
|
||||
networks:
|
||||
- disinto-net
|
||||
|
||||
agents-llama-all:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/agents/Dockerfile
|
||||
image: disinto/agents-llama:latest
|
||||
container_name: disinto-agents-llama-all
|
||||
restart: unless-stopped
|
||||
profiles: ["agents-llama-all"]
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
volumes:
|
||||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
environment:
|
||||
- FORGE_URL=http://forgejo:3000
|
||||
- FORGE_REPO=${FORGE_REPO:-disinto-admin/disinto}
|
||||
- FORGE_TOKEN=${FORGE_TOKEN_LLAMA:-}
|
||||
- FORGE_PASS=${FORGE_PASS_LLAMA:-}
|
||||
- FORGE_REVIEW_TOKEN=${FORGE_REVIEW_TOKEN:-}
|
||||
- FORGE_PLANNER_TOKEN=${FORGE_PLANNER_TOKEN:-}
|
||||
- FORGE_GARDENER_TOKEN=${FORGE_GARDENER_TOKEN:-}
|
||||
- FORGE_VAULT_TOKEN=${FORGE_VAULT_TOKEN:-}
|
||||
- FORGE_SUPERVISOR_TOKEN=${FORGE_SUPERVISOR_TOKEN:-}
|
||||
- FORGE_PREDICTOR_TOKEN=${FORGE_PREDICTOR_TOKEN:-}
|
||||
- FORGE_ARCHITECT_TOKEN=${FORGE_ARCHITECT_TOKEN:-}
|
||||
- FORGE_FILER_TOKEN=${FORGE_FILER_TOKEN:-}
|
||||
- FORGE_BOT_USERNAMES=${FORGE_BOT_USERNAMES:-}
|
||||
- WOODPECKER_TOKEN=${WOODPECKER_TOKEN:-}
|
||||
- CLAUDE_TIMEOUT=${CLAUDE_TIMEOUT:-7200}
|
||||
- CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}
|
||||
- CLAUDE_AUTOCOMPACT_PCT_OVERRIDE=60
|
||||
- CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
|
||||
- ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-}
|
||||
- FORGE_ADMIN_PASS=${FORGE_ADMIN_PASS:-}
|
||||
- DISINTO_CONTAINER=1
|
||||
- PROJECT_TOML=projects/disinto.toml
|
||||
- PROJECT_NAME=${PROJECT_NAME:-project}
|
||||
- PROJECT_REPO_ROOT=/home/agent/repos/${PROJECT_NAME:-project}
|
||||
- WOODPECKER_DATA_DIR=/woodpecker-data
|
||||
- WOODPECKER_REPO_ID=${WOODPECKER_REPO_ID:-}
|
||||
- CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}
|
||||
- POLL_INTERVAL=${POLL_INTERVAL:-300}
|
||||
- GARDENER_INTERVAL=${GARDENER_INTERVAL:-21600}
|
||||
- ARCHITECT_INTERVAL=${ARCHITECT_INTERVAL:-21600}
|
||||
- PLANNER_INTERVAL=${PLANNER_INTERVAL:-43200}
|
||||
- SUPERVISOR_INTERVAL=${SUPERVISOR_INTERVAL:-1200}
|
||||
- AGENT_ROLES=review,dev,gardener,architect,planner,predictor,supervisor
|
||||
healthcheck:
|
||||
test: ["CMD", "pgrep", "-f", "entrypoint.sh"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
|
|
@ -122,9 +203,9 @@ services:
|
|||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${HOME}/.claude:/home/agent/.claude
|
||||
- /usr/local/bin/claude:/usr/local/bin/claude:ro
|
||||
- ${HOME}/.ssh:/home/agent/.ssh:ro
|
||||
- ${CLAUDE_DIR:-${HOME}/.claude}:/home/agent/.claude
|
||||
- ${CLAUDE_BIN_DIR:-/usr/local/bin/claude}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
env_file:
|
||||
- .env
|
||||
|
||||
|
|
@ -138,9 +219,9 @@ services:
|
|||
- apparmor=unconfined
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /usr/local/bin/claude:/usr/local/bin/claude:ro
|
||||
- ${HOME}/.claude.json:/root/.claude.json:ro
|
||||
- ${HOME}/.claude:/root/.claude:ro
|
||||
- ${CLAUDE_BIN_DIR:-/usr/local/bin/claude}:/usr/local/bin/claude:ro
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/root/.claude.json:ro
|
||||
- ${CLAUDE_DIR:-${HOME}/.claude}:/root/.claude:ro
|
||||
- disinto-logs:/opt/disinto-logs
|
||||
environment:
|
||||
- FORGE_SUPERVISOR_TOKEN=${FORGE_SUPERVISOR_TOKEN:-}
|
||||
|
|
@ -156,6 +237,12 @@ services:
|
|||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://localhost:2019/config/"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
depends_on:
|
||||
- forgejo
|
||||
networks:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ RUN chmod +x /entrypoint.sh
|
|||
|
||||
# Entrypoint runs polling loop directly, dropping to agent user via gosu.
|
||||
# All scripts execute as the agent user (UID 1000) while preserving env vars.
|
||||
VOLUME /home/agent/data
|
||||
VOLUME /home/agent/repos
|
||||
|
||||
WORKDIR /home/agent/disinto
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
|
|
|||
|
|
@ -7,14 +7,15 @@ set -euo pipefail
|
|||
# poll scripts. All Docker Compose env vars are inherited (PATH, FORGE_TOKEN,
|
||||
# ANTHROPIC_API_KEY, etc.).
|
||||
#
|
||||
# AGENT_ROLES env var controls which scripts run: "review,dev,gardener,architect,planner,predictor"
|
||||
# (default: all six). Uses while-true loop with staggered intervals:
|
||||
# AGENT_ROLES env var controls which scripts run: "review,dev,gardener,architect,planner,predictor,supervisor"
|
||||
# (default: all seven). Uses while-true loop with staggered intervals:
|
||||
# - review-poll: every 5 minutes (offset by 0s)
|
||||
# - dev-poll: every 5 minutes (offset by 2 minutes)
|
||||
# - gardener: every GARDENER_INTERVAL seconds (default: 21600 = 6 hours)
|
||||
# - architect: every ARCHITECT_INTERVAL seconds (default: 21600 = 6 hours)
|
||||
# - planner: every PLANNER_INTERVAL seconds (default: 43200 = 12 hours)
|
||||
# - predictor: every 24 hours (288 iterations * 5 min)
|
||||
# - supervisor: every SUPERVISOR_INTERVAL seconds (default: 1200 = 20 min)
|
||||
|
||||
DISINTO_BAKED="/home/agent/disinto"
|
||||
DISINTO_LIVE="/home/agent/repos/_factory"
|
||||
|
|
@ -328,7 +329,7 @@ init_state_dir
|
|||
|
||||
# Parse AGENT_ROLES env var (default: all agents)
|
||||
# Expected format: comma-separated list like "review,dev,gardener"
|
||||
AGENT_ROLES="${AGENT_ROLES:-review,dev,gardener,architect,planner,predictor}"
|
||||
AGENT_ROLES="${AGENT_ROLES:-review,dev,gardener,architect,planner,predictor,supervisor}"
|
||||
log "Agent roles configured: ${AGENT_ROLES}"
|
||||
|
||||
# Poll interval in seconds (5 minutes default)
|
||||
|
|
@ -338,9 +339,10 @@ POLL_INTERVAL="${POLL_INTERVAL:-300}"
|
|||
GARDENER_INTERVAL="${GARDENER_INTERVAL:-21600}"
|
||||
ARCHITECT_INTERVAL="${ARCHITECT_INTERVAL:-21600}"
|
||||
PLANNER_INTERVAL="${PLANNER_INTERVAL:-43200}"
|
||||
SUPERVISOR_INTERVAL="${SUPERVISOR_INTERVAL:-1200}"
|
||||
|
||||
log "Entering polling loop (interval: ${POLL_INTERVAL}s, roles: ${AGENT_ROLES})"
|
||||
log "Gardener interval: ${GARDENER_INTERVAL}s, Architect interval: ${ARCHITECT_INTERVAL}s, Planner interval: ${PLANNER_INTERVAL}s"
|
||||
log "Gardener interval: ${GARDENER_INTERVAL}s, Architect interval: ${ARCHITECT_INTERVAL}s, Planner interval: ${PLANNER_INTERVAL}s, Supervisor interval: ${SUPERVISOR_INTERVAL}s"
|
||||
|
||||
# Main polling loop using iteration counter for gardener scheduling
|
||||
iteration=0
|
||||
|
|
@ -463,6 +465,19 @@ print(cfg.get('primary_branch', 'main'))
|
|||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Supervisor (interval configurable via SUPERVISOR_INTERVAL env var, default 20 min)
|
||||
if [[ ",${AGENT_ROLES}," == *",supervisor,"* ]]; then
|
||||
supervisor_iteration=$((iteration * POLL_INTERVAL))
|
||||
if [ $((supervisor_iteration % SUPERVISOR_INTERVAL)) -eq 0 ] && [ "$now" -ge "$supervisor_iteration" ]; then
|
||||
if ! pgrep -f "supervisor-run.sh" >/dev/null; then
|
||||
log "Running supervisor (iteration ${iteration}, ${SUPERVISOR_INTERVAL}s interval) for ${toml}"
|
||||
gosu agent bash -c "cd ${DISINTO_DIR} && bash supervisor/supervisor-run.sh \"${toml}\"" >> "${DISINTO_LOG_DIR}/supervisor.log" 2>&1 &
|
||||
else
|
||||
log "Skipping supervisor — already running"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
sleep "${POLL_INTERVAL}"
|
||||
|
|
|
|||
|
|
@ -30,6 +30,6 @@ WORKDIR /var/chat
|
|||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/')" || exit 1
|
||||
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')" || exit 1
|
||||
|
||||
ENTRYPOINT ["/entrypoint-chat.sh"]
|
||||
|
|
|
|||
|
|
@ -481,6 +481,14 @@ class ChatHandler(BaseHTTPRequestHandler):
|
|||
parsed = urlparse(self.path)
|
||||
path = parsed.path
|
||||
|
||||
# Health endpoint (no auth required) — used by Docker healthcheck
|
||||
if path == "/health":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"ok\n")
|
||||
return
|
||||
|
||||
# Verify endpoint for Caddy forward_auth (#709)
|
||||
if path == "/chat/auth/verify":
|
||||
self.handle_auth_verify()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
FROM caddy:latest
|
||||
RUN apk add --no-cache bash jq curl git docker-cli python3 openssh-client autossh
|
||||
COPY entrypoint-edge.sh /usr/local/bin/entrypoint-edge.sh
|
||||
|
||||
VOLUME /data
|
||||
|
||||
ENTRYPOINT ["bash", "/usr/local/bin/entrypoint-edge.sh"]
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@
|
|||
# 2. Scan vault/actions/ for TOML files without .result.json
|
||||
# 3. Verify TOML arrived via merged PR with admin merger (Forgejo API)
|
||||
# 4. Validate TOML using vault-env.sh validator
|
||||
# 5. Decrypt .env.vault.enc and extract only declared secrets
|
||||
# 6. Launch: docker run --rm disinto/agents:latest <action-id>
|
||||
# 5. Decrypt declared secrets via load_secret (lib/env.sh)
|
||||
# 6. Launch: delegate to _launch_runner_{docker,nomad} backend
|
||||
# 7. Write <action-id>.result.json with exit code, timestamp, logs summary
|
||||
#
|
||||
# Part of #76.
|
||||
|
|
@ -19,7 +19,7 @@ set -euo pipefail
|
|||
# Resolve script root (parent of lib/)
|
||||
SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
# Source shared environment
|
||||
# Source shared environment (provides load_secret, log helpers, etc.)
|
||||
source "${SCRIPT_ROOT}/../lib/env.sh"
|
||||
|
||||
# Project TOML location: prefer mounted path, fall back to cloned path
|
||||
|
|
@ -27,26 +27,18 @@ source "${SCRIPT_ROOT}/../lib/env.sh"
|
|||
# the shallow clone only has .toml.example files.
|
||||
PROJECTS_DIR="${PROJECTS_DIR:-${FACTORY_ROOT:-/opt/disinto}-projects}"
|
||||
|
||||
# Load vault secrets after env.sh (env.sh unsets them for agent security)
|
||||
# Vault secrets must be available to the dispatcher
|
||||
if [ -f "$FACTORY_ROOT/.env.vault.enc" ] && command -v sops &>/dev/null; then
|
||||
set -a
|
||||
eval "$(sops -d --output-type dotenv "$FACTORY_ROOT/.env.vault.enc" 2>/dev/null)" \
|
||||
|| echo "Warning: failed to decrypt .env.vault.enc — vault secrets not loaded" >&2
|
||||
set +a
|
||||
elif [ -f "$FACTORY_ROOT/.env.vault" ]; then
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$FACTORY_ROOT/.env.vault"
|
||||
set +a
|
||||
fi
|
||||
# -----------------------------------------------------------------------------
|
||||
# Backend selection: DISPATCHER_BACKEND={docker,nomad}
|
||||
# Default: docker. nomad lands as a pure addition during migration Step 5.
|
||||
# -----------------------------------------------------------------------------
|
||||
DISPATCHER_BACKEND="${DISPATCHER_BACKEND:-docker}"
|
||||
|
||||
# Ops repo location (vault/actions directory)
|
||||
OPS_REPO_ROOT="${OPS_REPO_ROOT:-/home/debian/disinto-ops}"
|
||||
VAULT_ACTIONS_DIR="${OPS_REPO_ROOT}/vault/actions"
|
||||
|
||||
# Vault action validation
|
||||
VAULT_ENV="${SCRIPT_ROOT}/../vault/vault-env.sh"
|
||||
VAULT_ENV="${SCRIPT_ROOT}/../action-vault/vault-env.sh"
|
||||
|
||||
# Admin users who can merge vault PRs (from issue #77)
|
||||
# Comma-separated list of Forgejo usernames with admin role
|
||||
|
|
@ -350,73 +342,113 @@ get_dispatch_mode() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Write result file for an action
|
||||
# Usage: write_result <action_id> <exit_code> <logs>
|
||||
write_result() {
|
||||
# Commit result.json to the ops repo via git push (portable, no bind-mount).
|
||||
#
|
||||
# Clones the ops repo into a scratch directory, writes the result file,
|
||||
# commits as vault-bot, and pushes to the primary branch.
|
||||
# Idempotent: skips if result.json already exists upstream.
|
||||
# Retries on push conflict with rebase-and-push (handles concurrent merges).
|
||||
#
|
||||
# Usage: commit_result_via_git <action_id> <exit_code> <logs>
|
||||
commit_result_via_git() {
|
||||
local action_id="$1"
|
||||
local exit_code="$2"
|
||||
local logs="$3"
|
||||
|
||||
local result_file="${VAULT_ACTIONS_DIR}/${action_id}.result.json"
|
||||
local result_relpath="vault/actions/${action_id}.result.json"
|
||||
local ops_clone_url="${FORGE_URL}/${FORGE_OPS_REPO}.git"
|
||||
local branch="${PRIMARY_BRANCH:-main}"
|
||||
local scratch_dir
|
||||
scratch_dir=$(mktemp -d /tmp/dispatcher-result-XXXXXX)
|
||||
# shellcheck disable=SC2064
|
||||
trap "rm -rf '${scratch_dir}'" RETURN
|
||||
|
||||
# Shallow clone of the ops repo — only the primary branch
|
||||
if ! git clone --depth 1 --branch "$branch" \
|
||||
"$ops_clone_url" "$scratch_dir" 2>/dev/null; then
|
||||
log "ERROR: Failed to clone ops repo for result commit (action ${action_id})"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Idempotency: skip if result.json already exists upstream
|
||||
if [ -f "${scratch_dir}/${result_relpath}" ]; then
|
||||
log "Result already exists upstream for ${action_id} — skipping commit"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Configure git identity as vault-bot
|
||||
git -C "$scratch_dir" config user.name "vault-bot"
|
||||
git -C "$scratch_dir" config user.email "vault-bot@disinto.local"
|
||||
|
||||
# Truncate logs if too long (keep last 1000 chars)
|
||||
if [ ${#logs} -gt 1000 ]; then
|
||||
logs="${logs: -1000}"
|
||||
fi
|
||||
|
||||
# Write result JSON
|
||||
# Write result JSON via jq (never string-interpolate into JSON)
|
||||
mkdir -p "$(dirname "${scratch_dir}/${result_relpath}")"
|
||||
jq -n \
|
||||
--arg id "$action_id" \
|
||||
--argjson exit_code "$exit_code" \
|
||||
--arg timestamp "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||
--arg logs "$logs" \
|
||||
'{id: $id, exit_code: $exit_code, timestamp: $timestamp, logs: $logs}' \
|
||||
> "$result_file"
|
||||
> "${scratch_dir}/${result_relpath}"
|
||||
|
||||
log "Result written: ${result_file}"
|
||||
git -C "$scratch_dir" add "$result_relpath"
|
||||
git -C "$scratch_dir" commit -q -m "vault: result for ${action_id}"
|
||||
|
||||
# Push with retry on conflict (rebase-and-push pattern).
|
||||
# Common case: admin merges another action PR between our clone and push.
|
||||
local attempt
|
||||
for attempt in 1 2 3; do
|
||||
if git -C "$scratch_dir" push origin "$branch" 2>/dev/null; then
|
||||
log "Result committed and pushed for ${action_id} (attempt ${attempt})"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Push conflict for ${action_id} (attempt ${attempt}/3) — rebasing"
|
||||
|
||||
if ! git -C "$scratch_dir" pull --rebase origin "$branch" 2>/dev/null; then
|
||||
# Rebase conflict — check if result was pushed by another process
|
||||
git -C "$scratch_dir" rebase --abort 2>/dev/null || true
|
||||
if git -C "$scratch_dir" fetch origin "$branch" 2>/dev/null && \
|
||||
git -C "$scratch_dir" show "origin/${branch}:${result_relpath}" >/dev/null 2>&1; then
|
||||
log "Result already exists upstream for ${action_id} (pushed by another process)"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
log "ERROR: Failed to push result for ${action_id} after 3 attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Launch runner for the given action
|
||||
# Usage: launch_runner <toml_file>
|
||||
launch_runner() {
|
||||
local toml_file="$1"
|
||||
local action_id
|
||||
action_id=$(basename "$toml_file" .toml)
|
||||
# Write result file for an action via git push to the ops repo.
|
||||
# Usage: write_result <action_id> <exit_code> <logs>
|
||||
write_result() {
|
||||
local action_id="$1"
|
||||
local exit_code="$2"
|
||||
local logs="$3"
|
||||
|
||||
log "Launching runner for action: ${action_id}"
|
||||
commit_result_via_git "$action_id" "$exit_code" "$logs"
|
||||
}
|
||||
|
||||
# Validate TOML
|
||||
if ! validate_action "$toml_file"; then
|
||||
log "ERROR: Action validation failed for ${action_id}"
|
||||
write_result "$action_id" 1 "Validation failed: see logs above"
|
||||
return 1
|
||||
fi
|
||||
# -----------------------------------------------------------------------------
|
||||
# Pluggable launcher backends
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# Check dispatch mode to determine if admin verification is needed
|
||||
local dispatch_mode
|
||||
dispatch_mode=$(get_dispatch_mode "$toml_file")
|
||||
# _launch_runner_docker ACTION_ID SECRETS_CSV MOUNTS_CSV
|
||||
#
|
||||
# Builds and executes a `docker run` command for the vault runner.
|
||||
# Secrets are resolved via load_secret (lib/env.sh).
|
||||
# Returns: exit code of the docker run. Stdout/stderr are captured to a temp
|
||||
# log file whose path is printed to stdout (caller reads it).
|
||||
_launch_runner_docker() {
|
||||
local action_id="$1"
|
||||
local secrets_csv="$2"
|
||||
local mounts_csv="$3"
|
||||
|
||||
if [ "$dispatch_mode" = "direct" ]; then
|
||||
log "Action ${action_id}: tier=${VAULT_TIER:-unknown}, dispatch_mode=${dispatch_mode} — skipping admin merge verification (direct commit)"
|
||||
else
|
||||
# Verify admin merge for PR-based actions
|
||||
log "Action ${action_id}: tier=${VAULT_TIER:-unknown}, dispatch_mode=${dispatch_mode} — verifying admin merge"
|
||||
if ! verify_admin_merged "$toml_file"; then
|
||||
log "ERROR: Admin merge verification failed for ${action_id}"
|
||||
write_result "$action_id" 1 "Admin merge verification failed: see logs above"
|
||||
return 1
|
||||
fi
|
||||
log "Action ${action_id}: admin merge verified"
|
||||
fi
|
||||
|
||||
# Extract secrets from validated action
|
||||
local secrets_array
|
||||
secrets_array="${VAULT_ACTION_SECRETS:-}"
|
||||
|
||||
# Build docker run command (self-contained, no compose context needed).
|
||||
# The edge container has the Docker socket but not the host's compose project,
|
||||
# so docker compose run would fail with exit 125. docker run is self-contained:
|
||||
# the dispatcher knows the image, network, env vars, and entrypoint.
|
||||
local -a cmd=(docker run --rm
|
||||
--name "vault-runner-${action_id}"
|
||||
--network host
|
||||
|
|
@ -451,29 +483,27 @@ launch_runner() {
|
|||
cmd+=(-v "${runtime_home}/.claude.json:/home/agent/.claude.json:ro")
|
||||
fi
|
||||
|
||||
# Add environment variables for secrets (if any declared)
|
||||
if [ -n "$secrets_array" ]; then
|
||||
for secret in $secrets_array; do
|
||||
# Add environment variables for secrets (resolved via load_secret)
|
||||
if [ -n "$secrets_csv" ]; then
|
||||
local secret
|
||||
for secret in $(echo "$secrets_csv" | tr ',' ' '); do
|
||||
secret=$(echo "$secret" | xargs)
|
||||
if [ -n "$secret" ]; then
|
||||
# Verify secret exists in vault
|
||||
if [ -z "${!secret:-}" ]; then
|
||||
log "ERROR: Secret '${secret}' not found in vault for action ${action_id}"
|
||||
write_result "$action_id" 1 "Secret not found in vault: ${secret}"
|
||||
return 1
|
||||
fi
|
||||
cmd+=(-e "${secret}=${!secret}")
|
||||
[ -n "$secret" ] || continue
|
||||
local secret_val
|
||||
secret_val=$(load_secret "$secret") || true
|
||||
if [ -z "$secret_val" ]; then
|
||||
log "ERROR: Secret '${secret}' could not be resolved for action ${action_id}"
|
||||
write_result "$action_id" 1 "Secret not found: ${secret}"
|
||||
return 1
|
||||
fi
|
||||
cmd+=(-e "${secret}=${secret_val}")
|
||||
done
|
||||
else
|
||||
log "Action ${action_id} has no secrets declared — runner will execute without extra env vars"
|
||||
fi
|
||||
|
||||
# Add volume mounts for file-based credentials (if any declared)
|
||||
local mounts_array
|
||||
mounts_array="${VAULT_ACTION_MOUNTS:-}"
|
||||
if [ -n "$mounts_array" ]; then
|
||||
for mount_alias in $mounts_array; do
|
||||
# Add volume mounts for file-based credentials
|
||||
if [ -n "$mounts_csv" ]; then
|
||||
local mount_alias
|
||||
for mount_alias in $(echo "$mounts_csv" | tr ',' ' '); do
|
||||
mount_alias=$(echo "$mount_alias" | xargs)
|
||||
[ -n "$mount_alias" ] || continue
|
||||
case "$mount_alias" in
|
||||
|
|
@ -501,7 +531,7 @@ launch_runner() {
|
|||
# Image and entrypoint arguments: runner entrypoint + action-id
|
||||
cmd+=(disinto/agents:latest /home/agent/disinto/docker/runner/entrypoint-runner.sh "$action_id")
|
||||
|
||||
log "Running: docker run --rm vault-runner-${action_id} (secrets: ${secrets_array:-none}, mounts: ${mounts_array:-none})"
|
||||
log "Running: docker run --rm vault-runner-${action_id} (secrets: ${secrets_csv:-none}, mounts: ${mounts_csv:-none})"
|
||||
|
||||
# Create temp file for logs
|
||||
local log_file
|
||||
|
|
@ -509,7 +539,6 @@ launch_runner() {
|
|||
trap 'rm -f "$log_file"' RETURN
|
||||
|
||||
# Execute with array expansion (safe from shell injection)
|
||||
# Capture stdout and stderr to log file
|
||||
"${cmd[@]}" > "$log_file" 2>&1
|
||||
local exit_code=$?
|
||||
|
||||
|
|
@ -529,6 +558,137 @@ launch_runner() {
|
|||
return $exit_code
|
||||
}
|
||||
|
||||
# _launch_runner_nomad ACTION_ID SECRETS_CSV MOUNTS_CSV
|
||||
#
|
||||
# Nomad backend stub — will be implemented in migration Step 5.
|
||||
_launch_runner_nomad() {
|
||||
echo "nomad backend not yet implemented" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Launch runner for the given action (backend-agnostic orchestrator)
|
||||
# Usage: launch_runner <toml_file>
|
||||
launch_runner() {
|
||||
local toml_file="$1"
|
||||
local action_id
|
||||
action_id=$(basename "$toml_file" .toml)
|
||||
|
||||
log "Launching runner for action: ${action_id}"
|
||||
|
||||
# Validate TOML
|
||||
if ! validate_action "$toml_file"; then
|
||||
log "ERROR: Action validation failed for ${action_id}"
|
||||
write_result "$action_id" 1 "Validation failed: see logs above"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check dispatch mode to determine if admin verification is needed
|
||||
local dispatch_mode
|
||||
dispatch_mode=$(get_dispatch_mode "$toml_file")
|
||||
|
||||
if [ "$dispatch_mode" = "direct" ]; then
|
||||
log "Action ${action_id}: tier=${VAULT_TIER:-unknown}, dispatch_mode=${dispatch_mode} — skipping admin merge verification (direct commit)"
|
||||
else
|
||||
# Verify admin merge for PR-based actions
|
||||
log "Action ${action_id}: tier=${VAULT_TIER:-unknown}, dispatch_mode=${dispatch_mode} — verifying admin merge"
|
||||
if ! verify_admin_merged "$toml_file"; then
|
||||
log "ERROR: Admin merge verification failed for ${action_id}"
|
||||
write_result "$action_id" 1 "Admin merge verification failed: see logs above"
|
||||
return 1
|
||||
fi
|
||||
log "Action ${action_id}: admin merge verified"
|
||||
fi
|
||||
|
||||
# Build CSV lists from validated action metadata
|
||||
local secrets_csv=""
|
||||
if [ -n "${VAULT_ACTION_SECRETS:-}" ]; then
|
||||
# Convert space-separated to comma-separated
|
||||
secrets_csv=$(echo "${VAULT_ACTION_SECRETS}" | xargs | tr ' ' ',')
|
||||
fi
|
||||
|
||||
local mounts_csv=""
|
||||
if [ -n "${VAULT_ACTION_MOUNTS:-}" ]; then
|
||||
mounts_csv=$(echo "${VAULT_ACTION_MOUNTS}" | xargs | tr ' ' ',')
|
||||
fi
|
||||
|
||||
# Delegate to the selected backend
|
||||
"_launch_runner_${DISPATCHER_BACKEND}" "$action_id" "$secrets_csv" "$mounts_csv"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Pluggable sidecar launcher (reproduce / triage / verify)
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# _dispatch_sidecar_docker CONTAINER_NAME ISSUE_NUM PROJECT_TOML IMAGE [FORMULA]
|
||||
#
|
||||
# Launches a sidecar container via docker run (background, pid-tracked).
|
||||
# Prints the background PID to stdout.
|
||||
_dispatch_sidecar_docker() {
|
||||
local container_name="$1"
|
||||
local issue_number="$2"
|
||||
local project_toml="$3"
|
||||
local image="$4"
|
||||
local formula="${5:-}"
|
||||
|
||||
local -a cmd=(docker run --rm
|
||||
--name "${container_name}"
|
||||
--network host
|
||||
--security-opt apparmor=unconfined
|
||||
-v /var/run/docker.sock:/var/run/docker.sock
|
||||
-v agent-data:/home/agent/data
|
||||
-v project-repos:/home/agent/repos
|
||||
-e "FORGE_URL=${FORGE_URL}"
|
||||
-e "FORGE_TOKEN=${FORGE_TOKEN}"
|
||||
-e "FORGE_REPO=${FORGE_REPO}"
|
||||
-e "PRIMARY_BRANCH=${PRIMARY_BRANCH:-main}"
|
||||
-e DISINTO_CONTAINER=1
|
||||
)
|
||||
|
||||
# Set formula if provided
|
||||
if [ -n "$formula" ]; then
|
||||
cmd+=(-e "DISINTO_FORMULA=${formula}")
|
||||
fi
|
||||
|
||||
# Pass through ANTHROPIC_API_KEY if set
|
||||
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
|
||||
cmd+=(-e "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}")
|
||||
fi
|
||||
|
||||
# Mount shared Claude config dir and ~/.ssh from the runtime user's home
|
||||
local runtime_home="${HOME:-/home/debian}"
|
||||
if [ -d "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}" ]; then
|
||||
cmd+=(-v "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}")
|
||||
cmd+=(-e "CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}")
|
||||
fi
|
||||
if [ -f "${runtime_home}/.claude.json" ]; then
|
||||
cmd+=(-v "${runtime_home}/.claude.json:/home/agent/.claude.json:ro")
|
||||
fi
|
||||
if [ -d "${runtime_home}/.ssh" ]; then
|
||||
cmd+=(-v "${runtime_home}/.ssh:/home/agent/.ssh:ro")
|
||||
fi
|
||||
if [ -f /usr/local/bin/claude ]; then
|
||||
cmd+=(-v /usr/local/bin/claude:/usr/local/bin/claude:ro)
|
||||
fi
|
||||
|
||||
# Mount the project TOML into the container at a stable path
|
||||
local container_toml="/home/agent/project.toml"
|
||||
cmd+=(-v "${project_toml}:${container_toml}:ro")
|
||||
|
||||
cmd+=("${image}" "$container_toml" "$issue_number")
|
||||
|
||||
# Launch in background
|
||||
"${cmd[@]}" &
|
||||
echo $!
|
||||
}
|
||||
|
||||
# _dispatch_sidecar_nomad CONTAINER_NAME ISSUE_NUM PROJECT_TOML IMAGE [FORMULA]
|
||||
#
|
||||
# Nomad sidecar backend stub — will be implemented in migration Step 5.
|
||||
_dispatch_sidecar_nomad() {
|
||||
echo "nomad backend not yet implemented" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Reproduce dispatch — launch sidecar for bug-report issues
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -607,52 +767,13 @@ dispatch_reproduce() {
|
|||
|
||||
log "Dispatching reproduce-agent for issue #${issue_number} (project: ${project_toml})"
|
||||
|
||||
# Build docker run command using array (safe from injection)
|
||||
local -a cmd=(docker run --rm
|
||||
--name "disinto-reproduce-${issue_number}"
|
||||
--network host
|
||||
--security-opt apparmor=unconfined
|
||||
-v /var/run/docker.sock:/var/run/docker.sock
|
||||
-v agent-data:/home/agent/data
|
||||
-v project-repos:/home/agent/repos
|
||||
-e "FORGE_URL=${FORGE_URL}"
|
||||
-e "FORGE_TOKEN=${FORGE_TOKEN}"
|
||||
-e "FORGE_REPO=${FORGE_REPO}"
|
||||
-e "PRIMARY_BRANCH=${PRIMARY_BRANCH:-main}"
|
||||
-e DISINTO_CONTAINER=1
|
||||
)
|
||||
local bg_pid
|
||||
bg_pid=$("_dispatch_sidecar_${DISPATCHER_BACKEND}" \
|
||||
"disinto-reproduce-${issue_number}" \
|
||||
"$issue_number" \
|
||||
"$project_toml" \
|
||||
"disinto-reproduce:latest")
|
||||
|
||||
# Pass through ANTHROPIC_API_KEY if set
|
||||
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
|
||||
cmd+=(-e "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}")
|
||||
fi
|
||||
|
||||
# Mount shared Claude config dir and ~/.ssh from the runtime user's home if available
|
||||
local runtime_home="${HOME:-/home/debian}"
|
||||
if [ -d "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}" ]; then
|
||||
cmd+=(-v "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}")
|
||||
cmd+=(-e "CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}")
|
||||
fi
|
||||
if [ -f "${runtime_home}/.claude.json" ]; then
|
||||
cmd+=(-v "${runtime_home}/.claude.json:/home/agent/.claude.json:ro")
|
||||
fi
|
||||
if [ -d "${runtime_home}/.ssh" ]; then
|
||||
cmd+=(-v "${runtime_home}/.ssh:/home/agent/.ssh:ro")
|
||||
fi
|
||||
# Mount claude CLI binary if present on host
|
||||
if [ -f /usr/local/bin/claude ]; then
|
||||
cmd+=(-v /usr/local/bin/claude:/usr/local/bin/claude:ro)
|
||||
fi
|
||||
|
||||
# Mount the project TOML into the container at a stable path
|
||||
local container_toml="/home/agent/project.toml"
|
||||
cmd+=(-v "${project_toml}:${container_toml}:ro")
|
||||
|
||||
cmd+=(disinto-reproduce:latest "$container_toml" "$issue_number")
|
||||
|
||||
# Launch in background; write pid-file so we don't double-launch
|
||||
"${cmd[@]}" &
|
||||
local bg_pid=$!
|
||||
echo "$bg_pid" > "$(_reproduce_lockfile "$issue_number")"
|
||||
log "Reproduce container launched (pid ${bg_pid}) for issue #${issue_number}"
|
||||
}
|
||||
|
|
@ -732,53 +853,14 @@ dispatch_triage() {
|
|||
|
||||
log "Dispatching triage-agent for issue #${issue_number} (project: ${project_toml})"
|
||||
|
||||
# Build docker run command using array (safe from injection)
|
||||
local -a cmd=(docker run --rm
|
||||
--name "disinto-triage-${issue_number}"
|
||||
--network host
|
||||
--security-opt apparmor=unconfined
|
||||
-v /var/run/docker.sock:/var/run/docker.sock
|
||||
-v agent-data:/home/agent/data
|
||||
-v project-repos:/home/agent/repos
|
||||
-e "FORGE_URL=${FORGE_URL}"
|
||||
-e "FORGE_TOKEN=${FORGE_TOKEN}"
|
||||
-e "FORGE_REPO=${FORGE_REPO}"
|
||||
-e "PRIMARY_BRANCH=${PRIMARY_BRANCH:-main}"
|
||||
-e DISINTO_CONTAINER=1
|
||||
-e DISINTO_FORMULA=triage
|
||||
)
|
||||
local bg_pid
|
||||
bg_pid=$("_dispatch_sidecar_${DISPATCHER_BACKEND}" \
|
||||
"disinto-triage-${issue_number}" \
|
||||
"$issue_number" \
|
||||
"$project_toml" \
|
||||
"disinto-reproduce:latest" \
|
||||
"triage")
|
||||
|
||||
# Pass through ANTHROPIC_API_KEY if set
|
||||
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
|
||||
cmd+=(-e "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}")
|
||||
fi
|
||||
|
||||
# Mount shared Claude config dir and ~/.ssh from the runtime user's home if available
|
||||
local runtime_home="${HOME:-/home/debian}"
|
||||
if [ -d "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}" ]; then
|
||||
cmd+=(-v "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}")
|
||||
cmd+=(-e "CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}")
|
||||
fi
|
||||
if [ -f "${runtime_home}/.claude.json" ]; then
|
||||
cmd+=(-v "${runtime_home}/.claude.json:/home/agent/.claude.json:ro")
|
||||
fi
|
||||
if [ -d "${runtime_home}/.ssh" ]; then
|
||||
cmd+=(-v "${runtime_home}/.ssh:/home/agent/.ssh:ro")
|
||||
fi
|
||||
# Mount claude CLI binary if present on host
|
||||
if [ -f /usr/local/bin/claude ]; then
|
||||
cmd+=(-v /usr/local/bin/claude:/usr/local/bin/claude:ro)
|
||||
fi
|
||||
|
||||
# Mount the project TOML into the container at a stable path
|
||||
local container_toml="/home/agent/project.toml"
|
||||
cmd+=(-v "${project_toml}:${container_toml}:ro")
|
||||
|
||||
cmd+=(disinto-reproduce:latest "$container_toml" "$issue_number")
|
||||
|
||||
# Launch in background; write pid-file so we don't double-launch
|
||||
"${cmd[@]}" &
|
||||
local bg_pid=$!
|
||||
echo "$bg_pid" > "$(_triage_lockfile "$issue_number")"
|
||||
log "Triage container launched (pid ${bg_pid}) for issue #${issue_number}"
|
||||
}
|
||||
|
|
@ -934,53 +1016,14 @@ dispatch_verify() {
|
|||
|
||||
log "Dispatching verification-agent for issue #${issue_number} (project: ${project_toml})"
|
||||
|
||||
# Build docker run command using array (safe from injection)
|
||||
local -a cmd=(docker run --rm
|
||||
--name "disinto-verify-${issue_number}"
|
||||
--network host
|
||||
--security-opt apparmor=unconfined
|
||||
-v /var/run/docker.sock:/var/run/docker.sock
|
||||
-v agent-data:/home/agent/data
|
||||
-v project-repos:/home/agent/repos
|
||||
-e "FORGE_URL=${FORGE_URL}"
|
||||
-e "FORGE_TOKEN=${FORGE_TOKEN}"
|
||||
-e "FORGE_REPO=${FORGE_REPO}"
|
||||
-e "PRIMARY_BRANCH=${PRIMARY_BRANCH:-main}"
|
||||
-e DISINTO_CONTAINER=1
|
||||
-e DISINTO_FORMULA=verify
|
||||
)
|
||||
local bg_pid
|
||||
bg_pid=$("_dispatch_sidecar_${DISPATCHER_BACKEND}" \
|
||||
"disinto-verify-${issue_number}" \
|
||||
"$issue_number" \
|
||||
"$project_toml" \
|
||||
"disinto-reproduce:latest" \
|
||||
"verify")
|
||||
|
||||
# Pass through ANTHROPIC_API_KEY if set
|
||||
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
|
||||
cmd+=(-e "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}")
|
||||
fi
|
||||
|
||||
# Mount shared Claude config dir and ~/.ssh from the runtime user's home if available
|
||||
local runtime_home="${HOME:-/home/debian}"
|
||||
if [ -d "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}" ]; then
|
||||
cmd+=(-v "${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}")
|
||||
cmd+=(-e "CLAUDE_CONFIG_DIR=${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}")
|
||||
fi
|
||||
if [ -f "${runtime_home}/.claude.json" ]; then
|
||||
cmd+=(-v "${runtime_home}/.claude.json:/home/agent/.claude.json:ro")
|
||||
fi
|
||||
if [ -d "${runtime_home}/.ssh" ]; then
|
||||
cmd+=(-v "${runtime_home}/.ssh:/home/agent/.ssh:ro")
|
||||
fi
|
||||
# Mount claude CLI binary if present on host
|
||||
if [ -f /usr/local/bin/claude ]; then
|
||||
cmd+=(-v /usr/local/bin/claude:/usr/local/bin/claude:ro)
|
||||
fi
|
||||
|
||||
# Mount the project TOML into the container at a stable path
|
||||
local container_toml="/home/agent/project.toml"
|
||||
cmd+=(-v "${project_toml}:${container_toml}:ro")
|
||||
|
||||
cmd+=(disinto-reproduce:latest "$container_toml" "$issue_number")
|
||||
|
||||
# Launch in background; write pid-file so we don't double-launch
|
||||
"${cmd[@]}" &
|
||||
local bg_pid=$!
|
||||
echo "$bg_pid" > "$(_verify_lockfile "$issue_number")"
|
||||
log "Verification container launched (pid ${bg_pid}) for issue #${issue_number}"
|
||||
}
|
||||
|
|
@ -1002,10 +1045,25 @@ ensure_ops_repo() {
|
|||
|
||||
# Main dispatcher loop
|
||||
main() {
|
||||
log "Starting dispatcher..."
|
||||
log "Starting dispatcher (backend=${DISPATCHER_BACKEND})..."
|
||||
log "Polling ops repo: ${VAULT_ACTIONS_DIR}"
|
||||
log "Admin users: ${ADMIN_USERS}"
|
||||
|
||||
# Validate backend selection at startup
|
||||
case "$DISPATCHER_BACKEND" in
|
||||
docker) ;;
|
||||
nomad)
|
||||
log "ERROR: nomad backend not yet implemented"
|
||||
echo "nomad backend not yet implemented" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
log "ERROR: unknown DISPATCHER_BACKEND=${DISPATCHER_BACKEND}"
|
||||
echo "unknown DISPATCHER_BACKEND=${DISPATCHER_BACKEND} (expected: docker, nomad)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
while true; do
|
||||
# Refresh ops repo at the start of each poll cycle
|
||||
ensure_ops_repo
|
||||
|
|
|
|||
|
|
@ -173,9 +173,40 @@ PROJECT_TOML="${PROJECT_TOML:-projects/disinto.toml}"
|
|||
sleep 1200 # 20 minutes
|
||||
done) &
|
||||
|
||||
# ── Load required secrets from secrets/*.enc (#777) ────────────────────
|
||||
# Edge container declares its required secrets; missing ones cause a hard fail.
|
||||
_AGE_KEY_FILE="${HOME}/.config/sops/age/keys.txt"
|
||||
_SECRETS_DIR="/opt/disinto/secrets"
|
||||
EDGE_REQUIRED_SECRETS="CADDY_SSH_KEY CADDY_SSH_HOST CADDY_SSH_USER CADDY_ACCESS_LOG"
|
||||
|
||||
_edge_decrypt_secret() {
|
||||
local enc_path="${_SECRETS_DIR}/${1}.enc"
|
||||
[ -f "$enc_path" ] || return 1
|
||||
age -d -i "$_AGE_KEY_FILE" "$enc_path" 2>/dev/null
|
||||
}
|
||||
|
||||
if [ -f "$_AGE_KEY_FILE" ] && [ -d "$_SECRETS_DIR" ]; then
|
||||
_missing=""
|
||||
for _secret_name in $EDGE_REQUIRED_SECRETS; do
|
||||
_val=$(_edge_decrypt_secret "$_secret_name") || { _missing="${_missing} ${_secret_name}"; continue; }
|
||||
export "$_secret_name=$_val"
|
||||
done
|
||||
if [ -n "$_missing" ]; then
|
||||
echo "FATAL: required secrets missing from secrets/*.enc:${_missing}" >&2
|
||||
echo " Run 'disinto secrets add <NAME>' for each missing secret." >&2
|
||||
echo " If migrating from .env.vault.enc, run 'disinto secrets migrate-from-vault' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "edge: loaded required secrets: ${EDGE_REQUIRED_SECRETS}" >&2
|
||||
else
|
||||
echo "FATAL: age key (${_AGE_KEY_FILE}) or secrets dir (${_SECRETS_DIR}) not found — cannot load required secrets" >&2
|
||||
echo " Ensure age is installed and secrets/*.enc files are present." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Start daily engagement collection cron loop in background (#745)
|
||||
# Runs collect-engagement.sh daily at ~23:50 UTC via a sleep loop that
|
||||
# calculates seconds until the next 23:50 window. SSH key from .env.vault.enc.
|
||||
# calculates seconds until the next 23:50 window. SSH key from secrets/*.enc (#777).
|
||||
(while true; do
|
||||
# Calculate seconds until next 23:50 UTC
|
||||
_now=$(date -u +%s)
|
||||
|
|
@ -186,26 +217,21 @@ done) &
|
|||
_sleep_secs=$(( _target - _now ))
|
||||
echo "edge: collect-engagement scheduled in ${_sleep_secs}s (next 23:50 UTC)" >&2
|
||||
sleep "$_sleep_secs"
|
||||
# Set CADDY_ACCESS_LOG so the script reads from the fetched local copy
|
||||
_fetch_log="/tmp/caddy-access-log-fetch.log"
|
||||
if [ -n "${CADDY_SSH_KEY:-}" ]; then
|
||||
_ssh_key_file=$(mktemp)
|
||||
printf '%s\n' "$CADDY_SSH_KEY" > "$_ssh_key_file"
|
||||
chmod 0600 "$_ssh_key_file"
|
||||
scp -i "$_ssh_key_file" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
"${CADDY_SSH_USER:-debian}@${CADDY_SSH_HOST:-disinto.ai}:${CADDY_ACCESS_LOG:-/var/log/caddy/access.log}" \
|
||||
"$_fetch_log" 2>&1 | tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
rm -f "$_ssh_key_file"
|
||||
if [ -s "$_fetch_log" ]; then
|
||||
CADDY_ACCESS_LOG="$_fetch_log" bash /opt/disinto/site/collect-engagement.sh 2>&1 \
|
||||
| tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
else
|
||||
echo "edge: collect-engagement: fetched log is empty, skipping parse" >&2
|
||||
fi
|
||||
rm -f "$_fetch_log"
|
||||
_ssh_key_file=$(mktemp)
|
||||
printf '%s\n' "$CADDY_SSH_KEY" > "$_ssh_key_file"
|
||||
chmod 0600 "$_ssh_key_file"
|
||||
scp -i "$_ssh_key_file" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
"${CADDY_SSH_USER}@${CADDY_SSH_HOST}:${CADDY_ACCESS_LOG}" \
|
||||
"$_fetch_log" 2>&1 | tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
rm -f "$_ssh_key_file"
|
||||
if [ -s "$_fetch_log" ]; then
|
||||
CADDY_ACCESS_LOG="$_fetch_log" bash /opt/disinto/site/collect-engagement.sh 2>&1 \
|
||||
| tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
else
|
||||
echo "edge: collect-engagement: CADDY_SSH_KEY not set, skipping" >&2
|
||||
echo "edge: collect-engagement: fetched log is empty, skipping parse" >&2
|
||||
fi
|
||||
rm -f "$_fetch_log"
|
||||
done) &
|
||||
|
||||
# Caddy as main process — run in foreground via wait so background jobs survive
|
||||
|
|
|
|||
|
|
@ -7,5 +7,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
RUN useradd -m -u 1000 -s /bin/bash agent
|
||||
COPY docker/reproduce/entrypoint-reproduce.sh /entrypoint-reproduce.sh
|
||||
RUN chmod +x /entrypoint-reproduce.sh
|
||||
VOLUME /home/agent/data
|
||||
VOLUME /home/agent/repos
|
||||
|
||||
WORKDIR /home/agent
|
||||
ENTRYPOINT ["/entrypoint-reproduce.sh"]
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ The `main` branch on the ops repo (`johba/disinto-ops`) is protected via Forgejo
|
|||
|
||||
## Vault PR Lifecycle
|
||||
|
||||
1. **Request** — Agent calls `lib/vault.sh:vault_request()` with action TOML content
|
||||
2. **Validation** — TOML is validated against the schema in `vault/vault-env.sh`
|
||||
1. **Request** — Agent calls `lib/action-vault.sh:vault_request()` with action TOML content
|
||||
2. **Validation** — TOML is validated against the schema in `action-vault/vault-env.sh`
|
||||
3. **PR Creation** — A PR is created on `disinto-ops` with:
|
||||
- Branch: `vault/<action-id>`
|
||||
- Title: `vault: <action-id>`
|
||||
|
|
@ -90,12 +90,12 @@ To verify the protection is working:
|
|||
|
||||
- #73 — Vault redesign proposal
|
||||
- #74 — Vault action TOML schema
|
||||
- #75 — Vault PR creation helper (`lib/vault.sh`)
|
||||
- #75 — Vault PR creation helper (`lib/action-vault.sh`)
|
||||
- #76 — Dispatcher rewrite (poll for merged vault PRs)
|
||||
- #77 — Branch protection on ops repo (this issue)
|
||||
|
||||
## See Also
|
||||
|
||||
- [`lib/vault.sh`](../lib/vault.sh) — Vault PR creation helper
|
||||
- [`vault/vault-env.sh`](../vault/vault-env.sh) — TOML validation
|
||||
- [`lib/action-vault.sh`](../lib/action-vault.sh) — Vault PR creation helper
|
||||
- [`action-vault/vault-env.sh`](../action-vault/vault-env.sh) — TOML validation
|
||||
- [`lib/branch-protection.sh`](../lib/branch-protection.sh) — Branch protection helper
|
||||
|
|
|
|||
59
docs/agents-llama.md
Normal file
59
docs/agents-llama.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# agents-llama — Local-Qwen Agents
|
||||
|
||||
The `agents-llama` service is an optional compose service that runs agents
|
||||
backed by a local llama-server instance (e.g. Qwen) instead of the Anthropic
|
||||
API. It uses the same Docker image as the main `agents` service but connects to
|
||||
a local inference endpoint via `ANTHROPIC_BASE_URL`.
|
||||
|
||||
Two profiles are available:
|
||||
|
||||
| Profile | Service | Roles | Use case |
|
||||
|---------|---------|-------|----------|
|
||||
| _(default)_ | `agents-llama` | `dev` only | Conservative: single-role soak test |
|
||||
| `agents-llama-all` | `agents-llama-all` | all 7 (review, dev, gardener, architect, planner, predictor, supervisor) | Pre-migration: validate every role on llama before Nomad cutover |
|
||||
|
||||
## Enabling
|
||||
|
||||
Set `ENABLE_LLAMA_AGENT=1` in `.env` (or `.env.enc`) and provide the required
|
||||
credentials:
|
||||
|
||||
```env
|
||||
ENABLE_LLAMA_AGENT=1
|
||||
FORGE_TOKEN_LLAMA=<dev-qwen API token>
|
||||
FORGE_PASS_LLAMA=<dev-qwen password>
|
||||
ANTHROPIC_BASE_URL=http://host.docker.internal:8081 # llama-server endpoint
|
||||
```
|
||||
|
||||
Then regenerate the compose file (`disinto init ...`) and bring the stack up.
|
||||
|
||||
### Running all 7 roles (agents-llama-all)
|
||||
|
||||
```bash
|
||||
docker compose --profile agents-llama-all up -d
|
||||
```
|
||||
|
||||
This starts the `agents-llama-all` container with all 7 bot roles against the
|
||||
local llama endpoint. The per-role forge tokens (`FORGE_REVIEW_TOKEN`,
|
||||
`FORGE_GARDENER_TOKEN`, etc.) must be set in `.env` — they are the same tokens
|
||||
used by the Claude-backed `agents` container.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **llama-server** (or compatible OpenAI-API endpoint) running on the host,
|
||||
reachable from inside Docker at the URL set in `ANTHROPIC_BASE_URL`.
|
||||
- A Forgejo bot user (e.g. `dev-qwen`) with its own API token and password,
|
||||
stored as `FORGE_TOKEN_LLAMA` / `FORGE_PASS_LLAMA`.
|
||||
|
||||
## Behaviour
|
||||
|
||||
- `agents-llama`: `AGENT_ROLES=dev` — only picks up dev work.
|
||||
- `agents-llama-all`: `AGENT_ROLES=review,dev,gardener,architect,planner,predictor,supervisor` — runs all 7 roles.
|
||||
- `CLAUDE_AUTOCOMPACT_PCT_OVERRIDE=60` — more aggressive compaction for smaller
|
||||
context windows.
|
||||
- Serialises on the llama-server's single KV cache (AD-002).
|
||||
|
||||
## Disabling
|
||||
|
||||
Set `ENABLE_LLAMA_AGENT=0` (or leave it unset) and regenerate. The service
|
||||
block is omitted entirely from `docker-compose.yml`; the stack starts cleanly
|
||||
without it.
|
||||
59
docs/mirror-bootstrap.md
Normal file
59
docs/mirror-bootstrap.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# Mirror Bootstrap — Pull-Mirror Cutover Path
|
||||
|
||||
How to populate an empty Forgejo repo from an external source using
|
||||
`lib/mirrors.sh`'s `mirror_pull_register()`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
| Variable | Example | Purpose |
|
||||
|---|---|---|
|
||||
| `FORGE_URL` | `http://forgejo:3000` | Forgejo instance base URL |
|
||||
| `FORGE_API_BASE` | `${FORGE_URL}/api/v1` | Global API base (set by `lib/env.sh`) |
|
||||
| `FORGE_TOKEN` | (admin or org-owner token) | Must have `repo:create` scope |
|
||||
|
||||
The target org/user must already exist on the Forgejo instance.
|
||||
|
||||
## Command
|
||||
|
||||
```bash
|
||||
source lib/env.sh
|
||||
source lib/mirrors.sh
|
||||
|
||||
# Register a pull mirror — creates the repo and starts the first sync.
|
||||
mirror_pull_register \
|
||||
"https://codeberg.org/johba/disinto.git" \ # source URL
|
||||
"disinto-admin" \ # target owner
|
||||
"disinto" \ # target repo name
|
||||
"8h0m0s" # sync interval (optional, default 8h)
|
||||
```
|
||||
|
||||
The function calls `POST /api/v1/repos/migrate` with `mirror: true`.
|
||||
Forgejo creates the repo and immediately queues the first sync.
|
||||
|
||||
## Verifying the sync
|
||||
|
||||
```bash
|
||||
# Check mirror status via API
|
||||
forge_api GET "/repos/disinto-admin/disinto" | jq '.mirror, .mirror_interval'
|
||||
|
||||
# Confirm content arrived — should list branches
|
||||
forge_api GET "/repos/disinto-admin/disinto/branches" | jq '.[].name'
|
||||
```
|
||||
|
||||
The first sync typically completes within a few seconds for small-to-medium
|
||||
repos. For large repos, poll the branches endpoint until content appears.
|
||||
|
||||
## Cutover scenario (Nomad migration)
|
||||
|
||||
At cutover to the Nomad box:
|
||||
|
||||
1. Stand up fresh Forgejo on the Nomad cluster (empty instance).
|
||||
2. Create the `disinto-admin` org via `disinto init` or API.
|
||||
3. Run `mirror_pull_register` pointing at the Codeberg source.
|
||||
4. Wait for sync to complete (check branches endpoint).
|
||||
5. Once content is confirmed, proceed with `disinto init` against the
|
||||
now-populated repo — all subsequent `mirror_push` calls will push
|
||||
to any additional mirrors configured in `projects/*.toml`.
|
||||
|
||||
No manual `git clone` + `git push` step is needed. The Forgejo pull-mirror
|
||||
handles the entire transfer.
|
||||
|
|
@ -50,7 +50,7 @@ description = """
|
|||
Fetch today's Caddy access log segment from the remote host using SCP.
|
||||
|
||||
The SSH key is read from the environment (CADDY_SSH_KEY), which is
|
||||
decrypted from .env.vault.enc by the dispatcher. It is NEVER hardcoded.
|
||||
decrypted from secrets/CADDY_SSH_KEY.enc by the edge entrypoint. It is NEVER hardcoded.
|
||||
|
||||
1. Write the SSH key to a temporary file with restricted permissions:
|
||||
_ssh_key_file=$(mktemp)
|
||||
|
|
|
|||
|
|
@ -79,28 +79,23 @@ AND set CADDY_ACCESS_LOG in the factory environment to match.
|
|||
|
||||
[[steps]]
|
||||
id = "store-private-key"
|
||||
title = "Add the private key to .env.vault.enc as CADDY_SSH_KEY"
|
||||
title = "Add the private key as CADDY_SSH_KEY secret"
|
||||
needs = ["generate-keypair"]
|
||||
description = """
|
||||
Store the private key in the factory's encrypted vault secrets.
|
||||
Store the private key in the factory's encrypted secrets store.
|
||||
|
||||
1. Read the private key:
|
||||
cat caddy-collect
|
||||
1. Add the private key using `disinto secrets add`:
|
||||
|
||||
2. Add it to .env.vault.enc (or .env.vault for plaintext fallback) as
|
||||
CADDY_SSH_KEY. The key is multi-line, so use the base64-encoded form:
|
||||
cat caddy-collect | disinto secrets add CADDY_SSH_KEY
|
||||
|
||||
echo "CADDY_SSH_KEY=$(base64 -w0 caddy-collect)" >> .env.vault.enc
|
||||
This encrypts the key with age and stores it as secrets/CADDY_SSH_KEY.enc.
|
||||
|
||||
Or, if using SOPS-encrypted vault, decrypt first, add the variable,
|
||||
then re-encrypt.
|
||||
|
||||
3. IMPORTANT: After storing, securely delete the local private key file:
|
||||
2. IMPORTANT: After storing, securely delete the local private key file:
|
||||
shred -u caddy-collect 2>/dev/null || rm -f caddy-collect
|
||||
rm -f caddy-collect.pub
|
||||
|
||||
The public key is already installed on the Caddy host; the private key
|
||||
now lives only in the vault.
|
||||
now lives only in secrets/CADDY_SSH_KEY.enc.
|
||||
|
||||
Never commit the private key to any git repository.
|
||||
"""
|
||||
|
|
@ -109,20 +104,19 @@ Never commit the private key to any git repository.
|
|||
|
||||
[[steps]]
|
||||
id = "store-caddy-host"
|
||||
title = "Add the Caddy host address to .env.vault.enc as CADDY_HOST"
|
||||
title = "Add the Caddy host details as secrets"
|
||||
needs = ["install-public-key"]
|
||||
description = """
|
||||
Store the Caddy host connection string so collect-engagement.sh knows
|
||||
Store the Caddy connection details so collect-engagement.sh knows
|
||||
where to SSH.
|
||||
|
||||
1. Add to .env.vault.enc (or .env.vault for plaintext fallback):
|
||||
1. Add each value using `disinto secrets add`:
|
||||
|
||||
echo "CADDY_HOST=user@caddy-host-ip-or-domain" >> .env.vault.enc
|
||||
echo 'disinto.ai' | disinto secrets add CADDY_SSH_HOST
|
||||
echo 'debian' | disinto secrets add CADDY_SSH_USER
|
||||
echo '/var/log/caddy/access.log' | disinto secrets add CADDY_ACCESS_LOG
|
||||
|
||||
Replace user@caddy-host-ip-or-domain with the actual SSH user and host
|
||||
(e.g. debian@203.0.113.42 or deploy@caddy.disinto.ai).
|
||||
|
||||
2. If using SOPS, decrypt/add/re-encrypt as above.
|
||||
Replace values with the actual SSH host, user, and log path for your setup.
|
||||
"""
|
||||
|
||||
# ── Step 5: Test the connection ──────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ should file a vault item instead of executing directly.
|
|||
**Exceptions** (do NOT flag these):
|
||||
- Code inside `vault/` — the vault system itself is allowed to handle secrets
|
||||
- References in comments or documentation explaining the architecture
|
||||
- `bin/disinto` setup commands that manage `.env.vault.enc` and the `run` subcommand
|
||||
- `bin/disinto` setup commands that manage `secrets/*.enc` and the `run` subcommand
|
||||
- Local operations (git push to forge, forge API calls with `FORGE_TOKEN`)
|
||||
|
||||
## 6. Re-review (if previous review is provided)
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ DUST (trivial — single-line edit, rename, comment, style, whitespace):
|
|||
|
||||
VAULT (needs human decision or external resource):
|
||||
File a vault procurement item using vault_request():
|
||||
source "$(dirname "$0")/../lib/vault.sh"
|
||||
source "$(dirname "$0")/../lib/action-vault.sh"
|
||||
TOML_CONTENT="# Vault action: <action_id>
|
||||
context = \"<description of what decision/resource is needed>\"
|
||||
unblocks = [\"#NNN\"]
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ needs = ["preflight"]
|
|||
|
||||
[[steps]]
|
||||
id = "commit-ops-changes"
|
||||
title = "Write tree, memory, and journal; commit and push"
|
||||
title = "Write tree, memory, and journal; commit and push branch"
|
||||
description = """
|
||||
### 1. Write prerequisite tree
|
||||
Write to: $OPS_REPO_ROOT/prerequisites.md
|
||||
|
|
@ -256,14 +256,16 @@ If (count - N) >= 5 or planner-memory.md missing, write to:
|
|||
Include: run counter marker, date, constraint focus, patterns, direction.
|
||||
Keep under 100 lines. Replace entire file.
|
||||
|
||||
### 3. Commit ops repo changes
|
||||
Commit the ops repo changes (prerequisites, memory, vault items):
|
||||
### 3. Commit ops repo changes to the planner branch
|
||||
Commit the ops repo changes (prerequisites, memory, vault items) and push the
|
||||
branch. Do NOT push directly to $PRIMARY_BRANCH — planner-run.sh will create a
|
||||
PR and walk it to merge via review-bot.
|
||||
cd "$OPS_REPO_ROOT"
|
||||
git add prerequisites.md knowledge/planner-memory.md vault/pending/
|
||||
git add -u
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "chore: planner run $(date -u +%Y-%m-%d)"
|
||||
git push origin "$PRIMARY_BRANCH"
|
||||
git push origin HEAD
|
||||
fi
|
||||
cd "$PROJECT_REPO_ROOT"
|
||||
|
||||
|
|
|
|||
|
|
@ -125,8 +125,8 @@ For each weakness you identify, choose one:
|
|||
The prediction explains the theory. The vault PR triggers the proof
|
||||
after human approval. When the planner runs next, evidence is already there.
|
||||
|
||||
Vault dispatch (requires lib/vault.sh):
|
||||
source "$PROJECT_REPO_ROOT/lib/vault.sh"
|
||||
Vault dispatch (requires lib/action-vault.sh):
|
||||
source "$PROJECT_REPO_ROOT/lib/action-vault.sh"
|
||||
|
||||
TOML_CONTENT="id = \"predict-<prediction_number>-<formula>\"
|
||||
context = \"Test prediction #<prediction_number>: <theory summary> — focus: <specific test>\"
|
||||
|
|
@ -154,7 +154,7 @@ tea is pre-configured with login "$TEA_LOGIN" and repo "$FORGE_REPO".
|
|||
--title "<title>" --body "<body>" --labels "prediction/unreviewed"
|
||||
|
||||
2. Dispatch formula via vault (if exploiting):
|
||||
source "$PROJECT_REPO_ROOT/lib/vault.sh"
|
||||
source "$PROJECT_REPO_ROOT/lib/action-vault.sh"
|
||||
PR_NUM=$(vault_request "predict-NNN-<formula>" "$TOML_CONTENT")
|
||||
# See EXPLOIT section above for TOML_CONTENT format
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Gardener Agent
|
||||
|
||||
**Role**: Backlog grooming — detect duplicate issues, missing acceptance
|
||||
|
|
@ -32,7 +32,7 @@ the gardener runs as part of the polling loop alongside the planner, predictor,
|
|||
PR, reviewed alongside AGENTS.md changes, executed by gardener-run.sh after merge.
|
||||
|
||||
**Environment variables consumed**:
|
||||
- `FORGE_TOKEN`, `FORGE_GARDENER_TOKEN` (falls back to FORGE_TOKEN), `FORGE_REPO`, `FORGE_API`, `PROJECT_NAME`, `PROJECT_REPO_ROOT`
|
||||
- `FORGE_TOKEN`, `FORGE_GARDENER_TOKEN` (falls back to FORGE_TOKEN), `FORGE_REPO`, `FORGE_API`, `PROJECT_NAME`, `PROJECT_REPO_ROOT`. `FORGE_TOKEN_OVERRIDE` is exported to `$FORGE_GARDENER_TOKEN` before sourcing env.sh so the gardener-bot identity survives re-sourcing (#762).
|
||||
- `PRIMARY_BRANCH`, `CLAUDE_MODEL` (set to sonnet by gardener-run.sh)
|
||||
|
||||
**Lifecycle**: gardener-run.sh (invoked by polling loop every 6h, `check_active gardener`) →
|
||||
|
|
|
|||
|
|
@ -1 +1,12 @@
|
|||
[]
|
||||
[
|
||||
{
|
||||
"action": "comment",
|
||||
"issue": 623,
|
||||
"body": "**Dependency check:** All blocking dependencies are now closed:\n- #620 ✓ closed\n- #621 ✓ closed \n- #622 ✓ closed\n\nPer the issue description: *\"Once #620/#621/#622 are green, this issue should fork into at least three backlog children: subpath routing + Forgejo ROOT_URL / Woodpecker HOST, disinto-chat container scaffold with OAuth gate, and Claude Code sandbox envelope + working-dir scoping.\"*\n\nThis vision issue is ready for the planner to decompose into backlog children."
|
||||
},
|
||||
{
|
||||
"action": "comment",
|
||||
"issue": 758,
|
||||
"body": "**Gardener flag:** This issue requires human admin action on Forgejo to resolve — changing branch protection settings on the ops repo. No automated formula can fix Forgejo admin settings.\n\nProposed options (from issue body):\n1. Add `planner-bot` to the merge whitelist in ops repo branch protection\n2. Remove branch protection from the ops repo (agents are primary writers)\n3. Create an admin-level service token for agents\n\nThis is blocking all ops repo writes (planner knowledge, sprint artifacts, vault items)."
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Shared Helpers (`lib/`)
|
||||
|
||||
All agents source `lib/env.sh` as their first action. Additional helpers are
|
||||
|
|
@ -6,7 +6,7 @@ sourced as needed.
|
|||
|
||||
| File | What it provides | Sourced by |
|
||||
|---|---|---|
|
||||
| `lib/env.sh` | Loads `.env`, sets `FACTORY_ROOT`, exports project config (`FORGE_REPO`, `PROJECT_NAME`, etc.), defines `log()`, `forge_api()`, `forge_api_all()` (paginates all pages; accepts optional second TOKEN parameter, defaults to `$FORGE_TOKEN`; handles invalid/empty JSON responses gracefully — returns empty on parse error instead of crashing), `woodpecker_api()`, `wpdb()`, `memory_guard()` (skips agent if RAM < threshold). Auto-loads project TOML if `PROJECT_TOML` is set. Exports per-agent tokens (`FORGE_PLANNER_TOKEN`, `FORGE_GARDENER_TOKEN`, `FORGE_VAULT_TOKEN`, `FORGE_SUPERVISOR_TOKEN`, `FORGE_PREDICTOR_TOKEN`) — each falls back to `$FORGE_TOKEN` if not set. **Vault-only token guard (AD-006)**: `unset GITHUB_TOKEN CLAWHUB_TOKEN` so agents never hold external-action tokens — only the runner container receives them. **Container note**: when `DISINTO_CONTAINER=1`, `.env` is NOT re-sourced — compose already injects env vars (including `FORGE_URL=http://forgejo:3000`) and re-sourcing would clobber them. **Save/restore scope (#364)**: only `FORGE_URL` is preserved across `.env` re-sourcing (compose injects `http://forgejo:3000`, `.env` has `http://localhost:3000`). `FORGE_TOKEN` is NOT preserved so refreshed tokens in `.env` take effect immediately. **Required env var**: `FORGE_PASS` — bot password for git HTTP push (Forgejo 11.x rejects API tokens for `git push`, #361). **Hard preconditions (#674)**: `USER` and `HOME` must be exported by the entrypoint before sourcing. When `PROJECT_TOML` is set, `PROJECT_REPO_ROOT`, `PRIMARY_BRANCH`, and `OPS_REPO_ROOT` must also be set (by entrypoint or TOML). | Every agent |
|
||||
| `lib/env.sh` | Loads `.env`, sets `FACTORY_ROOT`, exports project config (`FORGE_REPO`, `PROJECT_NAME`, etc.), defines `log()`, `forge_api()`, `forge_api_all()` (paginates all pages; accepts optional second TOKEN parameter, defaults to `$FORGE_TOKEN`; handles invalid/empty JSON responses gracefully — returns empty on parse error instead of crashing), `woodpecker_api()`, `wpdb()`, `memory_guard()` (skips agent if RAM < threshold), `load_secret()` (secret-source abstraction — see below). Auto-loads project TOML if `PROJECT_TOML` is set. Exports per-agent tokens (`FORGE_PLANNER_TOKEN`, `FORGE_GARDENER_TOKEN`, `FORGE_VAULT_TOKEN`, `FORGE_SUPERVISOR_TOKEN`, `FORGE_PREDICTOR_TOKEN`) — each falls back to `$FORGE_TOKEN` if not set. **Vault-only token guard (AD-006)**: `unset GITHUB_TOKEN CLAWHUB_TOKEN` so agents never hold external-action tokens — only the runner container receives them. **Container note**: when `DISINTO_CONTAINER=1`, `.env` is NOT re-sourced — compose already injects env vars (including `FORGE_URL=http://forgejo:3000`) and re-sourcing would clobber them. **Save/restore scope (#364)**: only `FORGE_URL` is preserved across `.env` re-sourcing (compose injects `http://forgejo:3000`, `.env` has `http://localhost:3000`). `FORGE_TOKEN` is NOT preserved so refreshed tokens in `.env` take effect immediately. **Per-agent token override (#762)**: agent run scripts export `FORGE_TOKEN_OVERRIDE=<agent-specific-token>` BEFORE sourcing `env.sh`; `env.sh` applies this override at lines 98-100, ensuring the correct identity survives any re-sourcing of `env.sh` by nested shells or `claude -p` invocations. **Required env var**: `FORGE_PASS` — bot password for git HTTP push (Forgejo 11.x rejects API tokens for `git push`, #361). **Hard preconditions (#674)**: `USER` and `HOME` must be exported by the entrypoint before sourcing. When `PROJECT_TOML` is set, `PROJECT_REPO_ROOT`, `PRIMARY_BRANCH`, and `OPS_REPO_ROOT` must also be set (by entrypoint or TOML). **`load_secret NAME [DEFAULT]` (#793)**: backend-agnostic secret resolution. Precedence: (1) `/secrets/<NAME>.env` — Nomad-rendered template, (2) current environment — already set by `.env.enc` / compose, (3) `secrets/<NAME>.enc` — age-encrypted per-key file (decrypted on demand, cached in process env), (4) DEFAULT or empty. Consumers call `$(load_secret GITHUB_TOKEN)` instead of `${GITHUB_TOKEN}` — identical behavior whether secrets come from Docker compose injection or Nomad Vault templates. | Every agent |
|
||||
| `lib/ci-helpers.sh` | `ci_passed()` — returns 0 if CI state is "success" (or no CI configured). `ci_required_for_pr()` — returns 0 if PR has code files (CI required), 1 if non-code only (CI not required). `is_infra_step()` — returns 0 if a single CI step failure matches infra heuristics (clone/git exit 128, any exit 137, log timeout patterns). `classify_pipeline_failure()` — returns "infra \<reason>" if any failed Woodpecker step matches infra heuristics via `is_infra_step()`, else "code". `ensure_priority_label()` — looks up (or creates) the `priority` label and returns its ID; caches in `_PRIORITY_LABEL_ID`. `ci_commit_status <sha>` — queries Woodpecker directly for CI state, falls back to forge commit status API. `ci_pipeline_number <sha>` — returns the Woodpecker pipeline number for a commit, falls back to parsing forge status `target_url`. `ci_promote <repo_id> <pipeline_num> <environment>` — promotes a pipeline to a named Woodpecker environment (vault-gated deployment: vault approves, vault-fire calls this — vault redesign in progress, see #73-#77). `ci_get_logs <pipeline_number> [--step <name>]` — reads CI logs from Woodpecker SQLite database via `lib/ci-log-reader.py`; outputs last 200 lines to stdout. Requires mounted woodpecker-data volume at /woodpecker-data. | dev-poll, review-poll, review-pr |
|
||||
| `lib/ci-debug.sh` | CLI tool for Woodpecker CI: `list`, `status`, `logs`, `failures` subcommands. Not sourced — run directly. | Humans / dev-agent (tool access) |
|
||||
| `lib/ci-log-reader.py` | Python tool: reads CI logs from Woodpecker SQLite database. `<pipeline_number> [--step <name>]` — returns last 200 lines from failed steps (or specified step). Used by `ci_get_logs()` in ci-helpers.sh. Requires `WOODPECKER_DATA_DIR` (default: /woodpecker-data). | ci-helpers.sh |
|
||||
|
|
@ -14,7 +14,7 @@ sourced as needed.
|
|||
| `lib/parse-deps.sh` | Extracts dependency issue numbers from an issue body (stdin → stdout, one number per line). Matches `## Dependencies` / `## Depends on` / `## Blocked by` sections and inline `depends on #N` / `blocked by #N` patterns. Inline scan skips fenced code blocks to prevent false positives from code examples in issue bodies. Not sourced — executed via `bash lib/parse-deps.sh`. | dev-poll |
|
||||
| `lib/formula-session.sh` | `acquire_run_lock()`, `load_formula()`, `load_formula_or_profile()`, `build_context_block()`, `ensure_ops_repo()`, `ops_commit_and_push()`, `build_prompt_footer()`, `build_sdk_prompt_footer()`, `formula_worktree_setup()`, `formula_prepare_profile_context()`, `formula_lessons_block()`, `profile_write_journal()`, `profile_load_lessons()`, `ensure_profile_repo()`, `_profile_has_repo()`, `_count_undigested_journals()`, `_profile_digest_journals()`, `_profile_restore_lessons()`, `_profile_commit_and_push()`, `resolve_agent_identity()`, `build_graph_section()`, `build_scratch_instruction()`, `read_scratch_context()`, `cleanup_stale_crashed_worktrees()` — shared helpers for formula-driven polling-loop agents (lock, .profile repo management, prompt assembly, worktree setup). Memory guard is provided by `memory_guard()` in `lib/env.sh` (not duplicated here). `resolve_agent_identity()` — sets `FORGE_TOKEN`, `AGENT_IDENTITY`, `FORGE_REMOTE` from per-agent token env vars and FORGE_URL remote detection. `build_graph_section()` generates the structural-analysis section (runs `lib/build-graph.py`, formats JSON output) — previously duplicated in planner-run.sh and predictor-run.sh, now shared here. `cleanup_stale_crashed_worktrees()` — thin wrapper around `worktree_cleanup_stale()` from `lib/worktree.sh` (kept for backwards compatibility). **Journal digestion guards (#702)**: `_profile_digest_journals()` respects `PROFILE_DIGEST_TIMEOUT` (default 300s) and `PROFILE_DIGEST_MAX_BATCH` (default 5 journals per run); `_profile_restore_lessons()` restores the previous lessons-learned.md on digest failure. | planner-run.sh, predictor-run.sh, gardener-run.sh, supervisor-run.sh, dev-agent.sh |
|
||||
| `lib/guard.sh` | `check_active(agent_name)` — reads `$FACTORY_ROOT/state/.{agent_name}-active`; exits 0 (skip) if the file is absent. Factory is off by default — state files must be created to enable each agent. **Logs a message to stderr** when skipping (`[check_active] SKIP: state file not found`), so agent dropout is visible in loop logs. Sourced by dev-poll.sh, review-poll.sh, predictor-run.sh, supervisor-run.sh. | polling-loop entry points |
|
||||
| `lib/mirrors.sh` | `mirror_push()` — pushes `$PRIMARY_BRANCH` + tags to all configured mirror remotes (fire-and-forget background pushes). Reads `MIRROR_NAMES` and `MIRROR_*` vars exported by `load-project.sh` from the `[mirrors]` TOML section. Failures are logged but never block the pipeline. Sourced by dev-poll.sh — called after every successful merge. | dev-poll.sh |
|
||||
| `lib/mirrors.sh` | `mirror_push()` — pushes `$PRIMARY_BRANCH` + tags to all configured mirror remotes (fire-and-forget background pushes). Reads `MIRROR_NAMES` and `MIRROR_*` vars exported by `load-project.sh` from the `[mirrors]` TOML section. Failures are logged but never block the pipeline. `mirror_pull_register(clone_url, owner, repo_name, [interval])` — registers a Forgejo pull mirror via `POST /repos/migrate` with `mirror: true`. Creates the target repo and queues the first sync automatically. Works against empty Forgejo instances — no pre-existing content required. Used for Nomad migration cutover: point at Codeberg source, wait for sync, then proceed with `disinto init`. See [docs/mirror-bootstrap.md](../docs/mirror-bootstrap.md) for the full cutover path. Sourced by dev-poll.sh — called after every successful merge. | dev-poll.sh |
|
||||
| `lib/build-graph.py` | Python tool: parses VISION.md, prerequisites.md (from ops repo), AGENTS.md, formulas/*.toml, evidence/ (from ops repo), and forge issues/labels into a NetworkX DiGraph. Runs structural analyses (orphaned objectives, stale prerequisites, thin evidence, circular deps) and outputs a JSON report. Used by `review-pr.sh` (per-PR changed-file analysis) and `predictor-run.sh` (full-project analysis) to provide structural context to Claude. | review-pr.sh, predictor-run.sh |
|
||||
| `lib/secret-scan.sh` | `scan_for_secrets()` — detects potential secrets (API keys, bearer tokens, private keys, URLs with embedded credentials) in text; returns 1 if secrets found. `redact_secrets()` — replaces detected secret patterns with `[REDACTED]`. | issue-lifecycle.sh |
|
||||
| `lib/stack-lock.sh` | File-based lock protocol for singleton project stack access. `stack_lock_acquire(holder, project)` — polls until free, breaks stale heartbeats (>10 min old), claims lock. `stack_lock_release(project)` — deletes lock file. `stack_lock_check(project)` — inspect current lock state. `stack_lock_heartbeat(project)` — update heartbeat timestamp (callers must call every 2 min while holding). Lock files at `~/data/locks/<project>-stack.lock`. | docker/edge/dispatcher.sh, reproduce formula |
|
||||
|
|
@ -22,7 +22,7 @@ sourced as needed.
|
|||
| `lib/worktree.sh` | Reusable git worktree management: `worktree_create(path, branch, [base_ref])` — create worktree, checkout base, fetch submodules. `worktree_recover(path, branch, [remote])` — detect existing worktree, reuse if on correct branch (sets `_WORKTREE_REUSED`), otherwise clean and recreate. `worktree_cleanup(path)` — `git worktree remove --force`, clear Claude Code project cache (`~/.claude/projects/` matching path). `worktree_cleanup_stale([max_age_hours])` — scan `/tmp` for orphaned worktrees older than threshold, skip preserved and active tmux worktrees, prune. `worktree_preserve(path, reason)` — mark worktree as preserved for debugging (writes `.worktree-preserved` marker, skipped by stale cleanup). | dev-agent.sh, supervisor-run.sh, planner-run.sh, predictor-run.sh, gardener-run.sh |
|
||||
| `lib/pr-lifecycle.sh` | Reusable PR lifecycle library: `pr_create()`, `pr_find_by_branch()`, `pr_poll_ci()`, `pr_poll_review()`, `pr_merge()`, `pr_is_merged()`, `pr_walk_to_merge()`, `build_phase_protocol_prompt()`. Requires `lib/ci-helpers.sh`. | dev-agent.sh (future) |
|
||||
| `lib/issue-lifecycle.sh` | Reusable issue lifecycle library: `issue_claim()` (add in-progress, remove backlog), `issue_release()` (remove in-progress, add backlog), `issue_block()` (post diagnostic comment with secret redaction, add blocked label), `issue_close()`, `issue_check_deps()` (parse deps, check transitive closure; sets `_ISSUE_BLOCKED_BY`, `_ISSUE_SUGGESTION`), `issue_suggest_next()` (find next unblocked backlog issue; sets `_ISSUE_NEXT`), `issue_post_refusal()` (structured refusal comment with dedup). Label IDs cached in globals on first lookup. Sources `lib/secret-scan.sh`. | dev-agent.sh (future) |
|
||||
| `lib/vault.sh` | **Vault PR helper** — create vault action PRs on ops repo via Forgejo API (works from containers without SSH). `vault_request <action_id> <toml_content>` validates TOML (using `validate_vault_action` from `vault/vault-env.sh`), creates branch `vault/<action-id>`, writes `vault/actions/<action-id>.toml`, creates PR targeting `main` with title `vault: <action-id>` and body from context field, returns PR number. Idempotent: if PR exists, returns existing number. **Low-tier bypass**: if the action's `blast_radius` classifies as `low` (via `vault/classify.sh`), `vault_request` calls `_vault_commit_direct()` which commits directly to ops `main` using `FORGE_ADMIN_TOKEN` — no PR, no approval wait. Returns `0` (not a PR number) for direct commits. Requires `FORGE_TOKEN`, `FORGE_ADMIN_TOKEN` (low-tier only), `FORGE_URL`, `FORGE_REPO`, `FORGE_OPS_REPO`. Uses the calling agent's own token (saves/restores `FORGE_TOKEN` around sourcing `vault-env.sh`), so approval workflow respects individual agent identities. | dev-agent (vault actions), future vault dispatcher |
|
||||
| `lib/action-vault.sh` | **Vault PR helper** — create vault action PRs on ops repo via Forgejo API (works from containers without SSH). `vault_request <action_id> <toml_content>` validates TOML (using `validate_vault_action` from `action-vault/vault-env.sh`), creates branch `vault/<action-id>`, writes `vault/actions/<action-id>.toml`, creates PR targeting `main` with title `vault: <action-id>` and body from context field, returns PR number. Idempotent: if PR exists, returns existing number. **Low-tier bypass**: if the action's `blast_radius` classifies as `low` (via `action-vault/classify.sh`), `vault_request` calls `_vault_commit_direct()` which commits directly to ops `main` using `FORGE_ADMIN_TOKEN` — no PR, no approval wait. Returns `0` (not a PR number) for direct commits. Requires `FORGE_TOKEN`, `FORGE_ADMIN_TOKEN` (low-tier only), `FORGE_URL`, `FORGE_REPO`, `FORGE_OPS_REPO`. Uses the calling agent's own token (saves/restores `FORGE_TOKEN` around sourcing `vault-env.sh`), so approval workflow respects individual agent identities. | dev-agent (vault actions), future vault dispatcher |
|
||||
| `lib/branch-protection.sh` | Branch protection helpers for Forgejo repos. `setup_vault_branch_protection()` — configures admin-only merge protection on main (require 1 approval, restrict merge to admin role, block direct pushes). `setup_profile_branch_protection()` — same protection for `.profile` repos. `verify_branch_protection()` — checks protection is correctly configured. `remove_branch_protection()` — removes protection (cleanup/testing). Handles race condition after initial push: retries with backoff if Forgejo hasn't processed the branch yet. Requires `FORGE_TOKEN`, `FORGE_URL`, `FORGE_OPS_REPO`. | bin/disinto (hire-an-agent) |
|
||||
| `lib/agent-sdk.sh` | `agent_run([--resume SESSION_ID] [--worktree DIR] PROMPT)` — one-shot `claude -p` invocation with session persistence. Saves session ID to `SID_FILE`, reads it back on resume. `agent_recover_session()` — restore previous session ID from `SID_FILE` on startup. **Nudge guard**: skips nudge injection if the worktree is clean and no push is expected, preventing spurious re-invocations. Callers must define `SID_FILE`, `LOGFILE`, and `log()` before sourcing. **Concurrency**: external `flock` on `session.lock` is gated behind `CLAUDE_EXTERNAL_LOCK=1` (default off). When unset, each container's per-session `CLAUDE_CONFIG_DIR` isolation lets Claude Code's native lockfile handle OAuth refresh — no external serialization needed. Set `CLAUDE_EXTERNAL_LOCK=1` to re-enable the old flock wrapper as a rollback mechanism. See [`docs/CLAUDE-AUTH-CONCURRENCY.md`](../docs/CLAUDE-AUTH-CONCURRENCY.md) and AD-002 (#647). | formula-driven agents (dev-agent, planner-run, predictor-run, gardener-run) |
|
||||
| `lib/forge-setup.sh` | `setup_forge()` — Forgejo instance provisioning: creates admin user, bot accounts, org, repos (code + ops), configures webhooks, sets repo topics. Extracted from `bin/disinto`. Requires `FORGE_URL`, `FORGE_TOKEN`, `FACTORY_ROOT`. **Password storage (#361)**: after creating each bot account, stores its password in `.env` as `FORGE_<BOT>_PASS` (e.g. `FORGE_PASS`, `FORGE_REVIEW_PASS`, etc.) for use by `forge-push.sh`. | bin/disinto (init) |
|
||||
|
|
@ -30,6 +30,8 @@ sourced as needed.
|
|||
| `lib/git-creds.sh` | Shared git credential helper configuration. `configure_git_creds([HOME_DIR] [RUN_AS_CMD])` — writes a static credential helper script and configures git globally to use password-based HTTP auth (Forgejo 11.x rejects API tokens for `git push`, #361). **Retry on cold boot (#741)**: resolves bot username from `FORGE_TOKEN` with 5 retries (exponential backoff 1-5s); fails loudly and returns 1 if Forgejo is unreachable — never falls back to a wrong hardcoded default (exports `BOT_USER` on success). `repair_baked_cred_urls([--as RUN_AS_CMD] DIR ...)` — rewrites any git remote URLs that have credentials baked in to use clean URLs instead; uses `safe.directory` bypass for root-owned repos (#671). Requires `FORGE_PASS`, `FORGE_URL`, `FORGE_TOKEN`. | entrypoints (agents, edge) |
|
||||
| `lib/ops-setup.sh` | `setup_ops_repo()` — creates ops repo on Forgejo if it doesn't exist, configures bot collaborators, clones/initializes ops repo locally, seeds directory structure (vault, knowledge, evidence, sprints). Evidence subdirectories seeded: engagement/, red-team/, holdout/, evolution/, user-test/. Also seeds sprints/ for architect output. Exports `_ACTUAL_OPS_SLUG`. `migrate_ops_repo(ops_root, [primary_branch])` — idempotent migration helper that seeds missing directories and .gitkeep files on existing ops repos (pre-#407 deployments). | bin/disinto (init) |
|
||||
| `lib/ci-setup.sh` | `_install_cron_impl()` — installs crontab entries for bare-metal deployments (compose mode uses polling loop instead). `_create_forgejo_oauth_app()` — generic helper to create an OAuth2 app on Forgejo (shared by Woodpecker and chat). `_create_woodpecker_oauth_impl()` — creates Woodpecker OAuth2 app (thin wrapper). `_create_chat_oauth_impl()` — creates disinto-chat OAuth2 app, writes `CHAT_OAUTH_CLIENT_ID`/`CHAT_OAUTH_CLIENT_SECRET` to `.env` (#708). `_generate_woodpecker_token_impl()` — auto-generates WOODPECKER_TOKEN via OAuth2 flow. `_activate_woodpecker_repo_impl()` — activates repo in Woodpecker. All gated by `_load_ci_context()` which validates required env vars. | bin/disinto (init) |
|
||||
| `lib/generators.sh` | Template generation for `disinto init`: `generate_compose()` — docker-compose.yml (uses `codeberg.org/forgejo/forgejo:11.0` tag; adds `security_opt: [apparmor:unconfined]` to all services for rootless container compatibility; Forgejo includes a healthcheck so dependent services use `condition: service_healthy` — fixes cold-start races, #665; adds `chat` service block with isolated `chat-config` named volume and `CHAT_HISTORY_DIR` bind-mount for per-user NDJSON history persistence (#710); injects `FORWARD_AUTH_SECRET` for Caddy↔chat defense-in-depth auth (#709); cost-cap env vars `CHAT_MAX_REQUESTS_PER_HOUR`, `CHAT_MAX_REQUESTS_PER_DAY`, `CHAT_MAX_TOKENS_PER_DAY` (#711); subdomain fallback comment for `EDGE_TUNNEL_FQDN_*` vars (#713); all `depends_on` now use `condition: service_healthy/started` instead of bare service names), `generate_caddyfile()` — Caddyfile (routes: `/forge/*` → forgejo:3000, `/woodpecker/*` → woodpecker:8000, `/staging/*` → staging:80; `/chat/login` and `/chat/oauth/callback` bypass `forward_auth` so unauthenticated users can reach the OAuth flow; `/chat/*` gated by `forward_auth` on `chat:8080/chat/auth/verify` which stamps `X-Forwarded-User` (#709); root `/` redirects to `/forge/`), `generate_staging_index()` — staging index, `generate_deploy_pipelines()` — Woodpecker deployment pipeline configs. Requires `FACTORY_ROOT`, `PROJECT_NAME`, `PRIMARY_BRANCH`. | bin/disinto (init) |
|
||||
| `lib/generators.sh` | Template generation for `disinto init`: `generate_compose()` — docker-compose.yml (uses `codeberg.org/forgejo/forgejo:11.0` tag; adds `security_opt: [apparmor:unconfined]` to all services for rootless container compatibility; Forgejo includes a healthcheck so dependent services use `condition: service_healthy` — fixes cold-start races, #665; adds `chat` service block with isolated `chat-config` named volume and `CHAT_HISTORY_DIR` bind-mount for per-user NDJSON history persistence (#710); injects `FORWARD_AUTH_SECRET` for Caddy↔chat defense-in-depth auth (#709); cost-cap env vars `CHAT_MAX_REQUESTS_PER_HOUR`, `CHAT_MAX_REQUESTS_PER_DAY`, `CHAT_MAX_TOKENS_PER_DAY` (#711); subdomain fallback comment for `EDGE_TUNNEL_FQDN_*` vars (#713); all `depends_on` now use `condition: service_healthy/started` instead of bare service names; all services now include `restart: unless-stopped` including the edge service — #768; agents service now uses `image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}` instead of `build:` (#429); `WOODPECKER_PLUGINS_PRIVILEGED` env var added to woodpecker service (#779); agents-llama conditional block gated on `ENABLE_LLAMA_AGENT=1` (#769); `agents-llama-all` compose service (profile `agents-llama-all`, all 7 roles: review,dev,gardener,architect,planner,predictor,supervisor) added by #801; agents service gains volume mounts for `./projects`, `./.env`, `./state`), `generate_caddyfile()` — Caddyfile (routes: `/forge/*` → forgejo:3000, `/woodpecker/*` → woodpecker:8000, `/staging/*` → staging:80; `/chat/login` and `/chat/oauth/callback` bypass `forward_auth` so unauthenticated users can reach the OAuth flow; `/chat/*` gated by `forward_auth` on `chat:8080/chat/auth/verify` which stamps `X-Forwarded-User` (#709); root `/` redirects to `/forge/`), `generate_staging_index()` — staging index, `generate_deploy_pipelines()` — Woodpecker deployment pipeline configs. Requires `FACTORY_ROOT`, `PROJECT_NAME`, `PRIMARY_BRANCH`. | bin/disinto (init) |
|
||||
| `lib/sprint-filer.sh` | Post-merge sub-issue filer for sprint PRs. Invoked by the `.woodpecker/ops-filer.yml` pipeline after a sprint PR merges to ops repo `main`. Parses `<!-- filer:begin --> ... <!-- filer:end -->` blocks from sprint PR bodies to extract sub-issue definitions, creates them on the project repo using `FORGE_FILER_TOKEN` (narrow-scope `filer-bot` identity with `issues:write` only), adds `in-progress` label to the parent vision issue, and handles vision lifecycle closure when all sub-issues are closed. Uses `filer_api_all()` for paginated fetches. Idempotent: uses `<!-- decomposed-from: #<vision>, sprint: <slug>, id: <id> -->` markers to skip already-filed issues. Requires `FORGE_FILER_TOKEN`, `FORGE_API`, `FORGE_API_BASE`, `FORGE_OPS_REPO`. | `.woodpecker/ops-filer.yml` (CI pipeline on ops repo) |
|
||||
| `lib/hire-agent.sh` | `disinto_hire_an_agent()` — user creation, `.profile` repo setup, formula copying, branch protection, and state marker creation for hiring a new agent. Requires `FORGE_URL`, `FORGE_TOKEN`, `FACTORY_ROOT`, `PROJECT_NAME`. Extracted from `bin/disinto`. | bin/disinto (hire) |
|
||||
| `lib/release.sh` | `disinto_release()` — vault TOML creation, branch setup on ops repo, PR creation, and auto-merge request for a versioned release. `_assert_release_globals()` validates required env vars. Requires `FORGE_URL`, `FORGE_TOKEN`, `FORGE_OPS_REPO`, `FACTORY_ROOT`, `PRIMARY_BRANCH`. Extracted from `bin/disinto`. | bin/disinto (release) |
|
||||
| `lib/hvault.sh` | HashiCorp Vault helper module. `hvault_kv_get(PATH, [KEY])` — read KV v2 secret, optionally extract one key. `hvault_kv_put(PATH, KEY=VAL ...)` — write KV v2 secret. `hvault_kv_list(PATH)` — list keys at a KV path. `hvault_policy_apply(NAME, FILE)` — idempotent policy upsert. `hvault_jwt_login(ROLE, JWT)` — exchange JWT for short-lived token. `hvault_token_lookup()` — returns TTL/policies/accessor for current token. All functions use `VAULT_ADDR` + `VAULT_TOKEN` from env (fallback: `/etc/vault.d/root.token`), emit structured JSON errors to stderr on failure. Tests: `tests/lib-hvault.bats` (requires `vault server -dev`). | Not sourced at runtime yet — pure scaffolding for Nomad+Vault migration (#799) |
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
# vault.sh — Helper for agents to create vault PRs on ops repo
|
||||
# action-vault.sh — Helper for agents to create vault PRs on ops repo
|
||||
#
|
||||
# Source after lib/env.sh:
|
||||
# source "$(dirname "$0")/../lib/env.sh"
|
||||
# source "$(dirname "$0")/lib/vault.sh"
|
||||
# source "$(dirname "$0")/lib/action-vault.sh"
|
||||
#
|
||||
# Required globals: FORGE_TOKEN, FORGE_URL, FORGE_REPO, FORGE_OPS_REPO
|
||||
# Optional: OPS_REPO_ROOT (local path for ops repo)
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
# vault_request <action_id> <toml_content> — Create vault PR, return PR number
|
||||
#
|
||||
# The function:
|
||||
# 1. Validates TOML content using validate_vault_action() from vault/vault-env.sh
|
||||
# 1. Validates TOML content using validate_vault_action() from action-vault/vault-env.sh
|
||||
# 2. Creates a branch on the ops repo: vault/<action-id>
|
||||
# 3. Writes TOML to vault/actions/<action-id>.toml on that branch
|
||||
# 4. Creates PR targeting main with title "vault: <action-id>"
|
||||
|
|
@ -133,7 +133,7 @@ vault_request() {
|
|||
printf '%s' "$toml_content" > "$tmp_toml"
|
||||
|
||||
# Source vault-env.sh for validate_vault_action
|
||||
local vault_env="${FACTORY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}/vault/vault-env.sh"
|
||||
local vault_env="${FACTORY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}/action-vault/vault-env.sh"
|
||||
if [ ! -f "$vault_env" ]; then
|
||||
echo "ERROR: vault-env.sh not found at $vault_env" >&2
|
||||
return 1
|
||||
|
|
@ -161,7 +161,7 @@ vault_request() {
|
|||
ops_api="$(_vault_ops_api)"
|
||||
|
||||
# Classify the action to determine if PR bypass is allowed
|
||||
local classify_script="${FACTORY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}/vault/classify.sh"
|
||||
local classify_script="${FACTORY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}/action-vault/classify.sh"
|
||||
local vault_tier
|
||||
vault_tier=$("$classify_script" "${VAULT_ACTION_FORMULA:-}" "${VAULT_BLAST_RADIUS_OVERRIDE:-}") || {
|
||||
# Classification failed, default to high tier (require PR)
|
||||
66
lib/env.sh
66
lib/env.sh
|
|
@ -158,8 +158,8 @@ export WOODPECKER_SERVER="${WOODPECKER_SERVER:-http://localhost:8000}"
|
|||
export CLAUDE_TIMEOUT="${CLAUDE_TIMEOUT:-7200}"
|
||||
|
||||
# Vault-only token guard (#745): external-action tokens (GITHUB_TOKEN, CLAWHUB_TOKEN)
|
||||
# must NEVER be available to agents. They live in .env.vault.enc and are injected
|
||||
# only into the ephemeral runner container at fire time. Unset them here so
|
||||
# must NEVER be available to agents. They live in secrets/*.enc and are decrypted
|
||||
# only into the ephemeral runner container at fire time (#777). Unset them here so
|
||||
# even an accidental .env inclusion cannot leak them into agent sessions.
|
||||
unset GITHUB_TOKEN 2>/dev/null || true
|
||||
unset CLAWHUB_TOKEN 2>/dev/null || true
|
||||
|
|
@ -313,6 +313,68 @@ memory_guard() {
|
|||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# SECRET LOADING ABSTRACTION
|
||||
# =============================================================================
|
||||
# load_secret NAME [DEFAULT]
|
||||
#
|
||||
# Resolves a secret value using the following precedence:
|
||||
# 1. /secrets/<NAME>.env — Nomad-rendered template (future)
|
||||
# 2. Current environment — already set by .env.enc, compose, etc.
|
||||
# 3. secrets/<NAME>.enc — age-encrypted per-key file (decrypted on demand)
|
||||
# 4. DEFAULT (or empty)
|
||||
#
|
||||
# Prints the resolved value to stdout. Caches age-decrypted values in the
|
||||
# process environment so subsequent calls are free.
|
||||
# =============================================================================
|
||||
load_secret() {
|
||||
local name="$1"
|
||||
local default="${2:-}"
|
||||
|
||||
# 1. Nomad-rendered template (future: Nomad writes /secrets/<NAME>.env)
|
||||
local nomad_path="/secrets/${name}.env"
|
||||
if [ -f "$nomad_path" ]; then
|
||||
# Source into a subshell to extract just the value
|
||||
local _nomad_val
|
||||
_nomad_val=$(
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$nomad_path"
|
||||
set +a
|
||||
printf '%s' "${!name:-}"
|
||||
)
|
||||
if [ -n "$_nomad_val" ]; then
|
||||
export "$name=$_nomad_val"
|
||||
printf '%s' "$_nomad_val"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Already in environment (set by .env.enc, compose injection, etc.)
|
||||
if [ -n "${!name:-}" ]; then
|
||||
printf '%s' "${!name}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 3. Age-encrypted per-key file: secrets/<NAME>.enc (#777)
|
||||
local _age_key="${HOME}/.config/sops/age/keys.txt"
|
||||
local _enc_path="${FACTORY_ROOT}/secrets/${name}.enc"
|
||||
if [ -f "$_enc_path" ] && [ -f "$_age_key" ] && command -v age &>/dev/null; then
|
||||
local _dec_val
|
||||
if _dec_val=$(age -d -i "$_age_key" "$_enc_path" 2>/dev/null) && [ -n "$_dec_val" ]; then
|
||||
export "$name=$_dec_val"
|
||||
printf '%s' "$_dec_val"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Default (or empty)
|
||||
if [ -n "$default" ]; then
|
||||
printf '%s' "$default"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Source tea helpers (available when tea binary is installed)
|
||||
if command -v tea &>/dev/null; then
|
||||
# shellcheck source=tea-helpers.sh
|
||||
|
|
|
|||
|
|
@ -31,8 +31,9 @@ _load_init_context() {
|
|||
# Execute a command in the Forgejo container (for admin operations)
|
||||
_forgejo_exec() {
|
||||
local use_bare="${DISINTO_BARE:-false}"
|
||||
local cname="${FORGEJO_CONTAINER_NAME:-disinto-forgejo}"
|
||||
if [ "$use_bare" = true ]; then
|
||||
docker exec -u git disinto-forgejo "$@"
|
||||
docker exec -u git "$cname" "$@"
|
||||
else
|
||||
docker compose -f "${FACTORY_ROOT}/docker-compose.yml" exec -T -u git forgejo "$@"
|
||||
fi
|
||||
|
|
@ -94,11 +95,12 @@ setup_forge() {
|
|||
# Bare-metal mode: standalone docker run
|
||||
mkdir -p "${FORGEJO_DATA_DIR}"
|
||||
|
||||
if docker ps -a --format '{{.Names}}' | grep -q '^disinto-forgejo$'; then
|
||||
docker start disinto-forgejo >/dev/null 2>&1 || true
|
||||
local cname="${FORGEJO_CONTAINER_NAME:-disinto-forgejo}"
|
||||
if docker ps -a --format '{{.Names}}' | grep -q "^${cname}$"; then
|
||||
docker start "$cname" >/dev/null 2>&1 || true
|
||||
else
|
||||
docker run -d \
|
||||
--name disinto-forgejo \
|
||||
--name "$cname" \
|
||||
--restart unless-stopped \
|
||||
-p "${forge_port}:3000" \
|
||||
-p 2222:22 \
|
||||
|
|
@ -210,8 +212,8 @@ setup_forge() {
|
|||
|
||||
# Create human user (disinto-admin) as site admin if it doesn't exist
|
||||
local human_user="disinto-admin"
|
||||
local human_pass
|
||||
human_pass="admin-$(head -c 16 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9' | head -c 20)"
|
||||
# human_user == admin_user; reuse admin_pass for basic-auth operations
|
||||
local human_pass="$admin_pass"
|
||||
|
||||
if ! curl -sf --max-time 5 -H "Authorization: token ${FORGE_TOKEN:-}" "${forge_url}/api/v1/users/${human_user}" >/dev/null 2>&1; then
|
||||
echo "Creating human user: ${human_user}"
|
||||
|
|
@ -243,63 +245,89 @@ setup_forge() {
|
|||
echo "Human user: ${human_user} (already exists)"
|
||||
fi
|
||||
|
||||
# Delete existing admin token if present (token sha1 is only returned at creation time)
|
||||
local existing_token_id
|
||||
existing_token_id=$(curl -sf \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
"${forge_url}/api/v1/users/${admin_user}/tokens" 2>/dev/null \
|
||||
| jq -r '.[] | select(.name == "disinto-admin-token") | .id') || existing_token_id=""
|
||||
if [ -n "$existing_token_id" ]; then
|
||||
curl -sf -X DELETE \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
"${forge_url}/api/v1/users/${admin_user}/tokens/${existing_token_id}" >/dev/null 2>&1 || true
|
||||
# Preserve admin token if already stored in .env (idempotent re-run)
|
||||
local admin_token=""
|
||||
if _token_exists_in_env "FORGE_ADMIN_TOKEN" "$env_file" && [ "$rotate_tokens" = false ]; then
|
||||
admin_token=$(grep '^FORGE_ADMIN_TOKEN=' "$env_file" | head -1 | cut -d= -f2-)
|
||||
[ -n "$admin_token" ] && echo "Admin token: preserved (use --rotate-tokens to force)"
|
||||
fi
|
||||
|
||||
# Create admin token (fresh, so sha1 is returned)
|
||||
local admin_token
|
||||
admin_token=$(curl -sf -X POST \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${forge_url}/api/v1/users/${admin_user}/tokens" \
|
||||
-d '{"name":"disinto-admin-token","scopes":["all"]}' 2>/dev/null \
|
||||
| jq -r '.sha1 // empty') || admin_token=""
|
||||
|
||||
if [ -z "$admin_token" ]; then
|
||||
echo "Error: failed to obtain admin API token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get or create human user token
|
||||
local human_token=""
|
||||
# Delete existing human token if present (token sha1 is only returned at creation time)
|
||||
local existing_human_token_id
|
||||
existing_human_token_id=$(curl -sf \
|
||||
-u "${human_user}:${human_pass}" \
|
||||
"${forge_url}/api/v1/users/${human_user}/tokens" 2>/dev/null \
|
||||
| jq -r '.[] | select(.name == "disinto-human-token") | .id') || existing_human_token_id=""
|
||||
if [ -n "$existing_human_token_id" ]; then
|
||||
curl -sf -X DELETE \
|
||||
-u "${human_user}:${human_pass}" \
|
||||
"${forge_url}/api/v1/users/${human_user}/tokens/${existing_human_token_id}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Create human token (fresh, so sha1 is returned)
|
||||
human_token=$(curl -sf -X POST \
|
||||
-u "${human_user}:${human_pass}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${forge_url}/api/v1/users/${human_user}/tokens" \
|
||||
-d '{"name":"disinto-human-token","scopes":["all"]}' 2>/dev/null \
|
||||
| jq -r '.sha1 // empty') || human_token=""
|
||||
|
||||
if [ -n "$human_token" ]; then
|
||||
# Store human token in .env
|
||||
if grep -q '^HUMAN_TOKEN=' "$env_file" 2>/dev/null; then
|
||||
sed -i "s|^HUMAN_TOKEN=.*|HUMAN_TOKEN=${human_token}|" "$env_file"
|
||||
else
|
||||
printf 'HUMAN_TOKEN=%s\n' "$human_token" >> "$env_file"
|
||||
# Delete existing admin token if present (token sha1 is only returned at creation time)
|
||||
local existing_token_id
|
||||
existing_token_id=$(curl -sf \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
"${forge_url}/api/v1/users/${admin_user}/tokens" 2>/dev/null \
|
||||
| jq -r '.[] | select(.name == "disinto-admin-token") | .id') || existing_token_id=""
|
||||
if [ -n "$existing_token_id" ]; then
|
||||
curl -sf -X DELETE \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
"${forge_url}/api/v1/users/${admin_user}/tokens/${existing_token_id}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Create admin token (fresh, so sha1 is returned)
|
||||
admin_token=$(curl -sf -X POST \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${forge_url}/api/v1/users/${admin_user}/tokens" \
|
||||
-d '{"name":"disinto-admin-token","scopes":["all"]}' 2>/dev/null \
|
||||
| jq -r '.sha1 // empty') || admin_token=""
|
||||
|
||||
if [ -z "$admin_token" ]; then
|
||||
echo "Error: failed to obtain admin API token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Store admin token for idempotent re-runs
|
||||
if grep -q '^FORGE_ADMIN_TOKEN=' "$env_file" 2>/dev/null; then
|
||||
sed -i "s|^FORGE_ADMIN_TOKEN=.*|FORGE_ADMIN_TOKEN=${admin_token}|" "$env_file"
|
||||
else
|
||||
printf 'FORGE_ADMIN_TOKEN=%s\n' "$admin_token" >> "$env_file"
|
||||
fi
|
||||
echo "Admin token: generated and saved (FORGE_ADMIN_TOKEN)"
|
||||
fi
|
||||
|
||||
# Get or create human user token (human_user == admin_user; use admin_pass)
|
||||
local human_token=""
|
||||
if _token_exists_in_env "HUMAN_TOKEN" "$env_file" && [ "$rotate_tokens" = false ]; then
|
||||
human_token=$(grep '^HUMAN_TOKEN=' "$env_file" | head -1 | cut -d= -f2-)
|
||||
if [ -n "$human_token" ]; then
|
||||
export HUMAN_TOKEN="$human_token"
|
||||
echo " Human token preserved (use --rotate-tokens to force)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$human_token" ]; then
|
||||
# Delete existing human token if present (token sha1 is only returned at creation time)
|
||||
local existing_human_token_id
|
||||
existing_human_token_id=$(curl -sf \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
"${forge_url}/api/v1/users/${human_user}/tokens" 2>/dev/null \
|
||||
| jq -r '.[] | select(.name == "disinto-human-token") | .id') || existing_human_token_id=""
|
||||
if [ -n "$existing_human_token_id" ]; then
|
||||
curl -sf -X DELETE \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
"${forge_url}/api/v1/users/${human_user}/tokens/${existing_human_token_id}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Create human token (use admin_pass since human_user == admin_user)
|
||||
human_token=$(curl -sf -X POST \
|
||||
-u "${admin_user}:${admin_pass}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${forge_url}/api/v1/users/${human_user}/tokens" \
|
||||
-d '{"name":"disinto-human-token","scopes":["all"]}' 2>/dev/null \
|
||||
| jq -r '.sha1 // empty') || human_token=""
|
||||
|
||||
if [ -n "$human_token" ]; then
|
||||
# Store human token in .env
|
||||
if grep -q '^HUMAN_TOKEN=' "$env_file" 2>/dev/null; then
|
||||
sed -i "s|^HUMAN_TOKEN=.*|HUMAN_TOKEN=${human_token}|" "$env_file"
|
||||
else
|
||||
printf 'HUMAN_TOKEN=%s\n' "$human_token" >> "$env_file"
|
||||
fi
|
||||
export HUMAN_TOKEN="$human_token"
|
||||
echo " Human token generated and saved (HUMAN_TOKEN)"
|
||||
fi
|
||||
export HUMAN_TOKEN="$human_token"
|
||||
echo " Human token saved (HUMAN_TOKEN)"
|
||||
fi
|
||||
|
||||
# Create bot users and tokens
|
||||
|
|
@ -719,7 +747,7 @@ setup_forge() {
|
|||
fi
|
||||
|
||||
# Add all bot users as collaborators with appropriate permissions
|
||||
# dev-bot: write (PR creation via lib/vault.sh)
|
||||
# dev-bot: write (PR creation via lib/action-vault.sh)
|
||||
# review-bot: read (PR review)
|
||||
# planner-bot: write (prerequisites.md, memory)
|
||||
# gardener-bot: write (backlog grooming)
|
||||
|
|
|
|||
|
|
@ -100,9 +100,7 @@ _generate_local_model_services() {
|
|||
cat >> "$temp_file" <<EOF
|
||||
|
||||
agents-${service_name}:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/agents/Dockerfile
|
||||
image: ghcr.io/disinto/agents:\${DISINTO_IMAGE_TAG:-latest}
|
||||
container_name: disinto-agents-${service_name}
|
||||
restart: unless-stopped
|
||||
security_opt:
|
||||
|
|
@ -111,9 +109,9 @@ _generate_local_model_services() {
|
|||
- agents-${service_name}-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- \${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:\${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- \${HOME}/.claude.json:/home/agent/.claude.json:ro
|
||||
- CLAUDE_BIN_PLACEHOLDER:/usr/local/bin/claude:ro
|
||||
- \${HOME}/.ssh:/home/agent/.ssh:ro
|
||||
- \${CLAUDE_CONFIG_FILE:-\${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- \${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- \${AGENT_SSH_DIR:-\${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
environment:
|
||||
FORGE_URL: http://forgejo:3000
|
||||
FORGE_REPO: ${FORGE_REPO:-disinto-admin/disinto}
|
||||
|
|
@ -142,6 +140,7 @@ _generate_local_model_services() {
|
|||
GARDENER_INTERVAL: "${GARDENER_INTERVAL:-21600}"
|
||||
ARCHITECT_INTERVAL: "${ARCHITECT_INTERVAL:-21600}"
|
||||
PLANNER_INTERVAL: "${PLANNER_INTERVAL:-43200}"
|
||||
SUPERVISOR_INTERVAL: "${SUPERVISOR_INTERVAL:-1200}"
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
|
|
@ -233,6 +232,7 @@ for name, config in agents.items():
|
|||
# to materialize a working stack on a fresh checkout.
|
||||
_generate_compose_impl() {
|
||||
local forge_port="${1:-3000}"
|
||||
local use_build="${2:-false}"
|
||||
local compose_file="${FACTORY_ROOT}/docker-compose.yml"
|
||||
|
||||
# Check if compose file already exists
|
||||
|
|
@ -296,6 +296,7 @@ services:
|
|||
WOODPECKER_AGENT_SECRET: ${WOODPECKER_AGENT_SECRET:-}
|
||||
WOODPECKER_DATABASE_DRIVER: sqlite3
|
||||
WOODPECKER_DATABASE_DATASOURCE: /var/lib/woodpecker/woodpecker.sqlite
|
||||
WOODPECKER_PLUGINS_PRIVILEGED: ${WOODPECKER_PLUGINS_PRIVILEGED:-plugins/docker}
|
||||
WOODPECKER_ENVIRONMENT: "FORGE_TOKEN:${FORGE_TOKEN}"
|
||||
depends_on:
|
||||
forgejo:
|
||||
|
|
@ -318,15 +319,19 @@ services:
|
|||
WOODPECKER_AGENT_SECRET: ${WOODPECKER_AGENT_SECRET:-}
|
||||
WOODPECKER_GRPC_SECURE: "false"
|
||||
WOODPECKER_HEALTHCHECK_ADDR: ":3333"
|
||||
WOODPECKER_BACKEND_DOCKER_NETWORK: disinto_disinto-net
|
||||
WOODPECKER_BACKEND_DOCKER_NETWORK: ${WOODPECKER_CI_NETWORK:-disinto_disinto-net}
|
||||
WOODPECKER_MAX_WORKFLOWS: 1
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3333/healthz"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
depends_on:
|
||||
- woodpecker
|
||||
|
||||
agents:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/agents/Dockerfile
|
||||
image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}
|
||||
container_name: disinto-agents
|
||||
restart: unless-stopped
|
||||
security_opt:
|
||||
|
|
@ -335,11 +340,14 @@ services:
|
|||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${HOME}/.claude.json:/home/agent/.claude.json:ro
|
||||
- CLAUDE_BIN_PLACEHOLDER:/usr/local/bin/claude:ro
|
||||
- ${HOME}/.ssh:/home/agent/.ssh:ro
|
||||
- ${HOME}/.config/sops/age:/home/agent/.config/sops/age:ro
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
- ./projects:/home/agent/disinto/projects:ro
|
||||
- ./.env:/home/agent/disinto/.env:ro
|
||||
- ./state:/home/agent/disinto/state
|
||||
environment:
|
||||
FORGE_URL: http://forgejo:3000
|
||||
FORGE_REPO: ${FORGE_REPO:-disinto-admin/disinto}
|
||||
|
|
@ -371,8 +379,14 @@ services:
|
|||
PLANNER_INTERVAL: ${PLANNER_INTERVAL:-43200}
|
||||
# IMPORTANT: agents get explicit environment variables (forge tokens, CI tokens, config).
|
||||
# Vault-only secrets (GITHUB_TOKEN, CLAWHUB_TOKEN, deploy keys) live in
|
||||
# .env.vault.enc and are NEVER injected here — only the runner
|
||||
# container receives them at fire time (AD-006, #745).
|
||||
# secrets/*.enc and are NEVER injected here — only the runner
|
||||
# container receives them at fire time (AD-006, #745, #777).
|
||||
healthcheck:
|
||||
test: ["CMD", "pgrep", "-f", "entrypoint.sh"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
|
|
@ -381,10 +395,137 @@ services:
|
|||
networks:
|
||||
- disinto-net
|
||||
|
||||
runner:
|
||||
COMPOSEEOF
|
||||
|
||||
# ── Conditional agents-llama block (ENABLE_LLAMA_AGENT=1) ──────────────
|
||||
# Local-Qwen dev agent — gated on ENABLE_LLAMA_AGENT so factories without
|
||||
# a local llama endpoint don't try to start it. See docs/agents-llama.md.
|
||||
if [ "${ENABLE_LLAMA_AGENT:-0}" = "1" ]; then
|
||||
cat >> "$compose_file" <<'LLAMAEOF'
|
||||
|
||||
agents-llama:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/agents/Dockerfile
|
||||
container_name: disinto-agents-llama
|
||||
restart: unless-stopped
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
volumes:
|
||||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
environment:
|
||||
FORGE_URL: http://forgejo:3000
|
||||
FORGE_REPO: ${FORGE_REPO:-disinto-admin/disinto}
|
||||
FORGE_TOKEN: ${FORGE_TOKEN_LLAMA:-}
|
||||
FORGE_PASS: ${FORGE_PASS_LLAMA:-}
|
||||
FORGE_BOT_USERNAMES: ${FORGE_BOT_USERNAMES:-}
|
||||
WOODPECKER_TOKEN: ${WOODPECKER_TOKEN:-}
|
||||
CLAUDE_TIMEOUT: ${CLAUDE_TIMEOUT:-7200}
|
||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: ${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}
|
||||
CLAUDE_AUTOCOMPACT_PCT_OVERRIDE: "60"
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-}
|
||||
FORGE_ADMIN_PASS: ${FORGE_ADMIN_PASS:-}
|
||||
DISINTO_CONTAINER: "1"
|
||||
PROJECT_NAME: ${PROJECT_NAME:-project}
|
||||
PROJECT_REPO_ROOT: /home/agent/repos/${PROJECT_NAME:-project}
|
||||
WOODPECKER_DATA_DIR: /woodpecker-data
|
||||
WOODPECKER_REPO_ID: "PLACEHOLDER_WP_REPO_ID"
|
||||
CLAUDE_CONFIG_DIR: ${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}
|
||||
POLL_INTERVAL: ${POLL_INTERVAL:-300}
|
||||
AGENT_ROLES: dev
|
||||
healthcheck:
|
||||
test: ["CMD", "pgrep", "-f", "entrypoint.sh"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- disinto-net
|
||||
|
||||
agents-llama-all:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/agents/Dockerfile
|
||||
container_name: disinto-agents-llama-all
|
||||
restart: unless-stopped
|
||||
profiles: ["agents-llama-all"]
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
volumes:
|
||||
- agent-data:/home/agent/data
|
||||
- project-repos:/home/agent/repos
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
- ${AGENT_SSH_DIR:-${HOME}/.ssh}:/home/agent/.ssh:ro
|
||||
- ${SOPS_AGE_DIR:-${HOME}/.config/sops/age}:/home/agent/.config/sops/age:ro
|
||||
- woodpecker-data:/woodpecker-data:ro
|
||||
environment:
|
||||
FORGE_URL: http://forgejo:3000
|
||||
FORGE_REPO: ${FORGE_REPO:-disinto-admin/disinto}
|
||||
FORGE_TOKEN: ${FORGE_TOKEN_LLAMA:-}
|
||||
FORGE_PASS: ${FORGE_PASS_LLAMA:-}
|
||||
FORGE_REVIEW_TOKEN: ${FORGE_REVIEW_TOKEN:-}
|
||||
FORGE_PLANNER_TOKEN: ${FORGE_PLANNER_TOKEN:-}
|
||||
FORGE_GARDENER_TOKEN: ${FORGE_GARDENER_TOKEN:-}
|
||||
FORGE_VAULT_TOKEN: ${FORGE_VAULT_TOKEN:-}
|
||||
FORGE_SUPERVISOR_TOKEN: ${FORGE_SUPERVISOR_TOKEN:-}
|
||||
FORGE_PREDICTOR_TOKEN: ${FORGE_PREDICTOR_TOKEN:-}
|
||||
FORGE_ARCHITECT_TOKEN: ${FORGE_ARCHITECT_TOKEN:-}
|
||||
FORGE_FILER_TOKEN: ${FORGE_FILER_TOKEN:-}
|
||||
FORGE_BOT_USERNAMES: ${FORGE_BOT_USERNAMES:-}
|
||||
WOODPECKER_TOKEN: ${WOODPECKER_TOKEN:-}
|
||||
CLAUDE_TIMEOUT: ${CLAUDE_TIMEOUT:-7200}
|
||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: ${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}
|
||||
CLAUDE_AUTOCOMPACT_PCT_OVERRIDE: "60"
|
||||
CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS: "1"
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-}
|
||||
FORGE_ADMIN_PASS: ${FORGE_ADMIN_PASS:-}
|
||||
DISINTO_CONTAINER: "1"
|
||||
PROJECT_NAME: ${PROJECT_NAME:-project}
|
||||
PROJECT_REPO_ROOT: /home/agent/repos/${PROJECT_NAME:-project}
|
||||
WOODPECKER_DATA_DIR: /woodpecker-data
|
||||
WOODPECKER_REPO_ID: "PLACEHOLDER_WP_REPO_ID"
|
||||
CLAUDE_CONFIG_DIR: ${CLAUDE_CONFIG_DIR:-/var/lib/disinto/claude-shared/config}
|
||||
POLL_INTERVAL: ${POLL_INTERVAL:-300}
|
||||
GARDENER_INTERVAL: ${GARDENER_INTERVAL:-21600}
|
||||
ARCHITECT_INTERVAL: ${ARCHITECT_INTERVAL:-21600}
|
||||
PLANNER_INTERVAL: ${PLANNER_INTERVAL:-43200}
|
||||
SUPERVISOR_INTERVAL: ${SUPERVISOR_INTERVAL:-1200}
|
||||
AGENT_ROLES: review,dev,gardener,architect,planner,predictor,supervisor
|
||||
healthcheck:
|
||||
test: ["CMD", "pgrep", "-f", "entrypoint.sh"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
woodpecker:
|
||||
condition: service_started
|
||||
networks:
|
||||
- disinto-net
|
||||
LLAMAEOF
|
||||
fi
|
||||
|
||||
# Resume the rest of the compose file (runner onward)
|
||||
cat >> "$compose_file" <<'COMPOSEEOF'
|
||||
|
||||
runner:
|
||||
image: ghcr.io/disinto/agents:${DISINTO_IMAGE_TAG:-latest}
|
||||
profiles: ["vault"]
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
|
|
@ -405,8 +546,9 @@ services:
|
|||
# Edge proxy — reverse proxy to Forgejo, Woodpecker, and staging
|
||||
# Serves on ports 80/443, routes based on path
|
||||
edge:
|
||||
build: ./docker/edge
|
||||
image: ghcr.io/disinto/edge:${DISINTO_IMAGE_TAG:-latest}
|
||||
container_name: disinto-edge
|
||||
restart: unless-stopped
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
ports:
|
||||
|
|
@ -441,7 +583,13 @@ services:
|
|||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ./secrets/tunnel_key:/run/secrets/tunnel_key:ro
|
||||
- ${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}:${CLAUDE_SHARED_DIR:-/var/lib/disinto/claude-shared}
|
||||
- ${HOME}/.claude.json:/home/agent/.claude.json:ro
|
||||
- ${CLAUDE_CONFIG_FILE:-${HOME}/.claude.json}:/home/agent/.claude.json:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://localhost:2019/config/"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
depends_on:
|
||||
forgejo:
|
||||
condition: service_healthy
|
||||
|
|
@ -459,6 +607,12 @@ services:
|
|||
command: ["caddy", "file-server", "--root", "/srv/site"]
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:2019/config/"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
volumes:
|
||||
- ./docker:/srv/site:ro
|
||||
networks:
|
||||
|
|
@ -499,7 +653,7 @@ services:
|
|||
memswap_limit: 512m
|
||||
volumes:
|
||||
# Mount claude binary from host (same as agents)
|
||||
- CLAUDE_BIN_PLACEHOLDER:/usr/local/bin/claude:ro
|
||||
- ${CLAUDE_BIN_DIR}:/usr/local/bin/claude:ro
|
||||
# Throwaway named volume for chat config (isolated from host ~/.claude)
|
||||
- chat-config:/var/chat/config
|
||||
# Chat history persistence: per-user NDJSON files on bind-mounted host volume
|
||||
|
|
@ -518,6 +672,12 @@ services:
|
|||
CHAT_MAX_REQUESTS_PER_HOUR: ${CHAT_MAX_REQUESTS_PER_HOUR:-60}
|
||||
CHAT_MAX_REQUESTS_PER_DAY: ${CHAT_MAX_REQUESTS_PER_DAY:-500}
|
||||
CHAT_MAX_TOKENS_PER_DAY: ${CHAT_MAX_TOKENS_PER_DAY:-1000000}
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
networks:
|
||||
- disinto-net
|
||||
|
||||
|
|
@ -556,20 +716,35 @@ COMPOSEEOF
|
|||
fi
|
||||
|
||||
# Append local-model agent services if any are configured
|
||||
# (must run before CLAUDE_BIN_PLACEHOLDER substitution so the placeholder
|
||||
# in local-model services is also resolved)
|
||||
_generate_local_model_services "$compose_file"
|
||||
|
||||
# Patch the Claude CLI binary path — resolve from host PATH at init time.
|
||||
# Resolve the Claude CLI binary path and persist as CLAUDE_BIN_DIR in .env.
|
||||
# docker-compose.yml references ${CLAUDE_BIN_DIR} so the value must be set.
|
||||
local claude_bin
|
||||
claude_bin="$(command -v claude 2>/dev/null || true)"
|
||||
if [ -n "$claude_bin" ]; then
|
||||
# Resolve symlinks to get the real binary path
|
||||
claude_bin="$(readlink -f "$claude_bin")"
|
||||
sed -i "s|CLAUDE_BIN_PLACEHOLDER|${claude_bin}|g" "$compose_file"
|
||||
else
|
||||
echo "Warning: claude CLI not found in PATH — update docker-compose.yml volumes manually" >&2
|
||||
sed -i "s|CLAUDE_BIN_PLACEHOLDER|/usr/local/bin/claude|g" "$compose_file"
|
||||
echo "Warning: claude CLI not found in PATH — set CLAUDE_BIN_DIR in .env manually" >&2
|
||||
claude_bin="/usr/local/bin/claude"
|
||||
fi
|
||||
# Persist CLAUDE_BIN_DIR into .env so docker-compose can resolve it.
|
||||
local env_file="${FACTORY_ROOT}/.env"
|
||||
if [ -f "$env_file" ]; then
|
||||
if grep -q "^CLAUDE_BIN_DIR=" "$env_file" 2>/dev/null; then
|
||||
sed -i "s|^CLAUDE_BIN_DIR=.*|CLAUDE_BIN_DIR=${claude_bin}|" "$env_file"
|
||||
else
|
||||
printf 'CLAUDE_BIN_DIR=%s\n' "$claude_bin" >> "$env_file"
|
||||
fi
|
||||
else
|
||||
printf 'CLAUDE_BIN_DIR=%s\n' "$claude_bin" > "$env_file"
|
||||
fi
|
||||
|
||||
# In build mode, replace image: with build: for locally-built images
|
||||
if [ "$use_build" = true ]; then
|
||||
sed -i 's|^\( agents:\)|\1|' "$compose_file"
|
||||
sed -i '/^ image: ghcr\.io\/disinto\/agents:/{s|image: ghcr\.io/disinto/agents:.*|build:\n context: .\n dockerfile: docker/agents/Dockerfile|}' "$compose_file"
|
||||
sed -i '/^ image: ghcr\.io\/disinto\/edge:/{s|image: ghcr\.io/disinto/edge:.*|build: ./docker/edge|}' "$compose_file"
|
||||
fi
|
||||
|
||||
echo "Created: ${compose_file}"
|
||||
|
|
@ -588,7 +763,11 @@ _generate_agent_docker_impl() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Generate docker/Caddyfile template for edge proxy.
|
||||
# Generate docker/Caddyfile for the edge proxy.
|
||||
# **CANONICAL SOURCE**: This generator is the single source of truth for the Caddyfile.
|
||||
# Output path: ${FACTORY_ROOT}/docker/Caddyfile (gitignored — generated artifact).
|
||||
# The edge compose service mounts this path as /etc/caddy/Caddyfile.
|
||||
# On a fresh clone, `disinto init` calls generate_caddyfile before first `disinto up`.
|
||||
_generate_caddyfile_impl() {
|
||||
local docker_dir="${FACTORY_ROOT}/docker"
|
||||
local caddyfile="${docker_dir}/Caddyfile"
|
||||
|
|
|
|||
279
lib/hvault.sh
Normal file
279
lib/hvault.sh
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
#!/usr/bin/env bash
|
||||
# hvault.sh — HashiCorp Vault helper module
|
||||
#
|
||||
# Typed, audited helpers for Vault KV v2 access so no script re-implements
|
||||
# `curl -H "X-Vault-Token: ..."` ad-hoc.
|
||||
#
|
||||
# Usage: source this file, then call any hvault_* function.
|
||||
#
|
||||
# Environment:
|
||||
# VAULT_ADDR — Vault server address (required, no default)
|
||||
# VAULT_TOKEN — auth token (precedence: env > /etc/vault.d/root.token)
|
||||
#
|
||||
# All functions emit structured JSON errors to stderr on failure.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Internal helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
# _hvault_err — emit structured JSON error to stderr
|
||||
# Args: func_name, message, [detail]
|
||||
_hvault_err() {
|
||||
local func="$1" msg="$2" detail="${3:-}"
|
||||
jq -n --arg func "$func" --arg msg "$msg" --arg detail "$detail" \
|
||||
'{error:true,function:$func,message:$msg,detail:$detail}' >&2
|
||||
}
|
||||
|
||||
# _hvault_resolve_token — resolve VAULT_TOKEN from env or token file
|
||||
_hvault_resolve_token() {
|
||||
if [ -n "${VAULT_TOKEN:-}" ]; then
|
||||
return 0
|
||||
fi
|
||||
local token_file="/etc/vault.d/root.token"
|
||||
if [ -f "$token_file" ]; then
|
||||
VAULT_TOKEN="$(cat "$token_file")"
|
||||
export VAULT_TOKEN
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# _hvault_check_prereqs — validate VAULT_ADDR and VAULT_TOKEN are set
|
||||
# Args: caller function name
|
||||
_hvault_check_prereqs() {
|
||||
local caller="$1"
|
||||
if [ -z "${VAULT_ADDR:-}" ]; then
|
||||
_hvault_err "$caller" "VAULT_ADDR is not set" "export VAULT_ADDR before calling $caller"
|
||||
return 1
|
||||
fi
|
||||
if ! _hvault_resolve_token; then
|
||||
_hvault_err "$caller" "VAULT_TOKEN is not set and /etc/vault.d/root.token not found" \
|
||||
"export VAULT_TOKEN or write token to /etc/vault.d/root.token"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# _hvault_request — execute a Vault API request
|
||||
# Args: method, path, [data]
|
||||
# Outputs: response body to stdout
|
||||
# Returns: 0 on 2xx, 1 otherwise (error JSON to stderr)
|
||||
_hvault_request() {
|
||||
local method="$1" path="$2" data="${3:-}"
|
||||
local url="${VAULT_ADDR}/v1/${path}"
|
||||
local http_code body
|
||||
local tmpfile
|
||||
tmpfile="$(mktemp)"
|
||||
|
||||
local curl_args=(
|
||||
-s
|
||||
-w '%{http_code}'
|
||||
-H "X-Vault-Token: ${VAULT_TOKEN}"
|
||||
-H "Content-Type: application/json"
|
||||
-X "$method"
|
||||
-o "$tmpfile"
|
||||
)
|
||||
if [ -n "$data" ]; then
|
||||
curl_args+=(-d "$data")
|
||||
fi
|
||||
|
||||
http_code="$(curl "${curl_args[@]}" "$url")" || {
|
||||
_hvault_err "_hvault_request" "curl failed" "url=$url"
|
||||
rm -f "$tmpfile"
|
||||
return 1
|
||||
}
|
||||
|
||||
body="$(cat "$tmpfile")"
|
||||
rm -f "$tmpfile"
|
||||
|
||||
# Check HTTP status — 2xx is success
|
||||
case "$http_code" in
|
||||
2[0-9][0-9])
|
||||
printf '%s' "$body"
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
_hvault_err "_hvault_request" "HTTP $http_code" "$body"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── Public API ───────────────────────────────────────────────────────────────
|
||||
|
||||
# hvault_kv_get PATH [KEY]
|
||||
# Read a KV v2 secret at PATH, optionally extract a single KEY.
|
||||
# Outputs: JSON value (full data object, or single key value)
|
||||
hvault_kv_get() {
|
||||
local path="${1:-}"
|
||||
local key="${2:-}"
|
||||
|
||||
if [ -z "$path" ]; then
|
||||
_hvault_err "hvault_kv_get" "PATH is required" "usage: hvault_kv_get PATH [KEY]"
|
||||
return 1
|
||||
fi
|
||||
_hvault_check_prereqs "hvault_kv_get" || return 1
|
||||
|
||||
local response
|
||||
response="$(_hvault_request GET "secret/data/${path}")" || return 1
|
||||
|
||||
if [ -n "$key" ]; then
|
||||
printf '%s' "$response" | jq -e -r --arg key "$key" '.data.data[$key]' 2>/dev/null || {
|
||||
_hvault_err "hvault_kv_get" "key not found" "key=$key path=$path"
|
||||
return 1
|
||||
}
|
||||
else
|
||||
printf '%s' "$response" | jq -e '.data.data' 2>/dev/null || {
|
||||
_hvault_err "hvault_kv_get" "failed to parse response" "path=$path"
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
}
|
||||
|
||||
# hvault_kv_put PATH KEY=VAL [KEY=VAL ...]
|
||||
# Write a KV v2 secret at PATH. Accepts one or more KEY=VAL pairs.
|
||||
hvault_kv_put() {
|
||||
local path="${1:-}"
|
||||
shift || true
|
||||
|
||||
if [ -z "$path" ] || [ $# -eq 0 ]; then
|
||||
_hvault_err "hvault_kv_put" "PATH and at least one KEY=VAL required" \
|
||||
"usage: hvault_kv_put PATH KEY=VAL [KEY=VAL ...]"
|
||||
return 1
|
||||
fi
|
||||
_hvault_check_prereqs "hvault_kv_put" || return 1
|
||||
|
||||
# Build JSON payload from KEY=VAL pairs entirely via jq
|
||||
local payload='{"data":{}}'
|
||||
for kv in "$@"; do
|
||||
local k="${kv%%=*}"
|
||||
local v="${kv#*=}"
|
||||
if [ "$k" = "$kv" ]; then
|
||||
_hvault_err "hvault_kv_put" "invalid KEY=VAL pair" "got: $kv"
|
||||
return 1
|
||||
fi
|
||||
payload="$(printf '%s' "$payload" | jq --arg k "$k" --arg v "$v" '.data[$k] = $v')"
|
||||
done
|
||||
|
||||
_hvault_request POST "secret/data/${path}" "$payload" >/dev/null
|
||||
}
|
||||
|
||||
# hvault_kv_list PATH
|
||||
# List keys at a KV v2 path.
|
||||
# Outputs: JSON array of key names
|
||||
hvault_kv_list() {
|
||||
local path="${1:-}"
|
||||
|
||||
if [ -z "$path" ]; then
|
||||
_hvault_err "hvault_kv_list" "PATH is required" "usage: hvault_kv_list PATH"
|
||||
return 1
|
||||
fi
|
||||
_hvault_check_prereqs "hvault_kv_list" || return 1
|
||||
|
||||
local response
|
||||
response="$(_hvault_request LIST "secret/metadata/${path}")" || return 1
|
||||
|
||||
printf '%s' "$response" | jq -e '.data.keys' 2>/dev/null || {
|
||||
_hvault_err "hvault_kv_list" "failed to parse response" "path=$path"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
# hvault_policy_apply NAME FILE
|
||||
# Idempotent policy upsert — create or update a Vault policy.
|
||||
hvault_policy_apply() {
|
||||
local name="${1:-}"
|
||||
local file="${2:-}"
|
||||
|
||||
if [ -z "$name" ] || [ -z "$file" ]; then
|
||||
_hvault_err "hvault_policy_apply" "NAME and FILE are required" \
|
||||
"usage: hvault_policy_apply NAME FILE"
|
||||
return 1
|
||||
fi
|
||||
if [ ! -f "$file" ]; then
|
||||
_hvault_err "hvault_policy_apply" "policy file not found" "file=$file"
|
||||
return 1
|
||||
fi
|
||||
_hvault_check_prereqs "hvault_policy_apply" || return 1
|
||||
|
||||
local policy_content
|
||||
policy_content="$(cat "$file")"
|
||||
local payload
|
||||
payload="$(jq -n --arg policy "$policy_content" '{"policy": $policy}')"
|
||||
|
||||
_hvault_request PUT "sys/policies/acl/${name}" "$payload" >/dev/null
|
||||
}
|
||||
|
||||
# hvault_jwt_login ROLE JWT
|
||||
# Exchange a JWT for a short-lived Vault token.
|
||||
# Outputs: client token string
|
||||
hvault_jwt_login() {
|
||||
local role="${1:-}"
|
||||
local jwt="${2:-}"
|
||||
|
||||
if [ -z "$role" ] || [ -z "$jwt" ]; then
|
||||
_hvault_err "hvault_jwt_login" "ROLE and JWT are required" \
|
||||
"usage: hvault_jwt_login ROLE JWT"
|
||||
return 1
|
||||
fi
|
||||
# Only need VAULT_ADDR, not VAULT_TOKEN (we're obtaining a token)
|
||||
if [ -z "${VAULT_ADDR:-}" ]; then
|
||||
_hvault_err "hvault_jwt_login" "VAULT_ADDR is not set"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local payload
|
||||
payload="$(jq -n --arg role "$role" --arg jwt "$jwt" \
|
||||
'{"role": $role, "jwt": $jwt}')"
|
||||
|
||||
local response
|
||||
# JWT login does not require an existing token — use curl directly
|
||||
local tmpfile http_code
|
||||
tmpfile="$(mktemp)"
|
||||
http_code="$(curl -s -w '%{http_code}' \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST \
|
||||
-d "$payload" \
|
||||
-o "$tmpfile" \
|
||||
"${VAULT_ADDR}/v1/auth/jwt/login")" || {
|
||||
_hvault_err "hvault_jwt_login" "curl failed"
|
||||
rm -f "$tmpfile"
|
||||
return 1
|
||||
}
|
||||
|
||||
local body
|
||||
body="$(cat "$tmpfile")"
|
||||
rm -f "$tmpfile"
|
||||
|
||||
case "$http_code" in
|
||||
2[0-9][0-9])
|
||||
printf '%s' "$body" | jq -e -r '.auth.client_token' 2>/dev/null || {
|
||||
_hvault_err "hvault_jwt_login" "failed to extract client_token" "$body"
|
||||
return 1
|
||||
}
|
||||
;;
|
||||
*)
|
||||
_hvault_err "hvault_jwt_login" "HTTP $http_code" "$body"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# hvault_token_lookup
|
||||
# Returns TTL, policies, and accessor for the current token.
|
||||
# Outputs: JSON object with ttl, policies, accessor fields
|
||||
hvault_token_lookup() {
|
||||
_hvault_check_prereqs "hvault_token_lookup" || return 1
|
||||
|
||||
local response
|
||||
response="$(_hvault_request GET "auth/token/lookup-self")" || return 1
|
||||
|
||||
printf '%s' "$response" | jq -e '{
|
||||
ttl: .data.ttl,
|
||||
policies: .data.policies,
|
||||
accessor: .data.accessor,
|
||||
display_name: .data.display_name
|
||||
}' 2>/dev/null || {
|
||||
_hvault_err "hvault_token_lookup" "failed to parse token info"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
338
lib/init/nomad/cluster-up.sh
Executable file
338
lib/init/nomad/cluster-up.sh
Executable file
|
|
@ -0,0 +1,338 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# lib/init/nomad/cluster-up.sh — Empty Nomad+Vault cluster orchestrator (S0.4)
|
||||
#
|
||||
# Wires together the S0.1–S0.3 building blocks into one idempotent
|
||||
# "bring up a single-node Nomad+Vault cluster" script:
|
||||
#
|
||||
# 1. install.sh (nomad + vault binaries)
|
||||
# 2. systemd-nomad.sh (nomad.service — unit + enable, not started)
|
||||
# 3. systemd-vault.sh (vault.service — unit + vault.hcl + enable)
|
||||
# 4. Host-volume dirs (/srv/disinto/* matching nomad/client.hcl)
|
||||
# 5. /etc/nomad.d/*.hcl (server.hcl + client.hcl from repo)
|
||||
# 6. vault-init.sh (first-run init + unseal + persist keys)
|
||||
# 7. systemctl start vault (auto-unseal via ExecStartPost; poll)
|
||||
# 8. systemctl start nomad (poll until ≥1 ready node)
|
||||
# 9. /etc/profile.d/disinto-nomad.sh (VAULT_ADDR + NOMAD_ADDR for shells)
|
||||
#
|
||||
# This is the "empty cluster" orchestrator — no jobs deployed. Subsequent
|
||||
# Step-1 issues layer job deployment on top of this checkpoint.
|
||||
#
|
||||
# Idempotency contract:
|
||||
# Running twice back-to-back on a healthy box is a no-op. Each sub-step
|
||||
# is itself idempotent — see install.sh / systemd-*.sh / vault-init.sh
|
||||
# headers for the per-step contract. Fast-paths in steps 7 and 8 skip
|
||||
# the systemctl start when the service is already active + healthy.
|
||||
#
|
||||
# Usage:
|
||||
# sudo lib/init/nomad/cluster-up.sh # bring cluster up
|
||||
# sudo lib/init/nomad/cluster-up.sh --dry-run # print step list, exit 0
|
||||
#
|
||||
# Environment (override polling for slow boxes):
|
||||
# VAULT_POLL_SECS max seconds to wait for vault to unseal (default: 30)
|
||||
# NOMAD_POLL_SECS max seconds to wait for nomad node=ready (default: 60)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (cluster up, or already up)
|
||||
# 1 precondition or step failure
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
|
||||
# Sub-scripts (siblings in this directory).
|
||||
INSTALL_SH="${SCRIPT_DIR}/install.sh"
|
||||
SYSTEMD_NOMAD_SH="${SCRIPT_DIR}/systemd-nomad.sh"
|
||||
SYSTEMD_VAULT_SH="${SCRIPT_DIR}/systemd-vault.sh"
|
||||
VAULT_INIT_SH="${SCRIPT_DIR}/vault-init.sh"
|
||||
|
||||
# In-repo Nomad configs copied to /etc/nomad.d/.
|
||||
NOMAD_CONFIG_DIR="/etc/nomad.d"
|
||||
NOMAD_SERVER_HCL_SRC="${REPO_ROOT}/nomad/server.hcl"
|
||||
NOMAD_CLIENT_HCL_SRC="${REPO_ROOT}/nomad/client.hcl"
|
||||
|
||||
# /etc/profile.d entry — makes VAULT_ADDR + NOMAD_ADDR available to
|
||||
# interactive shells without requiring the operator to source anything.
|
||||
PROFILE_D_FILE="/etc/profile.d/disinto-nomad.sh"
|
||||
|
||||
# Host-volume paths — MUST match the `host_volume "..."` declarations
|
||||
# in nomad/client.hcl. Adding a host_volume block there requires adding
|
||||
# its path here so the dir exists before nomad starts (otherwise client
|
||||
# fingerprinting fails and the node stays in "initializing").
|
||||
HOST_VOLUME_DIRS=(
|
||||
"/srv/disinto/forgejo-data"
|
||||
"/srv/disinto/woodpecker-data"
|
||||
"/srv/disinto/agent-data"
|
||||
"/srv/disinto/project-repos"
|
||||
"/srv/disinto/caddy-data"
|
||||
"/srv/disinto/chat-history"
|
||||
"/srv/disinto/ops-repo"
|
||||
)
|
||||
|
||||
# Default API addresses — matches the listener bindings in
|
||||
# nomad/server.hcl and nomad/vault.hcl. If either file ever moves
|
||||
# off 127.0.0.1 / default port, update both places together.
|
||||
VAULT_ADDR_DEFAULT="http://127.0.0.1:8200"
|
||||
NOMAD_ADDR_DEFAULT="http://127.0.0.1:4646"
|
||||
|
||||
VAULT_POLL_SECS="${VAULT_POLL_SECS:-30}"
|
||||
NOMAD_POLL_SECS="${NOMAD_POLL_SECS:-60}"
|
||||
|
||||
log() { printf '[cluster-up] %s\n' "$*"; }
|
||||
die() { printf '[cluster-up] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# ── Flag parsing ─────────────────────────────────────────────────────────────
|
||||
dry_run=false
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--dry-run) dry_run=true; shift ;;
|
||||
-h|--help)
|
||||
cat <<EOF
|
||||
Usage: sudo $(basename "$0") [--dry-run]
|
||||
|
||||
Brings up an empty single-node Nomad+Vault cluster (idempotent).
|
||||
|
||||
--dry-run Print the step list without performing any action.
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*) die "unknown flag: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Dry-run: print step list + exit ──────────────────────────────────────────
|
||||
if [ "$dry_run" = true ]; then
|
||||
cat <<EOF
|
||||
[dry-run] Step 1/9: install nomad + vault binaries
|
||||
→ sudo ${INSTALL_SH}
|
||||
|
||||
[dry-run] Step 2/9: write + enable nomad.service (NOT started)
|
||||
→ sudo ${SYSTEMD_NOMAD_SH}
|
||||
|
||||
[dry-run] Step 3/9: write + enable vault.service + vault.hcl (NOT started)
|
||||
→ sudo ${SYSTEMD_VAULT_SH}
|
||||
|
||||
[dry-run] Step 4/9: create host-volume dirs under /srv/disinto/
|
||||
EOF
|
||||
for d in "${HOST_VOLUME_DIRS[@]}"; do
|
||||
printf ' → install -d -m 0755 %s\n' "$d"
|
||||
done
|
||||
cat <<EOF
|
||||
|
||||
[dry-run] Step 5/9: install /etc/nomad.d/server.hcl + client.hcl from repo
|
||||
→ ${NOMAD_SERVER_HCL_SRC} → ${NOMAD_CONFIG_DIR}/server.hcl
|
||||
→ ${NOMAD_CLIENT_HCL_SRC} → ${NOMAD_CONFIG_DIR}/client.hcl
|
||||
|
||||
[dry-run] Step 6/9: first-run vault init + persist unseal.key + root.token
|
||||
→ sudo ${VAULT_INIT_SH}
|
||||
|
||||
[dry-run] Step 7/9: systemctl start vault + poll until unsealed (≤${VAULT_POLL_SECS}s)
|
||||
|
||||
[dry-run] Step 8/9: systemctl start nomad + poll until ≥1 node ready (≤${NOMAD_POLL_SECS}s)
|
||||
|
||||
[dry-run] Step 9/9: write ${PROFILE_D_FILE}
|
||||
→ export VAULT_ADDR=${VAULT_ADDR_DEFAULT}
|
||||
→ export NOMAD_ADDR=${NOMAD_ADDR_DEFAULT}
|
||||
|
||||
Dry run complete — no changes made.
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
die "must run as root (spawns install/systemd/vault-init sub-scripts)"
|
||||
fi
|
||||
|
||||
command -v systemctl >/dev/null 2>&1 \
|
||||
|| die "systemctl not found (systemd required)"
|
||||
|
||||
for f in "$INSTALL_SH" "$SYSTEMD_NOMAD_SH" "$SYSTEMD_VAULT_SH" "$VAULT_INIT_SH"; do
|
||||
[ -x "$f" ] || die "sub-script missing or non-executable: ${f}"
|
||||
done
|
||||
|
||||
[ -f "$NOMAD_SERVER_HCL_SRC" ] \
|
||||
|| die "source config not found: ${NOMAD_SERVER_HCL_SRC}"
|
||||
[ -f "$NOMAD_CLIENT_HCL_SRC" ] \
|
||||
|| die "source config not found: ${NOMAD_CLIENT_HCL_SRC}"
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# install_file_if_differs SRC DST MODE
|
||||
# Copy SRC to DST (root:root with MODE) iff on-disk content differs.
|
||||
# No-op + log otherwise — preserves mtime, avoids spurious reloads.
|
||||
install_file_if_differs() {
|
||||
local src="$1" dst="$2" mode="$3"
|
||||
if [ -f "$dst" ] && cmp -s "$src" "$dst"; then
|
||||
log "unchanged: ${dst}"
|
||||
return 0
|
||||
fi
|
||||
log "writing: ${dst}"
|
||||
install -m "$mode" -o root -g root "$src" "$dst"
|
||||
}
|
||||
|
||||
# vault_status_json — echo `vault status -format=json`, or '' on unreachable.
|
||||
# vault status exit codes: 0 = unsealed, 2 = sealed/uninit, 1 = unreachable.
|
||||
# We treat all of 0/2 as "reachable with state"; 1 yields empty output.
|
||||
# Wrapped in `|| true` so set -e doesn't abort on exit 2 (the expected
|
||||
# sealed-state case during first-boot polling).
|
||||
vault_status_json() {
|
||||
VAULT_ADDR="$VAULT_ADDR_DEFAULT" vault status -format=json 2>/dev/null || true
|
||||
}
|
||||
|
||||
# vault_is_unsealed — true iff vault reachable AND initialized AND unsealed.
|
||||
vault_is_unsealed() {
|
||||
local out init sealed
|
||||
out="$(vault_status_json)"
|
||||
[ -n "$out" ] || return 1
|
||||
init="$(printf '%s' "$out" | jq -r '.initialized' 2>/dev/null)" || init=""
|
||||
sealed="$(printf '%s' "$out" | jq -r '.sealed' 2>/dev/null)" || sealed=""
|
||||
[ "$init" = "true" ] && [ "$sealed" = "false" ]
|
||||
}
|
||||
|
||||
# nomad_ready_count — echo the number of ready nodes, or 0 on error.
|
||||
# `nomad node status -json` returns a JSON array of nodes, each with a
|
||||
# .Status field ("initializing" | "ready" | "down" | "disconnected").
|
||||
nomad_ready_count() {
|
||||
local out
|
||||
out="$(NOMAD_ADDR="$NOMAD_ADDR_DEFAULT" nomad node status -json 2>/dev/null || true)"
|
||||
if [ -z "$out" ]; then
|
||||
printf '0'
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "$out" \
|
||||
| jq '[.[] | select(.Status == "ready")] | length' 2>/dev/null \
|
||||
|| printf '0'
|
||||
}
|
||||
|
||||
# nomad_has_ready_node — true iff nomad_ready_count ≥ 1. Wrapper exists
|
||||
# so poll_until_healthy can call it as a single-arg command name.
|
||||
nomad_has_ready_node() { [ "$(nomad_ready_count)" -ge 1 ]; }
|
||||
|
||||
# _die_with_service_status SVC REASON
|
||||
# Log + dump `systemctl status SVC` to stderr + die with REASON. Factored
|
||||
# out so the poll helper doesn't carry three copies of the same dump.
|
||||
_die_with_service_status() {
|
||||
local svc="$1" reason="$2"
|
||||
log "${svc}.service ${reason} — systemctl status follows:"
|
||||
systemctl --no-pager --full status "$svc" >&2 || true
|
||||
die "${svc}.service ${reason}"
|
||||
}
|
||||
|
||||
# poll_until_healthy SVC CHECK_CMD TIMEOUT
|
||||
# Tick once per second for up to TIMEOUT seconds, invoking CHECK_CMD as a
|
||||
# command name (no arguments). Returns 0 on the first successful check.
|
||||
# Fails fast via _die_with_service_status if SVC enters systemd "failed"
|
||||
# state, and dies with a status dump if TIMEOUT elapses before CHECK_CMD
|
||||
# succeeds. Replaces the two in-line ready=1/break/sleep poll loops that
|
||||
# would otherwise each duplicate the same pattern already in vault-init.sh.
|
||||
poll_until_healthy() {
|
||||
local svc="$1" check="$2" timeout="$3"
|
||||
local waited=0
|
||||
until [ "$waited" -ge "$timeout" ]; do
|
||||
systemctl is-failed --quiet "$svc" \
|
||||
&& _die_with_service_status "$svc" "entered failed state during startup"
|
||||
if "$check"; then
|
||||
log "${svc} healthy after ${waited}s"
|
||||
return 0
|
||||
fi
|
||||
waited=$((waited + 1))
|
||||
sleep 1
|
||||
done
|
||||
_die_with_service_status "$svc" "not healthy within ${timeout}s"
|
||||
}
|
||||
|
||||
# ── Step 1/9: install.sh (nomad + vault binaries) ────────────────────────────
|
||||
log "── Step 1/9: install nomad + vault binaries ──"
|
||||
"$INSTALL_SH"
|
||||
|
||||
# ── Step 2/9: systemd-nomad.sh (unit + enable, not started) ──────────────────
|
||||
log "── Step 2/9: install nomad.service (enable, not start) ──"
|
||||
"$SYSTEMD_NOMAD_SH"
|
||||
|
||||
# ── Step 3/9: systemd-vault.sh (unit + vault.hcl + enable) ───────────────────
|
||||
log "── Step 3/9: install vault.service + vault.hcl (enable, not start) ──"
|
||||
"$SYSTEMD_VAULT_SH"
|
||||
|
||||
# ── Step 4/9: host-volume dirs matching nomad/client.hcl ─────────────────────
|
||||
log "── Step 4/9: host-volume dirs under /srv/disinto/ ──"
|
||||
# Parent /srv/disinto/ first (install -d handles missing parents, but being
|
||||
# explicit makes the log output read naturally as a top-down creation).
|
||||
install -d -m 0755 -o root -g root "/srv/disinto"
|
||||
for d in "${HOST_VOLUME_DIRS[@]}"; do
|
||||
if [ -d "$d" ]; then
|
||||
log "unchanged: ${d}"
|
||||
else
|
||||
log "creating: ${d}"
|
||||
install -d -m 0755 -o root -g root "$d"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Step 5/9: /etc/nomad.d/server.hcl + client.hcl ───────────────────────────
|
||||
log "── Step 5/9: install /etc/nomad.d/{server,client}.hcl ──"
|
||||
# systemd-nomad.sh already created /etc/nomad.d/. Re-assert for clarity +
|
||||
# in case someone runs cluster-up.sh with an exotic step ordering later.
|
||||
install -d -m 0755 -o root -g root "$NOMAD_CONFIG_DIR"
|
||||
install_file_if_differs "$NOMAD_SERVER_HCL_SRC" "${NOMAD_CONFIG_DIR}/server.hcl" 0644
|
||||
install_file_if_differs "$NOMAD_CLIENT_HCL_SRC" "${NOMAD_CONFIG_DIR}/client.hcl" 0644
|
||||
|
||||
# ── Step 6/9: vault-init (first-run init + unseal + persist keys) ────────────
|
||||
log "── Step 6/9: vault-init (no-op after first run) ──"
|
||||
# vault-init.sh spawns a temporary vault server if systemd isn't managing
|
||||
# one, runs `operator init`, writes unseal.key + root.token, unseals once,
|
||||
# then stops the temp server (EXIT trap). After it returns, port 8200 is
|
||||
# free for systemctl-managed vault to take in step 7.
|
||||
"$VAULT_INIT_SH"
|
||||
|
||||
# ── Step 7/9: systemctl start vault + poll until unsealed ────────────────────
|
||||
log "── Step 7/9: start vault + poll until unsealed ──"
|
||||
# Fast-path when vault.service is already active and Vault reports
|
||||
# initialized=true,sealed=false — re-runs are a no-op.
|
||||
if systemctl is-active --quiet vault && vault_is_unsealed; then
|
||||
log "vault already active + unsealed — skip start"
|
||||
else
|
||||
systemctl start vault
|
||||
poll_until_healthy vault vault_is_unsealed "$VAULT_POLL_SECS"
|
||||
fi
|
||||
|
||||
# ── Step 8/9: systemctl start nomad + poll until ≥1 node ready ───────────────
|
||||
log "── Step 8/9: start nomad + poll until ≥1 node ready ──"
|
||||
if systemctl is-active --quiet nomad && nomad_has_ready_node; then
|
||||
log "nomad already active + ≥1 node ready — skip start"
|
||||
else
|
||||
systemctl start nomad
|
||||
poll_until_healthy nomad nomad_has_ready_node "$NOMAD_POLL_SECS"
|
||||
fi
|
||||
|
||||
# ── Step 9/9: /etc/profile.d/disinto-nomad.sh ────────────────────────────────
|
||||
log "── Step 9/9: write ${PROFILE_D_FILE} ──"
|
||||
# Shell rc fragments in /etc/profile.d/ are sourced by /etc/profile for
|
||||
# every interactive login shell. Setting VAULT_ADDR + NOMAD_ADDR here means
|
||||
# the operator can run `vault status` / `nomad node status` straight after
|
||||
# `ssh factory-box` without fumbling env vars.
|
||||
desired_profile="# /etc/profile.d/disinto-nomad.sh — written by lib/init/nomad/cluster-up.sh
|
||||
# Interactive-shell defaults for Vault + Nomad clients on this box.
|
||||
export VAULT_ADDR=${VAULT_ADDR_DEFAULT}
|
||||
export NOMAD_ADDR=${NOMAD_ADDR_DEFAULT}
|
||||
"
|
||||
if [ -f "$PROFILE_D_FILE" ] \
|
||||
&& printf '%s' "$desired_profile" | cmp -s - "$PROFILE_D_FILE"; then
|
||||
log "unchanged: ${PROFILE_D_FILE}"
|
||||
else
|
||||
log "writing: ${PROFILE_D_FILE}"
|
||||
# Subshell + EXIT trap: guarantees the tempfile is cleaned up on both
|
||||
# success AND set-e-induced failure of `install`. A function-scoped
|
||||
# RETURN trap does NOT fire on errexit-abort in bash — the subshell is
|
||||
# the reliable cleanup boundary here.
|
||||
(
|
||||
tmp="$(mktemp)"
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
printf '%s' "$desired_profile" > "$tmp"
|
||||
install -m 0644 -o root -g root "$tmp" "$PROFILE_D_FILE"
|
||||
)
|
||||
fi
|
||||
|
||||
log "── done: empty nomad+vault cluster is up ──"
|
||||
log " Vault: ${VAULT_ADDR_DEFAULT} (Sealed=false Initialized=true)"
|
||||
log " Nomad: ${NOMAD_ADDR_DEFAULT} (≥1 node ready)"
|
||||
143
lib/init/nomad/install.sh
Executable file
143
lib/init/nomad/install.sh
Executable file
|
|
@ -0,0 +1,143 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# lib/init/nomad/install.sh — Idempotent apt install of HashiCorp Nomad + Vault
|
||||
#
|
||||
# Part of the Nomad+Vault migration. Installs both the `nomad` binary (S0.2,
|
||||
# issue #822) and the `vault` binary (S0.3, issue #823) from the same
|
||||
# HashiCorp apt repository. Does NOT configure, start, or enable any systemd
|
||||
# unit — lib/init/nomad/systemd-nomad.sh and lib/init/nomad/systemd-vault.sh
|
||||
# own that. Does NOT wire this script into `disinto init` — S0.4 owns that.
|
||||
#
|
||||
# Idempotency contract:
|
||||
# - Running twice back-to-back is a no-op once both target versions are
|
||||
# installed and the apt source is in place.
|
||||
# - Adds the HashiCorp apt keyring only if it is absent.
|
||||
# - Adds the HashiCorp apt sources list only if it is absent.
|
||||
# - Skips `apt-get install` for any package whose installed version already
|
||||
# matches the pin. If both are at pin, exits before touching apt.
|
||||
#
|
||||
# Configuration:
|
||||
# NOMAD_VERSION — pinned Nomad version (default: see below). Apt package
|
||||
# name is versioned as "nomad=<version>-1".
|
||||
# VAULT_VERSION — pinned Vault version (default: see below). Apt package
|
||||
# name is versioned as "vault=<version>-1".
|
||||
#
|
||||
# Usage:
|
||||
# sudo lib/init/nomad/install.sh
|
||||
# sudo NOMAD_VERSION=1.9.5 VAULT_VERSION=1.18.5 lib/init/nomad/install.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (installed or already present)
|
||||
# 1 precondition failure (not Debian/Ubuntu, missing tools, not root)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# Pin to specific 1.x releases. Bump here, not at call sites.
|
||||
NOMAD_VERSION="${NOMAD_VERSION:-1.9.5}"
|
||||
VAULT_VERSION="${VAULT_VERSION:-1.18.5}"
|
||||
|
||||
HASHICORP_KEYRING="/usr/share/keyrings/hashicorp-archive-keyring.gpg"
|
||||
HASHICORP_SOURCES="/etc/apt/sources.list.d/hashicorp.list"
|
||||
HASHICORP_GPG_URL="https://apt.releases.hashicorp.com/gpg"
|
||||
HASHICORP_REPO_URL="https://apt.releases.hashicorp.com"
|
||||
|
||||
log() { printf '[install] %s\n' "$*"; }
|
||||
die() { printf '[install] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# _installed_version BINARY
|
||||
# Echoes the installed semver for `nomad` or `vault` (e.g. "1.9.5").
|
||||
# Both tools print their version on the first line of `<bin> version` as
|
||||
# "<Name> v<semver>..." — the shared awk extracts $2 with the leading "v"
|
||||
# stripped. Empty string when the binary is absent or output is unexpected.
|
||||
_installed_version() {
|
||||
local bin="$1"
|
||||
command -v "$bin" >/dev/null 2>&1 || { printf ''; return 0; }
|
||||
"$bin" version 2>/dev/null \
|
||||
| awk 'NR==1 {sub(/^v/, "", $2); print $2; exit}'
|
||||
}
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
die "must run as root (needs apt-get + /usr/share/keyrings write access)"
|
||||
fi
|
||||
|
||||
for bin in apt-get gpg curl lsb_release; do
|
||||
command -v "$bin" >/dev/null 2>&1 \
|
||||
|| die "required binary not found: ${bin}"
|
||||
done
|
||||
|
||||
CODENAME="$(lsb_release -cs)"
|
||||
[ -n "$CODENAME" ] || die "lsb_release returned empty codename"
|
||||
|
||||
# ── Fast-path: are both already at desired versions? ─────────────────────────
|
||||
nomad_installed="$(_installed_version nomad)"
|
||||
vault_installed="$(_installed_version vault)"
|
||||
|
||||
need_pkgs=()
|
||||
if [ "$nomad_installed" = "$NOMAD_VERSION" ]; then
|
||||
log "nomad ${NOMAD_VERSION} already installed"
|
||||
else
|
||||
need_pkgs+=("nomad=${NOMAD_VERSION}-1")
|
||||
fi
|
||||
if [ "$vault_installed" = "$VAULT_VERSION" ]; then
|
||||
log "vault ${VAULT_VERSION} already installed"
|
||||
else
|
||||
need_pkgs+=("vault=${VAULT_VERSION}-1")
|
||||
fi
|
||||
|
||||
if [ "${#need_pkgs[@]}" -eq 0 ]; then
|
||||
log "nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Ensure HashiCorp apt keyring ─────────────────────────────────────────────
|
||||
if [ ! -f "$HASHICORP_KEYRING" ]; then
|
||||
log "adding HashiCorp apt keyring → ${HASHICORP_KEYRING}"
|
||||
tmpkey="$(mktemp)"
|
||||
trap 'rm -f "$tmpkey"' EXIT
|
||||
curl -fsSL "$HASHICORP_GPG_URL" -o "$tmpkey" \
|
||||
|| die "failed to fetch HashiCorp GPG key from ${HASHICORP_GPG_URL}"
|
||||
gpg --dearmor -o "$HASHICORP_KEYRING" < "$tmpkey" \
|
||||
|| die "failed to dearmor HashiCorp GPG key"
|
||||
chmod 0644 "$HASHICORP_KEYRING"
|
||||
rm -f "$tmpkey"
|
||||
trap - EXIT
|
||||
else
|
||||
log "HashiCorp apt keyring already present"
|
||||
fi
|
||||
|
||||
# ── Ensure HashiCorp apt sources list ────────────────────────────────────────
|
||||
desired_source="deb [signed-by=${HASHICORP_KEYRING}] ${HASHICORP_REPO_URL} ${CODENAME} main"
|
||||
if [ ! -f "$HASHICORP_SOURCES" ] \
|
||||
|| ! grep -qxF "$desired_source" "$HASHICORP_SOURCES"; then
|
||||
log "writing HashiCorp apt sources list → ${HASHICORP_SOURCES}"
|
||||
printf '%s\n' "$desired_source" > "$HASHICORP_SOURCES"
|
||||
apt_update_needed=1
|
||||
else
|
||||
log "HashiCorp apt sources list already present"
|
||||
apt_update_needed=0
|
||||
fi
|
||||
|
||||
# ── Install the pinned versions ──────────────────────────────────────────────
|
||||
if [ "$apt_update_needed" -eq 1 ]; then
|
||||
log "running apt-get update"
|
||||
DEBIAN_FRONTEND=noninteractive apt-get update -qq \
|
||||
|| die "apt-get update failed"
|
||||
fi
|
||||
|
||||
log "installing ${need_pkgs[*]}"
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
"${need_pkgs[@]}" \
|
||||
|| die "apt-get install ${need_pkgs[*]} failed"
|
||||
|
||||
# ── Verify ───────────────────────────────────────────────────────────────────
|
||||
final_nomad="$(_installed_version nomad)"
|
||||
if [ "$final_nomad" != "$NOMAD_VERSION" ]; then
|
||||
die "post-install check: expected nomad ${NOMAD_VERSION}, got '${final_nomad}'"
|
||||
fi
|
||||
final_vault="$(_installed_version vault)"
|
||||
if [ "$final_vault" != "$VAULT_VERSION" ]; then
|
||||
die "post-install check: expected vault ${VAULT_VERSION}, got '${final_vault}'"
|
||||
fi
|
||||
|
||||
log "nomad ${NOMAD_VERSION} + vault ${VAULT_VERSION} installed successfully"
|
||||
77
lib/init/nomad/lib-systemd.sh
Normal file
77
lib/init/nomad/lib-systemd.sh
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# lib/init/nomad/lib-systemd.sh — Shared idempotent systemd-unit installer
|
||||
#
|
||||
# Sourced by lib/init/nomad/systemd-nomad.sh and lib/init/nomad/systemd-vault.sh
|
||||
# (and any future sibling) to collapse the "write unit if content differs,
|
||||
# daemon-reload, enable (never start)" boilerplate.
|
||||
#
|
||||
# Install-but-don't-start is the invariant this helper enforces — mid-migration
|
||||
# installers land files and enable units; the orchestrator (S0.4) starts them.
|
||||
#
|
||||
# Public API (sourced into caller scope):
|
||||
#
|
||||
# systemd_require_preconditions UNIT_PATH
|
||||
# Asserts the caller is uid 0 and `systemctl` is on $PATH. Calls the
|
||||
# caller's die() with a UNIT_PATH-scoped message on failure.
|
||||
#
|
||||
# systemd_install_unit UNIT_PATH UNIT_NAME UNIT_CONTENT
|
||||
# Writes UNIT_CONTENT to UNIT_PATH (0644 root:root) only if on-disk
|
||||
# content differs. If written, runs `systemctl daemon-reload`. Then
|
||||
# enables UNIT_NAME (no-op if already enabled). Never starts the unit.
|
||||
#
|
||||
# Caller contract:
|
||||
# - Callers MUST define `log()` and `die()` before sourcing this file (we
|
||||
# call log() for status chatter and rely on the caller's error-handling
|
||||
# stance; `set -e` propagates install/cmp/systemctl failures).
|
||||
# =============================================================================
|
||||
|
||||
# systemd_require_preconditions UNIT_PATH
|
||||
systemd_require_preconditions() {
|
||||
local unit_path="$1"
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
die "must run as root (needs write access to ${unit_path})"
|
||||
fi
|
||||
command -v systemctl >/dev/null 2>&1 \
|
||||
|| die "systemctl not found (systemd is required)"
|
||||
}
|
||||
|
||||
# systemd_install_unit UNIT_PATH UNIT_NAME UNIT_CONTENT
|
||||
systemd_install_unit() {
|
||||
local unit_path="$1"
|
||||
local unit_name="$2"
|
||||
local unit_content="$3"
|
||||
|
||||
local needs_reload=0
|
||||
if [ ! -f "$unit_path" ] \
|
||||
|| ! printf '%s\n' "$unit_content" | cmp -s - "$unit_path"; then
|
||||
log "writing unit → ${unit_path}"
|
||||
# Subshell-scoped EXIT trap guarantees the temp file is removed on
|
||||
# both success AND set-e-induced failure of `install`. A function-
|
||||
# scoped RETURN trap does NOT fire on errexit-abort (bash only runs
|
||||
# RETURN on normal function exit), so the subshell is the reliable
|
||||
# cleanup boundary. It's also isolated from the caller's EXIT trap.
|
||||
(
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
printf '%s\n' "$unit_content" > "$tmp"
|
||||
install -m 0644 -o root -g root "$tmp" "$unit_path"
|
||||
)
|
||||
needs_reload=1
|
||||
else
|
||||
log "unit file already up to date"
|
||||
fi
|
||||
|
||||
if [ "$needs_reload" -eq 1 ]; then
|
||||
log "systemctl daemon-reload"
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
|
||||
if systemctl is-enabled --quiet "$unit_name" 2>/dev/null; then
|
||||
log "${unit_name} already enabled"
|
||||
else
|
||||
log "systemctl enable ${unit_name}"
|
||||
systemctl enable "$unit_name" >/dev/null
|
||||
fi
|
||||
}
|
||||
102
lib/init/nomad/systemd-nomad.sh
Executable file
102
lib/init/nomad/systemd-nomad.sh
Executable file
|
|
@ -0,0 +1,102 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# lib/init/nomad/systemd-nomad.sh — Idempotent systemd unit installer for Nomad
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.2, issue #822). Writes
|
||||
# /etc/systemd/system/nomad.service pointing at /etc/nomad.d/ and runs
|
||||
# `systemctl enable nomad` WITHOUT starting the service — we don't launch
|
||||
# the cluster until S0.4 wires everything together.
|
||||
#
|
||||
# Idempotency contract:
|
||||
# - Existing unit file is NOT rewritten when on-disk content already
|
||||
# matches the desired content (avoids spurious `daemon-reload`).
|
||||
# - `systemctl enable` on an already-enabled unit is a no-op.
|
||||
# - This script is safe to run unconditionally before every factory boot.
|
||||
#
|
||||
# Preconditions:
|
||||
# - nomad binary installed (see lib/init/nomad/install.sh)
|
||||
# - /etc/nomad.d/ will hold server.hcl / client.hcl (placed by S0.4)
|
||||
#
|
||||
# Usage:
|
||||
# sudo lib/init/nomad/systemd-nomad.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (unit installed + enabled, or already so)
|
||||
# 1 precondition failure (not root, no systemctl, no nomad binary)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
UNIT_PATH="/etc/systemd/system/nomad.service"
|
||||
NOMAD_CONFIG_DIR="/etc/nomad.d"
|
||||
NOMAD_DATA_DIR="/var/lib/nomad"
|
||||
|
||||
log() { printf '[systemd-nomad] %s\n' "$*"; }
|
||||
die() { printf '[systemd-nomad] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# shellcheck source=lib-systemd.sh
|
||||
. "$(dirname "${BASH_SOURCE[0]}")/lib-systemd.sh"
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
systemd_require_preconditions "$UNIT_PATH"
|
||||
|
||||
NOMAD_BIN="$(command -v nomad 2>/dev/null || true)"
|
||||
[ -n "$NOMAD_BIN" ] \
|
||||
|| die "nomad binary not found — run lib/init/nomad/install.sh first"
|
||||
|
||||
# ── Desired unit content ─────────────────────────────────────────────────────
|
||||
# Upstream-recommended baseline (https://developer.hashicorp.com/nomad/docs/install/production/deployment-guide)
|
||||
# trimmed for a single-node combined server+client dev box.
|
||||
# - Wants=/After= network-online: nomad must have networking up.
|
||||
# - User/Group=root: the Docker driver needs root to talk to dockerd.
|
||||
# - LimitNOFILE/LimitNPROC=infinity: avoid Nomad's startup warning.
|
||||
# - KillSignal=SIGINT: triggers Nomad's graceful shutdown path.
|
||||
# - Restart=on-failure with a bounded burst to avoid crash-loops eating the
|
||||
# journal when /etc/nomad.d/ is mis-configured.
|
||||
read -r -d '' DESIRED_UNIT <<EOF || true
|
||||
[Unit]
|
||||
Description=Nomad
|
||||
Documentation=https://developer.hashicorp.com/nomad/docs
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
# When Docker is present, ensure dockerd is up before nomad starts — the
|
||||
# Docker task driver needs the daemon socket available at startup.
|
||||
Wants=docker.service
|
||||
After=docker.service
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
User=root
|
||||
Group=root
|
||||
ExecReload=/bin/kill -HUP \$MAINPID
|
||||
ExecStart=${NOMAD_BIN} agent -config=${NOMAD_CONFIG_DIR}
|
||||
KillMode=process
|
||||
KillSignal=SIGINT
|
||||
LimitNOFILE=infinity
|
||||
LimitNPROC=infinity
|
||||
Restart=on-failure
|
||||
RestartSec=2
|
||||
StartLimitBurst=3
|
||||
StartLimitIntervalSec=10
|
||||
TasksMax=infinity
|
||||
OOMScoreAdjust=-1000
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
# ── Ensure config + data dirs exist ──────────────────────────────────────────
|
||||
# We do not populate /etc/nomad.d/ here (that's S0.4). We do create the
|
||||
# directory so `nomad agent -config=/etc/nomad.d` doesn't error if the unit
|
||||
# is started before hcl files are dropped in.
|
||||
for d in "$NOMAD_CONFIG_DIR" "$NOMAD_DATA_DIR"; do
|
||||
if [ ! -d "$d" ]; then
|
||||
log "creating ${d}"
|
||||
install -d -m 0755 "$d"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Install + reload + enable (shared with systemd-vault.sh via lib-systemd) ─
|
||||
systemd_install_unit "$UNIT_PATH" "nomad.service" "$DESIRED_UNIT"
|
||||
|
||||
log "done — unit installed and enabled (NOT started; S0.4 brings the cluster up)"
|
||||
151
lib/init/nomad/systemd-vault.sh
Executable file
151
lib/init/nomad/systemd-vault.sh
Executable file
|
|
@ -0,0 +1,151 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# lib/init/nomad/systemd-vault.sh — Idempotent systemd unit installer for Vault
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.3, issue #823). Lands three things:
|
||||
# 1. /etc/vault.d/ (0755 root:root)
|
||||
# 2. /etc/vault.d/vault.hcl (copy of nomad/vault.hcl, 0644 root:root)
|
||||
# 3. /var/lib/vault/data/ (0700 root:root, Vault file-storage backend)
|
||||
# 4. /etc/systemd/system/vault.service (0644 root:root)
|
||||
#
|
||||
# Then `systemctl enable vault` WITHOUT starting the service. Bootstrap
|
||||
# order is:
|
||||
# lib/init/nomad/install.sh (nomad + vault binaries)
|
||||
# lib/init/nomad/systemd-vault.sh (this script — unit + config + dirs)
|
||||
# lib/init/nomad/vault-init.sh (init + write unseal.key + unseal once)
|
||||
# systemctl start vault (ExecStartPost auto-unseals from file)
|
||||
#
|
||||
# The systemd unit's ExecStartPost reads /etc/vault.d/unseal.key and calls
|
||||
# `vault operator unseal`. That file is written by vault-init.sh on first
|
||||
# run; until it exists, `systemctl start vault` will leave Vault sealed
|
||||
# (ExecStartPost fails, unit goes into failed state — intentional, visible).
|
||||
#
|
||||
# Seal model:
|
||||
# The single unseal key lives at /etc/vault.d/unseal.key (0400 root).
|
||||
# Seal-key theft == vault theft. Factory-dev-box-acceptable tradeoff —
|
||||
# we avoid running a second Vault to auto-unseal the first.
|
||||
#
|
||||
# Idempotency contract:
|
||||
# - Unit file NOT rewritten when on-disk content already matches desired.
|
||||
# - vault.hcl NOT rewritten when on-disk content matches the repo copy.
|
||||
# - `systemctl enable` on an already-enabled unit is a no-op.
|
||||
# - Safe to run unconditionally before every factory boot.
|
||||
#
|
||||
# Preconditions:
|
||||
# - vault binary installed (lib/init/nomad/install.sh)
|
||||
# - nomad/vault.hcl present in the repo (relative to this script)
|
||||
#
|
||||
# Usage:
|
||||
# sudo lib/init/nomad/systemd-vault.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (unit+config installed + enabled, or already so)
|
||||
# 1 precondition failure (not root, no systemctl, no vault binary,
|
||||
# missing source config)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
UNIT_PATH="/etc/systemd/system/vault.service"
|
||||
VAULT_CONFIG_DIR="/etc/vault.d"
|
||||
VAULT_CONFIG_FILE="${VAULT_CONFIG_DIR}/vault.hcl"
|
||||
VAULT_DATA_DIR="/var/lib/vault/data"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
VAULT_HCL_SRC="${REPO_ROOT}/nomad/vault.hcl"
|
||||
|
||||
log() { printf '[systemd-vault] %s\n' "$*"; }
|
||||
die() { printf '[systemd-vault] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# shellcheck source=lib-systemd.sh
|
||||
. "${SCRIPT_DIR}/lib-systemd.sh"
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
systemd_require_preconditions "$UNIT_PATH"
|
||||
|
||||
VAULT_BIN="$(command -v vault 2>/dev/null || true)"
|
||||
[ -n "$VAULT_BIN" ] \
|
||||
|| die "vault binary not found — run lib/init/nomad/install.sh first"
|
||||
|
||||
[ -f "$VAULT_HCL_SRC" ] \
|
||||
|| die "source config not found: ${VAULT_HCL_SRC}"
|
||||
|
||||
# ── Desired unit content ─────────────────────────────────────────────────────
|
||||
# Adapted from HashiCorp's recommended vault.service template
|
||||
# (https://developer.hashicorp.com/vault/tutorials/getting-started-deploy/deploy)
|
||||
# for a single-node factory dev box:
|
||||
# - User=root keeps the seal-key read path simple (unseal.key is 0400 root).
|
||||
# - CAP_IPC_LOCK lets mlock() succeed so disable_mlock=false is honoured.
|
||||
# Harmless when running as root; required if this is ever flipped to a
|
||||
# dedicated `vault` user.
|
||||
# - ExecStartPost auto-unseals on every boot using the persisted key.
|
||||
# This is the dev-persisted-seal tradeoff — seal-key theft == vault
|
||||
# theft, but no second Vault to babysit.
|
||||
# - ConditionFileNotEmpty guards against starting without config — makes
|
||||
# a missing vault.hcl visible in systemctl status, not a crash loop.
|
||||
# - Type=notify so systemd waits for Vault's listener-ready notification
|
||||
# before running ExecStartPost (ExecStartPost also has `sleep 2` as a
|
||||
# belt-and-braces guard against Type=notify edge cases).
|
||||
# - \$MAINPID is escaped so bash doesn't expand it inside this heredoc.
|
||||
# - \$(cat ...) is escaped so the subshell runs at unit-execution time
|
||||
# (inside bash -c), not at heredoc-expansion time here.
|
||||
read -r -d '' DESIRED_UNIT <<EOF || true
|
||||
[Unit]
|
||||
Description=HashiCorp Vault
|
||||
Documentation=https://developer.hashicorp.com/vault/docs
|
||||
Requires=network-online.target
|
||||
After=network-online.target
|
||||
ConditionFileNotEmpty=${VAULT_CONFIG_FILE}
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
User=root
|
||||
Group=root
|
||||
Environment=VAULT_ADDR=http://127.0.0.1:8200
|
||||
SecureBits=keep-caps
|
||||
CapabilityBoundingSet=CAP_IPC_LOCK
|
||||
AmbientCapabilities=CAP_IPC_LOCK
|
||||
ExecStart=${VAULT_BIN} server -config=${VAULT_CONFIG_FILE}
|
||||
ExecStartPost=/bin/bash -c 'sleep 2 && ${VAULT_BIN} operator unseal \$(cat ${VAULT_CONFIG_DIR}/unseal.key)'
|
||||
ExecReload=/bin/kill --signal HUP \$MAINPID
|
||||
KillMode=process
|
||||
KillSignal=SIGINT
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=30
|
||||
LimitNOFILE=65536
|
||||
LimitMEMLOCK=infinity
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
# ── Ensure config + data dirs exist ──────────────────────────────────────────
|
||||
# /etc/vault.d is 0755 — vault.hcl is world-readable (no secrets in it);
|
||||
# the real secrets (unseal.key, root.token) get their own 0400 mode.
|
||||
# /var/lib/vault/data is 0700 — vault's on-disk state (encrypted-at-rest
|
||||
# by Vault itself, but an extra layer of "don't rely on that").
|
||||
if [ ! -d "$VAULT_CONFIG_DIR" ]; then
|
||||
log "creating ${VAULT_CONFIG_DIR}"
|
||||
install -d -m 0755 -o root -g root "$VAULT_CONFIG_DIR"
|
||||
fi
|
||||
if [ ! -d "$VAULT_DATA_DIR" ]; then
|
||||
log "creating ${VAULT_DATA_DIR}"
|
||||
install -d -m 0700 -o root -g root "$VAULT_DATA_DIR"
|
||||
fi
|
||||
|
||||
# ── Install vault.hcl only if content differs ────────────────────────────────
|
||||
if [ ! -f "$VAULT_CONFIG_FILE" ] \
|
||||
|| ! cmp -s "$VAULT_HCL_SRC" "$VAULT_CONFIG_FILE"; then
|
||||
log "writing config → ${VAULT_CONFIG_FILE}"
|
||||
install -m 0644 -o root -g root "$VAULT_HCL_SRC" "$VAULT_CONFIG_FILE"
|
||||
else
|
||||
log "config already up to date"
|
||||
fi
|
||||
|
||||
# ── Install + reload + enable (shared with systemd-nomad.sh via lib-systemd) ─
|
||||
systemd_install_unit "$UNIT_PATH" "vault.service" "$DESIRED_UNIT"
|
||||
|
||||
log "done — unit+config installed and enabled (NOT started; vault-init.sh next)"
|
||||
206
lib/init/nomad/vault-init.sh
Executable file
206
lib/init/nomad/vault-init.sh
Executable file
|
|
@ -0,0 +1,206 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# lib/init/nomad/vault-init.sh — Idempotent Vault first-run initializer
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.3, issue #823). Initializes Vault
|
||||
# in dev-persisted-seal mode (single unseal key on disk) and unseals once.
|
||||
# On re-run, becomes a no-op — never re-initializes or rotates the key.
|
||||
#
|
||||
# What it does (first run):
|
||||
# 1. Ensures Vault is reachable at ${VAULT_ADDR} — spawns a temporary
|
||||
# `vault server -config=/etc/vault.d/vault.hcl` if not already up.
|
||||
# 2. Runs `vault operator init -key-shares=1 -key-threshold=1` and
|
||||
# captures the resulting unseal key + root token.
|
||||
# 3. Writes /etc/vault.d/unseal.key (0400 root, no trailing newline).
|
||||
# 4. Writes /etc/vault.d/root.token (0400 root, no trailing newline).
|
||||
# 5. Unseals Vault once in the current process.
|
||||
# 6. Shuts down the temporary server if we started one (so a subsequent
|
||||
# `systemctl start vault` doesn't conflict on port 8200).
|
||||
#
|
||||
# Idempotency contract:
|
||||
# - /etc/vault.d/unseal.key exists AND `vault status` reports
|
||||
# initialized=true → exit 0, no mutation, no re-init.
|
||||
# - Initialized-but-unseal.key-missing is a hard failure (can't recover
|
||||
# the key without the existing storage; user must restore from backup).
|
||||
#
|
||||
# Bootstrap order:
|
||||
# lib/init/nomad/install.sh (installs vault binary)
|
||||
# lib/init/nomad/systemd-vault.sh (lands unit + config + dirs; enables)
|
||||
# lib/init/nomad/vault-init.sh (this script — init + unseal once)
|
||||
# systemctl start vault (ExecStartPost auto-unseals henceforth)
|
||||
#
|
||||
# Seal model:
|
||||
# Single unseal key persisted on disk at /etc/vault.d/unseal.key. Seal-key
|
||||
# theft == vault theft. Factory-dev-box-acceptable tradeoff — we avoid
|
||||
# running a second Vault to auto-unseal the first.
|
||||
#
|
||||
# Environment:
|
||||
# VAULT_ADDR — Vault API address (default: http://127.0.0.1:8200).
|
||||
#
|
||||
# Usage:
|
||||
# sudo lib/init/nomad/vault-init.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 success (initialized + unsealed + keys persisted; or already done)
|
||||
# 1 precondition / operational failure
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
VAULT_CONFIG_FILE="/etc/vault.d/vault.hcl"
|
||||
UNSEAL_KEY_FILE="/etc/vault.d/unseal.key"
|
||||
ROOT_TOKEN_FILE="/etc/vault.d/root.token"
|
||||
VAULT_ADDR="${VAULT_ADDR:-http://127.0.0.1:8200}"
|
||||
export VAULT_ADDR
|
||||
|
||||
# Track whether we spawned a temporary vault (for cleanup).
|
||||
spawned_pid=""
|
||||
spawned_log=""
|
||||
|
||||
log() { printf '[vault-init] %s\n' "$*"; }
|
||||
die() { printf '[vault-init] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# ── Cleanup: stop the temporary server (if we started one) on any exit ───────
|
||||
# EXIT trap fires on success AND failure AND signals — so we never leak a
|
||||
# background vault process holding port 8200 after this script returns.
|
||||
cleanup() {
|
||||
if [ -n "$spawned_pid" ] && kill -0 "$spawned_pid" 2>/dev/null; then
|
||||
log "stopping temporary vault (pid=${spawned_pid})"
|
||||
kill "$spawned_pid" 2>/dev/null || true
|
||||
wait "$spawned_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "$spawned_log" ] && [ -f "$spawned_log" ]; then
|
||||
rm -f "$spawned_log"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
die "must run as root (needs to write 0400 files under /etc/vault.d)"
|
||||
fi
|
||||
|
||||
for bin in vault jq; do
|
||||
command -v "$bin" >/dev/null 2>&1 \
|
||||
|| die "required binary not found: ${bin}"
|
||||
done
|
||||
|
||||
[ -f "$VAULT_CONFIG_FILE" ] \
|
||||
|| die "config not found: ${VAULT_CONFIG_FILE} — run systemd-vault.sh first"
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# vault_reachable — true iff `vault status` can reach the server.
|
||||
# Exit codes from `vault status`:
|
||||
# 0 = reachable, initialized, unsealed
|
||||
# 2 = reachable, sealed (or uninitialized)
|
||||
# 1 = unreachable / other error
|
||||
# We treat 0 and 2 as "reachable". `|| status=$?` avoids set -e tripping
|
||||
# on the expected sealed-is-also-fine case.
|
||||
vault_reachable() {
|
||||
local status=0
|
||||
vault status -format=json >/dev/null 2>&1 || status=$?
|
||||
[ "$status" -eq 0 ] || [ "$status" -eq 2 ]
|
||||
}
|
||||
|
||||
# vault_initialized — echoes "true" / "false" / "" (empty on parse failure
|
||||
# or unreachable vault). Always returns 0 so that `x="$(vault_initialized)"`
|
||||
# is safe under `set -euo pipefail`.
|
||||
#
|
||||
# Key subtlety: `vault status` exits 2 when Vault is sealed OR uninitialized
|
||||
# — the exact state we need to *observe* on first run. Without the
|
||||
# `|| true` guard, pipefail + set -e inside a standalone assignment would
|
||||
# propagate that exit 2 to the outer script and abort before we ever call
|
||||
# `vault operator init`. We capture `vault status`'s output to a variable
|
||||
# first (pipefail-safe), then feed it to jq separately.
|
||||
vault_initialized() {
|
||||
local out=""
|
||||
out="$(vault status -format=json 2>/dev/null || true)"
|
||||
[ -n "$out" ] || { printf ''; return 0; }
|
||||
printf '%s' "$out" | jq -r '.initialized' 2>/dev/null || printf ''
|
||||
}
|
||||
|
||||
# write_secret_file PATH CONTENT
|
||||
# Write CONTENT to PATH atomically with 0400 root:root and no trailing
|
||||
# newline. mktemp+install keeps perms tight for the whole lifetime of
|
||||
# the file on disk — no 0644-then-chmod window.
|
||||
write_secret_file() {
|
||||
local path="$1" content="$2"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
printf '%s' "$content" > "$tmp"
|
||||
install -m 0400 -o root -g root "$tmp" "$path"
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
# ── Ensure vault is reachable ────────────────────────────────────────────────
|
||||
if ! vault_reachable; then
|
||||
log "vault not reachable at ${VAULT_ADDR} — starting temporary server"
|
||||
spawned_log="$(mktemp)"
|
||||
vault server -config="$VAULT_CONFIG_FILE" >"$spawned_log" 2>&1 &
|
||||
spawned_pid=$!
|
||||
|
||||
# Poll for readiness. Vault's API listener comes up before notify-ready
|
||||
# in Type=notify mode, but well inside a few seconds even on cold boots.
|
||||
ready=0
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
|
||||
if vault_reachable; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$ready" -ne 1 ]; then
|
||||
log "vault did not become reachable within 15s — server log follows:"
|
||||
if [ -f "$spawned_log" ]; then
|
||||
sed 's/^/[vault-server] /' "$spawned_log" >&2 || true
|
||||
fi
|
||||
die "failed to start temporary vault server"
|
||||
fi
|
||||
log "temporary vault ready (pid=${spawned_pid})"
|
||||
fi
|
||||
|
||||
# ── Idempotency gate ─────────────────────────────────────────────────────────
|
||||
initialized="$(vault_initialized)"
|
||||
|
||||
if [ "$initialized" = "true" ] && [ -f "$UNSEAL_KEY_FILE" ]; then
|
||||
log "vault already initialized and unseal.key present — no-op"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$initialized" = "true" ] && [ ! -f "$UNSEAL_KEY_FILE" ]; then
|
||||
die "vault is initialized but ${UNSEAL_KEY_FILE} is missing — cannot recover the unseal key; restore from backup or wipe ${VAULT_CONFIG_FILE%/*}/data and re-run"
|
||||
fi
|
||||
|
||||
if [ "$initialized" != "false" ]; then
|
||||
die "unexpected initialized state: '${initialized}' (expected 'true' or 'false')"
|
||||
fi
|
||||
|
||||
# ── Initialize ───────────────────────────────────────────────────────────────
|
||||
log "initializing vault (key-shares=1, key-threshold=1)"
|
||||
init_json="$(vault operator init \
|
||||
-key-shares=1 \
|
||||
-key-threshold=1 \
|
||||
-format=json)" \
|
||||
|| die "vault operator init failed"
|
||||
|
||||
unseal_key="$(printf '%s' "$init_json" | jq -er '.unseal_keys_b64[0]')" \
|
||||
|| die "failed to extract unseal key from init response"
|
||||
root_token="$(printf '%s' "$init_json" | jq -er '.root_token')" \
|
||||
|| die "failed to extract root token from init response"
|
||||
|
||||
# Best-effort scrub of init_json from the env (the captured key+token still
|
||||
# sit in the local vars above — there's no clean way to wipe bash memory).
|
||||
unset init_json
|
||||
|
||||
# ── Persist keys ─────────────────────────────────────────────────────────────
|
||||
log "writing ${UNSEAL_KEY_FILE} (0400 root)"
|
||||
write_secret_file "$UNSEAL_KEY_FILE" "$unseal_key"
|
||||
log "writing ${ROOT_TOKEN_FILE} (0400 root)"
|
||||
write_secret_file "$ROOT_TOKEN_FILE" "$root_token"
|
||||
|
||||
# ── Unseal in the current process ────────────────────────────────────────────
|
||||
log "unsealing vault"
|
||||
vault operator unseal "$unseal_key" >/dev/null \
|
||||
|| die "vault operator unseal failed"
|
||||
|
||||
log "done — vault initialized + unsealed + keys persisted"
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
#!/usr/bin/env bash
|
||||
# mirrors.sh — Push primary branch + tags to configured mirror remotes.
|
||||
# mirrors.sh — Mirror helpers: push to remotes + register pull mirrors via API.
|
||||
#
|
||||
# Usage: source lib/mirrors.sh; mirror_push
|
||||
# source lib/mirrors.sh; mirror_pull_register <clone_url> <owner> <repo_name> [interval]
|
||||
# Requires: PROJECT_REPO_ROOT, PRIMARY_BRANCH, MIRROR_* vars from load-project.sh
|
||||
# FORGE_API_BASE, FORGE_TOKEN for pull-mirror registration
|
||||
|
||||
# shellcheck disable=SC2154 # globals set by load-project.sh / calling script
|
||||
|
||||
|
|
@ -37,3 +39,73 @@ mirror_push() {
|
|||
log "mirror: pushed to ${name} (pid $!)"
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# mirror_pull_register — register a Forgejo pull mirror via the /repos/migrate API.
|
||||
#
|
||||
# Creates a new repo as a pull mirror of an external source. Works against
|
||||
# empty target repos (the repo is created by the API call itself).
|
||||
#
|
||||
# Usage:
|
||||
# mirror_pull_register <clone_url> <owner> <repo_name> [interval]
|
||||
#
|
||||
# Args:
|
||||
# clone_url — HTTPS URL of the source repo (e.g. https://codeberg.org/johba/disinto.git)
|
||||
# owner — Forgejo org or user that will own the mirror repo
|
||||
# repo_name — name of the new mirror repo on Forgejo
|
||||
# interval — sync interval (default: "8h0m0s"; Forgejo duration format)
|
||||
#
|
||||
# Requires:
|
||||
# FORGE_API_BASE, FORGE_TOKEN (from env.sh)
|
||||
#
|
||||
# Returns 0 on success, 1 on failure. Prints the new repo JSON to stdout.
|
||||
# ---------------------------------------------------------------------------
|
||||
mirror_pull_register() {
|
||||
local clone_url="$1"
|
||||
local owner="$2"
|
||||
local repo_name="$3"
|
||||
local interval="${4:-8h0m0s}"
|
||||
|
||||
if [ -z "${FORGE_API_BASE:-}" ] || [ -z "${FORGE_TOKEN:-}" ]; then
|
||||
echo "ERROR: FORGE_API_BASE and FORGE_TOKEN must be set" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -z "$clone_url" ] || [ -z "$owner" ] || [ -z "$repo_name" ]; then
|
||||
echo "Usage: mirror_pull_register <clone_url> <owner> <repo_name> [interval]" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local payload
|
||||
payload=$(jq -n \
|
||||
--arg clone_addr "$clone_url" \
|
||||
--arg repo_name "$repo_name" \
|
||||
--arg repo_owner "$owner" \
|
||||
--arg interval "$interval" \
|
||||
'{
|
||||
clone_addr: $clone_addr,
|
||||
repo_name: $repo_name,
|
||||
repo_owner: $repo_owner,
|
||||
mirror: true,
|
||||
mirror_interval: $interval,
|
||||
service: "git"
|
||||
}')
|
||||
|
||||
local http_code body
|
||||
body=$(curl -s -w "\n%{http_code}" -X POST \
|
||||
-H "Authorization: token ${FORGE_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${FORGE_API_BASE}/repos/migrate" \
|
||||
-d "$payload")
|
||||
|
||||
http_code=$(printf '%s' "$body" | tail -n1)
|
||||
body=$(printf '%s' "$body" | sed '$d')
|
||||
|
||||
if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
|
||||
printf '%s\n' "$body"
|
||||
return 0
|
||||
else
|
||||
echo "ERROR: mirror_pull_register failed (HTTP ${http_code}): ${body}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@
|
|||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# Source vault.sh for _vault_log helper
|
||||
source "${FACTORY_ROOT}/lib/vault.sh"
|
||||
# Source action-vault.sh for _vault_log helper
|
||||
source "${FACTORY_ROOT}/lib/action-vault.sh"
|
||||
|
||||
# Assert required globals are set before using this module.
|
||||
_assert_release_globals() {
|
||||
|
|
|
|||
92
nomad/AGENTS.md
Normal file
92
nomad/AGENTS.md
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
# nomad/ — Agent Instructions
|
||||
|
||||
Nomad + Vault HCL for the factory's single-node cluster. These files are
|
||||
the source of truth that `lib/init/nomad/cluster-up.sh` copies onto a
|
||||
factory box under `/etc/nomad.d/` and `/etc/vault.d/` at init time.
|
||||
|
||||
This directory is part of the **Nomad+Vault migration (Step 0)** —
|
||||
see issues #821–#825 for the step breakdown. Jobspecs land in Step 1.
|
||||
|
||||
## What lives here
|
||||
|
||||
| File | Deployed to | Owned by |
|
||||
|---|---|---|
|
||||
| `server.hcl` | `/etc/nomad.d/server.hcl` | agent role, bind, ports, `data_dir` (S0.2) |
|
||||
| `client.hcl` | `/etc/nomad.d/client.hcl` | Docker driver cfg + `host_volume` declarations (S0.2) |
|
||||
| `vault.hcl` | `/etc/vault.d/vault.hcl` | Vault storage, listener, UI, `disable_mlock` (S0.3) |
|
||||
|
||||
Nomad auto-merges every `*.hcl` under `-config=/etc/nomad.d/`, so the
|
||||
split between `server.hcl` and `client.hcl` is for readability, not
|
||||
semantics. The top-of-file header in each config documents which blocks
|
||||
it owns.
|
||||
|
||||
## What does NOT live here yet
|
||||
|
||||
- **Jobspecs.** Step 0 brings up an *empty* cluster. Step 1 (and later)
|
||||
adds `*.nomad.hcl` job files for forgejo, woodpecker, agents, caddy,
|
||||
etc. When that lands, jobspecs will live in `nomad/jobs/` and each
|
||||
will get its own header comment pointing to the `host_volume` names
|
||||
it consumes (`volume = "forgejo-data"`, etc. — declared in
|
||||
`client.hcl`).
|
||||
- **TLS, ACLs, gossip encryption.** Deliberately absent in Step 0 —
|
||||
factory traffic stays on localhost. These land in later migration
|
||||
steps alongside multi-node support.
|
||||
|
||||
## Adding a jobspec (Step 1 and later)
|
||||
|
||||
1. Drop a file in `nomad/jobs/<service>.nomad.hcl`.
|
||||
2. If it needs persistent state, reference a `host_volume` already
|
||||
declared in `client.hcl` — *don't* add ad-hoc host paths in the
|
||||
jobspec. If a new volume is needed, add it to **both**:
|
||||
- `nomad/client.hcl` — the `host_volume "<name>" { path = … }` block
|
||||
- `lib/init/nomad/cluster-up.sh` — the `HOST_VOLUME_DIRS` array
|
||||
The two must stay in sync or nomad fingerprinting will fail and the
|
||||
node stays in "initializing".
|
||||
3. Pin image tags — `image = "forgejo/forgejo:1.22.5"`, not `:latest`.
|
||||
4. Add the jobspec path to `.woodpecker/nomad-validate.yml`'s trigger
|
||||
list so CI validates it.
|
||||
|
||||
## How CI validates these files
|
||||
|
||||
`.woodpecker/nomad-validate.yml` runs on every PR that touches `nomad/`,
|
||||
`lib/init/nomad/`, or `bin/disinto`. Four fail-closed steps:
|
||||
|
||||
1. **`nomad config validate nomad/server.hcl nomad/client.hcl`**
|
||||
— parses the HCL, fails on unknown blocks, bad port ranges, invalid
|
||||
driver config. Vault HCL is excluded (different tool).
|
||||
2. **`vault operator diagnose -config=nomad/vault.hcl -skip=storage -skip=listener`**
|
||||
— Vault's equivalent syntax + schema check. `-skip=storage/listener`
|
||||
disables the runtime checks (CI containers don't have
|
||||
`/var/lib/vault/data` or port 8200).
|
||||
3. **`shellcheck --severity=warning lib/init/nomad/*.sh bin/disinto`**
|
||||
— all init/dispatcher shell clean. `bin/disinto` has no `.sh`
|
||||
extension so the repo-wide shellcheck in `.woodpecker/ci.yml` skips
|
||||
it — this is the one place it gets checked.
|
||||
4. **`bats tests/disinto-init-nomad.bats`**
|
||||
— exercises the dispatcher: `disinto init --backend=nomad --dry-run`,
|
||||
`… --empty --dry-run`, and the `--backend=docker` regression guard.
|
||||
|
||||
If a PR breaks `nomad/server.hcl` (e.g. typo in a block name), step 1
|
||||
fails with a clear error; the fix makes it pass. PRs that don't touch
|
||||
any of the trigger paths skip this pipeline entirely.
|
||||
|
||||
## Version pinning
|
||||
|
||||
Nomad + Vault versions are pinned in **two** places — bumping one
|
||||
without the other is a CI-caught drift:
|
||||
|
||||
- `lib/init/nomad/install.sh` — the apt-installed versions on factory
|
||||
boxes (`NOMAD_VERSION`, `VAULT_VERSION`).
|
||||
- `.woodpecker/nomad-validate.yml` — the `hashicorp/nomad:…` and
|
||||
`hashicorp/vault:…` image tags used for static validation.
|
||||
|
||||
Bump both in the same PR. The CI pipeline will fail if the pinned
|
||||
image's `config validate` rejects syntax the installed runtime would
|
||||
accept (or vice versa).
|
||||
|
||||
## Related
|
||||
|
||||
- `lib/init/nomad/` — installer + systemd units + cluster-up orchestrator.
|
||||
- `.woodpecker/nomad-validate.yml` — this directory's CI pipeline.
|
||||
- Top-of-file headers in `server.hcl` / `client.hcl` / `vault.hcl`
|
||||
document the per-file ownership contract.
|
||||
88
nomad/client.hcl
Normal file
88
nomad/client.hcl
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
# =============================================================================
|
||||
# nomad/client.hcl — Docker driver + host_volume declarations
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.2, issue #822). Deployed to
|
||||
# /etc/nomad.d/client.hcl on the factory dev box alongside server.hcl.
|
||||
#
|
||||
# This file owns: Docker driver plugin config + host_volume pre-wiring.
|
||||
# server.hcl owns: agent role, bind, ports, data_dir.
|
||||
#
|
||||
# NOTE: Nomad merges every *.hcl under -config=/etc/nomad.d, so declaring
|
||||
# a second `client { ... }` block here augments (not replaces) the one in
|
||||
# server.hcl. On a single-node setup this file could be inlined into
|
||||
# server.hcl — the split is for readability, not semantics.
|
||||
#
|
||||
# host_volume declarations let Nomad jobspecs mount factory state by name
|
||||
# (volume = "forgejo-data", etc.) without coupling host paths into jobspec
|
||||
# HCL. Host paths under /srv/disinto/* are created out-of-band by the
|
||||
# orchestrator (S0.4) before any job references them.
|
||||
# =============================================================================
|
||||
|
||||
client {
|
||||
# forgejo git server data (repos, avatars, attachments).
|
||||
host_volume "forgejo-data" {
|
||||
path = "/srv/disinto/forgejo-data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# woodpecker CI data (pipeline artifacts, sqlite db).
|
||||
host_volume "woodpecker-data" {
|
||||
path = "/srv/disinto/woodpecker-data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# agent runtime data (claude config, logs, phase files).
|
||||
host_volume "agent-data" {
|
||||
path = "/srv/disinto/agent-data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# per-project git clones and worktrees.
|
||||
host_volume "project-repos" {
|
||||
path = "/srv/disinto/project-repos"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# caddy config + ACME state.
|
||||
host_volume "caddy-data" {
|
||||
path = "/srv/disinto/caddy-data"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# disinto chat transcripts + attachments.
|
||||
host_volume "chat-history" {
|
||||
path = "/srv/disinto/chat-history"
|
||||
read_only = false
|
||||
}
|
||||
|
||||
# ops repo clone (vault actions, sprint artifacts, knowledge).
|
||||
host_volume "ops-repo" {
|
||||
path = "/srv/disinto/ops-repo"
|
||||
read_only = false
|
||||
}
|
||||
}
|
||||
|
||||
# Docker task driver. `volumes.enabled = true` is required so jobspecs
|
||||
# can mount host_volume declarations defined above. `allow_privileged`
|
||||
# stays false — no factory workload needs privileged containers today,
|
||||
# and flipping it is an audit-worthy change.
|
||||
plugin "docker" {
|
||||
config {
|
||||
allow_privileged = false
|
||||
|
||||
volumes {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
# Leave images behind when jobs stop, so short job churn doesn't thrash
|
||||
# the image cache. Factory disk is not constrained; `docker system prune`
|
||||
# is the escape hatch.
|
||||
gc {
|
||||
image = false
|
||||
container = true
|
||||
dangling_containers {
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
53
nomad/server.hcl
Normal file
53
nomad/server.hcl
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# =============================================================================
|
||||
# nomad/server.hcl — Single-node combined server+client configuration
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.2, issue #822). Deployed to
|
||||
# /etc/nomad.d/server.hcl on the factory dev box alongside client.hcl.
|
||||
#
|
||||
# This file owns: agent role, ports, bind, data directory.
|
||||
# client.hcl owns: Docker driver plugin config + host_volume declarations.
|
||||
#
|
||||
# NOTE: On single-node setups these two files could be merged into one
|
||||
# (Nomad auto-merges every *.hcl under -config=/etc/nomad.d). The split is
|
||||
# purely for readability — role/bind/port vs. plugin/volume wiring.
|
||||
#
|
||||
# This is a factory dev-box baseline — TLS, ACLs, gossip encryption, and
|
||||
# consul/vault integration are deliberately absent and land in later steps.
|
||||
# =============================================================================
|
||||
|
||||
data_dir = "/var/lib/nomad"
|
||||
bind_addr = "127.0.0.1"
|
||||
log_level = "INFO"
|
||||
|
||||
# All Nomad agent traffic stays on localhost — the factory box does not
|
||||
# federate with peers. Ports are the Nomad defaults, pinned here so that
|
||||
# future changes to these numbers are a visible diff.
|
||||
ports {
|
||||
http = 4646
|
||||
rpc = 4647
|
||||
serf = 4648
|
||||
}
|
||||
|
||||
# Single-node combined mode: this agent is both the only server and the
|
||||
# only client. bootstrap_expect=1 makes the server quorum-of-one.
|
||||
server {
|
||||
enabled = true
|
||||
bootstrap_expect = 1
|
||||
}
|
||||
|
||||
client {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
# Advertise localhost to self to avoid surprises if the default IP
|
||||
# autodetection picks a transient interface (e.g. docker0, wg0).
|
||||
advertise {
|
||||
http = "127.0.0.1"
|
||||
rpc = "127.0.0.1"
|
||||
serf = "127.0.0.1"
|
||||
}
|
||||
|
||||
# UI on by default — same bind as http, no TLS (localhost only).
|
||||
ui {
|
||||
enabled = true
|
||||
}
|
||||
41
nomad/vault.hcl
Normal file
41
nomad/vault.hcl
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# =============================================================================
|
||||
# nomad/vault.hcl — Single-node Vault configuration (dev-persisted seal)
|
||||
#
|
||||
# Part of the Nomad+Vault migration (S0.3, issue #823). Deployed to
|
||||
# /etc/vault.d/vault.hcl on the factory dev box.
|
||||
#
|
||||
# Seal model: the single unseal key lives on disk at /etc/vault.d/unseal.key
|
||||
# (0400 root) and is read by systemd ExecStartPost on every boot. This is
|
||||
# the factory-dev-box-acceptable tradeoff — seal-key theft equals vault
|
||||
# theft, but we avoid running a second Vault to auto-unseal the first.
|
||||
#
|
||||
# This is a factory dev-box baseline — TLS, HA, Raft storage, and audit
|
||||
# devices are deliberately absent. Storage is the `file` backend (single
|
||||
# node only). Listener is localhost-only, so no external TLS is needed.
|
||||
# =============================================================================
|
||||
|
||||
# File storage backend — single-node only, no HA, no raft. State lives in
|
||||
# /var/lib/vault/data which is created (root:root 0700) by
|
||||
# lib/init/nomad/systemd-vault.sh before the unit starts.
|
||||
storage "file" {
|
||||
path = "/var/lib/vault/data"
|
||||
}
|
||||
|
||||
# Localhost-only listener. TLS is disabled because all callers are on the
|
||||
# same box — flipping this to tls_disable=false is an audit-worthy change
|
||||
# paired with cert provisioning.
|
||||
listener "tcp" {
|
||||
address = "127.0.0.1:8200"
|
||||
tls_disable = true
|
||||
}
|
||||
|
||||
# mlock prevents Vault's in-memory secrets from being swapped to disk. We
|
||||
# keep it enabled; the systemd unit grants CAP_IPC_LOCK so mlock() succeeds.
|
||||
disable_mlock = false
|
||||
|
||||
# Advertised API address — used by Vault clients on this host. Matches
|
||||
# the listener above.
|
||||
api_addr = "http://127.0.0.1:8200"
|
||||
|
||||
# UI on by default — same bind as listener, no TLS (localhost only).
|
||||
ui = true
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Planner Agent
|
||||
|
||||
**Role**: Strategic planning using a Prerequisite Tree (Theory of Constraints),
|
||||
|
|
@ -34,7 +34,9 @@ will then sections) and marks the prerequisite as blocked-on-vault in the tree.
|
|||
Deduplication: checks pending/ + approved/ + fired/ before creating.
|
||||
Phase 4 (journal-and-memory): write updated prerequisite tree + daily journal
|
||||
entry (committed to ops repo) and update `$OPS_REPO_ROOT/knowledge/planner-memory.md`.
|
||||
Phase 5 (commit-ops): commit all ops repo changes, push directly.
|
||||
Phase 5 (commit-ops): commit all ops repo changes to a `planner/run-YYYY-MM-DD`
|
||||
branch, then create a PR and walk it to merge via review-bot (`pr_create` →
|
||||
`pr_walk_to_merge`), mirroring the architect's ops flow. No direct push to main.
|
||||
AGENTS.md maintenance is handled by the Gardener.
|
||||
|
||||
**Artifacts use `$OPS_REPO_ROOT`**: All planner artifacts (journal,
|
||||
|
|
@ -55,7 +57,7 @@ nervous system component, not work.
|
|||
creates tmux session, injects formula prompt, monitors phase file, handles crash recovery, cleans up
|
||||
- `formulas/run-planner.toml` — Execution spec: six steps (preflight,
|
||||
prediction-triage, update-prerequisite-tree, file-at-constraints,
|
||||
journal-and-memory, commit-and-pr) with `needs` dependencies. Claude
|
||||
journal-and-memory, commit-ops-changes) with `needs` dependencies. Claude
|
||||
executes all steps in a single interactive session with tool access
|
||||
- `formulas/groom-backlog.toml` — Grooming formula for backlog triage and
|
||||
grooming. (Note: the planner no longer dispatches breakdown mode — complex
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@
|
|||
# 2. Load formula (formulas/run-planner.toml)
|
||||
# 3. Context: VISION.md, AGENTS.md, ops:RESOURCES.md, structural graph,
|
||||
# planner memory, journal entries
|
||||
# 4. agent_run(worktree, prompt) → Claude plans, may push knowledge updates
|
||||
# 4. Create ops branch planner/run-YYYY-MM-DD for changes
|
||||
# 5. agent_run(worktree, prompt) → Claude plans, commits to ops branch
|
||||
# 6. If ops branch has commits: pr_create → pr_walk_to_merge (review-bot)
|
||||
#
|
||||
# Usage:
|
||||
# planner-run.sh [projects/disinto.toml] # project config (default: disinto)
|
||||
|
|
@ -35,6 +37,10 @@ source "$FACTORY_ROOT/lib/worktree.sh"
|
|||
source "$FACTORY_ROOT/lib/guard.sh"
|
||||
# shellcheck source=../lib/agent-sdk.sh
|
||||
source "$FACTORY_ROOT/lib/agent-sdk.sh"
|
||||
# shellcheck source=../lib/ci-helpers.sh
|
||||
source "$FACTORY_ROOT/lib/ci-helpers.sh"
|
||||
# shellcheck source=../lib/pr-lifecycle.sh
|
||||
source "$FACTORY_ROOT/lib/pr-lifecycle.sh"
|
||||
|
||||
LOG_FILE="${DISINTO_LOG_DIR}/planner/planner.log"
|
||||
# shellcheck disable=SC2034 # consumed by agent-sdk.sh
|
||||
|
|
@ -146,12 +152,69 @@ ${PROMPT_FOOTER}"
|
|||
# ── Create worktree ──────────────────────────────────────────────────────
|
||||
formula_worktree_setup "$WORKTREE"
|
||||
|
||||
# ── Prepare ops branch for PR-based merge (#765) ────────────────────────
|
||||
PLANNER_OPS_BRANCH="planner/run-$(date -u +%Y-%m-%d)"
|
||||
(
|
||||
cd "$OPS_REPO_ROOT"
|
||||
git fetch origin "${PRIMARY_BRANCH}" --quiet 2>/dev/null || true
|
||||
git checkout "${PRIMARY_BRANCH}" --quiet 2>/dev/null || true
|
||||
git pull --ff-only origin "${PRIMARY_BRANCH}" --quiet 2>/dev/null || true
|
||||
# Create (or reset to) a fresh branch from PRIMARY_BRANCH
|
||||
git checkout -B "$PLANNER_OPS_BRANCH" "origin/${PRIMARY_BRANCH}" --quiet 2>/dev/null || \
|
||||
git checkout -b "$PLANNER_OPS_BRANCH" --quiet 2>/dev/null || true
|
||||
)
|
||||
log "ops branch: ${PLANNER_OPS_BRANCH}"
|
||||
|
||||
# ── Run agent ─────────────────────────────────────────────────────────────
|
||||
export CLAUDE_MODEL="opus"
|
||||
|
||||
agent_run --worktree "$WORKTREE" "$PROMPT"
|
||||
log "agent_run complete"
|
||||
|
||||
# ── PR lifecycle: create PR on ops repo and walk to merge (#765) ─────────
|
||||
OPS_FORGE_API="${FORGE_API_BASE}/repos/${FORGE_OPS_REPO}"
|
||||
ops_has_commits=false
|
||||
if ! git -C "$OPS_REPO_ROOT" diff --quiet "origin/${PRIMARY_BRANCH}..${PLANNER_OPS_BRANCH}" 2>/dev/null; then
|
||||
ops_has_commits=true
|
||||
fi
|
||||
|
||||
if [ "$ops_has_commits" = "true" ]; then
|
||||
log "ops branch has commits — creating PR"
|
||||
# Push the branch to the ops remote
|
||||
git -C "$OPS_REPO_ROOT" push origin "$PLANNER_OPS_BRANCH" --quiet 2>/dev/null || \
|
||||
git -C "$OPS_REPO_ROOT" push --force-with-lease origin "$PLANNER_OPS_BRANCH" 2>/dev/null
|
||||
|
||||
# Temporarily point FORGE_API at the ops repo for pr-lifecycle functions
|
||||
ORIG_FORGE_API="$FORGE_API"
|
||||
export FORGE_API="$OPS_FORGE_API"
|
||||
# Ops repo typically has no Woodpecker CI — skip CI polling
|
||||
ORIG_WOODPECKER_REPO_ID="${WOODPECKER_REPO_ID:-2}"
|
||||
export WOODPECKER_REPO_ID="0"
|
||||
|
||||
PR_NUM=$(pr_create "$PLANNER_OPS_BRANCH" \
|
||||
"chore: planner run $(date -u +%Y-%m-%d)" \
|
||||
"Automated planner run — updates prerequisite tree, memory, and vault items." \
|
||||
"${PRIMARY_BRANCH}" \
|
||||
"$OPS_FORGE_API") || true
|
||||
|
||||
if [ -n "$PR_NUM" ]; then
|
||||
log "ops PR #${PR_NUM} created — walking to merge"
|
||||
SESSION_ID=$(cat "$SID_FILE" 2>/dev/null || echo "planner-$$")
|
||||
pr_walk_to_merge "$PR_NUM" "$SESSION_ID" "$OPS_REPO_ROOT" 1 2 || {
|
||||
log "ops PR #${PR_NUM} walk finished: ${_PR_WALK_EXIT_REASON:-unknown}"
|
||||
}
|
||||
log "ops PR #${PR_NUM} result: ${_PR_WALK_EXIT_REASON:-unknown}"
|
||||
else
|
||||
log "WARNING: failed to create ops PR for branch ${PLANNER_OPS_BRANCH}"
|
||||
fi
|
||||
|
||||
# Restore original FORGE_API
|
||||
export FORGE_API="$ORIG_FORGE_API"
|
||||
export WOODPECKER_REPO_ID="$ORIG_WOODPECKER_REPO_ID"
|
||||
else
|
||||
log "no ops changes — skipping PR creation"
|
||||
fi
|
||||
|
||||
# Persist watermarks so next run can skip if nothing changed
|
||||
mkdir -p "$FACTORY_ROOT/state"
|
||||
echo "$CURRENT_SHA" > "$LAST_SHA_FILE"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Predictor Agent
|
||||
|
||||
**Role**: Abstract adversary (the "goblin"). Runs a 2-step formula
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Review Agent
|
||||
|
||||
**Role**: AI-powered PR review — post structured findings and formal
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<!-- last-reviewed: cdb0408466d875cdd893c712e3b3f0d0b909d4ef -->
|
||||
<!-- last-reviewed: c363ee0aea2ae447daab28c2c850d6abefc8c6b5 -->
|
||||
# Supervisor Agent
|
||||
|
||||
**Role**: Health monitoring and auto-remediation, executed as a formula-driven
|
||||
|
|
@ -7,13 +7,11 @@ then runs an interactive Claude session (sonnet) that assesses health, auto-fixe
|
|||
issues, and writes a daily journal. When blocked on external
|
||||
resources or human decisions, files vault items instead of escalating directly.
|
||||
|
||||
**Trigger**: `supervisor-run.sh` is invoked by the polling loop in `docker/edge/entrypoint-edge.sh`
|
||||
every 20 minutes (line 50-53). Sources `lib/guard.sh` and calls `check_active supervisor` first
|
||||
— skips if `$FACTORY_ROOT/state/.supervisor-active` is absent. Then runs `claude -p` via
|
||||
`agent-sdk.sh`, injects `formulas/run-supervisor.toml` with pre-collected metrics as context,
|
||||
and cleans up on completion or timeout (20 min max session). Note: the supervisor runs in the
|
||||
**edge container** (`entrypoint-edge.sh`), not the agent container — this distinction matters
|
||||
for operators debugging the factory.
|
||||
**Trigger**: `supervisor-run.sh` is invoked by two polling loops:
|
||||
- **Agents container** (`docker/agents/entrypoint.sh`): every `SUPERVISOR_INTERVAL` seconds (default 1200 = 20 min). Controlled by the `supervisor` role in `AGENT_ROLES` (included in the default seven-role set since P1/#801). Logs to `supervisor.log` in the agents container.
|
||||
- **Edge container** (`docker/edge/entrypoint-edge.sh`): separate loop in the edge container (line 169-172). Runs independently of the agents container's polling schedule.
|
||||
|
||||
Both invoke the same `supervisor-run.sh`. Sources `lib/guard.sh` and calls `check_active supervisor` first — skips if `$FACTORY_ROOT/state/.supervisor-active` is absent. Then runs `claude -p` via `agent-sdk.sh`, injects `formulas/run-supervisor.toml` with pre-collected metrics as context, and cleans up on completion or timeout.
|
||||
|
||||
**Key files**:
|
||||
- `supervisor/supervisor-run.sh` — Polling loop participant + orchestrator: lock, memory guard,
|
||||
|
|
@ -39,6 +37,7 @@ P3 (degraded PRs, circular deps, stale deps), P4 (housekeeping).
|
|||
**Environment variables consumed**:
|
||||
- `FORGE_TOKEN`, `FORGE_SUPERVISOR_TOKEN` (falls back to FORGE_TOKEN), `FORGE_REPO`, `FORGE_API`, `PROJECT_NAME`, `PROJECT_REPO_ROOT`, `OPS_REPO_ROOT`
|
||||
- `PRIMARY_BRANCH`, `CLAUDE_MODEL` (set to sonnet by supervisor-run.sh)
|
||||
- `SUPERVISOR_INTERVAL` — polling interval in seconds for agents container (default 1200 = 20 min)
|
||||
- `WOODPECKER_TOKEN`, `WOODPECKER_SERVER`, `WOODPECKER_DB_PASSWORD`, `WOODPECKER_DB_USER`, `WOODPECKER_DB_HOST`, `WOODPECKER_DB_NAME` — CI database queries
|
||||
|
||||
**Degraded mode (Issue #544)**: When `OPS_REPO_ROOT` is not set or the directory doesn't exist, the supervisor runs in degraded mode:
|
||||
|
|
|
|||
106
tests/disinto-init-nomad.bats
Normal file
106
tests/disinto-init-nomad.bats
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env bats
|
||||
# =============================================================================
|
||||
# tests/disinto-init-nomad.bats — Regression guard for `disinto init`
|
||||
# backend dispatch (S0.5, issue #825).
|
||||
#
|
||||
# Exercises the three CLI paths the Nomad+Vault migration cares about:
|
||||
# 1. --backend=nomad --dry-run → cluster-up step list
|
||||
# 2. --backend=nomad --empty --dry-run → same, with "--empty" banner
|
||||
# 3. --backend=docker --dry-run → docker path unaffected
|
||||
#
|
||||
# A throw-away `placeholder/repo` slug satisfies the CLI's positional-arg
|
||||
# requirement (the nomad dispatcher never touches it). --dry-run on both
|
||||
# backends short-circuits before any network/filesystem mutation, so the
|
||||
# suite is hermetic — no Forgejo, no sudo, no real cluster.
|
||||
# =============================================================================
|
||||
|
||||
setup_file() {
|
||||
export DISINTO_ROOT
|
||||
DISINTO_ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
||||
export DISINTO_BIN="${DISINTO_ROOT}/bin/disinto"
|
||||
[ -x "$DISINTO_BIN" ] || {
|
||||
echo "disinto binary not executable: $DISINTO_BIN" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
# ── --backend=nomad --dry-run ────────────────────────────────────────────────
|
||||
|
||||
@test "disinto init --backend=nomad --dry-run exits 0 and prints the step list" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
# Dispatcher banner (cluster-up mode, no --empty).
|
||||
[[ "$output" == *"nomad backend: default (cluster-up; jobs deferred to Step 1)"* ]]
|
||||
|
||||
# All nine cluster-up dry-run steps, in order.
|
||||
[[ "$output" == *"[dry-run] Step 1/9: install nomad + vault binaries"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 2/9: write + enable nomad.service (NOT started)"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 3/9: write + enable vault.service + vault.hcl (NOT started)"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 4/9: create host-volume dirs under /srv/disinto/"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 5/9: install /etc/nomad.d/server.hcl + client.hcl from repo"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 6/9: first-run vault init + persist unseal.key + root.token"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 7/9: systemctl start vault + poll until unsealed"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 8/9: systemctl start nomad + poll until ≥1 node ready"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 9/9: write /etc/profile.d/disinto-nomad.sh"* ]]
|
||||
|
||||
[[ "$output" == *"Dry run complete — no changes made."* ]]
|
||||
}
|
||||
|
||||
# ── --backend=nomad --empty --dry-run ────────────────────────────────────────
|
||||
|
||||
@test "disinto init --backend=nomad --empty --dry-run prints the --empty banner + step list" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=nomad --empty --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
# --empty changes the dispatcher banner but not the step list — Step 1
|
||||
# of the migration will branch on $empty to gate job deployment; today
|
||||
# both modes invoke the same cluster-up dry-run.
|
||||
[[ "$output" == *"nomad backend: --empty (cluster-up only, no jobs)"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 1/9: install nomad + vault binaries"* ]]
|
||||
[[ "$output" == *"Dry run complete — no changes made."* ]]
|
||||
}
|
||||
|
||||
# ── --backend=docker (regression guard) ──────────────────────────────────────
|
||||
|
||||
@test "disinto init --backend=docker does NOT dispatch to the nomad path" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=docker --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
# Negative assertion: the nomad dispatcher banners must be absent.
|
||||
[[ "$output" != *"nomad backend:"* ]]
|
||||
[[ "$output" != *"[dry-run] Step 1/9: install nomad + vault binaries"* ]]
|
||||
|
||||
# Positive assertion: docker-path output still appears — the existing
|
||||
# docker dry-run printed "=== disinto init ===" before listing the
|
||||
# intended forge/compose actions.
|
||||
[[ "$output" == *"=== disinto init ==="* ]]
|
||||
[[ "$output" == *"── Dry-run: intended actions ────"* ]]
|
||||
}
|
||||
|
||||
# ── Flag syntax: --flag=value vs --flag value ────────────────────────────────
|
||||
|
||||
# Both forms must work. The bin/disinto flag loop has separate cases for
|
||||
# `--backend value` and `--backend=value`; a regression in either would
|
||||
# silently route to the docker default, which is the worst failure mode
|
||||
# for a mid-migration dispatcher ("loud-failing stub" lesson from S0.4).
|
||||
@test "disinto init --backend nomad (space-separated) dispatches to nomad" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend nomad --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"nomad backend: default"* ]]
|
||||
[[ "$output" == *"[dry-run] Step 1/9: install nomad + vault binaries"* ]]
|
||||
}
|
||||
|
||||
# ── Flag validation ──────────────────────────────────────────────────────────
|
||||
|
||||
@test "--backend=bogus is rejected with a clear error" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=bogus --dry-run
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"invalid --backend value"* ]]
|
||||
}
|
||||
|
||||
@test "--empty without --backend=nomad is rejected" {
|
||||
run "$DISINTO_BIN" init placeholder/repo --backend=docker --empty --dry-run
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"--empty is only valid with --backend=nomad"* ]]
|
||||
}
|
||||
215
tests/lib-hvault.bats
Normal file
215
tests/lib-hvault.bats
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
#!/usr/bin/env bats
|
||||
# tests/lib-hvault.bats — Unit tests for lib/hvault.sh
|
||||
#
|
||||
# Runs against a dev-mode Vault server (single binary, no LXC needed).
|
||||
# CI launches vault server -dev inline before running these tests.
|
||||
|
||||
VAULT_BIN="${VAULT_BIN:-vault}"
|
||||
|
||||
setup_file() {
|
||||
export TEST_DIR
|
||||
TEST_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
||||
|
||||
# Start dev-mode vault on a random port
|
||||
export VAULT_DEV_PORT
|
||||
VAULT_DEV_PORT="$(shuf -i 18200-18299 -n 1)"
|
||||
export VAULT_ADDR="http://127.0.0.1:${VAULT_DEV_PORT}"
|
||||
|
||||
"$VAULT_BIN" server -dev \
|
||||
-dev-listen-address="127.0.0.1:${VAULT_DEV_PORT}" \
|
||||
-dev-root-token-id="test-root-token" \
|
||||
-dev-no-store-token \
|
||||
&>"${BATS_FILE_TMPDIR}/vault.log" &
|
||||
export VAULT_PID=$!
|
||||
|
||||
export VAULT_TOKEN="test-root-token"
|
||||
|
||||
# Wait for vault to be ready (up to 10s)
|
||||
local i=0
|
||||
while ! curl -sf "${VAULT_ADDR}/v1/sys/health" >/dev/null 2>&1; do
|
||||
sleep 0.5
|
||||
i=$((i + 1))
|
||||
if [ "$i" -ge 20 ]; then
|
||||
echo "Vault failed to start. Log:" >&2
|
||||
cat "${BATS_FILE_TMPDIR}/vault.log" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
teardown_file() {
|
||||
if [ -n "${VAULT_PID:-}" ]; then
|
||||
kill "$VAULT_PID" 2>/dev/null || true
|
||||
wait "$VAULT_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
setup() {
|
||||
# Source the module under test
|
||||
source "${TEST_DIR}/lib/hvault.sh"
|
||||
export VAULT_ADDR VAULT_TOKEN
|
||||
}
|
||||
|
||||
# ── hvault_kv_put + hvault_kv_get ────────────────────────────────────────────
|
||||
|
||||
@test "hvault_kv_put writes and hvault_kv_get reads a secret" {
|
||||
run hvault_kv_put "test/myapp" "username=admin" "password=s3cret"
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
run hvault_kv_get "test/myapp"
|
||||
[ "$status" -eq 0 ]
|
||||
echo "$output" | jq -e '.username == "admin"'
|
||||
echo "$output" | jq -e '.password == "s3cret"'
|
||||
}
|
||||
|
||||
@test "hvault_kv_get extracts a single key" {
|
||||
hvault_kv_put "test/single" "foo=bar" "baz=qux"
|
||||
|
||||
run hvault_kv_get "test/single" "foo"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "bar" ]
|
||||
}
|
||||
|
||||
@test "hvault_kv_get fails for missing key" {
|
||||
hvault_kv_put "test/keymiss" "exists=yes"
|
||||
|
||||
run hvault_kv_get "test/keymiss" "nope"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_kv_get fails for missing path" {
|
||||
run hvault_kv_get "test/does-not-exist-$(date +%s)"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_kv_put fails without KEY=VAL" {
|
||||
run hvault_kv_put "test/bad"
|
||||
[ "$status" -ne 0 ]
|
||||
echo "$output" | grep -q '"error":true' || echo "$stderr" | grep -q '"error":true'
|
||||
}
|
||||
|
||||
@test "hvault_kv_put rejects malformed pair (no =)" {
|
||||
run hvault_kv_put "test/bad2" "noequals"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_kv_get fails without PATH" {
|
||||
run hvault_kv_get
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
# ── hvault_kv_list ───────────────────────────────────────────────────────────
|
||||
|
||||
@test "hvault_kv_list lists keys at a path" {
|
||||
hvault_kv_put "test/listdir/a" "k=1"
|
||||
hvault_kv_put "test/listdir/b" "k=2"
|
||||
|
||||
run hvault_kv_list "test/listdir"
|
||||
[ "$status" -eq 0 ]
|
||||
echo "$output" | jq -e '. | length >= 2'
|
||||
echo "$output" | jq -e 'index("a")'
|
||||
echo "$output" | jq -e 'index("b")'
|
||||
}
|
||||
|
||||
@test "hvault_kv_list fails on nonexistent path" {
|
||||
run hvault_kv_list "test/no-such-path-$(date +%s)"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_kv_list fails without PATH" {
|
||||
run hvault_kv_list
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
# ── hvault_policy_apply ──────────────────────────────────────────────────────
|
||||
|
||||
@test "hvault_policy_apply creates a policy" {
|
||||
local pfile="${BATS_TEST_TMPDIR}/test-policy.hcl"
|
||||
cat > "$pfile" <<'HCL'
|
||||
path "secret/data/test/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
HCL
|
||||
|
||||
run hvault_policy_apply "test-reader" "$pfile"
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
# Verify the policy exists via Vault API
|
||||
run curl -sf -H "X-Vault-Token: ${VAULT_TOKEN}" \
|
||||
"${VAULT_ADDR}/v1/sys/policies/acl/test-reader"
|
||||
[ "$status" -eq 0 ]
|
||||
echo "$output" | jq -e '.data.policy' | grep -q "secret/data/test"
|
||||
}
|
||||
|
||||
@test "hvault_policy_apply is idempotent" {
|
||||
local pfile="${BATS_TEST_TMPDIR}/idem-policy.hcl"
|
||||
printf 'path "secret/*" { capabilities = ["list"] }\n' > "$pfile"
|
||||
|
||||
run hvault_policy_apply "idem-policy" "$pfile"
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
# Apply again — should succeed
|
||||
run hvault_policy_apply "idem-policy" "$pfile"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_policy_apply fails with missing file" {
|
||||
run hvault_policy_apply "bad-policy" "/nonexistent/policy.hcl"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_policy_apply fails without args" {
|
||||
run hvault_policy_apply
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
# ── hvault_token_lookup ──────────────────────────────────────────────────────
|
||||
|
||||
@test "hvault_token_lookup returns token info" {
|
||||
run hvault_token_lookup
|
||||
[ "$status" -eq 0 ]
|
||||
echo "$output" | jq -e '.policies'
|
||||
echo "$output" | jq -e '.accessor'
|
||||
echo "$output" | jq -e 'has("ttl")'
|
||||
}
|
||||
|
||||
@test "hvault_token_lookup fails without VAULT_TOKEN" {
|
||||
unset VAULT_TOKEN
|
||||
run hvault_token_lookup
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_token_lookup fails without VAULT_ADDR" {
|
||||
unset VAULT_ADDR
|
||||
run hvault_token_lookup
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
# ── hvault_jwt_login ─────────────────────────────────────────────────────────
|
||||
|
||||
@test "hvault_jwt_login fails without VAULT_ADDR" {
|
||||
unset VAULT_ADDR
|
||||
run hvault_jwt_login "myrole" "fakejwt"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_jwt_login fails without args" {
|
||||
run hvault_jwt_login
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "hvault_jwt_login returns error for unconfigured jwt auth" {
|
||||
# JWT auth backend is not enabled in dev mode by default — expect failure
|
||||
run hvault_jwt_login "myrole" "eyJhbGciOiJSUzI1NiJ9.fake.sig"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
# ── Env / prereq errors ─────────────────────────────────────────────────────
|
||||
|
||||
@test "all functions fail with structured JSON error when VAULT_ADDR unset" {
|
||||
unset VAULT_ADDR
|
||||
for fn in hvault_kv_get hvault_kv_put hvault_kv_list hvault_policy_apply hvault_token_lookup; do
|
||||
run $fn "dummy" "dummy"
|
||||
[ "$status" -ne 0 ]
|
||||
done
|
||||
}
|
||||
|
|
@ -29,7 +29,8 @@ cleanup() {
|
|||
pkill -f "mock-forgejo.py" 2>/dev/null || true
|
||||
rm -rf "$MOCK_BIN" /tmp/smoke-test-repo \
|
||||
"${FACTORY_ROOT}/projects/smoke-repo.toml" \
|
||||
/tmp/smoke-claude-shared /tmp/smoke-home-claude
|
||||
/tmp/smoke-claude-shared /tmp/smoke-home-claude \
|
||||
/tmp/smoke-env-before-rerun /tmp/smoke-env-before-dryrun
|
||||
# Restore .env only if we created the backup
|
||||
if [ -f "${FACTORY_ROOT}/.env.smoke-backup" ]; then
|
||||
mv "${FACTORY_ROOT}/.env.smoke-backup" "${FACTORY_ROOT}/.env"
|
||||
|
|
@ -178,8 +179,30 @@ else
|
|||
fail "disinto init exited non-zero"
|
||||
fi
|
||||
|
||||
# ── Idempotency test: run init again ───────────────────────────────────────
|
||||
# ── Dry-run test: must not modify state ────────────────────────────────────
|
||||
echo "=== Dry-run test ==="
|
||||
cp "${FACTORY_ROOT}/.env" /tmp/smoke-env-before-dryrun
|
||||
if bash "${FACTORY_ROOT}/bin/disinto" init \
|
||||
"${TEST_SLUG}" \
|
||||
--bare --yes --dry-run \
|
||||
--forge-url "$FORGE_URL" \
|
||||
--repo-root "/tmp/smoke-test-repo" 2>&1 | grep -q "Dry run complete"; then
|
||||
pass "disinto init --dry-run exited successfully"
|
||||
else
|
||||
fail "disinto init --dry-run did not complete"
|
||||
fi
|
||||
|
||||
# Verify --dry-run did not modify .env
|
||||
if diff -q /tmp/smoke-env-before-dryrun "${FACTORY_ROOT}/.env" >/dev/null 2>&1; then
|
||||
pass "dry-run: .env unchanged"
|
||||
else
|
||||
fail "dry-run: .env was modified (should be read-only)"
|
||||
fi
|
||||
rm -f /tmp/smoke-env-before-dryrun
|
||||
|
||||
# ── Idempotency test: run init again, verify .env is stable ────────────────
|
||||
echo "=== Idempotency test: running disinto init again ==="
|
||||
cp "${FACTORY_ROOT}/.env" /tmp/smoke-env-before-rerun
|
||||
if bash "${FACTORY_ROOT}/bin/disinto" init \
|
||||
"${TEST_SLUG}" \
|
||||
--bare --yes \
|
||||
|
|
@ -190,6 +213,29 @@ else
|
|||
fail "disinto init (re-run) exited non-zero"
|
||||
fi
|
||||
|
||||
# Verify .env is stable across re-runs (no token churn)
|
||||
if diff -q /tmp/smoke-env-before-rerun "${FACTORY_ROOT}/.env" >/dev/null 2>&1; then
|
||||
pass "idempotency: .env unchanged on re-run"
|
||||
else
|
||||
fail "idempotency: .env changed on re-run (token churn detected)"
|
||||
diff /tmp/smoke-env-before-rerun "${FACTORY_ROOT}/.env" >&2 || true
|
||||
fi
|
||||
rm -f /tmp/smoke-env-before-rerun
|
||||
|
||||
# Verify FORGE_ADMIN_TOKEN is stored in .env
|
||||
if grep -q '^FORGE_ADMIN_TOKEN=' "${FACTORY_ROOT}/.env"; then
|
||||
pass ".env contains FORGE_ADMIN_TOKEN"
|
||||
else
|
||||
fail ".env missing FORGE_ADMIN_TOKEN"
|
||||
fi
|
||||
|
||||
# Verify HUMAN_TOKEN is stored in .env
|
||||
if grep -q '^HUMAN_TOKEN=' "${FACTORY_ROOT}/.env"; then
|
||||
pass ".env contains HUMAN_TOKEN"
|
||||
else
|
||||
fail ".env missing HUMAN_TOKEN"
|
||||
fi
|
||||
|
||||
# ── 4. Verify Forgejo state ─────────────────────────────────────────────────
|
||||
echo "=== 4/6 Verifying Forgejo state ==="
|
||||
|
||||
|
|
|
|||
162
tests/smoke-load-secret.sh
Normal file
162
tests/smoke-load-secret.sh
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
#!/usr/bin/env bash
|
||||
# tests/smoke-load-secret.sh — Unit tests for load_secret() precedence chain
|
||||
#
|
||||
# Covers the 4 precedence cases:
|
||||
# 1. /secrets/<NAME>.env (Nomad template)
|
||||
# 2. Current environment
|
||||
# 3. secrets/<NAME>.enc (age-encrypted per-key file)
|
||||
# 4. Default / empty fallback
|
||||
#
|
||||
# Required tools: bash, age (for case 3)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
FACTORY_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
fail() { printf 'FAIL: %s\n' "$*" >&2; FAILED=1; }
|
||||
pass() { printf 'PASS: %s\n' "$*"; }
|
||||
FAILED=0
|
||||
|
||||
# Set up a temp workspace and fake HOME so age key paths work
|
||||
test_dir=$(mktemp -d)
|
||||
fake_home=$(mktemp -d)
|
||||
trap 'rm -rf "$test_dir" "$fake_home"' EXIT
|
||||
|
||||
# Minimal env for sourcing env.sh's load_secret function without the full boot
|
||||
# We source the function definition directly to isolate the unit under test.
|
||||
# shellcheck disable=SC2034
|
||||
export USER="${USER:-test}"
|
||||
export HOME="$fake_home"
|
||||
|
||||
# Source env.sh to get load_secret (and FACTORY_ROOT)
|
||||
source "${FACTORY_ROOT}/lib/env.sh"
|
||||
|
||||
# ── Case 4: Default / empty fallback ────────────────────────────────────────
|
||||
echo "=== 1/5 Case 4: default fallback ==="
|
||||
|
||||
unset TEST_SECRET_FALLBACK 2>/dev/null || true
|
||||
val=$(load_secret TEST_SECRET_FALLBACK "my-default")
|
||||
if [ "$val" = "my-default" ]; then
|
||||
pass "load_secret returns default when nothing is set"
|
||||
else
|
||||
fail "Expected 'my-default', got '${val}'"
|
||||
fi
|
||||
|
||||
val=$(load_secret TEST_SECRET_FALLBACK)
|
||||
if [ -z "$val" ]; then
|
||||
pass "load_secret returns empty when no default and nothing set"
|
||||
else
|
||||
fail "Expected empty, got '${val}'"
|
||||
fi
|
||||
|
||||
# ── Case 2: Environment variable already set ────────────────────────────────
|
||||
echo "=== 2/5 Case 2: environment variable ==="
|
||||
|
||||
export TEST_SECRET_ENV="from-environment"
|
||||
val=$(load_secret TEST_SECRET_ENV "ignored-default")
|
||||
if [ "$val" = "from-environment" ]; then
|
||||
pass "load_secret returns env value over default"
|
||||
else
|
||||
fail "Expected 'from-environment', got '${val}'"
|
||||
fi
|
||||
unset TEST_SECRET_ENV
|
||||
|
||||
# ── Case 3: Age-encrypted per-key file ──────────────────────────────────────
|
||||
echo "=== 3/5 Case 3: age-encrypted secret ==="
|
||||
|
||||
if command -v age &>/dev/null && command -v age-keygen &>/dev/null; then
|
||||
# Generate a test age key
|
||||
age_key_dir="${fake_home}/.config/sops/age"
|
||||
mkdir -p "$age_key_dir"
|
||||
age-keygen -o "${age_key_dir}/keys.txt" 2>/dev/null
|
||||
pub_key=$(age-keygen -y "${age_key_dir}/keys.txt")
|
||||
|
||||
# Create encrypted secret
|
||||
secrets_dir="${FACTORY_ROOT}/secrets"
|
||||
mkdir -p "$secrets_dir"
|
||||
printf 'age-test-value' | age -r "$pub_key" -o "${secrets_dir}/TEST_SECRET_AGE.enc"
|
||||
|
||||
unset TEST_SECRET_AGE 2>/dev/null || true
|
||||
val=$(load_secret TEST_SECRET_AGE "fallback")
|
||||
if [ "$val" = "age-test-value" ]; then
|
||||
pass "load_secret decrypts age-encrypted secret"
|
||||
else
|
||||
fail "Expected 'age-test-value', got '${val}'"
|
||||
fi
|
||||
|
||||
# Verify caching: call load_secret directly (not in subshell) so export propagates
|
||||
unset TEST_SECRET_AGE 2>/dev/null || true
|
||||
load_secret TEST_SECRET_AGE >/dev/null
|
||||
if [ "${TEST_SECRET_AGE:-}" = "age-test-value" ]; then
|
||||
pass "load_secret caches decrypted value in environment (direct call)"
|
||||
else
|
||||
fail "Decrypted value not cached in environment"
|
||||
fi
|
||||
|
||||
# Clean up test secret
|
||||
rm -f "${secrets_dir}/TEST_SECRET_AGE.enc"
|
||||
rmdir "$secrets_dir" 2>/dev/null || true
|
||||
unset TEST_SECRET_AGE
|
||||
else
|
||||
echo "SKIP: age/age-keygen not found — skipping age decryption test"
|
||||
fi
|
||||
|
||||
# ── Case 1: Nomad template path ────────────────────────────────────────────
|
||||
echo "=== 4/5 Case 1: Nomad template (/secrets/<NAME>.env) ==="
|
||||
|
||||
nomad_dir="/secrets"
|
||||
if [ -w "$(dirname "$nomad_dir")" ] 2>/dev/null || [ -w "$nomad_dir" ] 2>/dev/null; then
|
||||
mkdir -p "$nomad_dir"
|
||||
printf 'TEST_SECRET_NOMAD=from-nomad-template\n' > "${nomad_dir}/TEST_SECRET_NOMAD.env"
|
||||
|
||||
# Even with env set, Nomad path takes precedence
|
||||
export TEST_SECRET_NOMAD="from-env-should-lose"
|
||||
val=$(load_secret TEST_SECRET_NOMAD "default")
|
||||
if [ "$val" = "from-nomad-template" ]; then
|
||||
pass "load_secret prefers Nomad template over env"
|
||||
else
|
||||
fail "Expected 'from-nomad-template', got '${val}'"
|
||||
fi
|
||||
|
||||
rm -f "${nomad_dir}/TEST_SECRET_NOMAD.env"
|
||||
rmdir "$nomad_dir" 2>/dev/null || true
|
||||
unset TEST_SECRET_NOMAD
|
||||
else
|
||||
echo "SKIP: /secrets not writable — skipping Nomad template test (needs root or container)"
|
||||
fi
|
||||
|
||||
# ── Precedence: env beats age ────────────────────────────────────────────
|
||||
echo "=== 5/5 Precedence: env beats age-encrypted ==="
|
||||
|
||||
if command -v age &>/dev/null && command -v age-keygen &>/dev/null; then
|
||||
age_key_dir="${fake_home}/.config/sops/age"
|
||||
mkdir -p "$age_key_dir"
|
||||
[ -f "${age_key_dir}/keys.txt" ] || age-keygen -o "${age_key_dir}/keys.txt" 2>/dev/null
|
||||
pub_key=$(age-keygen -y "${age_key_dir}/keys.txt")
|
||||
|
||||
secrets_dir="${FACTORY_ROOT}/secrets"
|
||||
mkdir -p "$secrets_dir"
|
||||
printf 'age-value-should-lose' | age -r "$pub_key" -o "${secrets_dir}/TEST_SECRET_PREC.enc"
|
||||
|
||||
export TEST_SECRET_PREC="env-value-wins"
|
||||
val=$(load_secret TEST_SECRET_PREC "default")
|
||||
if [ "$val" = "env-value-wins" ]; then
|
||||
pass "load_secret prefers env over age-encrypted file"
|
||||
else
|
||||
fail "Expected 'env-value-wins', got '${val}'"
|
||||
fi
|
||||
|
||||
rm -f "${secrets_dir}/TEST_SECRET_PREC.enc"
|
||||
rmdir "$secrets_dir" 2>/dev/null || true
|
||||
unset TEST_SECRET_PREC
|
||||
else
|
||||
echo "SKIP: age not found — skipping precedence test"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo "=== SMOKE-LOAD-SECRET TEST FAILED ==="
|
||||
exit 1
|
||||
fi
|
||||
echo "=== SMOKE-LOAD-SECRET TEST PASSED ==="
|
||||
|
|
@ -83,9 +83,12 @@ curl -sL https://raw.githubusercontent.com/disinto-admin/disinto/fix/issue-621/t
|
|||
- Permissions: `root:disinto-register 0750`
|
||||
|
||||
3. **Installs Caddy**:
|
||||
- Backs up any pre-existing `/etc/caddy/Caddyfile` to `/etc/caddy/Caddyfile.pre-disinto`
|
||||
- Download Caddy with Gandi DNS plugin
|
||||
- Enable admin API on `127.0.0.1:2019`
|
||||
- Configure wildcard cert for `*.disinto.ai` via DNS-01
|
||||
- Creates `/etc/caddy/extra.d/` for operator-owned site blocks
|
||||
- Emitted Caddyfile ends with `import /etc/caddy/extra.d/*.caddy`
|
||||
|
||||
4. **Sets up SSH**:
|
||||
- Creates `disinto-register` authorized_keys with forced command
|
||||
|
|
@ -95,6 +98,27 @@ curl -sL https://raw.githubusercontent.com/disinto-admin/disinto/fix/issue-621/t
|
|||
- `/opt/disinto-edge/register.sh` — forced command handler
|
||||
- `/opt/disinto-edge/lib/*.sh` — helper libraries
|
||||
|
||||
## Operator-Owned Site Blocks
|
||||
|
||||
Edge-control owns the top-level `/etc/caddy/Caddyfile` and dynamic `<project>.<DOMAIN_SUFFIX>` routes injected via the Caddy admin API. Operators own everything under `/etc/caddy/extra.d/`.
|
||||
|
||||
To serve non-tunnel content (apex domain, www redirect, static sites), drop `.caddy` files into `/etc/caddy/extra.d/`:
|
||||
|
||||
```bash
|
||||
# Example: /etc/caddy/extra.d/landing.caddy
|
||||
disinto.ai {
|
||||
root * /home/debian/disinto-site
|
||||
file_server
|
||||
}
|
||||
|
||||
# Example: /etc/caddy/extra.d/www-redirect.caddy
|
||||
www.disinto.ai {
|
||||
redir https://disinto.ai{uri} permanent
|
||||
}
|
||||
```
|
||||
|
||||
These files survive across `install.sh` re-runs. The `--extra-caddyfile <path>` flag overrides the default import glob (`/etc/caddy/extra.d/*.caddy`) if needed.
|
||||
|
||||
## Usage
|
||||
|
||||
### Register a Tunnel (from dev box)
|
||||
|
|
|
|||
|
|
@ -43,18 +43,21 @@ INSTALL_DIR="/opt/disinto-edge"
|
|||
REGISTRY_DIR="/var/lib/disinto"
|
||||
CADDY_VERSION="2.8.4"
|
||||
DOMAIN_SUFFIX="disinto.ai"
|
||||
EXTRA_CADDYFILE="/etc/caddy/extra.d/*.caddy"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [options]
|
||||
|
||||
Options:
|
||||
--gandi-token <token> Gandi API token for wildcard cert (required)
|
||||
--install-dir <dir> Install directory (default: /opt/disinto-edge)
|
||||
--registry-dir <dir> Registry directory (default: /var/lib/disinto)
|
||||
--caddy-version <ver> Caddy version to install (default: ${CADDY_VERSION})
|
||||
--domain-suffix <suffix> Domain suffix for tunnels (default: disinto.ai)
|
||||
-h, --help Show this help
|
||||
--gandi-token <token> Gandi API token for wildcard cert (required)
|
||||
--install-dir <dir> Install directory (default: /opt/disinto-edge)
|
||||
--registry-dir <dir> Registry directory (default: /var/lib/disinto)
|
||||
--caddy-version <ver> Caddy version to install (default: ${CADDY_VERSION})
|
||||
--domain-suffix <suffix> Domain suffix for tunnels (default: disinto.ai)
|
||||
--extra-caddyfile <path> Import path for operator-owned Caddy config
|
||||
(default: /etc/caddy/extra.d/*.caddy)
|
||||
-h, --help Show this help
|
||||
|
||||
Example:
|
||||
$0 --gandi-token YOUR_GANDI_API_TOKEN
|
||||
|
|
@ -84,6 +87,10 @@ while [[ $# -gt 0 ]]; do
|
|||
DOMAIN_SUFFIX="$2"
|
||||
shift 2
|
||||
;;
|
||||
--extra-caddyfile)
|
||||
EXTRA_CADDYFILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
|
|
@ -225,8 +232,29 @@ EOF
|
|||
chmod 600 "$GANDI_ENV"
|
||||
|
||||
# Create Caddyfile with admin API and wildcard cert
|
||||
# Note: Caddy auto-generates server names (srv0, srv1, …). lib/caddy.sh
|
||||
# discovers the server name dynamically via _discover_server_name() so we
|
||||
# don't need to name the server here.
|
||||
CADDYFILE="/etc/caddy/Caddyfile"
|
||||
cat > "$CADDYFILE" <<EOF
|
||||
|
||||
# Back up existing Caddyfile before overwriting
|
||||
if [ -f "$CADDYFILE" ] && [ ! -f "${CADDYFILE}.pre-disinto" ]; then
|
||||
cp "$CADDYFILE" "${CADDYFILE}.pre-disinto"
|
||||
log_info "Backed up existing Caddyfile to ${CADDYFILE}.pre-disinto"
|
||||
fi
|
||||
|
||||
# Create extra.d directory for operator-owned site blocks
|
||||
EXTRA_DIR="/etc/caddy/extra.d"
|
||||
mkdir -p "$EXTRA_DIR"
|
||||
chmod 0755 "$EXTRA_DIR"
|
||||
if getent group caddy >/dev/null 2>&1; then
|
||||
chown root:caddy "$EXTRA_DIR"
|
||||
else
|
||||
log_warn "Group 'caddy' does not exist; extra.d owned by root:root"
|
||||
fi
|
||||
log_info "Created ${EXTRA_DIR} for operator-owned Caddy config"
|
||||
|
||||
cat > "$CADDYFILE" <<CADDYEOF
|
||||
# Caddy configuration for edge control plane
|
||||
# Admin API enabled on 127.0.0.1:2019
|
||||
|
||||
|
|
@ -240,7 +268,10 @@ cat > "$CADDYFILE" <<EOF
|
|||
dns gandi {env.GANDI_API_KEY}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# Operator-owned site blocks (apex, www, static content, etc.)
|
||||
import ${EXTRA_CADDYFILE}
|
||||
CADDYEOF
|
||||
|
||||
# Start Caddy
|
||||
systemctl restart caddy 2>/dev/null || {
|
||||
|
|
@ -359,6 +390,7 @@ echo "Configuration:"
|
|||
echo " Install directory: ${INSTALL_DIR}"
|
||||
echo " Registry: ${REGISTRY_FILE}"
|
||||
echo " Caddy admin API: http://127.0.0.1:2019"
|
||||
echo " Operator site blocks: ${EXTRA_DIR}/ (import ${EXTRA_CADDYFILE})"
|
||||
echo ""
|
||||
echo "Users:"
|
||||
echo " disinto-register - SSH forced command (runs ${INSTALL_DIR}/register.sh)"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,24 @@ CADDY_ADMIN_URL="${CADDY_ADMIN_URL:-http://127.0.0.1:2019}"
|
|||
# Domain suffix for projects
|
||||
DOMAIN_SUFFIX="${DOMAIN_SUFFIX:-disinto.ai}"
|
||||
|
||||
# Discover the Caddy server name that listens on :80/:443
|
||||
# Usage: _discover_server_name
|
||||
_discover_server_name() {
|
||||
local server_name
|
||||
server_name=$(curl -sS "${CADDY_ADMIN_URL}/config/apps/http/servers" \
|
||||
| jq -r 'to_entries | map(select(.value.listen[]? | test(":(80|443)$"))) | .[0].key // empty') || {
|
||||
echo "Error: could not query Caddy admin API for servers" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
if [ -z "$server_name" ]; then
|
||||
echo "Error: could not find a Caddy server listening on :80/:443" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$server_name"
|
||||
}
|
||||
|
||||
# Add a route for a project
|
||||
# Usage: add_route <project> <port>
|
||||
add_route() {
|
||||
|
|
@ -26,6 +44,9 @@ add_route() {
|
|||
local port="$2"
|
||||
local fqdn="${project}.${DOMAIN_SUFFIX}"
|
||||
|
||||
local server_name
|
||||
server_name=$(_discover_server_name) || return 1
|
||||
|
||||
# Build the route configuration (partial config)
|
||||
local route_config
|
||||
route_config=$(cat <<EOF
|
||||
|
|
@ -58,16 +79,21 @@ add_route() {
|
|||
EOF
|
||||
)
|
||||
|
||||
# Append route using POST /config/apps/http/servers/edge/routes
|
||||
local response
|
||||
response=$(curl -s -X POST \
|
||||
"${CADDY_ADMIN_URL}/config/apps/http/servers/edge/routes" \
|
||||
# Append route via admin API, checking HTTP status
|
||||
local response status body
|
||||
response=$(curl -sS -w '\n%{http_code}' -X POST \
|
||||
"${CADDY_ADMIN_URL}/config/apps/http/servers/${server_name}/routes" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$route_config" 2>&1) || {
|
||||
-d "$route_config") || {
|
||||
echo "Error: failed to add route for ${fqdn}" >&2
|
||||
echo "Response: ${response}" >&2
|
||||
return 1
|
||||
}
|
||||
status=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
if [ "$status" -ge 400 ]; then
|
||||
echo "Error: Caddy admin API returned ${status}: ${body}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Added route: ${fqdn} → 127.0.0.1:${port}" >&2
|
||||
}
|
||||
|
|
@ -78,31 +104,45 @@ remove_route() {
|
|||
local project="$1"
|
||||
local fqdn="${project}.${DOMAIN_SUFFIX}"
|
||||
|
||||
# First, get current routes
|
||||
local routes_json
|
||||
routes_json=$(curl -s "${CADDY_ADMIN_URL}/config/apps/http/servers/edge/routes" 2>&1) || {
|
||||
local server_name
|
||||
server_name=$(_discover_server_name) || return 1
|
||||
|
||||
# First, get current routes, checking HTTP status
|
||||
local response status body
|
||||
response=$(curl -sS -w '\n%{http_code}' \
|
||||
"${CADDY_ADMIN_URL}/config/apps/http/servers/${server_name}/routes") || {
|
||||
echo "Error: failed to get current routes" >&2
|
||||
return 1
|
||||
}
|
||||
status=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
if [ "$status" -ge 400 ]; then
|
||||
echo "Error: Caddy admin API returned ${status}: ${body}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Find the route index that matches our fqdn using jq
|
||||
local route_index
|
||||
route_index=$(echo "$routes_json" | jq -r "to_entries[] | select(.value.match[]?.host[]? == \"${fqdn}\") | .key" 2>/dev/null | head -1)
|
||||
route_index=$(echo "$body" | jq -r "to_entries[] | select(.value.match[]?.host[]? == \"${fqdn}\") | .key" 2>/dev/null | head -1)
|
||||
|
||||
if [ -z "$route_index" ] || [ "$route_index" = "null" ]; then
|
||||
echo "Warning: route for ${fqdn} not found" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Delete the route at the found index
|
||||
local response
|
||||
response=$(curl -s -X DELETE \
|
||||
"${CADDY_ADMIN_URL}/config/apps/http/servers/edge/routes/${route_index}" \
|
||||
-H "Content-Type: application/json" 2>&1) || {
|
||||
# Delete the route at the found index, checking HTTP status
|
||||
response=$(curl -sS -w '\n%{http_code}' -X DELETE \
|
||||
"${CADDY_ADMIN_URL}/config/apps/http/servers/${server_name}/routes/${route_index}" \
|
||||
-H "Content-Type: application/json") || {
|
||||
echo "Error: failed to remove route for ${fqdn}" >&2
|
||||
echo "Response: ${response}" >&2
|
||||
return 1
|
||||
}
|
||||
status=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
if [ "$status" -ge 400 ]; then
|
||||
echo "Error: Caddy admin API returned ${status}: ${body}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Removed route: ${fqdn}" >&2
|
||||
}
|
||||
|
|
@ -110,13 +150,18 @@ remove_route() {
|
|||
# Reload Caddy to apply configuration changes
|
||||
# Usage: reload_caddy
|
||||
reload_caddy() {
|
||||
local response
|
||||
response=$(curl -s -X POST \
|
||||
"${CADDY_ADMIN_URL}/reload" 2>&1) || {
|
||||
local response status body
|
||||
response=$(curl -sS -w '\n%{http_code}' -X POST \
|
||||
"${CADDY_ADMIN_URL}/reload") || {
|
||||
echo "Error: failed to reload Caddy" >&2
|
||||
echo "Response: ${response}" >&2
|
||||
return 1
|
||||
}
|
||||
status=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
if [ "$status" -ge 400 ]; then
|
||||
echo "Error: Caddy reload returned ${status}: ${body}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Caddy reloaded" >&2
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue