Merge pull request 'fix: Secure action runtime — ephemeral container with vault-injected secrets (#748)' (#762) from fix/issue-748 into main

This commit is contained in:
johba 2026-03-26 17:53:22 +01:00
commit 320236080e
6 changed files with 326 additions and 82 deletions

3
.gitignore vendored
View file

@ -1,8 +1,9 @@
# Plaintext secrets (never commit)
.env
# Encrypted secrets — safe to commit (.env.enc is SOPS-encrypted)
# Encrypted secrets — safe to commit (SOPS-encrypted with age)
!.env.enc
!.env.vault.enc
!.sops.yaml
# Per-box project config (generated by disinto init)

View file

@ -50,7 +50,7 @@ disinto/
- All scripts start with `#!/usr/bin/env bash` and `set -euo pipefail`
- Source shared environment: `source "$(dirname "$0")/../lib/env.sh"`
- Log to `$LOGFILE` using the `log()` function from env.sh or defined locally
- Never hardcode secrets — all come from `.env.enc` (or `.env` fallback) or TOML project files
- Never hardcode secrets — agent secrets come from `.env.enc`, vault secrets from `.env.vault.enc` (or `.env`/`.env.vault` fallback)
- Never embed secrets in issue bodies, PR descriptions, or comments — use env var references (e.g. `$BASE_RPC_URL`)
- ShellCheck must pass (CI runs `shellcheck` on all `.sh` files)
- Avoid duplicate code — shared helpers go in `lib/`
@ -151,7 +151,7 @@ Humans write these. Agents read and enforce them.
| AD-002 | Single-threaded pipeline per project. | One dev issue at a time. No new work while a PR awaits CI or review. Prevents merge conflicts and keeps context clear. |
| AD-003 | The runtime creates and destroys, the formula preserves. | Runtime manages worktrees/sessions/temp. Formulas commit knowledge to git before signaling done. |
| AD-004 | Event-driven > polling > fixed delays. | Never `waitForTimeout` or hardcoded sleep. Use phase files, webhooks, or poll loops with backoff. |
| AD-005 | Secrets via env var indirection, never in issue bodies. | Issue bodies become code. Secrets go in `.env.enc` (SOPS-encrypted) or fall back to `.env`, referenced as `$VAR_NAME`. |
| AD-005 | Secrets via env var indirection, never in issue bodies. | Issue bodies become code. Agent secrets go in `.env.enc`, vault secrets in `.env.vault.enc` (both SOPS-encrypted). Referenced as `$VAR_NAME`. Vault-runner gets only vault secrets; agents get only agent secrets. |
**Who enforces what:**
- **Gardener** checks open backlog issues against ADs during grooming; closes violations with a comment referencing the AD number.

View file

@ -9,7 +9,8 @@
# disinto logs [service] Tail service logs
# disinto shell Shell into the agent container
# disinto status Show factory status
# disinto secrets <edit|show|migrate> Manage encrypted secrets
# disinto secrets <subcommand> Manage encrypted secrets
# disinto vault-run <action-id> Run action in ephemeral vault container
#
# Usage:
# disinto init https://github.com/user/repo
@ -37,7 +38,8 @@ Usage:
disinto logs [service] Tail service logs
disinto shell Shell into the agent container
disinto status Show factory status
disinto secrets <edit|show|migrate> Manage encrypted secrets (.env.enc)
disinto secrets <subcommand> Manage encrypted secrets
disinto vault-run <action-id> Run action in ephemeral vault container
Init options:
--branch <name> Primary branch (default: auto-detect)
@ -107,7 +109,7 @@ write_sops_yaml() {
local pub_key="$1"
cat > "${FACTORY_ROOT}/.sops.yaml" <<EOF
creation_rules:
- path_regex: \.env\.enc$
- path_regex: \.env(\.vault)?\.enc$
age: "${pub_key}"
EOF
}
@ -237,6 +239,23 @@ services:
networks:
- disinto-net
vault-runner:
build: ./docker/agents
profiles: ["vault"]
security_opt:
- apparmor=unconfined
volumes:
- ./vault:/home/agent/disinto/vault
- ./lib:/home/agent/disinto/lib:ro
- ./formulas:/home/agent/disinto/formulas:ro
environment:
FORGE_URL: http://forgejo:3000
DISINTO_CONTAINER: "1"
# env_file set at runtime by: disinto vault-run --env-file <tmpfile>
entrypoint: ["bash", "/home/agent/disinto/vault/vault-run-action.sh"]
networks:
- disinto-net
volumes:
forgejo-data:
woodpecker-data:
@ -1370,6 +1389,26 @@ disinto_secrets() {
local subcmd="${1:-}"
local enc_file="${FACTORY_ROOT}/.env.enc"
local env_file="${FACTORY_ROOT}/.env"
local vault_enc_file="${FACTORY_ROOT}/.env.vault.enc"
local vault_env_file="${FACTORY_ROOT}/.env.vault"
# Shared helper: ensure sops+age and .sops.yaml exist
_secrets_ensure_sops() {
if ! command -v sops &>/dev/null || ! command -v age-keygen &>/dev/null; then
echo "Error: sops and age are required." >&2
echo " Install sops: https://github.com/getsops/sops/releases" >&2
echo " Install age: apt install age / brew install age" >&2
exit 1
fi
if ! ensure_age_key; then
echo "Error: failed to generate age key" >&2
exit 1
fi
if [ ! -f "${FACTORY_ROOT}/.sops.yaml" ]; then
write_sops_yaml "$AGE_PUBLIC_KEY"
echo "Created: .sops.yaml"
fi
}
case "$subcmd" in
edit)
@ -1391,31 +1430,110 @@ disinto_secrets() {
echo "Error: ${env_file} not found — nothing to migrate." >&2
exit 1
fi
if ! command -v sops &>/dev/null || ! command -v age-keygen &>/dev/null; then
echo "Error: sops and age are required for migration." >&2
echo " Install sops: https://github.com/getsops/sops/releases" >&2
echo " Install age: apt install age / brew install age" >&2
exit 1
fi
if ! ensure_age_key; then
echo "Error: failed to generate age key" >&2
exit 1
fi
if [ ! -f "${FACTORY_ROOT}/.sops.yaml" ]; then
write_sops_yaml "$AGE_PUBLIC_KEY"
echo "Created: .sops.yaml"
fi
_secrets_ensure_sops
encrypt_env_file "$env_file" "$enc_file"
rm -f "$env_file"
echo "Migrated: .env -> .env.enc (plaintext removed)"
;;
edit-vault)
if [ ! -f "$vault_enc_file" ]; then
echo "Error: ${vault_enc_file} not found. Run 'disinto secrets migrate-vault' first." >&2
exit 1
fi
sops "$vault_enc_file"
;;
show-vault)
if [ ! -f "$vault_enc_file" ]; then
echo "Error: ${vault_enc_file} not found." >&2
exit 1
fi
sops -d "$vault_enc_file"
;;
migrate-vault)
if [ ! -f "$vault_env_file" ]; then
echo "Error: ${vault_env_file} not found — nothing to migrate." >&2
echo " Create .env.vault with vault secrets (GITHUB_TOKEN, deploy keys, etc.)" >&2
exit 1
fi
_secrets_ensure_sops
encrypt_env_file "$vault_env_file" "$vault_enc_file"
rm -f "$vault_env_file"
echo "Migrated: .env.vault -> .env.vault.enc (plaintext removed)"
;;
*)
echo "Usage: disinto secrets <edit|show|migrate>" >&2
cat <<EOF >&2
Usage: disinto secrets <subcommand>
Agent secrets (.env.enc):
edit Edit agent secrets (FORGE_TOKEN, CLAUDE_API_KEY, etc.)
show Show decrypted agent secrets
migrate Encrypt .env -> .env.enc
Vault secrets (.env.vault.enc):
edit-vault Edit vault secrets (GITHUB_TOKEN, deploy keys, etc.)
show-vault Show decrypted vault secrets
migrate-vault Encrypt .env.vault -> .env.vault.enc
EOF
exit 1
;;
esac
}
# ── vault-run command ─────────────────────────────────────────────────────────
disinto_vault_run() {
local action_id="${1:?Usage: disinto vault-run <action-id>}"
local compose_file="${FACTORY_ROOT}/docker-compose.yml"
local vault_enc="${FACTORY_ROOT}/.env.vault.enc"
if [ ! -f "$compose_file" ]; then
echo "Error: docker-compose.yml not found" >&2
echo " Run 'disinto init <repo-url>' first (without --bare)" >&2
exit 1
fi
if [ ! -f "$vault_enc" ]; then
echo "Error: .env.vault.enc not found — create vault secrets first" >&2
echo " Run 'disinto secrets migrate-vault' after creating .env.vault" >&2
exit 1
fi
if ! command -v sops &>/dev/null; then
echo "Error: sops not found — required to decrypt vault secrets" >&2
exit 1
fi
# Decrypt vault secrets to temp file
local tmp_env
tmp_env=$(mktemp /tmp/disinto-vault-XXXXXX)
trap 'rm -f "$tmp_env"' EXIT
if ! sops -d --output-type dotenv "$vault_enc" > "$tmp_env" 2>/dev/null; then
rm -f "$tmp_env"
echo "Error: failed to decrypt .env.vault.enc" >&2
exit 1
fi
echo "Vault secrets decrypted to tmpfile"
# Run action in ephemeral vault-runner container
local rc=0
docker compose -f "$compose_file" \
run --rm --env-file "$tmp_env" \
vault-runner "$action_id" || rc=$?
# Clean up — secrets gone
rm -f "$tmp_env"
echo "Vault tmpfile removed"
if [ "$rc" -eq 0 ]; then
echo "Vault action ${action_id} completed successfully"
else
echo "Vault action ${action_id} failed (exit ${rc})" >&2
fi
return "$rc"
}
# ── up command ────────────────────────────────────────────────────────────────
disinto_up() {
@ -1484,13 +1602,14 @@ disinto_shell() {
# ── Main dispatch ────────────────────────────────────────────────────────────
case "${1:-}" in
init) shift; disinto_init "$@" ;;
up) shift; disinto_up "$@" ;;
down) shift; disinto_down "$@" ;;
logs) shift; disinto_logs "$@" ;;
shell) shift; disinto_shell ;;
status) shift; disinto_status "$@" ;;
secrets) shift; disinto_secrets "$@" ;;
init) shift; disinto_init "$@" ;;
up) shift; disinto_up "$@" ;;
down) shift; disinto_down "$@" ;;
logs) shift; disinto_logs "$@" ;;
shell) shift; disinto_shell ;;
status) shift; disinto_status "$@" ;;
secrets) shift; disinto_secrets "$@" ;;
vault-run) shift; disinto_vault_run "$@" ;;
-h|--help) usage ;;
*) usage ;;
esac

View file

@ -3,6 +3,8 @@
#
# Handles two pipelines:
# A. Action gating (*.json): pending/ → approved/ → fired/
# Execution delegated to ephemeral vault-runner container via disinto vault-run.
# The vault-runner gets vault secrets (.env.vault.enc); this script does NOT.
# B. Procurement (*.md): approved/ → fired/ (writes RESOURCES.md entry)
#
# If item is in pending/, moves to approved/ first.
@ -98,70 +100,30 @@ if [ "$IS_PROCUREMENT" = true ]; then
fi
# =============================================================================
# Pipeline B: Action gating — dispatch to handler
# Pipeline B: Action gating — delegate to ephemeral vault-runner container
# =============================================================================
ACTION_TYPE=$(jq -r '.type // ""' < "$ACTION_FILE")
ACTION_SOURCE=$(jq -r '.source // ""' < "$ACTION_FILE")
PAYLOAD=$(jq -c '.payload // {}' < "$ACTION_FILE")
if [ -z "$ACTION_TYPE" ]; then
log "ERROR: $ACTION_ID has no type field"
exit 1
fi
log "$ACTION_ID: firing type=$ACTION_TYPE source=$ACTION_SOURCE"
log "$ACTION_ID: firing type=$ACTION_TYPE source=$ACTION_SOURCE via vault-runner"
FIRE_EXIT=0
case "$ACTION_TYPE" in
webhook-call)
# Universal handler: HTTP call to endpoint with optional method/headers/body
ENDPOINT=$(echo "$PAYLOAD" | jq -r '.endpoint // ""')
METHOD=$(echo "$PAYLOAD" | jq -r '.method // "POST"')
REQ_BODY=$(echo "$PAYLOAD" | jq -r '.body // ""')
HEADERS=$(echo "$PAYLOAD" | jq -r '.headers // {} | to_entries[] | "-H\n\(.key): \(.value)"' 2>/dev/null || true)
if [ -z "$ENDPOINT" ]; then
log "ERROR: $ACTION_ID webhook-call missing endpoint"
exit 1
fi
# Build curl args
CURL_ARGS=(-sf -X "$METHOD" -o /dev/null -w "%{http_code}")
if [ -n "$HEADERS" ]; then
while IFS= read -r header; do
[ -n "$header" ] && CURL_ARGS+=(-H "$header")
done < <(echo "$PAYLOAD" | jq -r '.headers // {} | to_entries[] | "\(.key): \(.value)"' 2>/dev/null || true)
fi
if [ -n "$REQ_BODY" ] && [ "$REQ_BODY" != "null" ]; then
CURL_ARGS+=(-d "$REQ_BODY")
fi
HTTP_CODE=$(curl "${CURL_ARGS[@]}" "$ENDPOINT" 2>/dev/null) || HTTP_CODE="000"
if [[ "$HTTP_CODE" =~ ^2 ]]; then
log "$ACTION_ID: webhook-call → HTTP $HTTP_CODE OK"
else
log "ERROR: $ACTION_ID webhook-call → HTTP $HTTP_CODE"
FIRE_EXIT=1
fi
;;
blog-post|social-post|email-blast|pricing-change|dns-change|stripe-charge)
# Check for a handler script
HANDLER="${VAULT_DIR}/handlers/${ACTION_TYPE}.sh"
if [ -x "$HANDLER" ]; then
bash "$HANDLER" "$ACTION_ID" "$PAYLOAD" >> "$LOGFILE" 2>&1 || FIRE_EXIT=$?
else
log "ERROR: $ACTION_ID no handler for type '$ACTION_TYPE' (${HANDLER} not found)"
FIRE_EXIT=1
fi
;;
*)
log "ERROR: $ACTION_ID unknown action type '$ACTION_TYPE'"
FIRE_EXIT=1
;;
esac
# Delegate execution to the ephemeral vault-runner container.
# The vault-runner gets vault secrets (.env.vault.enc) injected at runtime;
# this host process never sees those secrets.
if [ -f "${FACTORY_ROOT}/.env.vault.enc" ] && [ -f "${FACTORY_ROOT}/docker-compose.yml" ]; then
bash "${FACTORY_ROOT}/bin/disinto" vault-run "$ACTION_ID" >> "$LOGFILE" 2>&1 || FIRE_EXIT=$?
else
# Fallback for bare-metal or pre-migration setups: run action handler directly
log "$ACTION_ID: no .env.vault.enc or docker-compose.yml — running action directly"
bash "${VAULT_DIR}/vault-run-action.sh" "$ACTION_ID" >> "$LOGFILE" 2>&1 || FIRE_EXIT=$?
fi
# =============================================================================
# Move to fired/ or leave in approved/ on failure

View file

@ -221,8 +221,80 @@ for req_file in "${VAULT_DIR}/pending/"*.md; do
unlock_action "$REQ_ID"
done
if [ "$PENDING_COUNT" -eq 0 ] && [ "$PROCURE_COUNT" -eq 0 ]; then
# =============================================================================
# PHASE 5: Detect vault-bot authorized comments on issues
# =============================================================================
status "phase 5: scanning for vault-bot authorized comments"
COMMENT_COUNT=0
if [ -n "${FORGE_REPO:-}" ] && [ -n "${FORGE_TOKEN:-}" ]; then
# Get open issues with action label
ACTION_ISSUES=$(curl -sf \
-H "Authorization: token ${FORGE_TOKEN}" \
"${FORGE_URL}/api/v1/repos/${FORGE_REPO}/issues?state=open&labels=action&limit=50" 2>/dev/null) || ACTION_ISSUES="[]"
ISSUE_COUNT=$(printf '%s' "$ACTION_ISSUES" | jq 'length')
for idx in $(seq 0 $((ISSUE_COUNT - 1))); do
ISSUE_NUM=$(printf '%s' "$ACTION_ISSUES" | jq -r ".[$idx].number")
# Skip if already processed
if [ -f "${VAULT_DIR}/.locks/issue-${ISSUE_NUM}.vault-fired" ]; then
continue
fi
# Get comments on this issue
COMMENTS=$(curl -sf \
-H "Authorization: token ${FORGE_TOKEN}" \
"${FORGE_URL}/api/v1/repos/${FORGE_REPO}/issues/${ISSUE_NUM}/comments?limit=50" 2>/dev/null) || continue
# Look for vault-bot comments containing VAULT:APPROVED with a JSON action spec
APPROVED_BODY=$(printf '%s' "$COMMENTS" | jq -r '
[.[] | select(.user.login == "vault-bot") | select(.body | test("VAULT:APPROVED"))] | last | .body // empty
' 2>/dev/null) || continue
[ -z "$APPROVED_BODY" ] && continue
# Extract JSON action spec from fenced code block in the comment
ACTION_JSON=$(printf '%s' "$APPROVED_BODY" | sed -n '/^```json$/,/^```$/p' | sed '1d;$d')
[ -z "$ACTION_JSON" ] && continue
# Validate JSON
if ! printf '%s' "$ACTION_JSON" | jq empty 2>/dev/null; then
log "malformed action JSON in vault-bot comment on issue #${ISSUE_NUM}"
continue
fi
ACTION_ID=$(printf '%s' "$ACTION_JSON" | jq -r '.id // empty')
if [ -z "$ACTION_ID" ]; then
ACTION_ID="issue-${ISSUE_NUM}-$(date +%s)"
ACTION_JSON=$(printf '%s' "$ACTION_JSON" | jq --arg id "$ACTION_ID" '.id = $id')
fi
# Skip if this action already exists in any stage
if [ -f "${VAULT_DIR}/approved/${ACTION_ID}.json" ] || \
[ -f "${VAULT_DIR}/fired/${ACTION_ID}.json" ] || \
[ -f "${VAULT_DIR}/rejected/${ACTION_ID}.json" ]; then
continue
fi
log "vault-bot authorized action on issue #${ISSUE_NUM}: ${ACTION_ID}"
printf '%s' "$ACTION_JSON" | jq '.status = "approved"' > "${VAULT_DIR}/approved/${ACTION_ID}.json"
COMMENT_COUNT=$((COMMENT_COUNT + 1))
# Fire the action
if bash "${VAULT_DIR}/vault-fire.sh" "$ACTION_ID" >> "$LOGFILE" 2>&1; then
log "fired ${ACTION_ID} from issue #${ISSUE_NUM}"
# Mark issue as processed
touch "${VAULT_DIR}/.locks/issue-${ISSUE_NUM}.vault-fired"
else
log "ERROR: fire failed for ${ACTION_ID} from issue #${ISSUE_NUM}"
fi
done
fi
if [ "$PENDING_COUNT" -eq 0 ] && [ "$PROCURE_COUNT" -eq 0 ] && [ "$COMMENT_COUNT" -eq 0 ]; then
status "all clear — no pending items"
else
status "poll complete — ${PENDING_COUNT} action(s), ${PROCURE_COUNT} procurement request(s)"
status "poll complete — ${PENDING_COUNT} action(s), ${PROCURE_COUNT} procurement(s), ${COMMENT_COUNT} comment-authorized"
fi

90
vault/vault-run-action.sh Executable file
View file

@ -0,0 +1,90 @@
#!/usr/bin/env bash
# vault-run-action.sh — Execute an action inside the ephemeral vault-runner container
#
# This script is the entrypoint for the vault-runner container. It runs with
# vault secrets injected as environment variables (GITHUB_TOKEN, CLAWHUB_TOKEN,
# deploy keys, etc.) and dispatches to the appropriate action handler.
#
# The vault-runner container is ephemeral: it starts, runs the action, and is
# destroyed. Secrets exist only in container memory, never on disk.
#
# Usage: vault-run-action.sh <action-id>
set -euo pipefail
VAULT_DIR="${DISINTO_VAULT_DIR:-/home/agent/disinto/vault}"
LOGFILE="${VAULT_DIR}/vault.log"
ACTION_ID="${1:?Usage: vault-run-action.sh <action-id>}"
log() {
printf '[%s] vault-runner: %s\n' "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" "$*" >> "$LOGFILE" 2>/dev/null || \
printf '[%s] vault-runner: %s\n' "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" "$*" >&2
}
# Find action file in approved/
ACTION_FILE="${VAULT_DIR}/approved/${ACTION_ID}.json"
if [ ! -f "$ACTION_FILE" ]; then
log "ERROR: action file not found: ${ACTION_FILE}"
echo "ERROR: action file not found: ${ACTION_FILE}" >&2
exit 1
fi
ACTION_TYPE=$(jq -r '.type // ""' < "$ACTION_FILE")
ACTION_SOURCE=$(jq -r '.source // ""' < "$ACTION_FILE")
PAYLOAD=$(jq -c '.payload // {}' < "$ACTION_FILE")
if [ -z "$ACTION_TYPE" ]; then
log "ERROR: ${ACTION_ID} has no type field"
exit 1
fi
log "${ACTION_ID}: executing type=${ACTION_TYPE} source=${ACTION_SOURCE}"
FIRE_EXIT=0
case "$ACTION_TYPE" in
webhook-call)
# HTTP call to endpoint with optional method/headers/body
ENDPOINT=$(echo "$PAYLOAD" | jq -r '.endpoint // ""')
METHOD=$(echo "$PAYLOAD" | jq -r '.method // "POST"')
REQ_BODY=$(echo "$PAYLOAD" | jq -r '.body // ""')
if [ -z "$ENDPOINT" ]; then
log "ERROR: ${ACTION_ID} webhook-call missing endpoint"
exit 1
fi
CURL_ARGS=(-sf -X "$METHOD" -o /dev/null -w "%{http_code}")
while IFS= read -r header; do
[ -n "$header" ] && CURL_ARGS+=(-H "$header")
done < <(echo "$PAYLOAD" | jq -r '.headers // {} | to_entries[] | "\(.key): \(.value)"' 2>/dev/null || true)
if [ -n "$REQ_BODY" ] && [ "$REQ_BODY" != "null" ]; then
CURL_ARGS+=(-d "$REQ_BODY")
fi
HTTP_CODE=$(curl "${CURL_ARGS[@]}" "$ENDPOINT" 2>/dev/null) || HTTP_CODE="000"
if [[ "$HTTP_CODE" =~ ^2 ]]; then
log "${ACTION_ID}: webhook-call -> HTTP ${HTTP_CODE} OK"
else
log "ERROR: ${ACTION_ID} webhook-call -> HTTP ${HTTP_CODE}"
FIRE_EXIT=1
fi
;;
blog-post|social-post|email-blast|pricing-change|dns-change|stripe-charge)
HANDLER="${VAULT_DIR}/handlers/${ACTION_TYPE}.sh"
if [ -x "$HANDLER" ]; then
bash "$HANDLER" "$ACTION_ID" "$PAYLOAD" 2>&1 || FIRE_EXIT=$?
else
log "ERROR: ${ACTION_ID} no handler for type '${ACTION_TYPE}' (${HANDLER} not found)"
FIRE_EXIT=1
fi
;;
*)
log "ERROR: ${ACTION_ID} unknown action type '${ACTION_TYPE}'"
FIRE_EXIT=1
;;
esac
exit "$FIRE_EXIT"