fix: feat: consolidate secret stores — single granular secrets/*.enc, deprecate .env.vault.enc (#777)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude
parent
a87dcdf40b
commit
88676e65ae
14 changed files with 254 additions and 130 deletions
|
|
@ -8,7 +8,7 @@
|
|||
# 2. Scan vault/actions/ for TOML files without .result.json
|
||||
# 3. Verify TOML arrived via merged PR with admin merger (Forgejo API)
|
||||
# 4. Validate TOML using vault-env.sh validator
|
||||
# 5. Decrypt .env.vault.enc and extract only declared secrets
|
||||
# 5. Decrypt declared secrets from secrets/<NAME>.enc (age-encrypted)
|
||||
# 6. Launch: docker run --rm disinto/agents:latest <action-id>
|
||||
# 7. Write <action-id>.result.json with exit code, timestamp, logs summary
|
||||
#
|
||||
|
|
@ -27,19 +27,34 @@ source "${SCRIPT_ROOT}/../lib/env.sh"
|
|||
# the shallow clone only has .toml.example files.
|
||||
PROJECTS_DIR="${PROJECTS_DIR:-${FACTORY_ROOT:-/opt/disinto}-projects}"
|
||||
|
||||
# Load vault secrets after env.sh (env.sh unsets them for agent security)
|
||||
# Vault secrets must be available to the dispatcher
|
||||
if [ -f "$FACTORY_ROOT/.env.vault.enc" ] && command -v sops &>/dev/null; then
|
||||
set -a
|
||||
eval "$(sops -d --output-type dotenv "$FACTORY_ROOT/.env.vault.enc" 2>/dev/null)" \
|
||||
|| echo "Warning: failed to decrypt .env.vault.enc — vault secrets not loaded" >&2
|
||||
set +a
|
||||
elif [ -f "$FACTORY_ROOT/.env.vault" ]; then
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$FACTORY_ROOT/.env.vault"
|
||||
set +a
|
||||
fi
|
||||
# Load granular secrets from secrets/*.enc (age-encrypted, one file per key).
|
||||
# These are decrypted on demand and exported so the dispatcher can pass them
|
||||
# to runner containers. Replaces the old monolithic .env.vault.enc store (#777).
|
||||
_AGE_KEY_FILE="${HOME}/.config/sops/age/keys.txt"
|
||||
_SECRETS_DIR="${FACTORY_ROOT}/secrets"
|
||||
|
||||
# decrypt_secret <NAME> — decrypt secrets/<NAME>.enc and print the plaintext value
|
||||
decrypt_secret() {
|
||||
local name="$1"
|
||||
local enc_path="${_SECRETS_DIR}/${name}.enc"
|
||||
if [ ! -f "$enc_path" ]; then
|
||||
return 1
|
||||
fi
|
||||
age -d -i "$_AGE_KEY_FILE" "$enc_path" 2>/dev/null
|
||||
}
|
||||
|
||||
# load_secrets <NAME ...> — decrypt each secret and export it
|
||||
load_secrets() {
|
||||
if [ ! -f "$_AGE_KEY_FILE" ]; then
|
||||
echo "Warning: age key not found at ${_AGE_KEY_FILE} — secrets not loaded" >&2
|
||||
return 1
|
||||
fi
|
||||
for name in "$@"; do
|
||||
local val
|
||||
val=$(decrypt_secret "$name") || continue
|
||||
export "$name=$val"
|
||||
done
|
||||
}
|
||||
|
||||
# Ops repo location (vault/actions directory)
|
||||
OPS_REPO_ROOT="${OPS_REPO_ROOT:-/home/debian/disinto-ops}"
|
||||
|
|
@ -452,17 +467,18 @@ launch_runner() {
|
|||
fi
|
||||
|
||||
# Add environment variables for secrets (if any declared)
|
||||
# Secrets are decrypted per-key from secrets/<NAME>.enc (#777)
|
||||
if [ -n "$secrets_array" ]; then
|
||||
for secret in $secrets_array; do
|
||||
secret=$(echo "$secret" | xargs)
|
||||
if [ -n "$secret" ]; then
|
||||
# Verify secret exists in vault
|
||||
if [ -z "${!secret:-}" ]; then
|
||||
log "ERROR: Secret '${secret}' not found in vault for action ${action_id}"
|
||||
write_result "$action_id" 1 "Secret not found in vault: ${secret}"
|
||||
local secret_val
|
||||
secret_val=$(decrypt_secret "$secret") || {
|
||||
log "ERROR: Secret '${secret}' not found in secrets/*.enc for action ${action_id}"
|
||||
write_result "$action_id" 1 "Secret not found: ${secret} (expected secrets/${secret}.enc)"
|
||||
return 1
|
||||
fi
|
||||
cmd+=(-e "${secret}=${!secret}")
|
||||
}
|
||||
cmd+=(-e "${secret}=${secret_val}")
|
||||
fi
|
||||
done
|
||||
else
|
||||
|
|
|
|||
|
|
@ -173,9 +173,40 @@ PROJECT_TOML="${PROJECT_TOML:-projects/disinto.toml}"
|
|||
sleep 1200 # 20 minutes
|
||||
done) &
|
||||
|
||||
# ── Load required secrets from secrets/*.enc (#777) ────────────────────
|
||||
# Edge container declares its required secrets; missing ones cause a hard fail.
|
||||
_AGE_KEY_FILE="${HOME}/.config/sops/age/keys.txt"
|
||||
_SECRETS_DIR="/opt/disinto/secrets"
|
||||
EDGE_REQUIRED_SECRETS="CADDY_SSH_KEY CADDY_SSH_HOST CADDY_SSH_USER CADDY_ACCESS_LOG"
|
||||
|
||||
_edge_decrypt_secret() {
|
||||
local enc_path="${_SECRETS_DIR}/${1}.enc"
|
||||
[ -f "$enc_path" ] || return 1
|
||||
age -d -i "$_AGE_KEY_FILE" "$enc_path" 2>/dev/null
|
||||
}
|
||||
|
||||
if [ -f "$_AGE_KEY_FILE" ] && [ -d "$_SECRETS_DIR" ]; then
|
||||
_missing=""
|
||||
for _secret_name in $EDGE_REQUIRED_SECRETS; do
|
||||
_val=$(_edge_decrypt_secret "$_secret_name") || { _missing="${_missing} ${_secret_name}"; continue; }
|
||||
export "$_secret_name=$_val"
|
||||
done
|
||||
if [ -n "$_missing" ]; then
|
||||
echo "FATAL: required secrets missing from secrets/*.enc:${_missing}" >&2
|
||||
echo " Run 'disinto secrets add <NAME>' for each missing secret." >&2
|
||||
echo " If migrating from .env.vault.enc, run 'disinto secrets migrate-from-vault' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "edge: loaded required secrets: ${EDGE_REQUIRED_SECRETS}" >&2
|
||||
else
|
||||
echo "FATAL: age key (${_AGE_KEY_FILE}) or secrets dir (${_SECRETS_DIR}) not found — cannot load required secrets" >&2
|
||||
echo " Ensure age is installed and secrets/*.enc files are present." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Start daily engagement collection cron loop in background (#745)
|
||||
# Runs collect-engagement.sh daily at ~23:50 UTC via a sleep loop that
|
||||
# calculates seconds until the next 23:50 window. SSH key from .env.vault.enc.
|
||||
# calculates seconds until the next 23:50 window. SSH key from secrets/*.enc (#777).
|
||||
(while true; do
|
||||
# Calculate seconds until next 23:50 UTC
|
||||
_now=$(date -u +%s)
|
||||
|
|
@ -186,26 +217,21 @@ done) &
|
|||
_sleep_secs=$(( _target - _now ))
|
||||
echo "edge: collect-engagement scheduled in ${_sleep_secs}s (next 23:50 UTC)" >&2
|
||||
sleep "$_sleep_secs"
|
||||
# Set CADDY_ACCESS_LOG so the script reads from the fetched local copy
|
||||
_fetch_log="/tmp/caddy-access-log-fetch.log"
|
||||
if [ -n "${CADDY_SSH_KEY:-}" ]; then
|
||||
_ssh_key_file=$(mktemp)
|
||||
printf '%s\n' "$CADDY_SSH_KEY" > "$_ssh_key_file"
|
||||
chmod 0600 "$_ssh_key_file"
|
||||
scp -i "$_ssh_key_file" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
"${CADDY_SSH_USER:-debian}@${CADDY_SSH_HOST:-disinto.ai}:${CADDY_ACCESS_LOG:-/var/log/caddy/access.log}" \
|
||||
"$_fetch_log" 2>&1 | tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
rm -f "$_ssh_key_file"
|
||||
if [ -s "$_fetch_log" ]; then
|
||||
CADDY_ACCESS_LOG="$_fetch_log" bash /opt/disinto/site/collect-engagement.sh 2>&1 \
|
||||
| tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
else
|
||||
echo "edge: collect-engagement: fetched log is empty, skipping parse" >&2
|
||||
fi
|
||||
rm -f "$_fetch_log"
|
||||
_ssh_key_file=$(mktemp)
|
||||
printf '%s\n' "$CADDY_SSH_KEY" > "$_ssh_key_file"
|
||||
chmod 0600 "$_ssh_key_file"
|
||||
scp -i "$_ssh_key_file" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
"${CADDY_SSH_USER}@${CADDY_SSH_HOST}:${CADDY_ACCESS_LOG}" \
|
||||
"$_fetch_log" 2>&1 | tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
rm -f "$_ssh_key_file"
|
||||
if [ -s "$_fetch_log" ]; then
|
||||
CADDY_ACCESS_LOG="$_fetch_log" bash /opt/disinto/site/collect-engagement.sh 2>&1 \
|
||||
| tee -a /opt/disinto-logs/collect-engagement.log || true
|
||||
else
|
||||
echo "edge: collect-engagement: CADDY_SSH_KEY not set, skipping" >&2
|
||||
echo "edge: collect-engagement: fetched log is empty, skipping parse" >&2
|
||||
fi
|
||||
rm -f "$_fetch_log"
|
||||
done) &
|
||||
|
||||
# Caddy as main process — run in foreground via wait so background jobs survive
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue