fix: [nomad-step-2] S2.2 — tools/vault-import.sh (import .env + sops into KV) (#880)
This commit is contained in:
dev-qwen2
parent
a34a478a8e
commit
c6691d5ee3
7 changed files with 887 additions and 0 deletions
477
tools/vault-import.sh
Executable file
477
tools/vault-import.sh
Executable file
|
|
@ -0,0 +1,477 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# vault-import.sh — Import .env and sops-decrypted secrets into Vault KV
|
||||
#
|
||||
# Reads existing .env and sops-encrypted .env.vault.enc from the old docker stack
|
||||
# and writes them to Vault KV paths matching the S2.1 policy layout.
|
||||
#
|
||||
# Usage:
|
||||
# vault-import.sh \
|
||||
# --env /path/to/.env \
|
||||
# --sops /path/to/.env.vault.enc \
|
||||
# --age-key /path/to/age/keys.txt
|
||||
#
|
||||
# Mapping:
|
||||
# From .env:
|
||||
# - FORGE_{ROLE}_TOKEN + FORGE_{ROLE}_PASS → kv/disinto/bots/<role>/{token,password}
|
||||
# (roles: review, dev, gardener, architect, planner, predictor, supervisor, vault)
|
||||
# - FORGE_TOKEN_LLAMA + FORGE_PASS_LLAMA → kv/disinto/bots/dev-qwen/{token,password}
|
||||
# - FORGE_TOKEN + FORGE_PASS → kv/disinto/shared/forge/{token,password}
|
||||
# - FORGE_ADMIN_TOKEN → kv/disinto/shared/forge/admin_token
|
||||
# - WOODPECKER_* → kv/disinto/shared/woodpecker/<lowercase_key>
|
||||
# - FORWARD_AUTH_SECRET, CHAT_OAUTH_* → kv/disinto/shared/chat/<lowercase_key>
|
||||
# From sops-decrypted .env.vault.enc:
|
||||
# - GITHUB_TOKEN, CODEBERG_TOKEN, CLAWHUB_TOKEN, DEPLOY_KEY, NPM_TOKEN, DOCKER_HUB_TOKEN
|
||||
# → kv/disinto/runner/<NAME>/value
|
||||
#
|
||||
# Security:
|
||||
# - Refuses to run if VAULT_ADDR is not localhost
|
||||
# - Writes to KV v2, not v1
|
||||
# - Validates sops age key file is mode 0400 before sourcing
|
||||
# - Never logs secret values — only key names
|
||||
#
|
||||
# Idempotency:
|
||||
# - Reports unchanged/updated/created per key via hvault_kv_get
|
||||
# - --dry-run prints the full import plan without writing
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Internal helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
# _log — emit a log message to stdout (never to stderr to avoid polluting diff)
|
||||
_log() {
|
||||
printf '[vault-import] %s\n' "$*"
|
||||
}
|
||||
|
||||
# _err — emit an error message to stderr
|
||||
_err() {
|
||||
printf '[vault-import] ERROR: %s\n' "$*" >&2
|
||||
}
|
||||
|
||||
# _die — log error and exit with status 1
|
||||
_die() {
|
||||
_err "$@"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# _check_vault_addr — ensure VAULT_ADDR is localhost (security check)
|
||||
_check_vault_addr() {
|
||||
local addr="${VAULT_ADDR:-}"
|
||||
if [[ ! "$addr" =~ ^https?://(localhost|127\.0\.0\.1)(:[0-9]+)?$ ]]; then
|
||||
_die "Security check failed: VAULT_ADDR must be localhost for safety. Got: $addr"
|
||||
fi
|
||||
}
|
||||
|
||||
# _validate_age_key_perms — ensure age key file is mode 0400
|
||||
_validate_age_key_perms() {
|
||||
local keyfile="$1"
|
||||
local perms
|
||||
perms="$(stat -c '%a' "$keyfile" 2>/dev/null)" || _die "Cannot stat age key file: $keyfile"
|
||||
if [ "$perms" != "400" ]; then
|
||||
_die "Age key file permissions are $perms, expected 400. Refusing to proceed for security."
|
||||
fi
|
||||
}
|
||||
|
||||
# _decrypt_sops — decrypt sops-encrypted file using SOPS_AGE_KEY_FILE
|
||||
_decrypt_sops() {
|
||||
local sops_file="$1"
|
||||
local age_key="$2"
|
||||
local output
|
||||
# sops outputs YAML format by default, extract KEY=VALUE lines
|
||||
output="$(SOPS_AGE_KEY_FILE="$age_key" sops -d "$sops_file" 2>/dev/null | \
|
||||
grep -E '^[A-Z_][A-Z0-9_]*=' | \
|
||||
sed 's/^\([^=]*\)=\(.*\)$/\1=\2/')" || \
|
||||
_die "Failed to decrypt sops file: $sops_file. Check age key and file integrity."
|
||||
printf '%s' "$output"
|
||||
}
|
||||
|
||||
# _load_env_file — source an environment file (safety: only KEY=value lines)
|
||||
_load_env_file() {
|
||||
local env_file="$1"
|
||||
local temp_env
|
||||
temp_env="$(mktemp)"
|
||||
# Extract only valid KEY=value lines (skip comments, blank lines, malformed)
|
||||
grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$env_file" 2>/dev/null > "$temp_env" || true
|
||||
# shellcheck source=/dev/null
|
||||
source "$temp_env"
|
||||
rm -f "$temp_env"
|
||||
}
|
||||
|
||||
# _kv_path_exists — check if a KV path exists (returns 0 if exists, 1 if not)
|
||||
_kv_path_exists() {
|
||||
local path="$1"
|
||||
# Use hvault_kv_get and check if it fails with "not found"
|
||||
if hvault_kv_get "$path" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
# Check if the error is specifically "not found"
|
||||
local err_output
|
||||
err_output="$(hvault_kv_get "$path" 2>&1)" || true
|
||||
if printf '%s' "$err_output" | grep -qi 'not found\|404'; then
|
||||
return 1
|
||||
fi
|
||||
# Some other error (e.g., auth failure) — treat as unknown
|
||||
return 1
|
||||
}
|
||||
|
||||
# _kv_get_value — get a single key value from a KV path
|
||||
_kv_get_value() {
|
||||
local path="$1"
|
||||
local key="$2"
|
||||
hvault_kv_get "$path" "$key"
|
||||
}
|
||||
|
||||
# _kv_put_secret — write a secret to KV v2
|
||||
_kv_put_secret() {
|
||||
local path="$1"
|
||||
shift
|
||||
local kv_pairs=("$@")
|
||||
local payload='{"data":{}}'
|
||||
|
||||
for kv in "${kv_pairs[@]}"; do
|
||||
local k="${kv%%=*}"
|
||||
local v="${kv#*=}"
|
||||
payload="$(printf '%s' "$payload" | jq -n --arg k "$k" --arg v "$v" '.data[$k] = $v')"
|
||||
done
|
||||
|
||||
# Use curl directly for KV v2 write with versioning
|
||||
curl -s -w '%{http_code}' \
|
||||
-H "X-Vault-Token: ${VAULT_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST \
|
||||
-d "$payload" \
|
||||
"${VAULT_ADDR}/v1/secret/data/${path}" >/dev/null
|
||||
}
|
||||
|
||||
# _format_status — format the status string for a key
|
||||
_format_status() {
|
||||
local status="$1"
|
||||
local path="$2"
|
||||
local key="$3"
|
||||
case "$status" in
|
||||
unchanged)
|
||||
printf ' %s: %s/%s (unchanged)' "$status" "$path" "$key"
|
||||
;;
|
||||
updated)
|
||||
printf ' %s: %s/%s (updated)' "$status" "$path" "$key"
|
||||
;;
|
||||
created)
|
||||
printf ' %s: %s/%s (created)' "$status" "$path" "$key"
|
||||
;;
|
||||
*)
|
||||
printf ' %s: %s/%s (unknown)' "$status" "$path" "$key"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── Mapping definitions ──────────────────────────────────────────────────────
|
||||
|
||||
# Bots mapping: FORGE_{ROLE}_TOKEN + FORGE_{ROLE}_PASS
|
||||
declare -a BOT_ROLES=(review dev gardener architect planner predictor supervisor vault)
|
||||
|
||||
# Runner tokens from sops-decrypted file
|
||||
declare -a RUNNER_TOKENS=(GITHUB_TOKEN CODEBERG_TOKEN CLAWHUB_TOKEN DEPLOY_KEY NPM_TOKEN DOCKER_HUB_TOKEN)
|
||||
|
||||
# ── Main logic ────────────────────────────────────────────────────────────────
|
||||
|
||||
main() {
|
||||
local env_file=""
|
||||
local sops_file=""
|
||||
local age_key_file=""
|
||||
local dry_run=false
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
env_file="$2"
|
||||
shift 2
|
||||
;;
|
||||
--sops)
|
||||
sops_file="$2"
|
||||
shift 2
|
||||
;;
|
||||
--age-key)
|
||||
age_key_file="$2"
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
dry_run=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
cat <<'EOF'
|
||||
vault-import.sh — Import .env and sops-decrypted secrets into Vault KV
|
||||
|
||||
Usage:
|
||||
vault-import.sh \
|
||||
--env /path/to/.env \
|
||||
--sops /path/to/.env.vault.enc \
|
||||
--age-key /path/to/age/keys.txt \
|
||||
[--dry-run]
|
||||
|
||||
Options:
|
||||
--env Path to .env file (required)
|
||||
--sops Path to sops-encrypted .env.vault.enc file (required)
|
||||
--age-key Path to age keys file (required)
|
||||
--dry-run Print import plan without writing to Vault (optional)
|
||||
--help Show this help message
|
||||
|
||||
Mapping:
|
||||
From .env:
|
||||
- FORGE_{ROLE}_TOKEN + FORGE_{ROLE}_PASS → kv/disinto/bots/<role>/{token,password}
|
||||
- FORGE_TOKEN_LLAMA + FORGE_PASS_LLAMA → kv/disinto/bots/dev-qwen/{token,password}
|
||||
- FORGE_TOKEN + FORGE_PASS → kv/disinto/shared/forge/{token,password}
|
||||
- FORGE_ADMIN_TOKEN → kv/disinto/shared/forge/admin_token
|
||||
- WOODPECKER_* → kv/disinto/shared/woodpecker/<lowercase_key>
|
||||
- FORWARD_AUTH_SECRET, CHAT_OAUTH_* → kv/disinto/shared/chat/<lowercase_key>
|
||||
|
||||
From sops-decrypted .env.vault.enc:
|
||||
- GITHUB_TOKEN, CODEBERG_TOKEN, CLAWHUB_TOKEN, DEPLOY_KEY, NPM_TOKEN, DOCKER_HUB_TOKEN
|
||||
→ kv/disinto/runner/<NAME>/value
|
||||
|
||||
Examples:
|
||||
vault-import.sh --env .env --sops .env.vault.enc --age-key age-keys.txt
|
||||
vault-import.sh --env .env --sops .env.vault.enc --age-key age-keys.txt --dry-run
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
_die "Unknown option: $1. Use --help for usage."
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate required arguments
|
||||
if [ -z "$env_file" ]; then
|
||||
_die "Missing required argument: --env"
|
||||
fi
|
||||
if [ -z "$sops_file" ]; then
|
||||
_die "Missing required argument: --sops"
|
||||
fi
|
||||
if [ -z "$age_key_file" ]; then
|
||||
_die "Missing required argument: --age-key"
|
||||
fi
|
||||
|
||||
# Validate files exist
|
||||
if [ ! -f "$env_file" ]; then
|
||||
_die "Environment file not found: $env_file"
|
||||
fi
|
||||
if [ ! -f "$sops_file" ]; then
|
||||
_die "Sops file not found: $sops_file"
|
||||
fi
|
||||
if [ ! -f "$age_key_file" ]; then
|
||||
_die "Age key file not found: $age_key_file"
|
||||
fi
|
||||
|
||||
# Security check: age key permissions
|
||||
_validate_age_key_perms "$age_key_file"
|
||||
|
||||
# Security check: VAULT_ADDR must be localhost
|
||||
_check_vault_addr
|
||||
|
||||
# Source the Vault helpers
|
||||
source "$(dirname "$0")/../lib/hvault.sh"
|
||||
|
||||
# Load .env file
|
||||
_log "Loading environment from: $env_file"
|
||||
_load_env_file "$env_file"
|
||||
|
||||
# Decrypt sops file
|
||||
_log "Decrypting sops file: $sops_file"
|
||||
local sops_env
|
||||
sops_env="$(_decrypt_sops "$sops_file" "$age_key_file")"
|
||||
# shellcheck disable=SC2086
|
||||
eval "$sops_env"
|
||||
|
||||
# Collect all import operations
|
||||
declare -a operations=()
|
||||
|
||||
# --- From .env ---
|
||||
|
||||
# Bots: FORGE_{ROLE}_TOKEN + FORGE_{ROLE}_PASS
|
||||
for role in "${BOT_ROLES[@]}"; do
|
||||
local token_var="FORGE_${role^^}_TOKEN"
|
||||
local pass_var="FORGE_${role^^}_PASS"
|
||||
local token_val="${!token_var:-}"
|
||||
local pass_val="${!pass_var:-}"
|
||||
|
||||
if [ -n "$token_val" ] && [ -n "$pass_val" ]; then
|
||||
operations+=("bots:$role:token:$env_file:$token_var")
|
||||
operations+=("bots:$role:pass:$env_file:$pass_var")
|
||||
elif [ -n "$token_val" ] || [ -n "$pass_val" ]; then
|
||||
_err "Warning: $role bot has token but no password (or vice versa), skipping"
|
||||
fi
|
||||
done
|
||||
|
||||
# Llama bot: FORGE_TOKEN_LLAMA + FORGE_PASS_LLAMA
|
||||
local llama_token="${FORGE_TOKEN_LLAMA:-}"
|
||||
local llama_pass="${FORGE_PASS_LLAMA:-}"
|
||||
if [ -n "$llama_token" ] && [ -n "$llama_pass" ]; then
|
||||
operations+=("bots:dev-qwen:token:$env_file:FORGE_TOKEN_LLAMA")
|
||||
operations+=("bots:dev-qwen:pass:$env_file:FORGE_PASS_LLAMA")
|
||||
elif [ -n "$llama_token" ] || [ -n "$llama_pass" ]; then
|
||||
_err "Warning: dev-qwen bot has token but no password (or vice versa), skipping"
|
||||
fi
|
||||
|
||||
# Generic forge creds: FORGE_TOKEN + FORGE_PASS
|
||||
local forge_token="${FORGE_TOKEN:-}"
|
||||
local forge_pass="${FORGE_PASS:-}"
|
||||
if [ -n "$forge_token" ] && [ -n "$forge_pass" ]; then
|
||||
operations+=("forge:token:$env_file:FORGE_TOKEN")
|
||||
operations+=("forge:pass:$env_file:FORGE_PASS")
|
||||
fi
|
||||
|
||||
# Forge admin token: FORGE_ADMIN_TOKEN
|
||||
local forge_admin_token="${FORGE_ADMIN_TOKEN:-}"
|
||||
if [ -n "$forge_admin_token" ]; then
|
||||
operations+=("forge:admin_token:$env_file:FORGE_ADMIN_TOKEN")
|
||||
fi
|
||||
|
||||
# Woodpecker secrets: WOODPECKER_*
|
||||
# Only read from the .env file, not shell environment
|
||||
local woodpecker_keys=()
|
||||
while IFS='=' read -r key _; do
|
||||
if [[ "$key" =~ ^WOODPECKER_ ]] || [[ "$key" =~ ^WP_[A-Z_]+$ ]]; then
|
||||
woodpecker_keys+=("$key")
|
||||
fi
|
||||
done < <(grep -E '^[A-Z_][A-Z0-9_]*=' "$env_file" 2>/dev/null || true)
|
||||
for key in "${woodpecker_keys[@]}"; do
|
||||
local val="${!key}"
|
||||
if [ -n "$val" ]; then
|
||||
local lowercase_key="${key,,}"
|
||||
operations+=("woodpecker:$lowercase_key:$env_file:$key")
|
||||
fi
|
||||
done
|
||||
|
||||
# Chat secrets: FORWARD_AUTH_SECRET, CHAT_OAUTH_CLIENT_ID, CHAT_OAUTH_CLIENT_SECRET
|
||||
for key in FORWARD_AUTH_SECRET CHAT_OAUTH_CLIENT_ID CHAT_OAUTH_CLIENT_SECRET; do
|
||||
local val="${!key:-}"
|
||||
if [ -n "$val" ]; then
|
||||
local lowercase_key="${key,,}"
|
||||
operations+=("chat:$lowercase_key:$env_file:$key")
|
||||
fi
|
||||
done
|
||||
|
||||
# --- From sops-decrypted .env.vault.enc ---
|
||||
|
||||
# Runner tokens
|
||||
for token_name in "${RUNNER_TOKENS[@]}"; do
|
||||
local token_val="${!token_name:-}"
|
||||
if [ -n "$token_val" ]; then
|
||||
operations+=("runner:${token_name}:value:$sops_file:$token_name")
|
||||
fi
|
||||
done
|
||||
|
||||
# If dry-run, just print the plan
|
||||
if $dry_run; then
|
||||
_log "=== DRY-RUN: Import plan ==="
|
||||
_log "Environment file: $env_file"
|
||||
_log "Sops file: $sops_file"
|
||||
_log "Age key: $age_key_file"
|
||||
_log ""
|
||||
_log "Planned operations:"
|
||||
for op in "${operations[@]}"; do
|
||||
_log " $op"
|
||||
done
|
||||
_log ""
|
||||
_log "Total: ${#operations[@]} operations"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Actual import with idempotency check ---
|
||||
|
||||
_log "=== Starting Vault import ==="
|
||||
_log "Environment file: $env_file"
|
||||
_log "Sops file: $sops_file"
|
||||
_log "Age key: $age_key_file"
|
||||
_log ""
|
||||
|
||||
local created=0
|
||||
local updated=0
|
||||
local unchanged=0
|
||||
|
||||
for op in "${operations[@]}"; do
|
||||
IFS=':' read -r category source_type source_file source_key <<< "$op"
|
||||
local source_value=""
|
||||
|
||||
if [ "$source_file" = "$env_file" ]; then
|
||||
source_value="${!source_key:-}"
|
||||
else
|
||||
# Source from sops-decrypted env
|
||||
# We need to extract just this key from the sops_env
|
||||
source_value="$(printf '%s' "$sops_env" | grep "^${source_key}=" | sed "s/^${source_key=}//" || true)"
|
||||
fi
|
||||
|
||||
# Determine Vault path
|
||||
local vault_path=""
|
||||
local vault_key=""
|
||||
|
||||
case "$category" in
|
||||
bots)
|
||||
vault_path="disinto/bots/${source_type}"
|
||||
vault_key="${source_file##*:}"
|
||||
;;
|
||||
forge)
|
||||
vault_path="disinto/shared/forge"
|
||||
vault_key="$source_type"
|
||||
;;
|
||||
woodpecker)
|
||||
vault_path="disinto/shared/woodpecker"
|
||||
vault_key="$source_type"
|
||||
;;
|
||||
chat)
|
||||
vault_path="disinto/shared/chat"
|
||||
vault_key="$source_type"
|
||||
;;
|
||||
runner)
|
||||
vault_path="disinto/runner"
|
||||
vault_key="$source_type"
|
||||
;;
|
||||
*)
|
||||
_err "Unknown category: $category"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Check if path exists
|
||||
local status="created"
|
||||
if _kv_path_exists "$vault_path"; then
|
||||
# Check if key exists in path
|
||||
local existing_value
|
||||
if existing_value="$(_kv_get_value "$vault_path" "$vault_key")" 2>/dev/null; then
|
||||
if [ "$existing_value" = "$source_value" ]; then
|
||||
status="unchanged"
|
||||
else
|
||||
status="updated"
|
||||
fi
|
||||
else
|
||||
status="created"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Output status
|
||||
_format_status "$status" "$vault_path" "$vault_key"
|
||||
printf '\n'
|
||||
|
||||
# Write if not unchanged
|
||||
if [ "$status" != "unchanged" ]; then
|
||||
_kv_put_secret "$vault_path" "${vault_key}=${source_value}"
|
||||
case "$status" in
|
||||
updated) ((updated++)) || true ;;
|
||||
created) ((created++)) || true ;;
|
||||
esac
|
||||
else
|
||||
((unchanged++)) || true
|
||||
fi
|
||||
done
|
||||
|
||||
_log ""
|
||||
_log "=== Import complete ==="
|
||||
_log "Created: $created"
|
||||
_log "Updated: $updated"
|
||||
_log "Unchanged: $unchanged"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Loading…
Add table
Add a link
Reference in a new issue