From 5ccf09b28daeda723ac64cc0808dcae9220ea1e7 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 23 Mar 2026 18:58:33 +0000 Subject: [PATCH] fix: Encrypt secrets at rest with SOPS + age (#613) - lib/env.sh: Two-tier secret loader (SOPS .env.enc > plaintext .env), remove ~/.netrc fallback - bin/disinto: Add age key generation and SOPS encryption during init, remove write_netrc(), add `disinto secrets` subcommand (edit/show/migrate), add sops+age to preflight warnings - .env.example: Annotate vars as [SECRET] or [CONFIG] - .gitignore: Allow .env.enc and .sops.yaml to be committed - BOOTSTRAP.md: Document SOPS + age setup, key backup, secret management - AGENTS.md: Update AD-005 and coding conventions for .env.enc Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 50 ++++++++--------- .gitignore | 6 +- AGENTS.md | 4 +- BOOTSTRAP.md | 52 ++++++++++++++---- bin/disinto | 151 ++++++++++++++++++++++++++++++++++++++++++++------- lib/env.sh | 13 +++-- 6 files changed, 210 insertions(+), 66 deletions(-) diff --git a/.env.example b/.env.example index 4fa2a15..65c20b7 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,12 @@ # Disinto — Environment Configuration # Copy to .env and fill in your values. # NEVER commit .env to the repo. +# +# With SOPS + age installed, `disinto init` encrypts secrets into .env.enc +# and removes plaintext .env. To migrate an existing .env: `disinto secrets migrate` +# +# Variables marked [SECRET] are credentials that grant access if leaked. +# Variables marked [CONFIG] are non-sensitive and safe in plaintext. # ── Per-project config ──────────────────────────────────────────────────── # Project-specific settings (FORGE_REPO, PROJECT_REPO_ROOT, PRIMARY_BRANCH, @@ -8,23 +14,12 @@ # for an example. Do NOT set them here; they leak into every session. # ── Forge (Forgejo) ───────────────────────────────────────────────────── -# Base URL for the local Forgejo instance. disinto init provisions this. -FORGE_URL=http://localhost:3000 +FORGE_URL=http://localhost:3000 # [CONFIG] local Forgejo instance # ── Auth tokens ─────────────────────────────────────────────────────────── -# Dev-agent token: push branches, create PRs, merge PRs. -# Use the dedicated bot account (e.g. dev-bot). -# Branch protection: this account must be in the merge whitelist. -FORGE_TOKEN= - -# Review-agent token: post review comments and submit formal approvals. -# Use the review bot account (e.g. review-bot). -# Branch protection: this account must be in the approvals whitelist. -FORGE_REVIEW_TOKEN= - -# Comma-separated forge usernames to filter from issue comments. -# The token owner is auto-detected; add extra bot accounts here if needed. -FORGE_BOT_USERNAMES= +FORGE_TOKEN= # [SECRET] dev-bot API token +FORGE_REVIEW_TOKEN= # [SECRET] review-bot API token +FORGE_BOT_USERNAMES= # [CONFIG] comma-separated bot usernames # ── Backwards compatibility ─────────────────────────────────────────────── # If CODEBERG_TOKEN is set but FORGE_TOKEN is not, env.sh falls back to @@ -32,26 +27,25 @@ FORGE_BOT_USERNAMES= # CODEBERG_BOT_USERNAMES). No action needed for existing deployments. # ── Woodpecker CI ───────────────────────────────────────────────────────── -WOODPECKER_TOKEN= -WOODPECKER_SERVER=http://localhost:8000 +WOODPECKER_TOKEN= # [SECRET] Woodpecker API token +WOODPECKER_SERVER=http://localhost:8000 # [CONFIG] Woodpecker server URL # WOODPECKER_REPO_ID — now per-project, set in projects/*.toml [ci] section # Woodpecker Postgres (for direct DB queries) -WOODPECKER_DB_PASSWORD= -WOODPECKER_DB_USER=woodpecker -WOODPECKER_DB_HOST=127.0.0.1 -WOODPECKER_DB_NAME=woodpecker +WOODPECKER_DB_PASSWORD= # [SECRET] Postgres password +WOODPECKER_DB_USER=woodpecker # [CONFIG] Postgres user +WOODPECKER_DB_HOST=127.0.0.1 # [CONFIG] Postgres host +WOODPECKER_DB_NAME=woodpecker # [CONFIG] Postgres database name # ── Matrix (optional — real-time notifications & escalation replies) ────── -MATRIX_HOMESERVER=http://localhost:8008 # Dendrite/Synapse URL -MATRIX_BOT_USER=@factory:your.server # bot's Matrix user ID -MATRIX_TOKEN= # bot's access token -MATRIX_ROOM_ID= # coordination room ID (!xxx:your.server) +MATRIX_HOMESERVER=http://localhost:8008 # [CONFIG] Dendrite/Synapse URL +MATRIX_BOT_USER=@factory:your.server # [CONFIG] bot's Matrix user ID +MATRIX_TOKEN= # [SECRET] bot's access token +MATRIX_ROOM_ID= # [CONFIG] coordination room ID # ── Project-specific secrets ────────────────────────────────────────────── # Store all project secrets here so formulas reference env vars, never hardcode. -# Example: BASE_RPC_URL for on-chain evolution scripts. -BASE_RPC_URL= +BASE_RPC_URL= # [SECRET] on-chain RPC endpoint # ── Tuning ──────────────────────────────────────────────────────────────── -CLAUDE_TIMEOUT=7200 # max seconds per Claude invocation +CLAUDE_TIMEOUT=7200 # [CONFIG] max seconds per Claude invocation diff --git a/.gitignore b/.gitignore index 313bb30..cd9c95e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ -# Secrets +# Plaintext secrets (never commit) .env +# Encrypted secrets — safe to commit (.env.enc is SOPS-encrypted) +!.env.enc +!.sops.yaml + # Per-box project config (generated by disinto init) projects/*.toml diff --git a/AGENTS.md b/AGENTS.md index a93c67e..51d34b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,7 +50,7 @@ disinto/ - All scripts start with `#!/usr/bin/env bash` and `set -euo pipefail` - Source shared environment: `source "$(dirname "$0")/../lib/env.sh"` - Log to `$LOGFILE` using the `log()` function from env.sh or defined locally -- Never hardcode secrets — all come from `.env` or TOML project files +- Never hardcode secrets — all come from `.env.enc` (or `.env` fallback) or TOML project files - Never embed secrets in issue bodies, PR descriptions, or comments — use env var references (e.g. `$BASE_RPC_URL`) - ShellCheck must pass (CI runs `shellcheck` on all `.sh` files) - Avoid duplicate code — shared helpers go in `lib/` @@ -129,7 +129,7 @@ Humans write these. Agents read and enforce them. | AD-002 | Single-threaded pipeline per project. | One dev issue at a time. No new work while a PR awaits CI or review. Prevents merge conflicts and keeps context clear. | | AD-003 | The runtime creates and destroys, the formula preserves. | Runtime manages worktrees/sessions/temp. Formulas commit knowledge to git before signaling done. | | AD-004 | Event-driven > polling > fixed delays. | Never `waitForTimeout` or hardcoded sleep. Use phase files, webhooks, or poll loops with backoff. | -| AD-005 | Secrets via env var indirection, never in issue bodies. | Issue bodies become code. Secrets go in `.env` or TOML project files, referenced as `$VAR_NAME`. | +| AD-005 | Secrets via env var indirection, never in issue bodies. | Issue bodies become code. Secrets go in `.env.enc` (SOPS-encrypted) or fall back to `.env`, referenced as `$VAR_NAME`. | **Who enforces what:** - **Gardener** checks open backlog issues against ADs during grooming; closes violations with a comment referencing the AD number. diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md index 1d55db3..74a4e3c 100644 --- a/BOOTSTRAP.md +++ b/BOOTSTRAP.md @@ -31,7 +31,39 @@ This will: No external accounts or tokens needed. -## 1. Configure `.env` +## 1. Secret Management (SOPS + age) + +Disinto encrypts secrets at rest using [SOPS](https://github.com/getsops/sops) with [age](https://age-encryption.org/) encryption. When `sops` and `age` are installed, `disinto init` automatically: + +1. Generates an age key at `~/.config/sops/age/keys.txt` (if none exists) +2. Creates `.sops.yaml` pinning the age public key +3. Encrypts all secrets into `.env.enc` (safe to commit) +4. Removes the plaintext `.env` + +**Install the tools:** + +```bash +# age (key generation) +apt install age # Debian/Ubuntu +brew install age # macOS + +# sops (encryption/decryption) +# Download from https://github.com/getsops/sops/releases +``` + +**The age private key** at `~/.config/sops/age/keys.txt` is the single file that must be protected. Back it up securely — without it, `.env.enc` cannot be decrypted. LUKS disk encryption on the VPS protects this key at rest. + +**Managing secrets after setup:** + +```bash +disinto secrets edit # Opens .env.enc in $EDITOR, re-encrypts on save +disinto secrets show # Prints decrypted secrets (for debugging) +disinto secrets migrate # Converts existing plaintext .env -> .env.enc +``` + +**Fallback:** If `sops`/`age` are not installed, `disinto init` writes secrets to a plaintext `.env` file with a warning. All agents load secrets transparently — `lib/env.sh` checks for `.env.enc` first, then falls back to `.env`. + +## 2. Configure `.env` ```bash cp .env.example .env @@ -70,7 +102,7 @@ CLAUDE_TIMEOUT=7200 # seconds per Claude invocation If you have an existing deployment using `CODEBERG_TOKEN` / `REVIEW_BOT_TOKEN` in `.env`, those still work — `env.sh` falls back to the old names automatically. No migration needed. -## 2. Configure Project TOML +## 3. Configure Project TOML Each project needs a `projects/.toml` file with box-specific settings (absolute paths, Woodpecker CI IDs, Matrix credentials, forge URL). These files are @@ -98,7 +130,7 @@ forge_url = "http://localhost:3000" The repo ships `projects/*.toml.example` templates showing the expected structure. See any `.toml.example` file for the full field reference. -## 3. Claude Code Global Settings +## 4. Claude Code Global Settings Configure `~/.claude/settings.json` with **only** permissions and `skipDangerousModePermissionPrompt`. Do not add hooks to the global settings — `agent-session.sh` injects per-worktree hooks automatically. @@ -124,7 +156,7 @@ claude --dangerously-skip-permissions # Exit after it initializes successfully ``` -## 4. File Ownership +## 5. File Ownership Everything under `/home/debian` must be owned by `debian:debian`. Root-owned files cause permission errors when agents run as the `debian` user. @@ -139,7 +171,7 @@ Verify no root-owned files exist in agent temp directories: find /tmp/dev-* /tmp/harb-* /tmp/review-* -not -user debian 2>/dev/null ``` -## 4b. Woodpecker CI + Forgejo Integration +## 5b. Woodpecker CI + Forgejo Integration `disinto init` automatically configures Woodpecker to use the local Forgejo instance as its forge backend if `WOODPECKER_SERVER` is set in `.env`. This includes: @@ -184,7 +216,7 @@ curl -X POST \ Woodpecker will now trigger pipelines on pushes to Forgejo and push commit status back. Disinto queries Woodpecker directly for CI status (with a forge API fallback), so pipeline results are visible even if Woodpecker's status push to Forgejo is delayed. -## 5. Prepare the Target Repo +## 6. Prepare the Target Repo ### Required: CI pipeline @@ -268,7 +300,7 @@ entire repo as "new", generating a noisy first-run diff. See `formulas/run-planner.toml` (agents-update step) for the full AGENTS.md conventions. -## 6. Write Good Issues +## 7. Write Good Issues Dev-agent works best with issues that have: @@ -283,7 +315,7 @@ Dev-agent works best with issues that have: Dev-agent checks that all referenced issues are closed (= merged) before starting work. If any are open, the issue is skipped and checked again next cycle. -## 7. Install Cron +## 8. Install Cron ```bash crontab -e @@ -342,7 +374,7 @@ FACTORY_ROOT=/home/you/disinto The staggered offsets prevent agents from competing for resources. Each project gets its own lock file (`/tmp/dev-agent-{name}.lock`) derived from the `name` field in its TOML, so concurrent runs across projects are safe. -## 8. Verify +## 9. Verify ```bash # Should complete with "all clear" (no problems to fix) @@ -363,7 +395,7 @@ tail -30 dev/dev-agent.log tail -30 review/review.log ``` -## 9. Optional: Matrix Notifications +## 10. Optional: Matrix Notifications If you want real-time notifications and human-in-the-loop escalation: diff --git a/bin/disinto b/bin/disinto index c73c8cd..eed7b80 100755 --- a/bin/disinto +++ b/bin/disinto @@ -5,6 +5,7 @@ # Commands: # disinto init [options] Bootstrap a new project # disinto status Show factory status +# disinto secrets Manage encrypted secrets # # Usage: # disinto init https://github.com/user/repo @@ -25,6 +26,7 @@ disinto — autonomous code factory CLI Usage: disinto init [options] Bootstrap a new project disinto status Show factory status + disinto secrets Manage encrypted secrets (.env.enc) Init options: --branch Primary branch (default: auto-detect) @@ -62,26 +64,73 @@ clone_url_from_slug() { printf '%s/%s.git' "$forge_url" "$slug" } -# Write (or update) credentials in ~/.netrc for a given host. -write_netrc() { - local host="$1" login="$2" token="$3" - local netrc="${HOME}/.netrc" +# Ensure an age key exists; generate one if missing. +# Exports AGE_PUBLIC_KEY on success. +ensure_age_key() { + local key_dir="${HOME}/.config/sops/age" + local key_file="${key_dir}/keys.txt" - # Remove existing entry for this host if present - if [ -f "$netrc" ]; then - local tmp - tmp=$(mktemp) - awk -v h="$host" ' - $0 ~ "^machine " h { skip=1; next } - /^machine / { skip=0 } - !skip - ' "$netrc" > "$tmp" - mv "$tmp" "$netrc" + if [ -f "$key_file" ]; then + AGE_PUBLIC_KEY="$(age-keygen -y "$key_file" 2>/dev/null)" + export AGE_PUBLIC_KEY + return 0 fi - # Append new entry - printf 'machine %s\nlogin %s\npassword %s\n' "$host" "$login" "$token" >> "$netrc" - chmod 600 "$netrc" + if ! command -v age-keygen &>/dev/null; then + return 1 + fi + + mkdir -p "$key_dir" + age-keygen -o "$key_file" 2>/dev/null + chmod 600 "$key_file" + AGE_PUBLIC_KEY="$(age-keygen -y "$key_file" 2>/dev/null)" + export AGE_PUBLIC_KEY + echo "Generated age key: ${key_file}" +} + +# Write .sops.yaml pinning the age recipient for .env.enc files. +write_sops_yaml() { + local pub_key="$1" + cat > "${FACTORY_ROOT}/.sops.yaml" < +encrypt_env_file() { + local input="$1" output="$2" + sops -e --input-type dotenv --output-type dotenv "$input" > "$output" +} + +# Store secrets into .env.enc (encrypted) if SOPS + age available, else .env (plaintext). +# Reads existing .env, updates/adds vars, writes back. +write_secrets_encrypted() { + local env_file="${FACTORY_ROOT}/.env" + local enc_file="${FACTORY_ROOT}/.env.enc" + + if command -v sops &>/dev/null && command -v age-keygen &>/dev/null; then + if ensure_age_key; then + # Write .sops.yaml if missing + if [ ! -f "${FACTORY_ROOT}/.sops.yaml" ]; then + write_sops_yaml "$AGE_PUBLIC_KEY" + fi + + # Encrypt the plaintext .env to .env.enc + if [ -f "$env_file" ]; then + encrypt_env_file "$env_file" "$enc_file" + rm -f "$env_file" + echo "Secrets encrypted to .env.enc (plaintext .env removed)" + return 0 + fi + fi + fi + + # Fallback: keep plaintext .env + echo "Warning: sops/age not available — secrets stored in plaintext .env" >&2 + return 0 } FORGEJO_DATA_DIR="${HOME}/.disinto/forgejo" @@ -385,6 +434,14 @@ preflight_check() { if ! command -v docker &>/dev/null; then echo "Warning: docker not found (needed for Forgejo provisioning)" >&2 fi + if ! command -v sops &>/dev/null; then + echo "Warning: sops not found (secrets will be stored in plaintext .env)" >&2 + echo " Install: https://github.com/getsops/sops/releases" >&2 + fi + if ! command -v age-keygen &>/dev/null; then + echo "Warning: age not found (needed for secret encryption with SOPS)" >&2 + echo " Install: apt install age / brew install age" >&2 + fi if [ "$errors" -gt 0 ]; then echo "" >&2 @@ -836,6 +893,9 @@ p.write_text(text) # Install cron jobs install_cron "$project_name" "$toml_path" "$auto_yes" + # Encrypt secrets if SOPS + age are available + write_secrets_encrypted + echo "" echo "Done. Project ${project_name} is ready." echo " Config: ${toml_path}" @@ -919,11 +979,64 @@ with open(sys.argv[1], 'rb') as f: fi } +# ── secrets command ──────────────────────────────────────────────────────────── + +disinto_secrets() { + local subcmd="${1:-}" + local enc_file="${FACTORY_ROOT}/.env.enc" + local env_file="${FACTORY_ROOT}/.env" + + case "$subcmd" in + edit) + if [ ! -f "$enc_file" ]; then + echo "Error: ${enc_file} not found. Run 'disinto secrets migrate' first." >&2 + exit 1 + fi + sops "$enc_file" + ;; + show) + if [ ! -f "$enc_file" ]; then + echo "Error: ${enc_file} not found." >&2 + exit 1 + fi + sops -d "$enc_file" + ;; + migrate) + if [ ! -f "$env_file" ]; then + echo "Error: ${env_file} not found — nothing to migrate." >&2 + exit 1 + fi + if ! command -v sops &>/dev/null || ! command -v age-keygen &>/dev/null; then + echo "Error: sops and age are required for migration." >&2 + echo " Install sops: https://github.com/getsops/sops/releases" >&2 + echo " Install age: apt install age / brew install age" >&2 + exit 1 + fi + if ! ensure_age_key; then + echo "Error: failed to generate age key" >&2 + exit 1 + fi + if [ ! -f "${FACTORY_ROOT}/.sops.yaml" ]; then + write_sops_yaml "$AGE_PUBLIC_KEY" + echo "Created: .sops.yaml" + fi + encrypt_env_file "$env_file" "$enc_file" + rm -f "$env_file" + echo "Migrated: .env -> .env.enc (plaintext removed)" + ;; + *) + echo "Usage: disinto secrets " >&2 + exit 1 + ;; + esac +} + # ── Main dispatch ──────────────────────────────────────────────────────────── case "${1:-}" in - init) shift; disinto_init "$@" ;; - status) shift; disinto_status "$@" ;; + init) shift; disinto_init "$@" ;; + status) shift; disinto_status "$@" ;; + secrets) shift; disinto_secrets "$@" ;; -h|--help) usage ;; *) usage ;; esac diff --git a/lib/env.sh b/lib/env.sh index 2543fc8..0c66282 100755 --- a/lib/env.sh +++ b/lib/env.sh @@ -7,8 +7,12 @@ set -euo pipefail # Resolve script root (parent of lib/) FACTORY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# Load .env if present -if [ -f "$FACTORY_ROOT/.env" ]; then +# Load secrets: prefer .env.enc (SOPS-encrypted), fall back to plaintext .env +if [ -f "$FACTORY_ROOT/.env.enc" ] && command -v sops &>/dev/null; then + set -a + eval "$(sops -d --output-type dotenv "$FACTORY_ROOT/.env.enc" 2>/dev/null)" || true + set +a +elif [ -f "$FACTORY_ROOT/.env" ]; then set -a # shellcheck source=/dev/null source "$FACTORY_ROOT/.env" @@ -24,13 +28,10 @@ if [ -n "${PROJECT_TOML:-}" ] && [ -f "$PROJECT_TOML" ]; then source "${FACTORY_ROOT}/lib/load-project.sh" "$PROJECT_TOML" fi -# Forge token: new FORGE_TOKEN > legacy CODEBERG_TOKEN > ~/.netrc +# Forge token: new FORGE_TOKEN > legacy CODEBERG_TOKEN if [ -z "${FORGE_TOKEN:-}" ]; then FORGE_TOKEN="${CODEBERG_TOKEN:-}" fi -if [ -z "${FORGE_TOKEN:-}" ]; then - FORGE_TOKEN="$(awk '/codeberg.org/{getline;getline;print $2}' ~/.netrc 2>/dev/null || true)" -fi export FORGE_TOKEN export CODEBERG_TOKEN="${FORGE_TOKEN}" # backwards compat