Merge pull request 'fix: [nomad-prep] P4 — scaffold lib/hvault.sh (HashiCorp Vault helper module) (#799)' (#814) from fix/issue-799 into main

AGENTS.md

@@ -35,7 +35,7 @@ disinto/                 (code repo)
 │                  SCHEMA.md — vault item schema documentation
 │                  validate.sh — vault item validator
 │                  examples/ — example vault action TOMLs (promote, publish, release, webhook-call)
-├── lib/           env.sh, agent-sdk.sh, ci-helpers.sh, ci-debug.sh, load-project.sh, parse-deps.sh, guard.sh, mirrors.sh, pr-lifecycle.sh, issue-lifecycle.sh, worktree.sh, formula-session.sh, stack-lock.sh, forge-setup.sh, forge-push.sh, ops-setup.sh, ci-setup.sh, generators.sh, hire-agent.sh, release.sh, build-graph.py, branch-protection.sh, secret-scan.sh, tea-helpers.sh, action-vault.sh, ci-log-reader.py, git-creds.sh, sprint-filer.sh
+├── lib/           env.sh, agent-sdk.sh, ci-helpers.sh, ci-debug.sh, load-project.sh, parse-deps.sh, guard.sh, mirrors.sh, pr-lifecycle.sh, issue-lifecycle.sh, worktree.sh, formula-session.sh, stack-lock.sh, forge-setup.sh, forge-push.sh, ops-setup.sh, ci-setup.sh, generators.sh, hire-agent.sh, release.sh, build-graph.py, branch-protection.sh, secret-scan.sh, tea-helpers.sh, action-vault.sh, ci-log-reader.py, git-creds.sh, sprint-filer.sh, hvault.sh
 │                  hooks/ — Claude Code session hooks (on-compact-reinject, on-idle-stop, on-phase-change, on-pretooluse-guard, on-session-end, on-stop-failure)
 ├── projects/      *.toml.example — templates; *.toml — local per-box config (gitignored)
 ├── formulas/      Issue templates (TOML specs for multi-step agent tasks)
@@ -43,7 +43,7 @@ disinto/                 (code repo)
 ├── tools/         Operational tools: edge-control/ (register.sh, install.sh, verify-chat-sandbox.sh)
 ├── docs/          Protocol docs (PHASE-PROTOCOL.md, EVIDENCE-ARCHITECTURE.md)
 ├── site/          disinto.ai website content
-├── tests/         Test files (mock-forgejo.py, smoke-init.sh)
+├── tests/         Test files (mock-forgejo.py, smoke-init.sh, lib-hvault.bats)
 ├── templates/     Issue templates
 ├── bin/           The `disinto` CLI script
 ├── disinto-factory/  Setup documentation and skill

lib/AGENTS.md

@@ -34,3 +34,4 @@ sourced as needed.
 | `lib/sprint-filer.sh` | Post-merge sub-issue filer for sprint PRs. Invoked by the `.woodpecker/ops-filer.yml` pipeline after a sprint PR merges to ops repo `main`. Parses `<!-- filer:begin --> ... <!-- filer:end -->` blocks from sprint PR bodies to extract sub-issue definitions, creates them on the project repo using `FORGE_FILER_TOKEN` (narrow-scope `filer-bot` identity with `issues:write` only), adds `in-progress` label to the parent vision issue, and handles vision lifecycle closure when all sub-issues are closed. Uses `filer_api_all()` for paginated fetches. Idempotent: uses `<!-- decomposed-from: #<vision>, sprint: <slug>, id: <id> -->` markers to skip already-filed issues. Requires `FORGE_FILER_TOKEN`, `FORGE_API`, `FORGE_API_BASE`, `FORGE_OPS_REPO`. | `.woodpecker/ops-filer.yml` (CI pipeline on ops repo) |
 | `lib/hire-agent.sh` | `disinto_hire_an_agent()` — user creation, `.profile` repo setup, formula copying, branch protection, and state marker creation for hiring a new agent. Requires `FORGE_URL`, `FORGE_TOKEN`, `FACTORY_ROOT`, `PROJECT_NAME`. Extracted from `bin/disinto`. | bin/disinto (hire) |
 | `lib/release.sh` | `disinto_release()` — vault TOML creation, branch setup on ops repo, PR creation, and auto-merge request for a versioned release. `_assert_release_globals()` validates required env vars. Requires `FORGE_URL`, `FORGE_TOKEN`, `FORGE_OPS_REPO`, `FACTORY_ROOT`, `PRIMARY_BRANCH`. Extracted from `bin/disinto`. | bin/disinto (release) |
+| `lib/hvault.sh` | HashiCorp Vault helper module. `hvault_kv_get(PATH, [KEY])` — read KV v2 secret, optionally extract one key. `hvault_kv_put(PATH, KEY=VAL ...)` — write KV v2 secret. `hvault_kv_list(PATH)` — list keys at a KV path. `hvault_policy_apply(NAME, FILE)` — idempotent policy upsert. `hvault_jwt_login(ROLE, JWT)` — exchange JWT for short-lived token. `hvault_token_lookup()` — returns TTL/policies/accessor for current token. All functions use `VAULT_ADDR` + `VAULT_TOKEN` from env (fallback: `/etc/vault.d/root.token`), emit structured JSON errors to stderr on failure. Tests: `tests/lib-hvault.bats` (requires `vault server -dev`). | Not sourced at runtime yet — pure scaffolding for Nomad+Vault migration (#799) |

lib/hvault.sh

@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+# hvault.sh — HashiCorp Vault helper module
+#
+# Typed, audited helpers for Vault KV v2 access so no script re-implements
+# `curl -H "X-Vault-Token: ..."` ad-hoc.
+#
+# Usage: source this file, then call any hvault_* function.
+#
+# Environment:
+#   VAULT_ADDR  — Vault server address (required, no default)
+#   VAULT_TOKEN — auth token (precedence: env > /etc/vault.d/root.token)
+#
+# All functions emit structured JSON errors to stderr on failure.
+
+set -euo pipefail
+
+# ── Internal helpers ─────────────────────────────────────────────────────────
+
+# _hvault_err — emit structured JSON error to stderr
+# Args: func_name, message, [detail]
+_hvault_err() {
+  local func="$1" msg="$2" detail="${3:-}"
+  jq -n --arg func "$func" --arg msg "$msg" --arg detail "$detail" \
+    '{error:true,function:$func,message:$msg,detail:$detail}' >&2
+}
+
+# _hvault_resolve_token — resolve VAULT_TOKEN from env or token file
+_hvault_resolve_token() {
+  if [ -n "${VAULT_TOKEN:-}" ]; then
+    return 0
+  fi
+  local token_file="/etc/vault.d/root.token"
+  if [ -f "$token_file" ]; then
+    VAULT_TOKEN="$(cat "$token_file")"
+    export VAULT_TOKEN
+    return 0
+  fi
+  return 1
+}
+
+# _hvault_check_prereqs — validate VAULT_ADDR and VAULT_TOKEN are set
+# Args: caller function name
+_hvault_check_prereqs() {
+  local caller="$1"
+  if [ -z "${VAULT_ADDR:-}" ]; then
+    _hvault_err "$caller" "VAULT_ADDR is not set" "export VAULT_ADDR before calling $caller"
+    return 1
+  fi
+  if ! _hvault_resolve_token; then
+    _hvault_err "$caller" "VAULT_TOKEN is not set and /etc/vault.d/root.token not found" \
+      "export VAULT_TOKEN or write token to /etc/vault.d/root.token"
+    return 1
+  fi
+}
+
+# _hvault_request — execute a Vault API request
+# Args: method, path, [data]
+# Outputs: response body to stdout
+# Returns: 0 on 2xx, 1 otherwise (error JSON to stderr)
+_hvault_request() {
+  local method="$1" path="$2" data="${3:-}"
+  local url="${VAULT_ADDR}/v1/${path}"
+  local http_code body
+  local tmpfile
+  tmpfile="$(mktemp)"
+
+  local curl_args=(
+    -s
+    -w '%{http_code}'
+    -H "X-Vault-Token: ${VAULT_TOKEN}"
+    -H "Content-Type: application/json"
+    -X "$method"
+    -o "$tmpfile"
+  )
+  if [ -n "$data" ]; then
+    curl_args+=(-d "$data")
+  fi
+
+  http_code="$(curl "${curl_args[@]}" "$url")" || {
+    _hvault_err "_hvault_request" "curl failed" "url=$url"
+    rm -f "$tmpfile"
+    return 1
+  }
+
+  body="$(cat "$tmpfile")"
+  rm -f "$tmpfile"
+
+  # Check HTTP status — 2xx is success
+  case "$http_code" in
+    2[0-9][0-9])
+      printf '%s' "$body"
+      return 0
+      ;;
+    *)
+      _hvault_err "_hvault_request" "HTTP $http_code" "$body"
+      return 1
+      ;;
+  esac
+}
+
+# ── Public API ───────────────────────────────────────────────────────────────
+
+# hvault_kv_get PATH [KEY]
+#   Read a KV v2 secret at PATH, optionally extract a single KEY.
+#   Outputs: JSON value (full data object, or single key value)
+hvault_kv_get() {
+  local path="${1:-}"
+  local key="${2:-}"
+
+  if [ -z "$path" ]; then
+    _hvault_err "hvault_kv_get" "PATH is required" "usage: hvault_kv_get PATH [KEY]"
+    return 1
+  fi
+  _hvault_check_prereqs "hvault_kv_get" || return 1
+
+  local response
+  response="$(_hvault_request GET "secret/data/${path}")" || return 1
+
+  if [ -n "$key" ]; then
+    printf '%s' "$response" | jq -e -r --arg key "$key" '.data.data[$key]' 2>/dev/null || {
+      _hvault_err "hvault_kv_get" "key not found" "key=$key path=$path"
+      return 1
+    }
+  else
+    printf '%s' "$response" | jq -e '.data.data' 2>/dev/null || {
+      _hvault_err "hvault_kv_get" "failed to parse response" "path=$path"
+      return 1
+    }
+  fi
+}
+
+# hvault_kv_put PATH KEY=VAL [KEY=VAL ...]
+#   Write a KV v2 secret at PATH. Accepts one or more KEY=VAL pairs.
+hvault_kv_put() {
+  local path="${1:-}"
+  shift || true
+
+  if [ -z "$path" ] || [ $# -eq 0 ]; then
+    _hvault_err "hvault_kv_put" "PATH and at least one KEY=VAL required" \
+      "usage: hvault_kv_put PATH KEY=VAL [KEY=VAL ...]"
+    return 1
+  fi
+  _hvault_check_prereqs "hvault_kv_put" || return 1
+
+  # Build JSON payload from KEY=VAL pairs entirely via jq
+  local payload='{"data":{}}'
+  for kv in "$@"; do
+    local k="${kv%%=*}"
+    local v="${kv#*=}"
+    if [ "$k" = "$kv" ]; then
+      _hvault_err "hvault_kv_put" "invalid KEY=VAL pair" "got: $kv"
+      return 1
+    fi
+    payload="$(printf '%s' "$payload" | jq --arg k "$k" --arg v "$v" '.data[$k] = $v')"
+  done
+
+  _hvault_request POST "secret/data/${path}" "$payload" >/dev/null
+}
+
+# hvault_kv_list PATH
+#   List keys at a KV v2 path.
+#   Outputs: JSON array of key names
+hvault_kv_list() {
+  local path="${1:-}"
+
+  if [ -z "$path" ]; then
+    _hvault_err "hvault_kv_list" "PATH is required" "usage: hvault_kv_list PATH"
+    return 1
+  fi
+  _hvault_check_prereqs "hvault_kv_list" || return 1
+
+  local response
+  response="$(_hvault_request LIST "secret/metadata/${path}")" || return 1
+
+  printf '%s' "$response" | jq -e '.data.keys' 2>/dev/null || {
+    _hvault_err "hvault_kv_list" "failed to parse response" "path=$path"
+    return 1
+  }
+}
+
+# hvault_policy_apply NAME FILE
+#   Idempotent policy upsert — create or update a Vault policy.
+hvault_policy_apply() {
+  local name="${1:-}"
+  local file="${2:-}"
+
+  if [ -z "$name" ] || [ -z "$file" ]; then
+    _hvault_err "hvault_policy_apply" "NAME and FILE are required" \
+      "usage: hvault_policy_apply NAME FILE"
+    return 1
+  fi
+  if [ ! -f "$file" ]; then
+    _hvault_err "hvault_policy_apply" "policy file not found" "file=$file"
+    return 1
+  fi
+  _hvault_check_prereqs "hvault_policy_apply" || return 1
+
+  local policy_content
+  policy_content="$(cat "$file")"
+  local payload
+  payload="$(jq -n --arg policy "$policy_content" '{"policy": $policy}')"
+
+  _hvault_request PUT "sys/policies/acl/${name}" "$payload" >/dev/null
+}
+
+# hvault_jwt_login ROLE JWT
+#   Exchange a JWT for a short-lived Vault token.
+#   Outputs: client token string
+hvault_jwt_login() {
+  local role="${1:-}"
+  local jwt="${2:-}"
+
+  if [ -z "$role" ] || [ -z "$jwt" ]; then
+    _hvault_err "hvault_jwt_login" "ROLE and JWT are required" \
+      "usage: hvault_jwt_login ROLE JWT"
+    return 1
+  fi
+  # Only need VAULT_ADDR, not VAULT_TOKEN (we're obtaining a token)
+  if [ -z "${VAULT_ADDR:-}" ]; then
+    _hvault_err "hvault_jwt_login" "VAULT_ADDR is not set"
+    return 1
+  fi
+
+  local payload
+  payload="$(jq -n --arg role "$role" --arg jwt "$jwt" \
+    '{"role": $role, "jwt": $jwt}')"
+
+  local response
+  # JWT login does not require an existing token — use curl directly
+  local tmpfile http_code
+  tmpfile="$(mktemp)"
+  http_code="$(curl -s -w '%{http_code}' \
+    -H "Content-Type: application/json" \
+    -X POST \
+    -d "$payload" \
+    -o "$tmpfile" \
+    "${VAULT_ADDR}/v1/auth/jwt/login")" || {
+    _hvault_err "hvault_jwt_login" "curl failed"
+    rm -f "$tmpfile"
+    return 1
+  }
+
+  local body
+  body="$(cat "$tmpfile")"
+  rm -f "$tmpfile"
+
+  case "$http_code" in
+    2[0-9][0-9])
+      printf '%s' "$body" | jq -e -r '.auth.client_token' 2>/dev/null || {
+        _hvault_err "hvault_jwt_login" "failed to extract client_token" "$body"
+        return 1
+      }
+      ;;
+    *)
+      _hvault_err "hvault_jwt_login" "HTTP $http_code" "$body"
+      return 1
+      ;;
+  esac
+}
+
+# hvault_token_lookup
+#   Returns TTL, policies, and accessor for the current token.
+#   Outputs: JSON object with ttl, policies, accessor fields
+hvault_token_lookup() {
+  _hvault_check_prereqs "hvault_token_lookup" || return 1
+
+  local response
+  response="$(_hvault_request GET "auth/token/lookup-self")" || return 1
+
+  printf '%s' "$response" | jq -e '{
+    ttl: .data.ttl,
+    policies: .data.policies,
+    accessor: .data.accessor,
+    display_name: .data.display_name
+  }' 2>/dev/null || {
+    _hvault_err "hvault_token_lookup" "failed to parse token info"
+    return 1
+  }
+}

tests/lib-hvault.bats

@@ -0,0 +1,215 @@
+#!/usr/bin/env bats
+# tests/lib-hvault.bats — Unit tests for lib/hvault.sh
+#
+# Runs against a dev-mode Vault server (single binary, no LXC needed).
+# CI launches vault server -dev inline before running these tests.
+
+VAULT_BIN="${VAULT_BIN:-vault}"
+
+setup_file() {
+  export TEST_DIR
+  TEST_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
+
+  # Start dev-mode vault on a random port
+  export VAULT_DEV_PORT
+  VAULT_DEV_PORT="$(shuf -i 18200-18299 -n 1)"
+  export VAULT_ADDR="http://127.0.0.1:${VAULT_DEV_PORT}"
+
+  "$VAULT_BIN" server -dev \
+    -dev-listen-address="127.0.0.1:${VAULT_DEV_PORT}" \
+    -dev-root-token-id="test-root-token" \
+    -dev-no-store-token \
+    &>"${BATS_FILE_TMPDIR}/vault.log" &
+  export VAULT_PID=$!
+
+  export VAULT_TOKEN="test-root-token"
+
+  # Wait for vault to be ready (up to 10s)
+  local i=0
+  while ! curl -sf "${VAULT_ADDR}/v1/sys/health" >/dev/null 2>&1; do
+    sleep 0.5
+    i=$((i + 1))
+    if [ "$i" -ge 20 ]; then
+      echo "Vault failed to start. Log:" >&2
+      cat "${BATS_FILE_TMPDIR}/vault.log" >&2
+      return 1
+    fi
+  done
+}
+
+teardown_file() {
+  if [ -n "${VAULT_PID:-}" ]; then
+    kill "$VAULT_PID" 2>/dev/null || true
+    wait "$VAULT_PID" 2>/dev/null || true
+  fi
+}
+
+setup() {
+  # Source the module under test
+  source "${TEST_DIR}/lib/hvault.sh"
+  export VAULT_ADDR VAULT_TOKEN
+}
+
+# ── hvault_kv_put + hvault_kv_get ────────────────────────────────────────────
+
+@test "hvault_kv_put writes and hvault_kv_get reads a secret" {
+  run hvault_kv_put "test/myapp" "username=admin" "password=s3cret"
+  [ "$status" -eq 0 ]
+
+  run hvault_kv_get "test/myapp"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.username == "admin"'
+  echo "$output" | jq -e '.password == "s3cret"'
+}
+
+@test "hvault_kv_get extracts a single key" {
+  hvault_kv_put "test/single" "foo=bar" "baz=qux"
+
+  run hvault_kv_get "test/single" "foo"
+  [ "$status" -eq 0 ]
+  [ "$output" = "bar" ]
+}
+
+@test "hvault_kv_get fails for missing key" {
+  hvault_kv_put "test/keymiss" "exists=yes"
+
+  run hvault_kv_get "test/keymiss" "nope"
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_kv_get fails for missing path" {
+  run hvault_kv_get "test/does-not-exist-$(date +%s)"
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_kv_put fails without KEY=VAL" {
+  run hvault_kv_put "test/bad"
+  [ "$status" -ne 0 ]
+  echo "$output" | grep -q '"error":true' || echo "$stderr" | grep -q '"error":true'
+}
+
+@test "hvault_kv_put rejects malformed pair (no =)" {
+  run hvault_kv_put "test/bad2" "noequals"
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_kv_get fails without PATH" {
+  run hvault_kv_get
+  [ "$status" -ne 0 ]
+}
+
+# ── hvault_kv_list ───────────────────────────────────────────────────────────
+
+@test "hvault_kv_list lists keys at a path" {
+  hvault_kv_put "test/listdir/a" "k=1"
+  hvault_kv_put "test/listdir/b" "k=2"
+
+  run hvault_kv_list "test/listdir"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '. | length >= 2'
+  echo "$output" | jq -e 'index("a")'
+  echo "$output" | jq -e 'index("b")'
+}
+
+@test "hvault_kv_list fails on nonexistent path" {
+  run hvault_kv_list "test/no-such-path-$(date +%s)"
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_kv_list fails without PATH" {
+  run hvault_kv_list
+  [ "$status" -ne 0 ]
+}
+
+# ── hvault_policy_apply ──────────────────────────────────────────────────────
+
+@test "hvault_policy_apply creates a policy" {
+  local pfile="${BATS_TEST_TMPDIR}/test-policy.hcl"
+  cat > "$pfile" <<'HCL'
+path "secret/data/test/*" {
+  capabilities = ["read"]
+}
+HCL
+
+  run hvault_policy_apply "test-reader" "$pfile"
+  [ "$status" -eq 0 ]
+
+  # Verify the policy exists via Vault API
+  run curl -sf -H "X-Vault-Token: ${VAULT_TOKEN}" \
+    "${VAULT_ADDR}/v1/sys/policies/acl/test-reader"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.data.policy' | grep -q "secret/data/test"
+}
+
+@test "hvault_policy_apply is idempotent" {
+  local pfile="${BATS_TEST_TMPDIR}/idem-policy.hcl"
+  printf 'path "secret/*" { capabilities = ["list"] }\n' > "$pfile"
+
+  run hvault_policy_apply "idem-policy" "$pfile"
+  [ "$status" -eq 0 ]
+
+  # Apply again — should succeed
+  run hvault_policy_apply "idem-policy" "$pfile"
+  [ "$status" -eq 0 ]
+}
+
+@test "hvault_policy_apply fails with missing file" {
+  run hvault_policy_apply "bad-policy" "/nonexistent/policy.hcl"
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_policy_apply fails without args" {
+  run hvault_policy_apply
+  [ "$status" -ne 0 ]
+}
+
+# ── hvault_token_lookup ──────────────────────────────────────────────────────
+
+@test "hvault_token_lookup returns token info" {
+  run hvault_token_lookup
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.policies'
+  echo "$output" | jq -e '.accessor'
+  echo "$output" | jq -e 'has("ttl")'
+}
+
+@test "hvault_token_lookup fails without VAULT_TOKEN" {
+  unset VAULT_TOKEN
+  run hvault_token_lookup
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_token_lookup fails without VAULT_ADDR" {
+  unset VAULT_ADDR
+  run hvault_token_lookup
+  [ "$status" -ne 0 ]
+}
+
+# ── hvault_jwt_login ─────────────────────────────────────────────────────────
+
+@test "hvault_jwt_login fails without VAULT_ADDR" {
+  unset VAULT_ADDR
+  run hvault_jwt_login "myrole" "fakejwt"
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_jwt_login fails without args" {
+  run hvault_jwt_login
+  [ "$status" -ne 0 ]
+}
+
+@test "hvault_jwt_login returns error for unconfigured jwt auth" {
+  # JWT auth backend is not enabled in dev mode by default — expect failure
+  run hvault_jwt_login "myrole" "eyJhbGciOiJSUzI1NiJ9.fake.sig"
+  [ "$status" -ne 0 ]
+}
+
+# ── Env / prereq errors ─────────────────────────────────────────────────────
+
+@test "all functions fail with structured JSON error when VAULT_ADDR unset" {
+  unset VAULT_ADDR
+  for fn in hvault_kv_get hvault_kv_put hvault_kv_list hvault_policy_apply hvault_token_lookup; do
+    run $fn "dummy" "dummy"
+    [ "$status" -ne 0 ]
+  done
+}