fix: [nomad-step-2] S2.6 — CI: vault policy fmt + validate + roles.yaml check (#884)
This commit is contained in:
parent
88e49b9e9d
commit
108b928cfc
5 changed files with 725 additions and 9 deletions
|
|
@ -6,11 +6,20 @@
|
||||||
# lib/init/nomad/, plus the `disinto init` dispatcher, gets checked
|
# lib/init/nomad/, plus the `disinto init` dispatcher, gets checked
|
||||||
# before it can land.
|
# before it can land.
|
||||||
#
|
#
|
||||||
|
# Also includes Vault policy validation (S2.6, issue #884):
|
||||||
|
# - vault policy fmt -check on vault/policies/*.hcl
|
||||||
|
# - vault policy validate on each policy file
|
||||||
|
# - roles.yaml validator (yamllint + policy reference check)
|
||||||
|
# - secret-scan gate on policy files
|
||||||
|
#
|
||||||
# Triggers on PRs (and pushes) that touch any of:
|
# Triggers on PRs (and pushes) that touch any of:
|
||||||
# nomad/** — HCL configs (server, client, vault)
|
# nomad/** — HCL configs (server, client, vault)
|
||||||
# lib/init/nomad/** — cluster-up / install / systemd / vault-init
|
# lib/init/nomad/** — cluster-up / install / systemd / vault-init
|
||||||
# bin/disinto — `disinto init --backend=nomad` dispatcher
|
# bin/disinto — `disinto init --backend=nomad` dispatcher
|
||||||
# tests/disinto-init-nomad.bats — the bats suite itself
|
# tests/disinto-init-nomad.bats — the bats suite itself
|
||||||
|
# vault/policies/*.hcl — Vault ACL policies (S2.6)
|
||||||
|
# vault/roles.yaml — JWT auth role definitions (S2.6)
|
||||||
|
# lib/init/nomad/vault-*.sh — Vault init scripts (S2.6)
|
||||||
# .woodpecker/nomad-validate.yml — the pipeline definition
|
# .woodpecker/nomad-validate.yml — the pipeline definition
|
||||||
#
|
#
|
||||||
# Steps (all fail-closed — any error blocks merge):
|
# Steps (all fail-closed — any error blocks merge):
|
||||||
|
|
@ -19,8 +28,12 @@
|
||||||
# nomad/jobs/*.hcl (new jobspecs get
|
# nomad/jobs/*.hcl (new jobspecs get
|
||||||
# CI coverage automatically)
|
# CI coverage automatically)
|
||||||
# 3. vault-operator-diagnose — `vault operator diagnose` syntax check on vault.hcl
|
# 3. vault-operator-diagnose — `vault operator diagnose` syntax check on vault.hcl
|
||||||
# 4. shellcheck-nomad — shellcheck the cluster-up + install scripts + disinto
|
# 4. vault-policy-fmt — `vault policy fmt -check` on vault/policies/*.hcl
|
||||||
# 5. bats-init-nomad — `disinto init --backend=nomad --dry-run` smoke tests
|
# 5. vault-policy-validate — `vault policy validate` on each policy file
|
||||||
|
# 6. vault-roles-validate — yamllint + policy reference check
|
||||||
|
# 7. vault-secret-scan — scan policy files for embedded secrets
|
||||||
|
# 8. shellcheck-nomad — shellcheck the cluster-up + install scripts + disinto
|
||||||
|
# 9. bats-init-nomad — `disinto init --backend=nomad --dry-run` smoke tests
|
||||||
#
|
#
|
||||||
# Pinned image versions match lib/init/nomad/install.sh (nomad 1.9.5 /
|
# Pinned image versions match lib/init/nomad/install.sh (nomad 1.9.5 /
|
||||||
# vault 1.18.5). Bump there AND here together — drift = CI passing on
|
# vault 1.18.5). Bump there AND here together — drift = CI passing on
|
||||||
|
|
@ -34,6 +47,9 @@ when:
|
||||||
- "lib/init/nomad/**"
|
- "lib/init/nomad/**"
|
||||||
- "bin/disinto"
|
- "bin/disinto"
|
||||||
- "tests/disinto-init-nomad.bats"
|
- "tests/disinto-init-nomad.bats"
|
||||||
|
- "vault/policies/*.hcl"
|
||||||
|
- "vault/roles.yaml"
|
||||||
|
- "lib/init/nomad/vault-*.sh"
|
||||||
- ".woodpecker/nomad-validate.yml"
|
- ".woodpecker/nomad-validate.yml"
|
||||||
|
|
||||||
# Authenticated clone — same pattern as .woodpecker/ci.yml. Forgejo is
|
# Authenticated clone — same pattern as .woodpecker/ci.yml. Forgejo is
|
||||||
|
|
@ -123,7 +139,231 @@ steps:
|
||||||
*) echo "vault config: hard failure (rc=$rc)" >&2; exit "$rc" ;;
|
*) echo "vault config: hard failure (rc=$rc)" >&2; exit "$rc" ;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
# ── 4. Shellcheck ────────────────────────────────────────────────────────
|
# ── 4. Vault policy fmt -check ───────────────────────────────────────────
|
||||||
|
# `vault policy fmt -check` is non-destructive; it reads each policy file
|
||||||
|
# and compares against the formatted version. Any difference means the file
|
||||||
|
# needs formatting. This enforces consistent indentation (2-space), no
|
||||||
|
# trailing whitespace, and proper HCL formatting conventions.
|
||||||
|
#
|
||||||
|
# CI runs this BEFORE vault policy validate because unformatted HCL can
|
||||||
|
# sometimes cause confusing validation errors.
|
||||||
|
- name: vault-policy-fmt
|
||||||
|
image: hashicorp/vault:1.18.5
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
failed=0
|
||||||
|
for f in vault/policies/*.hcl; do
|
||||||
|
[ -f "$f" ] || continue
|
||||||
|
echo "fmt-check: $f"
|
||||||
|
if ! vault policy fmt -check "$f" > /dev/null 2>&1; then
|
||||||
|
echo " ERROR: $f is not formatted correctly" >&2
|
||||||
|
vault policy fmt -check "$f" >&2 || true
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$failed" -gt 0 ]; then
|
||||||
|
echo "vault-policy-fmt: formatting errors found" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "vault-policy-fmt: all policies formatted correctly"
|
||||||
|
|
||||||
|
# ── 5. Vault policy validate ─────────────────────────────────────────────
|
||||||
|
# `vault policy validate` performs syntax + semantic validation:
|
||||||
|
# - Checks for unknown stanzas/blocks
|
||||||
|
# - Validates path patterns are valid
|
||||||
|
# - Validates capabilities are known (read, list, create, update, delete, sudo)
|
||||||
|
# - Checks for missing required fields
|
||||||
|
#
|
||||||
|
# Requires a running Vault instance (dev mode is sufficient for CI).
|
||||||
|
# Uses the default dev server at http://127.0.0.1:8200 with "root" token.
|
||||||
|
#
|
||||||
|
# Exit codes:
|
||||||
|
# 0 — policy is valid
|
||||||
|
# 1 — policy has errors (syntax or semantic)
|
||||||
|
#
|
||||||
|
# CI starts a Vault dev server inline for validation.
|
||||||
|
- name: vault-policy-validate
|
||||||
|
image: hashicorp/vault:1.18.5
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
# Start Vault dev server in background
|
||||||
|
vault server -dev -dev-root-token-id=root -dev-listen-address=0.0.0.0:8200 &
|
||||||
|
VAULT_PID=$!
|
||||||
|
trap "kill $VAULT_PID 2>/dev/null || true" EXIT
|
||||||
|
|
||||||
|
# Wait for Vault to be ready
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
if vault status > /dev/null 2>&1; then
|
||||||
|
echo "vault-policy-validate: Vault is ready"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
|
||||||
|
# Validate each policy
|
||||||
|
failed=0
|
||||||
|
for f in vault/policies/*.hcl; do
|
||||||
|
[ -f "$f" ] || continue
|
||||||
|
echo "validate: $f"
|
||||||
|
if ! VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=root vault policy validate "$f" > /dev/null 2>&1; then
|
||||||
|
echo " ERROR: $f validation failed" >&2
|
||||||
|
VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=root vault policy validate "$f" >&2 || true
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$failed" -gt 0 ]; then
|
||||||
|
echo "vault-policy-validate: validation errors found" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "vault-policy-validate: all policies valid"
|
||||||
|
|
||||||
|
# ── 6. Vault roles.yaml validate ─────────────────────────────────────────
|
||||||
|
# Validates vault/roles.yaml:
|
||||||
|
# 1. yamllint check — ensures YAML syntax is valid
|
||||||
|
# 2. Policy reference check — each role references a policy that exists
|
||||||
|
# 3. Required fields check — each role has name, policies, and auth fields
|
||||||
|
#
|
||||||
|
# If roles.yaml doesn't exist yet, this step is skipped (it will be added
|
||||||
|
# in S2.3 alongside JWT auth configuration).
|
||||||
|
- name: vault-roles-validate
|
||||||
|
image: python:3.12-alpine
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
apk add --no-cache yamllint jq
|
||||||
|
|
||||||
|
if [ ! -f vault/roles.yaml ]; then
|
||||||
|
echo "vault-roles-validate: roles.yaml not found, skipping"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "yamllint: vault/roles.yaml"
|
||||||
|
yamllint -q vault/roles.yaml || {
|
||||||
|
echo " ERROR: yamllint found issues" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
echo " OK"
|
||||||
|
|
||||||
|
# Extract and validate policy references
|
||||||
|
echo "policy-reference-check: validating policy references"
|
||||||
|
policy_dir="vault/policies"
|
||||||
|
failed=0
|
||||||
|
|
||||||
|
# Get referenced policies from roles.yaml
|
||||||
|
referenced=$(jq -r '.roles[].policies[]?' vault/roles.yaml 2>/dev/null | sort -u || true)
|
||||||
|
|
||||||
|
if [ -z "$referenced" ]; then
|
||||||
|
echo "vault-roles-validate: no policies referenced in roles.yaml" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get existing policy names
|
||||||
|
existing=$(find "$policy_dir" -maxdepth 1 -name '*.hcl' -type f -exec basename {} .hcl \; | sort)
|
||||||
|
|
||||||
|
for policy in $referenced; do
|
||||||
|
if ! echo "$existing" | grep -q "^${policy}$"; then
|
||||||
|
echo "vault-roles-validate: ERROR: policy '$policy' referenced but not found" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$failed" -gt 0 ]; then
|
||||||
|
echo "vault-roles-validate: policy reference errors found" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "vault-roles-validate: all policy references valid"
|
||||||
|
|
||||||
|
# ── 7. Vault secret-scan ─────────────────────────────────────────────────
|
||||||
|
# Scans policy HCL files for embedded secrets (rare but dangerous copy-paste
|
||||||
|
# mistake). Uses the same patterns as lib/secret-scan.sh:
|
||||||
|
# - Long hex strings (32+ chars)
|
||||||
|
# - API key patterns
|
||||||
|
# - URLs with embedded credentials
|
||||||
|
# - Bearer tokens
|
||||||
|
#
|
||||||
|
# Environment variables like $TOKEN or ${TOKEN} are excluded as safe.
|
||||||
|
- name: vault-secret-scan
|
||||||
|
image: alpine:3.19
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
apk add --no-cache bash
|
||||||
|
|
||||||
|
# Copy the secret-scan.sh script into the container
|
||||||
|
cat > /tmp/secret-scan.sh << 'EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Inline version of lib/secret-scan.sh for CI secret detection
|
||||||
|
|
||||||
|
_SECRET_PATTERNS=(
|
||||||
|
'[0-9a-fA-F]{32,}'
|
||||||
|
'Bearer [A-Za-z0-9_/+=-]{20,}'
|
||||||
|
'0x[0-9a-fA-F]{64}'
|
||||||
|
'https?://[^[:space:]]*[0-9a-fA-F]{20,}'
|
||||||
|
'AKIA[0-9A-Z]{16}'
|
||||||
|
'(API_KEY|SECRET|TOKEN|PRIVATE_KEY|PASSWORD|INFURA|ALCHEMY)=[^[:space:]"]{16,}'
|
||||||
|
)
|
||||||
|
|
||||||
|
_SAFE_PATTERNS=(
|
||||||
|
'\$\{?[A-Z_]+\}?'
|
||||||
|
'commit [0-9a-f]{40}'
|
||||||
|
'Merge [0-9a-f]{40}'
|
||||||
|
'last-reviewed: [0-9a-f]{40}'
|
||||||
|
'codeberg\.org/[^[:space:]]+'
|
||||||
|
'localhost:3000/[^[:space:]]+'
|
||||||
|
'SC[0-9]{4}'
|
||||||
|
)
|
||||||
|
|
||||||
|
scan_for_secrets() {
|
||||||
|
local text="${1:-$(cat)}"
|
||||||
|
local found=0
|
||||||
|
|
||||||
|
local cleaned="$text"
|
||||||
|
for safe in "${_SAFE_PATTERNS[@]}"; do
|
||||||
|
cleaned=$(printf '%s' "$cleaned" | sed -E "s/${safe}/__SAFE__/g" 2>/dev/null || printf '%s' "$cleaned")
|
||||||
|
done
|
||||||
|
|
||||||
|
for pattern in "${_SECRET_PATTERNS[@]}"; do
|
||||||
|
local matches
|
||||||
|
matches=$(printf '%s' "$cleaned" | grep -oE "$pattern" 2>/dev/null || true)
|
||||||
|
if [ -n "$matches" ]; then
|
||||||
|
while IFS= read -r match; do
|
||||||
|
[ "$match" = "__SAFE__" ] && continue
|
||||||
|
[ -z "$match" ] && continue
|
||||||
|
printf 'secret-scan: detected potential secret matching pattern [%s]: %s\n' \
|
||||||
|
"$pattern" "${match:0:8}...${match: -4}" >&2
|
||||||
|
found=1
|
||||||
|
done <<< "$matches"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return $found
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
chmod +x /tmp/secret-scan.sh
|
||||||
|
|
||||||
|
# Scan policy files
|
||||||
|
echo "secret-scan: vault/policies/*.hcl"
|
||||||
|
failed=0
|
||||||
|
for f in vault/policies/*.hcl; do
|
||||||
|
[ -f "$f" ] || continue
|
||||||
|
echo " scanning: $f"
|
||||||
|
if ! /tmp/secret-scan.sh < "$f"; then
|
||||||
|
echo " ERROR: potential secrets detected in $f" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$failed" -gt 0 ]; then
|
||||||
|
echo "vault-secret-scan: secrets detected" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "vault-secret-scan: no secrets detected"
|
||||||
|
|
||||||
|
# ── 8. Shellcheck ────────────────────────────────────────────────────────
|
||||||
# Covers the new lib/init/nomad/*.sh scripts plus bin/disinto (which owns
|
# Covers the new lib/init/nomad/*.sh scripts plus bin/disinto (which owns
|
||||||
# the backend dispatcher). bin/disinto has no .sh extension so the
|
# the backend dispatcher). bin/disinto has no .sh extension so the
|
||||||
# repo-wide shellcheck in .woodpecker/ci.yml skips it — this step is the
|
# repo-wide shellcheck in .woodpecker/ci.yml skips it — this step is the
|
||||||
|
|
@ -133,7 +373,7 @@ steps:
|
||||||
commands:
|
commands:
|
||||||
- shellcheck --severity=warning lib/init/nomad/*.sh bin/disinto
|
- shellcheck --severity=warning lib/init/nomad/*.sh bin/disinto
|
||||||
|
|
||||||
# ── 5. bats: `disinto init --backend=nomad --dry-run` ────────────────────
|
# ── 9. bats: `disinto init --backend=nomad --dry-run` ────────────────────
|
||||||
# Smoke-tests the CLI dispatcher: both --backend=nomad variants exit 0
|
# Smoke-tests the CLI dispatcher: both --backend=nomad variants exit 0
|
||||||
# with the expected step list, and --backend=docker stays on the docker
|
# with the expected step list, and --backend=docker stays on the docker
|
||||||
# path (regression guard). Pure dry-run — no sudo, no network.
|
# path (regression guard). Pure dry-run — no sudo, no network.
|
||||||
|
|
|
||||||
|
|
@ -117,5 +117,7 @@ accept (or vice versa).
|
||||||
|
|
||||||
- `lib/init/nomad/` — installer + systemd units + cluster-up orchestrator.
|
- `lib/init/nomad/` — installer + systemd units + cluster-up orchestrator.
|
||||||
- `.woodpecker/nomad-validate.yml` — this directory's CI pipeline.
|
- `.woodpecker/nomad-validate.yml` — this directory's CI pipeline.
|
||||||
|
- `vault/policies/` — Vault ACL policies (S2.6)
|
||||||
|
- `vault/policies/AGENTS.md` — policy lifecycle, CI enforcement, common failures
|
||||||
- Top-of-file headers in `server.hcl` / `client.hcl` / `vault.hcl`
|
- Top-of-file headers in `server.hcl` / `client.hcl` / `vault.hcl`
|
||||||
document the per-file ownership contract.
|
document the per-file ownership contract.
|
||||||
|
|
|
||||||
|
|
@ -48,12 +48,43 @@ validation.
|
||||||
1. Drop a file matching one of the four naming patterns above. Use an
|
1. Drop a file matching one of the four naming patterns above. Use an
|
||||||
existing file in the same family as the template — comment header,
|
existing file in the same family as the template — comment header,
|
||||||
capability list, and KV path layout should match the family.
|
capability list, and KV path layout should match the family.
|
||||||
2. Run `tools/vault-apply-policies.sh --dry-run` to confirm the new
|
2. Run `vault policy fmt -write <file>` to ensure consistent formatting.
|
||||||
|
3. Run `vault policy validate <file>` locally to check syntax + semantics.
|
||||||
|
4. Run `tools/vault-apply-policies.sh --dry-run` to confirm the new
|
||||||
basename appears in the planned-work list with the expected SHA.
|
basename appears in the planned-work list with the expected SHA.
|
||||||
3. Run `tools/vault-apply-policies.sh` against a Vault instance to
|
5. Run `tools/vault-apply-policies.sh` against a Vault instance to
|
||||||
create it; re-run to confirm it reports `unchanged`.
|
create it; re-run to confirm it reports `unchanged`.
|
||||||
4. The CI fmt + validate step lands in S2.6 (#884). Until then
|
|
||||||
`vault policy fmt <file>` locally is the fastest sanity check.
|
## Policy lifecycle
|
||||||
|
|
||||||
|
Adding a new policy is a three-step process:
|
||||||
|
|
||||||
|
1. **Add policy HCL** — Drop a file in `vault/policies/` matching one of the
|
||||||
|
naming patterns. Run `vault policy fmt <file>` locally to ensure consistent
|
||||||
|
formatting.
|
||||||
|
2. **Update roles.yaml** — Add a JWT auth role in `vault/roles.yaml` that
|
||||||
|
references the new policy name (basename without `.hcl`).
|
||||||
|
3. **Attach to Nomad job** — In S2.4, add the policy to a jobspec's
|
||||||
|
`template { vault { policies = ["<policy-name>"] } }` stanza.
|
||||||
|
|
||||||
|
CI enforces:
|
||||||
|
|
||||||
|
- `vault policy fmt -check` — all `.hcl` files must be formatted
|
||||||
|
- `vault policy validate` — syntax + semantic check (no unknown stanzas,
|
||||||
|
valid capabilities)
|
||||||
|
- `roles.yaml` validator — each role must reference a policy that exists
|
||||||
|
in `vault/policies/`
|
||||||
|
- secret-scan gate — no literal secrets in policy files (rare but
|
||||||
|
dangerous copy-paste mistake)
|
||||||
|
|
||||||
|
## Common failure modes
|
||||||
|
|
||||||
|
| Symptom | Cause | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| `vault policy fmt -check` fails | HCL not formatted (wrong indentation, trailing spaces) | Run `vault policy fmt -write <file>` |
|
||||||
|
| `vault policy validate` fails | Unknown stanza, invalid capability, missing required field | Check Vault docs; valid capabilities: `read`, `list`, `create`, `update`, `delete`, `sudo` |
|
||||||
|
| `roles.yaml` validator fails | Policy name in role doesn't match any `.hcl` basename | Ensure policy name = filename without `.hcl` |
|
||||||
|
| secret-scan fails | Literal secret value embedded (e.g., `token = "abc123..."`) | Use env var reference (`$TOKEN`) or sops/age-encrypted secret |
|
||||||
|
|
||||||
## What this directory does NOT own
|
## What this directory does NOT own
|
||||||
|
|
||||||
|
|
@ -63,4 +94,3 @@ validation.
|
||||||
(#881).
|
(#881).
|
||||||
- **Writing the secret values themselves.** That's S2.2 (#880) via
|
- **Writing the secret values themselves.** That's S2.2 (#880) via
|
||||||
`tools/vault-import.sh`.
|
`tools/vault-import.sh`.
|
||||||
- **CI policy fmt + validate + roles.yaml check.** That's S2.6 (#884).
|
|
||||||
|
|
|
||||||
224
vault/policies/validate.sh
Executable file
224
vault/policies/validate.sh
Executable file
|
|
@ -0,0 +1,224 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# vault/policies/validate.sh — Validate Vault policy HCL files
|
||||||
|
#
|
||||||
|
# Usage: vault/policies/validate.sh [--check-exists]
|
||||||
|
#
|
||||||
|
# This script provides CI validation for Vault policy files:
|
||||||
|
# 1. `vault policy fmt -check` — ensures consistent formatting (non-destructive)
|
||||||
|
# 2. `vault policy validate` — syntax + semantic validation (requires Vault dev mode)
|
||||||
|
# 3. Optional: check that referenced policies exist in roles.yaml
|
||||||
|
#
|
||||||
|
# Exit codes:
|
||||||
|
# 0 — all checks pass
|
||||||
|
# 1 — formatting or validation error
|
||||||
|
# 2 — policy reference validation error (roles.yaml check)
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# VAULT_ADDR — Vault server URL (defaults to http://127.0.0.1:8200 for dev mode)
|
||||||
|
# VAULT_TOKEN — Dev mode token (defaults to "root" for CI)
|
||||||
|
#
|
||||||
|
# CI usage:
|
||||||
|
# vault/policies/validate.sh
|
||||||
|
# vault/policies/validate.sh --check-exists # when roles.yaml exists
|
||||||
|
#
|
||||||
|
# Notes:
|
||||||
|
# - fmt -check is non-destructive; it only reports diff
|
||||||
|
# - validate requires a running Vault instance (dev mode is sufficient for CI)
|
||||||
|
# - Exit 2 is tolerated for advisory warnings (TLS-disabled listeners)
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ROLES_YAML="${SCRIPT_DIR}/../roles.yaml"
|
||||||
|
VAULT_ADDR="${VAULT_ADDR:-http://127.0.0.1:8200}"
|
||||||
|
VAULT_TOKEN="${VAULT_TOKEN:-root}"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage: $(basename "$0") [OPTIONS]
|
||||||
|
|
||||||
|
Validate Vault policy HCL files.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--check-exists Also validate that roles.yaml references exist policies
|
||||||
|
--help Show this help message
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
VAULT_ADDR Vault server URL (default: http://127.0.0.1:8200)
|
||||||
|
VAULT_TOKEN Dev mode token (default: root)
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
0 All checks pass
|
||||||
|
1 Formatting or validation error
|
||||||
|
2 Policy reference validation error (roles.yaml check)
|
||||||
|
EOF
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Check if Vault is available (dev mode is sufficient for CI)
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
check_vault_available() {
|
||||||
|
local rc=0
|
||||||
|
# Try to fetch version; exit 2 is advisory (TLS warning), which is OK for CI
|
||||||
|
if ! vault status > /dev/null 2>&1; then
|
||||||
|
rc=$?
|
||||||
|
case "$rc" in
|
||||||
|
0|2) return 0 ;; # OK for CI
|
||||||
|
*) echo "vault/policies/validate.sh: Vault not available (exit $rc)" >&2; return 1 ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 1: vault policy fmt -check
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
fmt_check() {
|
||||||
|
local failed=0
|
||||||
|
local hcl_files
|
||||||
|
hcl_files=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.hcl' -type f 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [ -z "$hcl_files" ]; then
|
||||||
|
echo "vault/policies/validate.sh: no .hcl files found in $SCRIPT_DIR" >&2
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
for f in $hcl_files; do
|
||||||
|
echo "fmt-check: $f"
|
||||||
|
if ! vault policy fmt -check "$f" > /dev/null 2>&1; then
|
||||||
|
echo " ERROR: file not formatted correctly" >&2
|
||||||
|
vault policy fmt -check "$f" 2>&1 | head -20 >&2 || true
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return $failed
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 2: vault policy validate (syntax + semantic)
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
validate_syntax() {
|
||||||
|
local failed=0
|
||||||
|
local hcl_files
|
||||||
|
hcl_files=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.hcl' -type f 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [ -z "$hcl_files" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check Vault is available first
|
||||||
|
if ! check_vault_available; then
|
||||||
|
echo "vault/policies/validate.sh: skipping validation (Vault unavailable)" >&2
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
for f in $hcl_files; do
|
||||||
|
echo "validate: $f"
|
||||||
|
local rc=0
|
||||||
|
if ! vault policy validate "$f" > /dev/null 2>&1; then
|
||||||
|
rc=$?
|
||||||
|
case "$rc" in
|
||||||
|
0) ;; # Should not happen, but be safe
|
||||||
|
1|2)
|
||||||
|
echo " ERROR: validation failed (exit $rc)" >&2
|
||||||
|
vault policy validate "$f" 2>&1 | head -20 >&2 || true
|
||||||
|
failed=1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo " ERROR: unexpected exit code $rc" >&2
|
||||||
|
failed=1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return $failed
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 3: Check that roles.yaml references exist
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
check_policy_references() {
|
||||||
|
if [ ! -f "$ROLES_YAML" ]; then
|
||||||
|
echo "vault/policies/validate.sh: roles.yaml not found, skipping reference check" >&2
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local failed=0
|
||||||
|
local policy_names
|
||||||
|
|
||||||
|
# Get list of policy names (basenames without .hcl)
|
||||||
|
policy_names=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.hcl' -type f -exec basename {} .hcl \; | sort)
|
||||||
|
|
||||||
|
# Extract policy names from roles.yaml using yq or grep+sed
|
||||||
|
local referenced_policies
|
||||||
|
if command -v yq > /dev/null 2>&1; then
|
||||||
|
# yq is available, use it
|
||||||
|
referenced_policies=$(yq -r '.roles[].policies[]?' "$ROLES_YAML" 2>/dev/null | sort -u || true)
|
||||||
|
else
|
||||||
|
# Fallback: grep for 'policies:' lines and extract values
|
||||||
|
referenced_policies=$(grep -E '^\s*policies:' "$ROLES_YAML" 2>/dev/null | \
|
||||||
|
sed -E 's/.*policies:\s*\[(.*)\].*/\1/' | \
|
||||||
|
tr ',' '\n' | \
|
||||||
|
sed 's/^[[:space:]]*"//;s/"[[:space:]]*$//' | \
|
||||||
|
sort -u || true)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$referenced_policies" ]; then
|
||||||
|
echo "vault/policies/validate.sh: no policies referenced in roles.yaml" >&2
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
for policy in $referenced_policies; do
|
||||||
|
if ! echo "$policy_names" | grep -q "^${policy}$"; then
|
||||||
|
echo "vault/policies/validate.sh: ERROR: policy '$policy' referenced in roles.yaml but not found" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return $failed
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Main
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
check_refs=0
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--check-exists)
|
||||||
|
check_refs=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--help|-h)
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option: $1" >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "vault/policies/validate.sh — validating policy HCL files"
|
||||||
|
echo " VAULT_ADDR: $VAULT_ADDR"
|
||||||
|
echo " roles.yaml: $ROLES_YAML (exists: $([ -f "$ROLES_YAML" ] && echo yes || echo no))"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Run fmt check
|
||||||
|
fmt_check || exit 1
|
||||||
|
|
||||||
|
# Run syntax validation
|
||||||
|
validate_syntax || exit 1
|
||||||
|
|
||||||
|
# Run reference check if requested
|
||||||
|
if [ "$check_refs" -eq 1 ]; then
|
||||||
|
check_policy_references || exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "vault/policies/validate.sh: all checks passed"
|
||||||
|
exit 0
|
||||||
220
vault/validate-roles.sh
Executable file
220
vault/validate-roles.sh
Executable file
|
|
@ -0,0 +1,220 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# vault/validate-roles.sh — Validate roles.yaml for Vault workload identity
|
||||||
|
#
|
||||||
|
# Usage: vault/validate-roles.sh
|
||||||
|
#
|
||||||
|
# This script validates the roles.yaml file for Nomad workload identity:
|
||||||
|
# 1. yamllint check — ensures YAML syntax is valid
|
||||||
|
# 2. Policy reference check — each role references a policy that exists
|
||||||
|
# 3. Required fields check — each role has required fields (name, policies, auth)
|
||||||
|
#
|
||||||
|
# Exit codes:
|
||||||
|
# 0 — all checks pass
|
||||||
|
# 1 — YAML syntax or validation error
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# VAULT_POLICY_DIR — Directory containing policy HCL files (default: vault/policies/)
|
||||||
|
#
|
||||||
|
# CI usage:
|
||||||
|
# vault/validate-roles.sh
|
||||||
|
#
|
||||||
|
# Notes:
|
||||||
|
# - Requires yamllint to be installed
|
||||||
|
# - Policy existence check requires Vault policy files to exist
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ROLES_YAML="${SCRIPT_DIR}/roles.yaml"
|
||||||
|
VAULT_POLICY_DIR="${VAULT_POLICY_DIR:-${SCRIPT_DIR}/policies}"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage: $(basename "$0")
|
||||||
|
|
||||||
|
Validate vault/roles.yaml for Vault workload identity configuration.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--help Show this help message
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
VAULT_POLICY_DIR — Directory containing policy HCL files (default: vault/policies/)
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
0 — all checks pass
|
||||||
|
1 — YAML syntax or validation error
|
||||||
|
|
||||||
|
Requires:
|
||||||
|
- yamllint (for YAML syntax check)
|
||||||
|
- yq (optional, for YAML parsing; falls back to grep/sed)
|
||||||
|
EOF
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Check if required tools are available
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
check_dependencies() {
|
||||||
|
if ! command -v yamllint > /dev/null 2>&1; then
|
||||||
|
echo "validate-roles.sh: yamllint not found in PATH" >&2
|
||||||
|
echo " Install with: pip install yamllint" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 1: yamllint check
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
yamllint_check() {
|
||||||
|
echo "yamllint: $ROLES_YAML"
|
||||||
|
if ! yamllint -q "$ROLES_YAML" 2>&1; then
|
||||||
|
echo " ERROR: yamllint found issues" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo " OK"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 2: Extract policy names from roles.yaml
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
get_referenced_policies() {
|
||||||
|
if command -v yq > /dev/null 2>&1; then
|
||||||
|
# yq is available, use it
|
||||||
|
yq -r '.roles[].policies[]?' "$ROLES_YAML" 2>/dev/null | sort -u || true
|
||||||
|
else
|
||||||
|
# Fallback: grep for 'policies:' lines and extract values
|
||||||
|
grep -E '^\s*policies:' "$ROLES_YAML" 2>/dev/null | \
|
||||||
|
sed -E 's/.*policies:\s*\[(.*)\].*/\1/' | \
|
||||||
|
tr ',' '\n' | \
|
||||||
|
sed 's/^[[:space:]]*"//;s/"[[:space:]]*$//' | \
|
||||||
|
grep -v '^$' | \
|
||||||
|
sort -u || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 3: Check that referenced policies exist
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
check_policy_references() {
|
||||||
|
local failed=0
|
||||||
|
|
||||||
|
if [ ! -d "$VAULT_POLICY_DIR" ]; then
|
||||||
|
echo "validate-roles.sh: policy directory not found: $VAULT_POLICY_DIR" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get list of policy names (basenames without .hcl)
|
||||||
|
local policy_names
|
||||||
|
policy_names=$(find "$VAULT_POLICY_DIR" -maxdepth 1 -name '*.hcl' -type f -exec basename {} .hcl \; | sort)
|
||||||
|
|
||||||
|
if [ -z "$policy_names" ]; then
|
||||||
|
echo "validate-roles.sh: no .hcl files found in $VAULT_POLICY_DIR" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local referenced_policies
|
||||||
|
referenced_policies=$(get_referenced_policies)
|
||||||
|
|
||||||
|
if [ -z "$referenced_policies" ]; then
|
||||||
|
echo "validate-roles.sh: no policies referenced in roles.yaml" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for policy in $referenced_policies; do
|
||||||
|
if ! echo "$policy_names" | grep -q "^${policy}$"; then
|
||||||
|
echo "validate-roles.sh: ERROR: policy '$policy' referenced in roles.yaml but not found" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return $failed
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Step 4: Check required fields in roles
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
check_required_fields() {
|
||||||
|
local failed=0
|
||||||
|
|
||||||
|
if command -v yq > /dev/null 2>&1; then
|
||||||
|
# Check each role has required fields
|
||||||
|
local roles_count
|
||||||
|
roles_count=$(yq '.roles | length' "$ROLES_YAML" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
if [ "$roles_count" -eq 0 ]; then
|
||||||
|
echo "validate-roles.sh: WARNING: no roles defined in roles.yaml" >&2
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
for ((i=0; i<roles_count; i++)); do
|
||||||
|
local role_name
|
||||||
|
role_name=$(yq -r ".roles[$i].name // \"<unnamed>\"" "$ROLES_YAML" 2>/dev/null || echo "<unnamed>")
|
||||||
|
|
||||||
|
# Check for name field
|
||||||
|
if [ "$role_name" = "<unnamed>" ]; then
|
||||||
|
echo "validate-roles.sh: ERROR: role missing 'name' field" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for policies field
|
||||||
|
local policies_count
|
||||||
|
policies_count=$(yq ".roles[$i].policies | length" "$ROLES_YAML" 2>/dev/null || echo "0")
|
||||||
|
if [ "$policies_count" -eq 0 ]; then
|
||||||
|
echo "validate-roles.sh: ERROR: role '$role_name' has no policies defined" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for auth field (JWT auth config)
|
||||||
|
local auth_method
|
||||||
|
auth_method=$(yq -r ".roles[$i].auth // \"<none>\"" "$ROLES_YAML" 2>/dev/null || echo "<none>")
|
||||||
|
if [ "$auth_method" = "<none>" ]; then
|
||||||
|
echo "validate-roles.sh: ERROR: role '$role_name' has no auth configuration" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
return $failed
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Main
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Parse arguments
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--help|-h)
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option: $1" >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "vault/validate-roles.sh — validating roles.yaml"
|
||||||
|
echo " roles.yaml: $ROLES_YAML (exists: $([ -f "$ROLES_YAML" ] && echo yes || echo no))"
|
||||||
|
echo " policy dir: $VAULT_POLICY_DIR"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Exit early if roles.yaml doesn't exist (it will be created in a future step)
|
||||||
|
if [ ! -f "$ROLES_YAML" ]; then
|
||||||
|
echo "vault/validate-roles.sh: roles.yaml not found, skipping validation" >&2
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check dependencies
|
||||||
|
check_dependencies
|
||||||
|
|
||||||
|
# Run validations
|
||||||
|
yamllint_check || exit 1
|
||||||
|
check_policy_references || exit 1
|
||||||
|
check_required_fields || exit 1
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "vault/validate-roles.sh: all checks passed"
|
||||||
|
exit 0
|
||||||
Loading…
Add table
Add a link
Reference in a new issue