fix: [nomad-step-0] S0.3 — install vault + systemd auto-unseal + vault-init.sh (dev-persisted seal) (#823)

Adds the Vault half of the factory-dev-box bringup, landed but not started
(per the install-but-don't-start pattern used for nomad in #822):

- lib/init/nomad/install.sh — now also installs vault from the shared
  HashiCorp apt repo. VAULT_VERSION pinned (1.18.5). Fast-path skips apt
  entirely when both binaries are at their pins; partial upgrades only
  touch the package that drifted.

- nomad/vault.hcl — single-node config: file storage backend at
  /var/lib/vault/data, localhost listener on :8200, ui on, mlock kept on.
  No TLS / HA / audit yet; those land in later steps.

- lib/init/nomad/systemd-vault.sh — writes /etc/systemd/system/vault.service
  (Type=notify, ExecStartPost auto-unseals from /etc/vault.d/unseal.key,
  CAP_IPC_LOCK granted for mlock), deploys nomad/vault.hcl to
  /etc/vault.d/, creates /var/lib/vault/data (0700 root), enables the
  unit without starting it. Idempotent via content-compare.

- lib/init/nomad/vault-init.sh — first-run init: spawns a temporary
  `vault server` if not already reachable, runs operator-init with
  key-shares=1/threshold=1, persists unseal.key + root.token (0400 root),
  unseals once in-process, shuts down the temp server. Re-run detects
  initialized + unseal.key present → no-op. Initialized but key missing
  is a hard failure (can't recover).

lib/hvault.sh already defaults VAULT_TOKEN to /etc/vault.d/root.token
when the env var is absent, so no change needed there.

Seal model: the single unseal key lives on disk; seal-key theft equals
vault theft. Factory-dev-box-acceptable tradeoff — avoids running a
second Vault to auto-unseal the first.

Blocks S0.4 (#824).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

lib/init/nomad/lib-systemd.sh

@@ -46,11 +46,18 @@ systemd_install_unit() {
   if [ ! -f "$unit_path" ] \
      || ! printf '%s\n' "$unit_content" | cmp -s - "$unit_path"; then
     log "writing unit → ${unit_path}"
-    local tmp
+    # Subshell-scoped EXIT trap guarantees the temp file is removed on
-    tmp="$(mktemp)"
+    # both success AND set-e-induced failure of `install`. A function-
-    printf '%s\n' "$unit_content" > "$tmp"
+    # scoped RETURN trap does NOT fire on errexit-abort (bash only runs
-    install -m 0644 -o root -g root "$tmp" "$unit_path"
+    # RETURN on normal function exit), so the subshell is the reliable
-    rm -f "$tmp"
+    # cleanup boundary. It's also isolated from the caller's EXIT trap.
+    (
+      local tmp
+      tmp="$(mktemp)"
+      trap 'rm -f "$tmp"' EXIT
+      printf '%s\n' "$unit_content" > "$tmp"
+      install -m 0644 -o root -g root "$tmp" "$unit_path"
+    )
     needs_reload=1
   else
     log "unit file already up to date"

lib/init/nomad/systemd-vault.sh

@@ -22,7 +22,8 @@
 #
 # Seal model:
 #   The single unseal key lives at /etc/vault.d/unseal.key (0400 root).
-#   Seal-key theft == vault theft. Dev-box acceptable; see docs/VAULT.md.
+#   Seal-key theft == vault theft. Factory-dev-box-acceptable tradeoff —
+#   we avoid running a second Vault to auto-unseal the first.
 #
 # Idempotency contract:
 #   - Unit file NOT rewritten when on-disk content already matches desired.

lib/init/nomad/vault-init.sh

@@ -31,7 +31,8 @@
 #
 # Seal model:
 #   Single unseal key persisted on disk at /etc/vault.d/unseal.key. Seal-key
-#   theft == vault theft. Factory-dev-box-acceptable; see docs/VAULT.md.
+#   theft == vault theft. Factory-dev-box-acceptable tradeoff — we avoid
+#   running a second Vault to auto-unseal the first.
 #
 # Environment:
 #   VAULT_ADDR  — Vault API address (default: http://127.0.0.1:8200).
@@ -101,9 +102,21 @@ vault_reachable() {
   [ "$status" -eq 0 ] || [ "$status" -eq 2 ]
 }
 
-# vault_initialized — echoes "true" / "false" / "" (empty on parse failure).
+# vault_initialized — echoes "true" / "false" / "" (empty on parse failure
+# or unreachable vault). Always returns 0 so that `x="$(vault_initialized)"`
+# is safe under `set -euo pipefail`.
+#
+# Key subtlety: `vault status` exits 2 when Vault is sealed OR uninitialized
+# — the exact state we need to *observe* on first run. Without the
+# `|| true` guard, pipefail + set -e inside a standalone assignment would
+# propagate that exit 2 to the outer script and abort before we ever call
+# `vault operator init`. We capture `vault status`'s output to a variable
+# first (pipefail-safe), then feed it to jq separately.
 vault_initialized() {
-  vault status -format=json 2>/dev/null | jq -r '.initialized' 2>/dev/null
+  local out=""
+  out="$(vault status -format=json 2>/dev/null || true)"
+  [ -n "$out" ] || { printf ''; return 0; }
+  printf '%s' "$out" | jq -r '.initialized' 2>/dev/null || printf ''
 }
 
 # write_secret_file PATH CONTENT