fix: [nomad-step-2] S2.2 — Fix bot/runner operation parsing and sops value extraction (#880)

Add `pull_policy: build` to every agent service emitted by the generator
that shares `docker/agents/Dockerfile` as its build context. Without it,
`docker compose up -d --force-recreate agents-<name>` reuses the cached
`disinto/agents:latest` image and silently keeps running stale
`docker/agents/entrypoint.sh` code even after the repo is updated. This
masked PR #864 (and likely earlier merges) — the fix landed on disk but
never reached the container.

#853 already paired `build:` with `image:` on hired-agent stanzas, which
was enough for first-time ups but not for re-ups. `pull_policy: build`
tells Compose to rebuild the image on every up; BuildKit's layer cache
makes the no-change case near-instant, and the change case picks up the
new source automatically. This covers:

- TOML-driven `agents-<name>` hired via `disinto hire-an-agent` — primary
  target of the issue.
- Legacy `agents-llama` and `agents-llama-all` stanzas — same Dockerfile,
  same staleness problem.

`bin/disinto up` already passed `--build`, so operators on the supported
UX path were already covered; this closes the gap for the direct
`docker compose` path the issue explicitly names in its acceptance.

Regression test added to `tests/lib-generators.bats` to pin the directive
alongside the existing #853 build/image invariants.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

docker/agents/Dockerfile

@@ -2,7 +2,7 @@ FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     bash curl git jq tmux python3 python3-pip openssh-client ca-certificates age shellcheck procps gosu \
-    && pip3 install --break-system-packages networkx \
+    && pip3 install --break-system-packages networkx tomlkit \
     && rm -rf /var/lib/apt/lists/*
 
 # Pre-built binaries (copied from docker/agents/bin/)

lib/generators.sh

@@ -123,6 +123,11 @@ _generate_local_model_services() {
       context: .
       dockerfile: docker/agents/Dockerfile
     image: disinto/agents:\${DISINTO_IMAGE_TAG:-latest}
+    # Rebuild on every up (#887): without this, \`docker compose up -d --force-recreate\`
+    # reuses the cached image and silently keeps running stale docker/agents/ code
+    # even after the repo is updated. \`pull_policy: build\` makes Compose rebuild
+    # the image on every up; BuildKit layer cache makes unchanged rebuilds fast.
+    pull_policy: build
     container_name: disinto-agents-${service_name}
     restart: unless-stopped
     security_opt:
@@ -443,6 +448,9 @@ COMPOSEEOF
     build:
       context: .
       dockerfile: docker/agents/Dockerfile
+    # Rebuild on every up (#887): makes docker/agents/ source changes reach this
+    # container without a manual \`docker compose build\`. Cache-fast when clean.
+    pull_policy: build
     container_name: disinto-agents-llama
     restart: unless-stopped
     security_opt:
@@ -493,6 +501,9 @@ COMPOSEEOF
     build:
       context: .
       dockerfile: docker/agents/Dockerfile
+    # Rebuild on every up (#887): makes docker/agents/ source changes reach this
+    # container without a manual \`docker compose build\`. Cache-fast when clean.
+    pull_policy: build
     container_name: disinto-agents-llama-all
     restart: unless-stopped
     profiles: ["agents-llama-all"]

lib/hire-agent.sh

@@ -535,7 +535,10 @@ EOF
     local interval="${poll_interval:-60}"
     echo "  Writing [agents.${section_name}] to ${toml_file}..."
     python3 -c '
-import sys, re, pathlib
+import sys
+import tomlkit
+import re
+import pathlib
 
 toml_path = sys.argv[1]
 section_name = sys.argv[2]
@@ -548,38 +551,39 @@ poll_interval = sys.argv[7]
 p = pathlib.Path(toml_path)
 text = p.read_text()
 
-# Build the new section
-new_section = f"""
-[agents.{section_name}]
-base_url = "{base_url}"
-model = "{model}"
-api_key = "sk-no-key-required"
-roles = ["{role}"]
-forge_user = "{agent_name}"
-compact_pct = 60
-poll_interval = {poll_interval}
-"""
+# Step 1: Remove any commented-out [agents.X] blocks (they cause parse issues)
+# Match # [agents.section_name] followed by lines that are not section headers
+# Use negative lookahead to stop before a real section header (# [ or [)
+commented_pattern = rf"(?:^|\n)# \[agents\.{re.escape(section_name)}\](?:\n(?!# \[|\[)[^\n]*)*"
+text = re.sub(commented_pattern, "", text, flags=re.DOTALL)
 
-# Check if section already exists and replace it
-pattern = rf"\[agents\.{re.escape(section_name)}\][^\[]*"
-if re.search(pattern, text):
-    text = re.sub(pattern, new_section.strip() + "\n", text)
-else:
-    # Remove commented-out example [agents.llama] block if present
-    text = re.sub(
-        r"\n# Local-model agents \(optional\).*?(?=\n# \[mirrors\]|\n\[mirrors\]|\Z)",
-        "",
-        text,
-        flags=re.DOTALL,
-    )
-    # Append before [mirrors] if it exists, otherwise at end
-    mirrors_match = re.search(r"\n(# )?\[mirrors\]", text)
-    if mirrors_match:
-        text = text[:mirrors_match.start()] + "\n" + new_section + text[mirrors_match.start():]
-    else:
-        text = text.rstrip() + "\n" + new_section
+# Step 2: Parse TOML with tomlkit (preserves comments and formatting)
+try:
+    doc = tomlkit.parse(text)
+except Exception as e:
+    print(f"Error: Invalid TOML in {toml_path}: {e}", file=sys.stderr)
+    sys.exit(1)
 
-p.write_text(text)
+# Step 3: Ensure agents table exists
+if "agents" not in doc:
+    doc.add("agents", tomlkit.table())
+
+# Step 4: Update the specific agent section
+doc["agents"][section_name] = {
+    "base_url": base_url,
+    "model": model,
+    "api_key": "sk-no-key-required",
+    "roles": [role],
+    "forge_user": agent_name,
+    "compact_pct": 60,
+    "poll_interval": int(poll_interval),
+}
+
+# Step 5: Serialize back to TOML (preserves comments)
+output = tomlkit.dumps(doc)
+
+# Step 6: Write back
+p.write_text(output)
 ' "$toml_file" "$section_name" "$local_model" "$model" "$agent_name" "$role" "$interval"
 
     echo "  Agent config written to TOML"

tests/lib-generators.bats

@@ -97,6 +97,38 @@ EOF
   [[ "$output" == *'dockerfile: docker/agents/Dockerfile'* ]]
 }
 
+@test "local-model agent service emits pull_policy: build so docker compose up rebuilds on source change (#887)" {
+  # Without pull_policy: build, `docker compose up -d --force-recreate` reuses
+  # the cached `disinto/agents:latest` image and silently runs stale
+  # docker/agents/entrypoint.sh even after the repo is updated. `pull_policy:
+  # build` forces a rebuild on every up; BuildKit layer cache makes unchanged
+  # rebuilds near-instant. The alternative was requiring every operator to
+  # remember `--build` on every invocation, which was the bug that prompted
+  # #887 (2h of debugging a fix that was merged but never reached the container).
+  cat > "${FACTORY_ROOT}/projects/test.toml" <<'EOF'
+name      = "test"
+repo      = "test-owner/test-repo"
+forge_url = "http://localhost:3000"
+
+[agents.dev-qwen2]
+base_url   = "http://10.10.10.1:8081"
+model      = "qwen"
+api_key    = "sk-no-key-required"
+roles      = ["dev"]
+forge_user = "dev-qwen2"
+EOF
+
+  run bash -c "
+    set -euo pipefail
+    source '${ROOT}/lib/generators.sh'
+    _generate_local_model_services '${FACTORY_ROOT}/docker-compose.yml'
+    cat '${FACTORY_ROOT}/docker-compose.yml'
+  "
+
+  [ "$status" -eq 0 ]
+  [[ "$output" == *'pull_policy: build'* ]]
+}
+
 @test "local-model agent service keys FORGE_BOT_USER to forge_user even when it differs from service name (#849)" {
   # Exercise the case the issue calls out: two agents in the same factory
   # whose service names are identical (`[agents.llama]`) but whose

tools/vault-import.sh

@@ -420,25 +420,38 @@ EOF
   local unchanged=0
 
   for op in "${operations[@]}"; do
-    # Parse operation: category|field|file|key (4 fields for most, 5 for bots/runner)
-    IFS='|' read -r category field file key <<< "$op"
-    local source_value=""
+    # Parse operation: category|field|subkey|file|envvar (5 fields for bots/runner)
+    # or category|field|file|envvar (4 fields for forge/woodpecker/chat)
+    local category field subkey file envvar=""
+    local field_count
+    field_count="$(printf '%s' "$op" | awk -F'|' '{print NF}')"
 
-    if [ "$file" = "$env_file" ]; then
-      source_value="${!key:-}"
+    if [ "$field_count" -eq 5 ]; then
+      # 5 fields: category|role|subkey|file|envvar
+      IFS='|' read -r category field subkey file envvar <<< "$op"
     else
-      # Source from sops-decrypted env
-      source_value="$(printf '%s' "$sops_env" | grep "^${key}=" | sed "s/^${key=}//" || true)"
+      # 4 fields: category|field|file|envvar
+      IFS='|' read -r category field file envvar <<< "$op"
+      subkey="$field"  # For 4-field ops, field is the vault key
     fi
 
     # Determine Vault path and key based on category
     local vault_path=""
-    local vault_key="$key"
+    local vault_key="$subkey"
+    local source_value=""
+
+    if [ "$file" = "$env_file" ]; then
+      # Source from environment file (envvar contains the variable name)
+      source_value="${!envvar:-}"
+    else
+      # Source from sops-decrypted env (envvar contains the variable name)
+      source_value="$(printf '%s' "$sops_env" | grep "^${envvar}=" | sed "s/^${envvar}=//" || true)"
+    fi
 
     case "$category" in
       bots)
         vault_path="disinto/bots/${field}"
-        vault_key="$field"
+        vault_key="$subkey"
         ;;
       forge)
         vault_path="disinto/shared/forge"