diff --git a/formulas/groom-backlog.toml b/formulas/groom-backlog.toml new file mode 100644 index 0000000..d561ec2 --- /dev/null +++ b/formulas/groom-backlog.toml @@ -0,0 +1,152 @@ +# formulas/groom-backlog.toml — Groom the backlog: triage all tech-debt with verify loop + +name = "groom-backlog" +description = "Triage and process all tech-debt issues — blockers first, then by impact score, verify to zero" +version = 1 + +[context] +files = ["README.md", "AGENTS.md", "VISION.md"] + +[[steps]] +id = "inventory" +title = "Fetch, score, and classify all tech-debt issues" +description = """ +Fetch all open tech-debt issues: + curl -sf -H "Authorization: token $CODEBERG_TOKEN" \ + "$CODEBERG_API/issues?type=issues&state=open&limit=50" | \ + jq '[.[] | select(.labels | map(.name) | any(. == "tech-debt"))]' + +For each issue compute a triage score: + impact: blocker=13 / velocity-drag=8 / quality=5 / cosmetic=2 + effort: trivial=1 / gardener-can-fix=3 / needs-human=8 / unknown=13 + score = impact / effort (higher = do first) + staleness: last update >90 days ago = stale candidate + +Flag likely duplicates (similar title/body, >70% word overlap). + +Separate into tiers: + tier-0 = blockers: issues blocking the factory pipeline (impact >= 13, or flagged as PRIORITY_blockers_starving_factory) + tier-1 = high-value: score >= 1.0, gardener can process + tier-2 = dust: score < 1.0, cosmetic, single-line, trivial + +Print tier counts before proceeding. +""" + +[[steps]] +id = "process-blockers" +title = "Resolve all tier-0 blockers — factory cannot proceed until these reach zero" +description = """ +Process EVERY tier-0 issue. No skipping. + +The bash pre-analysis above may have flagged PRIORITY_blockers_starving_factory issues. +These are issues that block backlog items but are not themselves labeled backlog. +The dev-agent is completely starved until they are promoted or resolved. + +For each tier-0 issue: + - Read the full body: curl -sf -H "Authorization: token $CODEBERG_TOKEN" "$CODEBERG_API/issues/{number}" + - If resolvable: promote to backlog — add acceptance criteria, affected files, relabel + - If needs human decision: add to ESCALATE block + - If invalid / wontfix: close with explanation comment + +After completing all tier-0, re-fetch to check for new blockers: + curl -sf -H "Authorization: token $CODEBERG_TOKEN" \ + "$CODEBERG_API/issues?type=issues&state=open&limit=50" | \ + jq '[.[] | select(.labels | map(.name) | any(. == "tech-debt"))]' + +If new tier-0 blockers appeared, process those too. +Tier-0 MUST reach zero before proceeding to tier-1. +""" +needs = ["inventory"] + +[[steps]] +id = "process-scored" +title = "Process tier-1 issues in descending score order" +description = """ +Work through tier-1 issues from highest score to lowest. + +For each issue choose ONE action and write its result to the result file: + +PROMOTE (substantial work — multi-file, behavioral, architectural, security): + 1. Read full body + 2. Add ## Acceptance criteria with checkboxes + 3. Add ## Affected files section + 4. Add ## Dependencies if needed + 5. Relabel from tech-debt to backlog + 6. Write: echo "ACTION: promoted #NNN to backlog — " >> "$RESULT_FILE" + +DUST (trivial — single-line edit, rename, comment, style, whitespace): + Write: echo 'DUST: {"issue": NNN, "group": "", "title": "...", "reason": "..."}' >> "$RESULT_FILE" + Group by file or subsystem: e.g. "gardener", "lib/env.sh", "dev-poll" + Do NOT close the issue — the script auto-bundles groups of 3+ into a backlog issue. + +DUPLICATE (>80% overlap after reading both bodies — confirm before closing): + Post comment: curl -X POST ... /issues/NNN/comments -d '{"body":"Duplicate of #OLDER"}' + Close: curl -X PATCH ... /issues/NNN -d '{"state":"closed"}' + Write: echo "ACTION: closed #NNN as duplicate of #OLDER" >> "$RESULT_FILE" + +ESCALATE (ambiguous scope, architectural question, needs human decision): + Collect into the ESCALATE block written to the result file at the end. + +Dust vs ore rules: + Dust: comment fix, variable rename, whitespace/formatting, single-line edit, trivial cleanup with no behavior change + Ore: multi-file changes, behavioral fixes, architectural improvements, security/correctness issues + +Sibling dependency rule (CRITICAL): + Issues created from the same PR review or code audit are SIBLINGS — independent work items. + NEVER add bidirectional ## Dependencies between siblings — this creates permanent deadlocks. + Use ## Related for cross-references: "## Related\n- #NNN (sibling)" + The dev-poll parser only reads ## Dependencies / ## Depends on / ## Blocked by headers. + +Every 5 issues processed, check for new tier-0 blockers. If found, stop and handle them before continuing. +""" +needs = ["process-blockers"] + +[[steps]] +id = "classify-dust" +title = "Triage tier-2 dust items" +description = """ +For tier-2 items (trivial, cosmetic, score < 1.0): + - Write DUST lines grouped by file/subsystem (script auto-bundles 3+ into one backlog issue) + - Close stale/invalid with explanation comment + - Close duplicates with cross-reference comment + +These do not need promotion — just classification so they leave the tech-debt queue. +The script handles the bundling; emit correct DUST lines for each item. +""" +needs = ["process-scored"] + +[[steps]] +id = "verify" +title = "Verify completion and loop until zero tech-debt" +description = """ +Re-fetch ALL open tech-debt issues and count them: + REMAINING=$(curl -sf -H "Authorization: token $CODEBERG_TOKEN" \ + "$CODEBERG_API/issues?type=issues&state=open&limit=50" | \ + jq '[.[] | select(.labels | map(.name) | any(. == "tech-debt"))] | length') + echo "Remaining tech-debt: $REMAINING" + +Check each tier: + tier-0 count == 0 (HARD REQUIREMENT — factory is blocked until zero) + tier-1 all processed or escalated + tier-2 all classified + +If tier-0 > 0: + Go back to process-blockers. Repeat until tier-0 == 0. + +If tier-1 has unprocessed issues: + Go back to process-scored. + +If tier-2 still has unclassified dust: + Go back to classify-dust. + +If all tiers clear, write the completion summary and signal done: + echo "ACTION: grooming complete — 0 tech-debt remaining" >> "$RESULT_FILE" + echo 'PHASE:done' > "$PHASE_FILE" + +Escalation format (for items needing human decision — write to result file): + printf 'ESCALATE\n1. #NNN "title" — reason (a) option1 (b) option2 (c) option3\n' >> "$RESULT_FILE" + +On unrecoverable error (API unavailable, repeated failures): + printf 'PHASE:failed\nReason: %s\n' 'describe what failed' > "$PHASE_FILE" +""" +needs = ["classify-dust"] diff --git a/gardener/gardener-agent.sh b/gardener/gardener-agent.sh index bb41ed2..ce710fe 100644 --- a/gardener/gardener-agent.sh +++ b/gardener/gardener-agent.sh @@ -31,22 +31,16 @@ source "$FACTORY_ROOT/lib/env.sh" source "$FACTORY_ROOT/lib/agent-session.sh" LOG_FILE="$SCRIPT_DIR/gardener.log" -CLAUDE_TIMEOUT="${CLAUDE_TIMEOUT:-3600}" - SESSION_NAME="gardener-${PROJECT_NAME}" PHASE_FILE="/tmp/gardener-session-${PROJECT_NAME}.phase" RESULT_FILE="/tmp/gardener-result-${PROJECT_NAME}.txt" DUST_FILE="$SCRIPT_DIR/dust.jsonl" +# shellcheck disable=SC2034 # read by monitor_phase_loop in lib/agent-session.sh PHASE_POLL_INTERVAL=15 -MAX_RUNTIME="${CLAUDE_TIMEOUT}" log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%S)Z] $*" >> "$LOG_FILE"; } -read_phase() { - { cat "$PHASE_FILE" 2>/dev/null || true; } | head -1 | tr -d '[:space:]' -} - log "--- gardener-agent start ---" # ── Read escalation reply (passed via env by gardener-poll.sh) ──────────── @@ -165,113 +159,94 @@ if [ "$PROBLEM_COUNT" -eq 0 ] && [ -z "$ESCALATION_REPLY" ]; then exit 0 fi -# ── Build prompt ────────────────────────────────────────────────────────── -log "Building gardener prompt" +# ── Load formula ───────────────────────────────────────────────────────── +log "Loading groom-backlog formula" +FORMULA_FILE="$FACTORY_ROOT/formulas/groom-backlog.toml" +if [ ! -f "$FORMULA_FILE" ]; then + log "ERROR: formula not found: $FORMULA_FILE" + exit 1 +fi +FORMULA_CONTENT=$(cat "$FORMULA_FILE") -# Build issue summary for context (titles + labels + deps) +# ── Read context files from project root ────────────────────────────────── +CONTEXT_BLOCK="" +for ctx in README.md AGENTS.md VISION.md; do + ctx_path="${PROJECT_REPO_ROOT}/${ctx}" + if [ -f "$ctx_path" ]; then + CONTEXT_BLOCK="${CONTEXT_BLOCK} +### ${ctx} +$(cat "$ctx_path") +" + fi +done + +# ── Build issue context ──────────────────────────────────────────────────── ISSUE_SUMMARY=$(echo "$ISSUES_JSON" | jq -r '.[] | "#\(.number) [\(.labels | map(.name) | join(","))] \(.title)"') -# Build list of issues already staged as dust (so LLM doesn't re-emit them) STAGED_DUST="" if [ -s "$DUST_FILE" ]; then STAGED_DUST=$(jq -r '"#\(.issue) (\(.group))"' "$DUST_FILE" 2>/dev/null | sort -u || true) fi -PROMPT="You are the issue gardener for ${CODEBERG_REPO}. Your job: keep the backlog clean, well-structured, and actionable. +# ── Build optional prompt sections ──────────────────────────────────────── +CONTEXT_SECTION="" +if [ -n "$CONTEXT_BLOCK" ]; then + CONTEXT_SECTION="## Project context +${CONTEXT_BLOCK}" +fi -## Current open issues -$ISSUE_SUMMARY +STAGED_DUST_SECTION="" +if [ -n "$STAGED_DUST" ]; then + STAGED_DUST_SECTION=" +### Already staged as dust — do NOT re-emit DUST for these +${STAGED_DUST}" +fi -## Problems detected -$(echo -e "$PROBLEMS") -## Tools available -- Codeberg API: use curl with the CODEBERG_TOKEN env var (already set in your environment) -- Base URL: ${CODEBERG_API} -- Read issue: \`curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" '${CODEBERG_API}/issues/{number}' | jq '.body'\` -- Relabel: \`curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PUT -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}/labels' -d '{\"labels\":[LABEL_ID]}'\` -- Comment: \`curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X POST -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}/comments' -d '{\"body\":\"...\"}'\` -- Close: \`curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PATCH -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}' -d '{\"state\":\"closed\"}'\` -- Edit body: \`curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PATCH -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}' -d '{\"body\":\"new body\"}'\` -- List labels: \`curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" '${CODEBERG_API}/labels'\` (to find label IDs) -- NEVER echo, log, or include the actual token value in any output — always reference \$CODEBERG_TOKEN -- You're running in the project repo root. Read README.md and any docs/ files before making decisions. +ESCALATION_SECTION="" +if [ -n "$ESCALATION_REPLY" ]; then + ESCALATION_SECTION=" +### Human response to previous escalation +Format: '1a 2c 3b' means question 1→option (a), 2→option (c), 3→option (b). +Execute each chosen option via the Codeberg API FIRST, before processing new items. +If a choice is unclear, re-escalate that single item with a clarifying question. -## Primary mission: unblock the factory -Issues prefixed with PRIORITY_blockers_starving_factory are your TOP priority. These are non-backlog issues that block existing backlog items — the dev-agent is completely starved until these are promoted. Process ALL of them before touching regular tech-debt. +${ESCALATION_REPLY}" +fi -## Your objective: zero tech-debt issues +# ── Build prompt from formula + dynamic context ──────────────────────────── +log "Building gardener prompt from formula" -Tech-debt is unprocessed work — it sits outside the factory pipeline -(dev-agent only pulls backlog). Every tech-debt issue is a decision -you haven't made yet: +PROMPT="You are the issue gardener for ${CODEBERG_REPO}. Work through the formula below — there is no time limit, run until PHASE:done. -- Substantial? → promote to backlog (add affected files, acceptance - criteria, dependencies) -- Dust? → bundle into an ore issue -- Duplicate? → close with cross-reference -- Invalid/wontfix? → close with explanation -- Needs human decision? → escalate +${CONTEXT_SECTION} +## Formula +${FORMULA_CONTENT} -Process ALL tech-debt issues every run. The goal is zero tech-debt -when you're done. If you can't reach zero (needs human input, -unclear scope), escalate those specifically and close out everything -else. +## Runtime context (bash pre-analysis) +### All open issues +${ISSUE_SUMMARY} -Tech-debt is your inbox. An empty inbox is a healthy factory. - -## Dust vs Ore — bundle trivial tech-debt -Don't promote trivial tech-debt individually — each costs a full factory cycle (CI + dev-agent + review + merge). If an issue is dust (comment fix, rename, style-only, single-line change, trivial cleanup), output a DUST line instead of promoting: - -DUST: {\"issue\": NNN, \"group\": \"\", \"title\": \"issue title\", \"reason\": \"why it's dust\"} - -Group by file or subsystem (e.g. \"gardener\", \"lib/env.sh\", \"dev-poll\"). The script collects dust items into a staging file. When a group accumulates 3+ items, the script bundles them into one backlog issue automatically. - -Only promote tech-debt that is substantial: multi-file changes, behavioral fixes, architectural improvements. Dust is any issue where the fix is a single-line edit, a rename, a comment tweak, or a style-only change. -$(if [ -n "$STAGED_DUST" ]; then echo " -These issues are ALREADY staged as dust — do NOT emit DUST lines for them again: -${STAGED_DUST}"; fi) - -## Other rules -1. **Duplicates**: If confident (>80% overlap + same scope after reading bodies), close the newer one with a comment referencing the older. If unsure, ESCALATE. -2. **Thin issues** (non-tech-debt): Add acceptance criteria. Read the body first. -3. **Stale issues**: If clearly superseded or no longer relevant, close with explanation. If unclear, ESCALATE. -4. **Oversized issues**: If >5 acceptance criteria touching different files/concerns, ESCALATE with suggested split. -5. **Dependencies**: If an issue references another that must land first, add a \`## Dependencies\n- #NNN\` section if missing. -6. **Sibling issues**: When creating multiple issues from the same source (PR review, code audit), NEVER add bidirectional dependencies between them. Siblings are independent work items, not parent/child. Use \`## Related\n- #NNN (sibling)\` for cross-references between siblings — NOT \`## Dependencies\`. The dev-poll \`get_deps()\` parser only reads \`## Dependencies\` / \`## Depends on\` / \`## Blocked by\` headers, so \`## Related\` is safely ignored. Bidirectional deps create permanent deadlocks that stall the entire factory. - -## Escalation format -For anything needing human decision, output EXACTLY this format (one block, all items): -\`\`\` -ESCALATE -1. #NNN \"title\" — reason (a) option1 (b) option2 (c) option3 -2. #NNN \"title\" — reason (a) option1 (b) option2 -\`\`\` +### Problems detected +$(echo -e "$PROBLEMS")${STAGED_DUST_SECTION}${ESCALATION_SECTION} +## Codeberg API reference +Base URL: ${CODEBERG_API} +Auth header: -H \"Authorization: token \$CODEBERG_TOKEN\" + Read issue: curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" '${CODEBERG_API}/issues/{number}' | jq '.body' + Relabel: curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PUT -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}/labels' -d '{\"labels\":[LABEL_ID]}' + Comment: curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X POST -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}/comments' -d '{\"body\":\"...\"}' + Close: curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PATCH -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}' -d '{\"state\":\"closed\"}' + Edit body: curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PATCH -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}' -d '{\"body\":\"new body\"}' + List labels: curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" '${CODEBERG_API}/labels' +NEVER echo or include the actual token value in output — always reference \$CODEBERG_TOKEN. ## Output format (MANDATORY — write each line to result file using bash) -Write your structured output to ${RESULT_FILE}. Use bash to append each line: echo \"ACTION: description of what you did\" >> '${RESULT_FILE}' echo 'DUST: {\"issue\": NNN, \"group\": \"...\", \"title\": \"...\", \"reason\": \"...\"}' >> '${RESULT_FILE}' -For escalations, write the full block to the result file: printf 'ESCALATE\n1. #NNN \"title\" — reason (a) option1 (b) option2\n' >> '${RESULT_FILE}' -If truly nothing to do: echo 'CLEAN' >> '${RESULT_FILE}' - -## Important -- You MUST process the tech_debt_promotion items listed above. Read each issue, add acceptance criteria + affected files, then relabel to backlog. -- If an issue is ambiguous or needs a design decision, ESCALATE it — don't skip it silently. -- Every tech-debt issue in the list above should result in either an ACTION (promoted) or an ESCALATE (needs decision). Never skip silently. -$(if [ -n "$ESCALATION_REPLY" ]; then echo " -## Human Response to Previous Escalation -The human replied with shorthand choices keyed to the previous ESCALATE block. -Format: '1a 2c 3b' means question 1→option (a), question 2→option (c), question 3→option (b). - -Raw reply: -${ESCALATION_REPLY} - -Execute each chosen option NOW via the Codeberg API before processing new items. -If a choice is unclear, re-escalate that single item with a clarifying question."; fi) + echo 'CLEAN' >> '${RESULT_FILE}' # only if truly nothing to do ## Phase protocol (REQUIRED) -When you have finished ALL work, write to the phase file: +When all work is done and verify confirms zero tech-debt: echo 'PHASE:done' > '${PHASE_FILE}' On unrecoverable error: printf 'PHASE:failed\nReason: %s\n' 'describe error' > '${PHASE_FILE}'" @@ -294,81 +269,36 @@ matrix_send "gardener" "🌱 Gardener session started for ${CODEBERG_REPO}" 2>/d # ── Phase monitoring loop ───────────────────────────────────────────────── log "Monitoring phase file: ${PHASE_FILE}" -LAST_PHASE_MTIME=0 -IDLE_ELAPSED=0 -CRASHED=false +GARDENER_CRASH_COUNT=0 -while true; do - sleep "$PHASE_POLL_INTERVAL" - IDLE_ELAPSED=$((IDLE_ELAPSED + PHASE_POLL_INTERVAL)) - - # --- Session health check --- - if ! tmux has-session -t "$SESSION_NAME" 2>/dev/null; then - CURRENT_PHASE=$(read_phase) - case "$CURRENT_PHASE" in - PHASE:done|PHASE:failed) - # Expected terminal phase — exit loop - break - ;; - *) - if [ "$CRASHED" = true ]; then - log "ERROR: session crashed again after recovery — giving up" - break - fi - CRASHED=true - log "WARNING: tmux session died unexpectedly (phase: ${CURRENT_PHASE:-none})" - # Attempt one crash recovery - RECOVERY_MSG="The previous gardener session was interrupted unexpectedly. - -Re-run your analysis from scratch: -1. Fetch open issues and identify problems using the Codeberg API -2. Take all necessary actions (close dupes, add criteria, promote tech-debt, etc.) -3. Write structured output to ${RESULT_FILE}: - - echo \"ACTION: ...\" >> '${RESULT_FILE}' - - echo 'DUST: {...}' >> '${RESULT_FILE}' - - printf 'ESCALATE\n1. ...\n' >> '${RESULT_FILE}' -4. When finished: echo 'PHASE:done' > '${PHASE_FILE}'" - - rm -f "$RESULT_FILE" - touch "$RESULT_FILE" - if create_agent_session "$SESSION_NAME" "$PROJECT_REPO_ROOT" 2>/dev/null; then - agent_inject_into_session "$SESSION_NAME" "$RECOVERY_MSG" - log "Recovery session started" - IDLE_ELAPSED=0 - else - log "ERROR: could not restart session after crash" - break - fi - continue - ;; - esac - fi - - # --- Check phase file for changes --- - PHASE_MTIME=$(stat -c %Y "$PHASE_FILE" 2>/dev/null || echo 0) - CURRENT_PHASE=$(read_phase) - - if [ -z "$CURRENT_PHASE" ] || [ "$PHASE_MTIME" -le "$LAST_PHASE_MTIME" ]; then - # No phase change — check idle timeout - if [ "$IDLE_ELAPSED" -ge "$MAX_RUNTIME" ]; then - log "TIMEOUT: gardener session idle for ${MAX_RUNTIME}s — killing" - matrix_send "gardener" "⚠️ Gardener session timed out after ${MAX_RUNTIME}s" 2>/dev/null || true +gardener_phase_callback() { + local phase="$1" + log "phase: ${phase}" + case "$phase" in + PHASE:crashed) + if [ "$GARDENER_CRASH_COUNT" -gt 0 ]; then + log "ERROR: session crashed again after recovery — giving up" + return 0 + fi + GARDENER_CRASH_COUNT=$((GARDENER_CRASH_COUNT + 1)) + log "WARNING: tmux session died unexpectedly — attempting recovery" + rm -f "$RESULT_FILE" + touch "$RESULT_FILE" + if create_agent_session "$SESSION_NAME" "$PROJECT_REPO_ROOT" 2>/dev/null; then + agent_inject_into_session "$SESSION_NAME" "$PROMPT" + log "Recovery session started" + else + log "ERROR: could not restart session after crash" + fi + ;; + PHASE:done|PHASE:failed|PHASE:needs_human|PHASE:merged) agent_kill_session "$SESSION_NAME" - break - fi - continue - fi + ;; + esac +} - # Phase changed - LAST_PHASE_MTIME="$PHASE_MTIME" - IDLE_ELAPSED=0 - log "phase: ${CURRENT_PHASE}" - - if [ "$CURRENT_PHASE" = "PHASE:done" ] || [ "$CURRENT_PHASE" = "PHASE:failed" ]; then - agent_kill_session "$SESSION_NAME" - break - fi -done +# No idle timeout — gardener runs until PHASE:done or PHASE:failed +monitor_phase_loop "$PHASE_FILE" 999999 "gardener_phase_callback" FINAL_PHASE=$(read_phase) log "Final phase: ${FINAL_PHASE:-none}"