fix: feat: agents flush context to scratch file before compaction (#262)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

action/action-agent.sh

@@ -41,6 +41,7 @@ WORKTREE="${PROJECT_REPO_ROOT}"
 PHASE_FILE="/tmp/action-session-${PROJECT_NAME:-harb}-${ISSUE}.phase"
 IMPL_SUMMARY_FILE="/tmp/action-impl-summary-${PROJECT_NAME:-harb}-${ISSUE}.txt"
 PREFLIGHT_RESULT="/tmp/action-preflight-${ISSUE}.json"
+SCRATCH_FILE="/tmp/action-${ISSUE}-scratch.md"
 
 log() {
   printf '[%s] action#%s %s\n' "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" "$ISSUE" "$*" >> "$LOGFILE"
@@ -86,7 +87,7 @@ cleanup() {
   agent_kill_session "$SESSION_NAME"
   # Best-effort docker cleanup for containers started during this action
   (cd "${PROJECT_REPO_ROOT}" 2>/dev/null && docker compose down 2>/dev/null) || true
-  rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$PREFLIGHT_RESULT"
+  rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$PREFLIGHT_RESULT" "$SCRATCH_FILE"
 }
 trap cleanup EXIT
 
@@ -166,6 +167,24 @@ if [ -n "${_thread_id:-}" ]; then
     >> "${MATRIX_THREAD_MAP:-/tmp/matrix-thread-map}" 2>/dev/null || true
 fi
 
+# --- Read scratch file (compaction survival) ---
+SCRATCH_CONTEXT=""
+if [ -f "$SCRATCH_FILE" ]; then
+  SCRATCH_CONTEXT="## Previous context (from scratch file)
+$(cat "$SCRATCH_FILE")
+"
+fi
+SCRATCH_INSTRUCTION="## Context scratch file (compaction survival)
+
+Periodically (every 10-15 tool calls), write a summary of:
+- What you have discovered so far
+- Decisions made and why
+- What remains to do
+to: ${SCRATCH_FILE}
+
+If you find this file exists when you start, read it first — it is your previous context.
+This file is ephemeral — not evidence or permanent memory, just a compaction survival mechanism."
+
 # --- Build initial prompt ---
 PRIOR_SECTION=""
 if [ -n "$PRIOR_COMMENTS" ]; then
@@ -193,7 +212,7 @@ in the issue below.
 ## Issue #${ISSUE}: ${ISSUE_TITLE}
 
 ${ISSUE_BODY}
-
+${SCRATCH_CONTEXT}
 ${PRIOR_SECTION}## Instructions
 
 1. Read the action formula steps in the issue body carefully.
@@ -235,6 +254,8 @@ ${PRIOR_SECTION}## Instructions
 If the prior comments above show work already completed, resume from where it
 left off.
 
+${SCRATCH_INSTRUCTION}
+
 ${PHASE_PROTOCOL_INSTRUCTIONS}"
 
 # --- Create tmux session ---
@@ -264,16 +285,16 @@ case "${_MONITOR_LOOP_EXIT:-}" in
     # Escalate to supervisor (idle_prompt already escalated via _on_phase_change callback)
     echo "{\"issue\":${ISSUE},\"pr\":${PR_NUMBER:-0},\"reason\":\"idle_timeout\",\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \
       >> "${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.jsonl"
-    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE"
+    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "$SCRATCH_FILE"
     ;;
   idle_prompt)
     # Notification + escalation already handled by _on_phase_change(PHASE:failed) callback
-    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE"
+    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "$SCRATCH_FILE"
     ;;
   done)
     # Belt-and-suspenders: callback handles primary cleanup,
     # but ensure sentinel files are removed if callback was interrupted
-    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE"
+    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "$SCRATCH_FILE"
     ;;
 esac
 

dev/dev-agent.sh

@@ -88,6 +88,9 @@ IMPL_SUMMARY_FILE="/tmp/dev-impl-summary-${PROJECT_NAME}-${ISSUE}.txt"
 # Matrix thread tracking — one thread per issue for conversational notifications
 THREAD_FILE="/tmp/dev-thread-${PROJECT_NAME}-${ISSUE}"
 
+# Scratch file for context compaction survival
+SCRATCH_FILE="/tmp/dev-${PROJECT_NAME}-${ISSUE}-scratch.md"
+
 # Timing
 export PHASE_POLL_INTERVAL=30    # seconds between phase checks (read by agent-session.sh)
 IDLE_TIMEOUT=7200         # 2h: kill session if phase stale this long
@@ -493,6 +496,26 @@ else
   done
 fi
 
+# =============================================================================
+# READ SCRATCH FILE (compaction survival)
+# =============================================================================
+SCRATCH_CONTEXT=""
+if [ -f "$SCRATCH_FILE" ]; then
+  SCRATCH_CONTEXT="## Previous context (from scratch file)
+$(cat "$SCRATCH_FILE")
+"
+fi
+SCRATCH_INSTRUCTION="## Context scratch file (compaction survival)
+
+Periodically (every 10-15 tool calls), write a summary of:
+- What you have discovered so far
+- Decisions made and why
+- What remains to do
+to: ${SCRATCH_FILE}
+
+If you find this file exists when you start, read it first — it is your previous context.
+This file is ephemeral — not evidence or permanent memory, just a compaction survival mechanism."
+
 # =============================================================================
 # BUILD PROMPT
 # =============================================================================
@@ -593,7 +616,7 @@ This is issue #${ISSUE} for the ${CODEBERG_REPO} project.
 ## Issue: ${ISSUE_TITLE}
 
 ${ISSUE_BODY}
-
+${SCRATCH_CONTEXT}
 ## CRASH RECOVERY
 
 Your previous session for this issue was interrupted. Resume from where you left off.
@@ -617,6 +640,8 @@ $(if [ -n "$CI_RESULT" ]; then printf '\n### Last CI result:\n%s\n' "$CI_RESULT"
 2. Resume from the last known phase.
 3. Follow the phase protocol below.
 
+${SCRATCH_INSTRUCTION}
+
 ${PHASE_PROTOCOL_INSTRUCTIONS}"
 else
   # Normal mode: initial implementation prompt
@@ -626,7 +651,7 @@ You have been assigned issue #${ISSUE} for the ${CODEBERG_REPO} project.
 ## Issue: ${ISSUE_TITLE}
 
 ${ISSUE_BODY}
-
+${SCRATCH_CONTEXT}
 ## Other open issues labeled 'backlog' (for context if you need to suggest alternatives):
 ${OPEN_ISSUES_SUMMARY}
 
@@ -678,6 +703,8 @@ printf 'PHASE:failed\nReason: refused\n' > \"${PHASE_FILE}\"
 
 **Do NOT invent dependencies that aren't real.** If the code compiles and tests pass, that's ready.
 
+${SCRATCH_INSTRUCTION}
+
 ${PHASE_PROTOCOL_INSTRUCTIONS}"
 fi
 
@@ -745,7 +772,7 @@ case "${_MONITOR_LOOP_EXIT:-}" in
     else
       cleanup_worktree
     fi
-    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" \
+    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "$SCRATCH_FILE" \
       "/tmp/ci-result-${PROJECT_NAME}-${ISSUE}.txt"
     [ -n "${PR_NUMBER:-}" ] && rm -f "/tmp/review-injected-${PROJECT_NAME}-${PR_NUMBER}"
     ;;
@@ -755,7 +782,7 @@ case "${_MONITOR_LOOP_EXIT:-}" in
   done)
     # Belt-and-suspenders: callback in phase-handler.sh handles primary cleanup,
     # but ensure sentinel files are removed if callback was interrupted
-    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" \
+    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "$SCRATCH_FILE" \
       "/tmp/ci-result-${PROJECT_NAME}-${ISSUE}.txt"
     [ -n "${PR_NUMBER:-}" ] && rm -f "/tmp/review-injected-${PROJECT_NAME}-${PR_NUMBER}"
     CLAIMED=false

dev/phase-handler.sh

@@ -525,7 +525,7 @@ Instructions:
           cleanup_labels
           agent_kill_session "$SESSION_NAME"
           cleanup_worktree
-          rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE"
+          rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "${SCRATCH_FILE:-}"
           exit 0
         else
           log "PR #${PR_NUMBER} was closed WITHOUT merge — NOT closing issue"
@@ -580,7 +580,7 @@ Instructions:
     # Local cleanup
     agent_kill_session "$SESSION_NAME"
     cleanup_worktree
-    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" \
+    rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "${SCRATCH_FILE:-}" \
       "/tmp/ci-result-${PROJECT_NAME}-${ISSUE}.txt"
     [ -n "${PR_NUMBER:-}" ] && rm -f "/tmp/review-injected-${PROJECT_NAME}-${PR_NUMBER}"
     CLAIMED=false  # Don't unclaim again in cleanup()
@@ -679,7 +679,7 @@ $(printf '%s' "$REFUSAL_JSON" | head -c 2000)
       CLAIMED=false  # Don't unclaim again in cleanup()
       agent_kill_session "$SESSION_NAME"
       cleanup_worktree
-      rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" \
+      rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "${SCRATCH_FILE:-}" \
         "/tmp/ci-result-${PROJECT_NAME}-${ISSUE}.txt"
       [ -n "${PR_NUMBER:-}" ] && rm -f "/tmp/review-injected-${PROJECT_NAME}-${PR_NUMBER}"
       return 1
@@ -708,7 +708,7 @@ $(printf '%s' "$REFUSAL_JSON" | head -c 2000)
       else
         cleanup_worktree
       fi
-      rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" \
+      rm -f "$PHASE_FILE" "$IMPL_SUMMARY_FILE" "$THREAD_FILE" "${SCRATCH_FILE:-}" \
         "/tmp/ci-result-${PROJECT_NAME}-${ISSUE}.txt"
       [ -n "${PR_NUMBER:-}" ] && rm -f "/tmp/review-injected-${PROJECT_NAME}-${PR_NUMBER}"
       return 1

gardener/gardener-agent.sh

@@ -37,6 +37,7 @@ SESSION_NAME="gardener-${PROJECT_NAME}"
 PHASE_FILE="/tmp/gardener-session-${PROJECT_NAME}.phase"
 RESULT_FILE="/tmp/gardener-result-${PROJECT_NAME}.txt"
 DUST_FILE="$SCRIPT_DIR/dust.jsonl"
+SCRATCH_FILE="/tmp/gardener-${PROJECT_NAME}-scratch.md"
 
 # shellcheck disable=SC2034  # read by monitor_phase_loop in lib/agent-session.sh
 PHASE_POLL_INTERVAL=15
@@ -221,13 +222,18 @@ If a choice is unclear, re-escalate that single item with a clarifying question.
 ${ESCALATION_REPLY}"
 fi
 
+# ── Read scratch file (compaction survival) ───────────────────────────────
+SCRATCH_CONTEXT=$(read_scratch_context "$SCRATCH_FILE")
+SCRATCH_INSTRUCTION=$(build_scratch_instruction "$SCRATCH_FILE")
+
 # ── Build prompt from formula + dynamic context ────────────────────────────
 log "Building gardener prompt from formula"
 
 PROMPT="You are the issue gardener for ${CODEBERG_REPO}. Work through the formula below. You MUST write PHASE:done to '${PHASE_FILE}' when finished — the orchestrator will time you out if you return to the prompt without signalling.
 
 ${CONTEXT_SECTION}
-## Formula
+${SCRATCH_CONTEXT:+${SCRATCH_CONTEXT}
+}## Formula
 ${FORMULA_CONTENT}
 
 ## Runtime context (bash pre-analysis)
@@ -253,6 +259,8 @@ NEVER echo or include the actual token value in output — always reference \$CO
   printf 'ESCALATE\n1. #NNN \"title\" — reason (a) option1 (b) option2\n' >> '${RESULT_FILE}'
   echo 'CLEAN' >> '${RESULT_FILE}'  # only if truly nothing to do
 
+${SCRATCH_INSTRUCTION}
+
 ## Phase protocol (REQUIRED)
 When all work is done and verify confirms zero tech-debt:
   echo 'PHASE:done' > '${PHASE_FILE}'
@@ -462,4 +470,9 @@ Fix all items above in a single PR. Each is a small change (rename, comment, sty
   done <<< "$DUST_GROUPS"
 fi
 
+# ── Cleanup scratch file on normal exit ──────────────────────────────────
+if [ "$FINAL_PHASE" = "PHASE:done" ]; then
+  rm -f "$SCRATCH_FILE"
+fi
+
 log "--- gardener-agent done ---"

lib/formula-session.sh

@@ -126,6 +126,37 @@ formula_phase_callback() {
   esac
 }
 
+# ── Scratch file helpers (compaction survival) ────────────────────────────
+
+# build_scratch_instruction SCRATCH_FILE
+# Returns a prompt block instructing Claude to periodically flush context
+# to a scratch file so understanding survives context compaction.
+build_scratch_instruction() {
+  local scratch_file="$1"
+  cat <<_SCRATCH_EOF_
+## Context scratch file (compaction survival)
+
+Periodically (every 10-15 tool calls), write a summary of:
+- What you have discovered so far
+- Decisions made and why
+- What remains to do
+to: ${scratch_file}
+
+If you find this file exists when you start, read it first — it is your previous context.
+This file is ephemeral — not evidence or permanent memory, just a compaction survival mechanism.
+_SCRATCH_EOF_
+}
+
+# read_scratch_context SCRATCH_FILE
+# If the scratch file exists, returns a context block for prompt injection.
+# Returns empty string if the file does not exist.
+read_scratch_context() {
+  local scratch_file="$1"
+  if [ -f "$scratch_file" ]; then
+    printf '## Previous context (from scratch file)\n%s\n' "$(cat "$scratch_file")"
+  fi
+}
+
 # ── Prompt + monitor helpers ──────────────────────────────────────────────
 
 # build_prompt_footer [EXTRA_API_LINES]

planner/planner-run.sh

@@ -31,6 +31,8 @@ PHASE_FILE="/tmp/planner-session-${PROJECT_NAME}.phase"
 # shellcheck disable=SC2034  # read by monitor_phase_loop in lib/agent-session.sh
 PHASE_POLL_INTERVAL=15
 
+SCRATCH_FILE="/tmp/planner-${PROJECT_NAME}-scratch.md"
+
 log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%S)Z] $*" >> "$LOG_FILE"; }
 
 # ── Guards ────────────────────────────────────────────────────────────────
@@ -53,6 +55,10 @@ $(cat "$MEMORY_FILE")
 "
 fi
 
+# ── Read scratch file (compaction survival) ───────────────────────────────
+SCRATCH_CONTEXT=$(read_scratch_context "$SCRATCH_FILE")
+SCRATCH_INSTRUCTION=$(build_scratch_instruction "$SCRATCH_FILE")
+
 # ── Build prompt ─────────────────────────────────────────────────────────
 build_prompt_footer "
   Relabel:     curl -sf -H \"Authorization: token \$CODEBERG_TOKEN\" -X PUT -H 'Content-Type: application/json' '${CODEBERG_API}/issues/{number}/labels' -d '{\"labels\":[LABEL_ID]}'
@@ -65,12 +71,21 @@ PROMPT="You are the strategic planner for ${CODEBERG_REPO}. Work through the for
 
 ## Project context
 ${CONTEXT_BLOCK}${MEMORY_BLOCK}
-
+${SCRATCH_CONTEXT:+${SCRATCH_CONTEXT}
+}
 ## Formula
 ${FORMULA_CONTENT}
 
+${SCRATCH_INSTRUCTION}
+
 ${PROMPT_FOOTER}"
 
 # ── Run session ──────────────────────────────────────────────────────────
 export CLAUDE_MODEL="opus"
 run_formula_and_monitor "planner"
+
+# ── Cleanup scratch file on normal exit ──────────────────────────────────
+FINAL_PHASE=$(read_phase "$PHASE_FILE")
+if [ "$FINAL_PHASE" = "PHASE:done" ]; then
+  rm -f "$SCRATCH_FILE"
+fi

predictor/predictor-run.sh

@@ -33,6 +33,8 @@ PHASE_FILE="/tmp/predictor-session-${PROJECT_NAME}.phase"
 # shellcheck disable=SC2034  # read by monitor_phase_loop in lib/agent-session.sh
 PHASE_POLL_INTERVAL=15
 
+SCRATCH_FILE="/tmp/predictor-${PROJECT_NAME}-scratch.md"
+
 log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%S)Z] $*" >> "$LOG_FILE"; }
 
 # ── Guards ────────────────────────────────────────────────────────────────
@@ -45,6 +47,10 @@ log "--- Predictor run start ---"
 load_formula "$FACTORY_ROOT/formulas/run-predictor.toml"
 build_context_block AGENTS.md RESOURCES.md
 
+# ── Read scratch file (compaction survival) ───────────────────────────────
+SCRATCH_CONTEXT=$(read_scratch_context "$SCRATCH_FILE")
+SCRATCH_INSTRUCTION=$(build_scratch_instruction "$SCRATCH_FILE")
+
 # ── Build prompt ─────────────────────────────────────────────────────────
 build_prompt_footer
 
@@ -58,12 +64,21 @@ about CI health, issue staleness, agent status, and system conditions.
 
 ## Project context
 ${CONTEXT_BLOCK}
-
+${SCRATCH_CONTEXT:+${SCRATCH_CONTEXT}
+}
 ## Formula
 ${FORMULA_CONTENT}
 
+${SCRATCH_INSTRUCTION}
+
 ${PROMPT_FOOTER}"
 
 # ── Run session ──────────────────────────────────────────────────────────
 export CLAUDE_MODEL="sonnet"
 run_formula_and_monitor "predictor"
+
+# ── Cleanup scratch file on normal exit ──────────────────────────────────
+FINAL_PHASE=$(read_phase "$PHASE_FILE")
+if [ "$FINAL_PHASE" = "PHASE:done" ]; then
+  rm -f "$SCRATCH_FILE"
+fi