diff --git a/dev/dev-poll.sh b/dev/dev-poll.sh index 6dd3678..03302d7 100755 --- a/dev/dev-poll.sh +++ b/dev/dev-poll.sh @@ -57,7 +57,7 @@ json.dump(d,open(f,'w')) " 2>/dev/null || true } -# Check whether an issue/PR has been escalated to supervisor (unprocessed or processed) +# Check whether an issue/PR has been escalated (unprocessed or processed) is_escalated() { local issue="$1" pr="$2" python3 -c " @@ -66,8 +66,8 @@ try: issue, pr = int('${issue}'), int('${pr}') except (ValueError, TypeError): sys.exit(1) -for path in ['${FACTORY_ROOT}/supervisor/escalations.jsonl', - '${FACTORY_ROOT}/supervisor/escalations.done.jsonl']: +for path in ['${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.jsonl', + '${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.done.jsonl']: try: with open(path) as fh: for line in fh: @@ -260,12 +260,12 @@ if [ "$ORPHAN_COUNT" -gt 0 ]; then FIX_ATTEMPTS=$(ci_fix_count "$HAS_PR") if [ "$FIX_ATTEMPTS" -ge 3 ] || is_escalated "$ISSUE_NUM" "$HAS_PR"; then # Already escalated — skip silently, let pipeline continue to backlog - log "issue #${ISSUE_NUM} PR #${HAS_PR} CI exhausted (${FIX_ATTEMPTS} attempts) — escalated to supervisor, skipping" + log "issue #${ISSUE_NUM} PR #${HAS_PR} CI exhausted (${FIX_ATTEMPTS} attempts) — escalated to gardener, skipping" # Only write escalation + alert once (first time hitting 3) if [ "$FIX_ATTEMPTS" -eq 3 ]; then - echo "{\"issue\":${ISSUE_NUM},\"pr\":${HAS_PR},\"reason\":\"ci_exhausted_poll\",\"attempts\":${FIX_ATTEMPTS},\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \ - >> "${FACTORY_ROOT}/supervisor/escalations.jsonl" - matrix_send "dev" "🚨 PR #${HAS_PR} (issue #${ISSUE_NUM}) CI failed after ${FIX_ATTEMPTS} attempts — escalated to supervisor" 2>/dev/null || true + echo "{\"issue\":${ISSUE_NUM},\"pr\":${HAS_PR},\"project\":\"${PROJECT_NAME}\",\"reason\":\"ci_exhausted_poll\",\"attempts\":${FIX_ATTEMPTS},\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \ + >> "${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.jsonl" + matrix_send "dev" "🚨 PR #${HAS_PR} (issue #${ISSUE_NUM}) CI failed after ${FIX_ATTEMPTS} attempts — escalated to gardener" 2>/dev/null || true ci_fix_increment "$HAS_PR" # bump to 4 so we don't re-alert fi # Fall through to backlog scan instead of exit @@ -341,10 +341,10 @@ for i in $(seq 0 $(($(echo "$OPEN_PRS" | jq 'length') - 1))); do FIX_ATTEMPTS=$(ci_fix_count "$PR_NUM") if [ "$FIX_ATTEMPTS" -ge 3 ] || is_escalated "$STUCK_ISSUE" "$PR_NUM"; then # Already escalated — skip to let pipeline continue - log "PR #${PR_NUM} (issue #${STUCK_ISSUE}) CI exhausted (${FIX_ATTEMPTS} attempts) — escalated to supervisor, skipping" + log "PR #${PR_NUM} (issue #${STUCK_ISSUE}) CI exhausted (${FIX_ATTEMPTS} attempts) — escalated to gardener, skipping" if [ "$FIX_ATTEMPTS" -eq 3 ]; then - echo "{\"issue\":${STUCK_ISSUE},\"pr\":${PR_NUM},\"reason\":\"ci_exhausted_poll\",\"attempts\":${FIX_ATTEMPTS},\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \ - >> "${FACTORY_ROOT}/supervisor/escalations.jsonl" + echo "{\"issue\":${STUCK_ISSUE},\"pr\":${PR_NUM},\"project\":\"${PROJECT_NAME}\",\"reason\":\"ci_exhausted_poll\",\"attempts\":${FIX_ATTEMPTS},\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \ + >> "${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.jsonl" matrix_send "dev" "🚨 PR #${PR_NUM} (issue #${STUCK_ISSUE}) CI failed after ${FIX_ATTEMPTS} attempts — escalated" 2>/dev/null || true ci_fix_increment "$PR_NUM" # bump to 4 to prevent re-alert fi @@ -417,10 +417,10 @@ for i in $(seq 0 $((BACKLOG_COUNT - 1))); do elif ! ci_passed "$CI_STATE" && [ "$CI_STATE" != "" ] && [ "$CI_STATE" != "pending" ] && [ "$CI_STATE" != "unknown" ]; then FIX_ATTEMPTS=$(ci_fix_count "$EXISTING_PR") if [ "$FIX_ATTEMPTS" -ge 3 ] || is_escalated "$ISSUE_NUM" "$EXISTING_PR"; then - log "#${ISSUE_NUM} PR #${EXISTING_PR} CI failed — escalated to supervisor, skipping (not blocking pipeline)" + log "#${ISSUE_NUM} PR #${EXISTING_PR} CI failed — escalated to gardener, skipping (not blocking pipeline)" if [ "$FIX_ATTEMPTS" -eq 3 ]; then - echo "{\"issue\":${ISSUE_NUM},\"pr\":${EXISTING_PR},\"reason\":\"ci_exhausted_poll\",\"attempts\":${FIX_ATTEMPTS},\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \ - >> "${FACTORY_ROOT}/supervisor/escalations.jsonl" + echo "{\"issue\":${ISSUE_NUM},\"pr\":${EXISTING_PR},\"project\":\"${PROJECT_NAME}\",\"reason\":\"ci_exhausted_poll\",\"attempts\":${FIX_ATTEMPTS},\"ts\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \ + >> "${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.jsonl" matrix_send "dev" "🚨 PR #${EXISTING_PR} (issue #${ISSUE_NUM}) CI failed after ${FIX_ATTEMPTS} attempts — escalated" 2>/dev/null || true ci_fix_increment "$EXISTING_PR" # bump to 4 to prevent re-alert fi diff --git a/gardener/gardener-poll.sh b/gardener/gardener-poll.sh index a820dba..f363257 100755 --- a/gardener/gardener-poll.sh +++ b/gardener/gardener-poll.sh @@ -305,4 +305,200 @@ if [ -n "$ACTIONS" ]; then done fi +# ── Process dev-agent escalations (per-project) ────────────────────────── +ESCALATION_FILE="${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.jsonl" +ESCALATION_DONE="${FACTORY_ROOT}/supervisor/escalations-${PROJECT_NAME}.done.jsonl" + +if [ -s "$ESCALATION_FILE" ]; then + # Atomically snapshot the file before processing to prevent race with + # concurrent dev-poll appends: new entries go to a fresh ESCALATION_FILE + # while we process the snapshot, so nothing is ever silently dropped. + ESCALATION_SNAP="${ESCALATION_FILE}.processing.$$" + mv "$ESCALATION_FILE" "$ESCALATION_SNAP" + + ESCALATION_COUNT=$(wc -l < "$ESCALATION_SNAP") + log "Processing ${ESCALATION_COUNT} escalation(s) for ${PROJECT_NAME}" + _esc_total_created=0 + + while IFS= read -r esc_entry; do + [ -z "$esc_entry" ] && continue + + ESC_ISSUE=$(echo "$esc_entry" | jq -r '.issue // empty') + ESC_PR=$(echo "$esc_entry" | jq -r '.pr // empty') + ESC_ATTEMPTS=$(echo "$esc_entry" | jq -r '.attempts // 3') + + if [ -z "$ESC_ISSUE" ] || [ -z "$ESC_PR" ]; then + echo "$esc_entry" >> "$ESCALATION_DONE" + continue + fi + + log "Escalation: issue #${ESC_ISSUE} PR #${ESC_PR} (${ESC_ATTEMPTS} CI attempt(s))" + + # Fetch the failing pipeline for this PR + ESC_PR_SHA=$(curl -sf -H "Authorization: token ${CODEBERG_TOKEN}" \ + "${CODEBERG_API}/pulls/${ESC_PR}" 2>/dev/null | jq -r '.head.sha // ""') || true + + ESC_PIPELINE="" + ESC_SUB_ISSUES_CREATED=0 + ESC_GENERIC_FAIL="" + ESC_LOGS_AVAILABLE=0 + + if [ -n "$ESC_PR_SHA" ]; then + # Validate SHA is a 40-char hex string before interpolating into SQL + if [[ "$ESC_PR_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then + ESC_PIPELINE=$(wpdb -c "SELECT number FROM pipelines WHERE repo_id=${WOODPECKER_REPO_ID} AND commit='${ESC_PR_SHA}' ORDER BY created DESC LIMIT 1;" 2>/dev/null | xargs || true) + else + log "WARNING: ESC_PR_SHA '${ESC_PR_SHA}' is not a valid hex SHA — skipping pipeline lookup" + fi + fi + + if [ -n "$ESC_PIPELINE" ]; then + FAILED_STEPS=$(curl -sf \ + -H "Authorization: Bearer ${WOODPECKER_TOKEN}" \ + "${WOODPECKER_SERVER}/api/repos/${WOODPECKER_REPO_ID}/pipelines/${ESC_PIPELINE}" 2>/dev/null | \ + jq -r '.workflows[]?.children[]? | select(.state=="failure") | "\(.pid)\t\(.name)"' 2>/dev/null || true) + + while IFS=$'\t' read -r step_pid step_name; do + [ -z "$step_pid" ] && continue + [[ "$step_pid" =~ ^[0-9]+$ ]] || { log "WARNING: invalid step_pid '${step_pid}' — skipping"; continue; } + step_logs=$(woodpecker-cli pipeline log show "${CODEBERG_REPO}" "${ESC_PIPELINE}" "${step_pid}" 2>/dev/null | tail -150 || true) + [ -z "$step_logs" ] && continue + ESC_LOGS_AVAILABLE=1 + + if echo "$step_name" | grep -qi "shellcheck"; then + # Create one sub-issue per file with ShellCheck errors + sc_files=$(echo "$step_logs" | grep -oP '(?<=In )\S+(?= line \d+:)' | sort -u || true) + + while IFS= read -r sc_file; do + [ -z "$sc_file" ] && continue + # grep -F for literal filename match (dots in filenames are regex wildcards) + file_errors=$(echo "$step_logs" | grep -F -A3 "In ${sc_file} line" | head -30) + # SC codes only from this file's errors, not the whole step log + sc_codes=$(echo "$file_errors" | grep -oP 'SC\d+' | sort -u | tr '\n' ' ' | sed 's/ $//' || true) + + sub_title="fix: ShellCheck errors in ${sc_file} (from PR #${ESC_PR})" + sub_body="## ShellCheck CI failure — \`${sc_file}\` + +Spawned by gardener from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). + +### Errors +\`\`\` +${file_errors} +\`\`\` + +Fix all ShellCheck errors${sc_codes:+ (${sc_codes})} in \`${sc_file}\` so PR #${ESC_PR} CI passes. + +### Context +- Parent issue: #${ESC_ISSUE} +- PR: #${ESC_PR} +- Pipeline: #${ESC_PIPELINE} (step: ${step_name})" + + new_issue=$(curl -sf -X POST \ + -H "Authorization: token ${CODEBERG_TOKEN}" \ + -H "Content-Type: application/json" \ + "${CODEBERG_API}/issues" \ + -d "$(jq -nc --arg t "$sub_title" --arg b "$sub_body" \ + '{"title":$t,"body":$b,"labels":["backlog"]}')" 2>/dev/null | jq -r '.number // ""') || true + + if [ -n "$new_issue" ]; then + log "Created sub-issue #${new_issue}: ShellCheck in ${sc_file} (from #${ESC_ISSUE})" + ESC_SUB_ISSUES_CREATED=$((ESC_SUB_ISSUES_CREATED + 1)) + _esc_total_created=$((_esc_total_created + 1)) + matrix_send "gardener" "📋 Created sub-issue #${new_issue}: ShellCheck in ${sc_file} (from escalated #${ESC_ISSUE})" 2>/dev/null || true + fi + done <<< "$sc_files" + + else + # Accumulate non-ShellCheck failures for one combined issue + esc_section="=== ${step_name} === +$(echo "$step_logs" | tail -50)" + if [ -z "$ESC_GENERIC_FAIL" ]; then + ESC_GENERIC_FAIL="$esc_section" + else + ESC_GENERIC_FAIL="${ESC_GENERIC_FAIL} +${esc_section}" + fi + fi + done <<< "$FAILED_STEPS" + fi + + # Create one sub-issue for all non-ShellCheck CI failures + if [ -n "$ESC_GENERIC_FAIL" ]; then + sub_title="fix: CI failures in PR #${ESC_PR} (from issue #${ESC_ISSUE})" + sub_body="## CI failure — fix required + +Spawned by gardener from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). + +### Failed step output +\`\`\` +${ESC_GENERIC_FAIL} +\`\`\` + +### Context +- Parent issue: #${ESC_ISSUE} +- PR: #${ESC_PR}${ESC_PIPELINE:+ +- Pipeline: #${ESC_PIPELINE}}" + + new_issue=$(curl -sf -X POST \ + -H "Authorization: token ${CODEBERG_TOKEN}" \ + -H "Content-Type: application/json" \ + "${CODEBERG_API}/issues" \ + -d "$(jq -nc --arg t "$sub_title" --arg b "$sub_body" \ + '{"title":$t,"body":$b,"labels":["backlog"]}')" 2>/dev/null | jq -r '.number // ""') || true + + if [ -n "$new_issue" ]; then + log "Created sub-issue #${new_issue}: CI failures for PR #${ESC_PR} (from #${ESC_ISSUE})" + ESC_SUB_ISSUES_CREATED=$((ESC_SUB_ISSUES_CREATED + 1)) + _esc_total_created=$((_esc_total_created + 1)) + matrix_send "gardener" "📋 Created sub-issue #${new_issue}: CI failures for PR #${ESC_PR} (from escalated #${ESC_ISSUE})" 2>/dev/null || true + fi + fi + + # Fallback: no sub-issues created — differentiate logs-unavailable from creation failure + if [ "$ESC_SUB_ISSUES_CREATED" -eq 0 ]; then + sub_title="fix: investigate CI failure for PR #${ESC_PR} (from issue #${ESC_ISSUE})" + if [ "$ESC_LOGS_AVAILABLE" -eq 1 ]; then + # Logs were fetched but all issue creation API calls failed + sub_body="## CI failure — investigation required + +Spawned by gardener from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). CI logs were retrieved but sub-issue creation failed (API error). + +Check PR #${ESC_PR} CI output, identify the failing checks, and fix them so the PR can merge." + else + # Could not retrieve CI logs at all + sub_body="## CI failure — investigation required + +Spawned by gardener from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). CI logs were unavailable at escalation time. + +Check PR #${ESC_PR} CI output, identify the failing checks, and fix them so the PR can merge." + fi + + new_issue=$(curl -sf -X POST \ + -H "Authorization: token ${CODEBERG_TOKEN}" \ + -H "Content-Type: application/json" \ + "${CODEBERG_API}/issues" \ + -d "$(jq -nc --arg t "$sub_title" --arg b "$sub_body" \ + '{"title":$t,"body":$b,"labels":["backlog"]}')" 2>/dev/null | jq -r '.number // ""') || true + + if [ -n "$new_issue" ]; then + log "Created fallback sub-issue #${new_issue} for escalated #${ESC_ISSUE}" + _esc_total_created=$((_esc_total_created + 1)) + matrix_send "gardener" "📋 Created sub-issue #${new_issue}: investigate CI for PR #${ESC_PR} (from escalated #${ESC_ISSUE})" 2>/dev/null || true + fi + fi + + # Mark as processed + echo "$esc_entry" >> "$ESCALATION_DONE" + done < "$ESCALATION_SNAP" + + rm -f "$ESCALATION_SNAP" + log "Escalations processed — moved to $(basename "$ESCALATION_DONE")" + + # Report resolution count to supervisor for its fixed() summary + if [ "${_esc_total_created:-0}" -gt 0 ]; then + printf '%d %s\n' "$_esc_total_created" "$PROJECT_NAME" \ + >> "${FACTORY_ROOT}/supervisor/gardener-esc-resolved.log" + fi +fi + log "--- Gardener poll done ---" diff --git a/supervisor/supervisor-poll.sh b/supervisor/supervisor-poll.sh index 48c2834..a62a1a1 100755 --- a/supervisor/supervisor-poll.sh +++ b/supervisor/supervisor-poll.sh @@ -216,193 +216,24 @@ for logfile in "${FACTORY_ROOT}"/{dev,review,supervisor}/*.log; do fi done -# Process dev-agent escalations — create sub-issues for each CI failure -ESCALATION_FILE="${FACTORY_ROOT}/supervisor/escalations.jsonl" -ESCALATION_DONE="${FACTORY_ROOT}/supervisor/escalations.done.jsonl" +# Report pending escalations (processing has moved to gardener-poll.sh per-project) +for _esc_file in "${FACTORY_ROOT}/supervisor/escalations-"*.jsonl; do + [ -f "$_esc_file" ] || continue + [[ "$_esc_file" == *.done.jsonl ]] && continue + _esc_count=$(wc -l < "$_esc_file" 2>/dev/null || true) + [ "${_esc_count:-0}" -gt 0 ] || continue + _esc_proj=$(basename "$_esc_file" .jsonl) + _esc_proj="${_esc_proj#escalations-}" + flog "${_esc_proj}: ${_esc_count} escalation(s) pending (gardener will process)" +done -if [ -s "$ESCALATION_FILE" ]; then - # Atomically snapshot the file before processing to prevent race with - # concurrent dev-poll appends: new entries go to a fresh ESCALATION_FILE - # while we process the snapshot, so nothing is ever silently dropped. - ESCALATION_SNAP="${ESCALATION_FILE}.processing.$$" - mv "$ESCALATION_FILE" "$ESCALATION_SNAP" - - ESCALATION_COUNT=$(wc -l < "$ESCALATION_SNAP") - flog "Processing ${ESCALATION_COUNT} escalation(s) from dev-agent" - - while IFS= read -r esc_entry; do - [ -z "$esc_entry" ] && continue - - ESC_ISSUE=$(echo "$esc_entry" | jq -r '.issue // empty') - ESC_PR=$(echo "$esc_entry" | jq -r '.pr // empty') - ESC_ATTEMPTS=$(echo "$esc_entry" | jq -r '.attempts // 3') - - if [ -z "$ESC_ISSUE" ] || [ -z "$ESC_PR" ]; then - echo "$esc_entry" >> "$ESCALATION_DONE" - continue - fi - - flog "Escalation: issue #${ESC_ISSUE} PR #${ESC_PR} (${ESC_ATTEMPTS} CI attempt(s))" - - # Fetch the failing pipeline for this PR - ESC_PR_SHA=$(curl -sf -H "Authorization: token ${CODEBERG_TOKEN}" \ - "${CODEBERG_API}/pulls/${ESC_PR}" 2>/dev/null | jq -r '.head.sha // ""') || true - - ESC_PIPELINE="" - ESC_SUB_ISSUES_CREATED=0 - ESC_GENERIC_FAIL="" - ESC_LOGS_AVAILABLE=0 - - if [ -n "$ESC_PR_SHA" ]; then - # Validate SHA is a 40-char hex string before interpolating into SQL - if [[ "$ESC_PR_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then - ESC_PIPELINE=$(wpdb -c "SELECT number FROM pipelines WHERE repo_id=${WOODPECKER_REPO_ID} AND commit='${ESC_PR_SHA}' ORDER BY created DESC LIMIT 1;" 2>/dev/null | xargs || true) - else - flog "WARNING: ESC_PR_SHA '${ESC_PR_SHA}' is not a valid hex SHA — skipping pipeline lookup" - fi - fi - - if [ -n "$ESC_PIPELINE" ]; then - FAILED_STEPS=$(curl -sf \ - -H "Authorization: Bearer ${WOODPECKER_TOKEN}" \ - "${WOODPECKER_SERVER}/api/repos/${WOODPECKER_REPO_ID}/pipelines/${ESC_PIPELINE}" 2>/dev/null | \ - jq -r '.workflows[]?.children[]? | select(.state=="failure") | "\(.pid)\t\(.name)"' 2>/dev/null || true) - - while IFS=$'\t' read -r step_pid step_name; do - [ -z "$step_pid" ] && continue - [[ "$step_pid" =~ ^[0-9]+$ ]] || { flog "WARNING: invalid step_pid '${step_pid}' — skipping"; continue; } - step_logs=$(woodpecker-cli pipeline log show "${CODEBERG_REPO}" "${ESC_PIPELINE}" "${step_pid}" 2>/dev/null | tail -150 || true) - [ -z "$step_logs" ] && continue - ESC_LOGS_AVAILABLE=1 - - if echo "$step_name" | grep -qi "shellcheck"; then - # Create one sub-issue per file with ShellCheck errors - sc_files=$(echo "$step_logs" | grep -oP '(?<=In )\S+(?= line \d+:)' | sort -u || true) - - while IFS= read -r sc_file; do - [ -z "$sc_file" ] && continue - # grep -F for literal filename match (dots in filenames are regex wildcards) - file_errors=$(echo "$step_logs" | grep -F -A3 "In ${sc_file} line" | head -30) - # SC codes only from this file's errors, not the whole step log - sc_codes=$(echo "$file_errors" | grep -oP 'SC\d+' | sort -u | tr '\n' ' ' | sed 's/ $//' || true) - - sub_title="fix: ShellCheck errors in ${sc_file} (from PR #${ESC_PR})" - sub_body="## ShellCheck CI failure — \`${sc_file}\` - -Spawned by supervisor from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). - -### Errors -\`\`\` -${file_errors} -\`\`\` - -Fix all ShellCheck errors${sc_codes:+ (${sc_codes})} in \`${sc_file}\` so PR #${ESC_PR} CI passes. - -### Context -- Parent issue: #${ESC_ISSUE} -- PR: #${ESC_PR} -- Pipeline: #${ESC_PIPELINE} (step: ${step_name})" - - new_issue=$(curl -sf -X POST \ - -H "Authorization: token ${CODEBERG_TOKEN}" \ - -H "Content-Type: application/json" \ - "${CODEBERG_API}/issues" \ - -d "$(jq -nc --arg t "$sub_title" --arg b "$sub_body" \ - '{"title":$t,"body":$b,"labels":["backlog"]}')" 2>/dev/null | jq -r '.number // ""') || true - - if [ -n "$new_issue" ]; then - flog "Created sub-issue #${new_issue}: ShellCheck in ${sc_file} (from #${ESC_ISSUE})" - fixed "Sub-issue #${new_issue}: ShellCheck errors in ${sc_file} (escalated from #${ESC_ISSUE})" - ESC_SUB_ISSUES_CREATED=$((ESC_SUB_ISSUES_CREATED + 1)) - matrix_send "supervisor" "📋 Created sub-issue #${new_issue}: ShellCheck in ${sc_file} (from escalated #${ESC_ISSUE})" 2>/dev/null || true - fi - done <<< "$sc_files" - - else - # Accumulate non-ShellCheck failures for one combined issue - esc_section="=== ${step_name} === -$(echo "$step_logs" | tail -50)" - if [ -z "$ESC_GENERIC_FAIL" ]; then - ESC_GENERIC_FAIL="$esc_section" - else - ESC_GENERIC_FAIL="${ESC_GENERIC_FAIL} -${esc_section}" - fi - fi - done <<< "$FAILED_STEPS" - fi - - # Create one sub-issue for all non-ShellCheck CI failures - if [ -n "$ESC_GENERIC_FAIL" ]; then - sub_title="fix: CI failures in PR #${ESC_PR} (from issue #${ESC_ISSUE})" - sub_body="## CI failure — fix required - -Spawned by supervisor from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). - -### Failed step output -\`\`\` -${ESC_GENERIC_FAIL} -\`\`\` - -### Context -- Parent issue: #${ESC_ISSUE} -- PR: #${ESC_PR}${ESC_PIPELINE:+ -- Pipeline: #${ESC_PIPELINE}}" - - new_issue=$(curl -sf -X POST \ - -H "Authorization: token ${CODEBERG_TOKEN}" \ - -H "Content-Type: application/json" \ - "${CODEBERG_API}/issues" \ - -d "$(jq -nc --arg t "$sub_title" --arg b "$sub_body" \ - '{"title":$t,"body":$b,"labels":["backlog"]}')" 2>/dev/null | jq -r '.number // ""') || true - - if [ -n "$new_issue" ]; then - flog "Created sub-issue #${new_issue}: CI failures for PR #${ESC_PR} (from #${ESC_ISSUE})" - fixed "Sub-issue #${new_issue}: CI failures for PR #${ESC_PR} (escalated from #${ESC_ISSUE})" - ESC_SUB_ISSUES_CREATED=$((ESC_SUB_ISSUES_CREATED + 1)) - matrix_send "supervisor" "📋 Created sub-issue #${new_issue}: CI failures for PR #${ESC_PR} (from escalated #${ESC_ISSUE})" 2>/dev/null || true - fi - fi - - # Fallback: no sub-issues created — differentiate logs-unavailable from creation failure - if [ "$ESC_SUB_ISSUES_CREATED" -eq 0 ]; then - sub_title="fix: investigate CI failure for PR #${ESC_PR} (from issue #${ESC_ISSUE})" - if [ "$ESC_LOGS_AVAILABLE" -eq 1 ]; then - # Logs were fetched but all issue creation API calls failed - sub_body="## CI failure — investigation required - -Spawned by supervisor from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). CI logs were retrieved but sub-issue creation failed (API error). - -Check PR #${ESC_PR} CI output, identify the failing checks, and fix them so the PR can merge." - else - # Could not retrieve CI logs at all - sub_body="## CI failure — investigation required - -Spawned by supervisor from escalated issue #${ESC_ISSUE} (PR #${ESC_PR} failed CI after ${ESC_ATTEMPTS} attempt(s)). CI logs were unavailable at escalation time. - -Check PR #${ESC_PR} CI output, identify the failing checks, and fix them so the PR can merge." - fi - - new_issue=$(curl -sf -X POST \ - -H "Authorization: token ${CODEBERG_TOKEN}" \ - -H "Content-Type: application/json" \ - "${CODEBERG_API}/issues" \ - -d "$(jq -nc --arg t "$sub_title" --arg b "$sub_body" \ - '{"title":$t,"body":$b,"labels":["backlog"]}')" 2>/dev/null | jq -r '.number // ""') || true - - if [ -n "$new_issue" ]; then - flog "Created fallback sub-issue #${new_issue} for escalated #${ESC_ISSUE}" - fixed "Fallback sub-issue #${new_issue}: investigate CI for PR #${ESC_PR} (escalated from #${ESC_ISSUE})" - matrix_send "supervisor" "📋 Created sub-issue #${new_issue}: investigate CI for PR #${ESC_PR} (from escalated #${ESC_ISSUE})" 2>/dev/null || true - fi - fi - - # Mark as processed - echo "$esc_entry" >> "$ESCALATION_DONE" - done < "$ESCALATION_SNAP" - - rm -f "$ESCALATION_SNAP" - flog "Escalations processed — moved to $(basename "$ESCALATION_DONE")" +# Pick up escalation resolutions handled by gardener +_gesc_log="${FACTORY_ROOT}/supervisor/gardener-esc-resolved.log" +if [ -f "$_gesc_log" ]; then + while IFS=' ' read -r _gn _gp; do + [ -n "${_gn:-}" ] && fixed "${_gp:-unknown}: gardener created ${_gn} sub-issue(s) from escalations" + done < "$_gesc_log" + rm -f "$_gesc_log" fi # #############################################################################