johba/disinto
1
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: bug: supervisor never delivers disk alerts — crashes during PR scan (#252)

Browse source 
Send P0 and P1 alerts to Matrix immediately after detection, before
per-project checks run. Also guard check_project calls with || flog so
any API timeout or jq parse failure inside the per-project scan cannot
kill the script before alert delivery.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
openhands 2026-03-19 09:16:56 +00:00
parent e12bbbc0f2
commit 5632138cc3
1 changed files with 14 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

16
supervisor/supervisor-poll.sh
View file

@ -141,6 +141,12 @@ if [ "${AVAIL_MB:-9999}" -lt 500 ] || { [ "${SWAP_USED_MB:-0}" -gt 3000 ] && [ "
fi fi
fi fi
# P0 is urgent — send immediately before per-project checks can crash the script
if [ -n "$P0_ALERTS" ]; then
matrix_send "supervisor" "🚨 Supervisor P0 alerts:
$(printf '%b' "$P0_ALERTS")" 2>/dev/null || true
fi
# ============================================================================= # =============================================================================
# P1: DISK # P1: DISK
# ============================================================================= # =============================================================================
@ -184,6 +190,12 @@ if [ "${DISK_PERCENT:-0}" -gt 80 ]; then
fi fi
fi fi
# P1 is urgent — send immediately before per-project checks can crash the script
if [ -n "$P1_ALERTS" ]; then
matrix_send "supervisor" "⚠️ Supervisor P1 alerts:
$(printf '%b' "$P1_ALERTS")" 2>/dev/null || true
fi
# Emit infra metric # Emit infra metric
_RAM_TOTAL_MB=$(free -m | awk '/Mem:/{print $2}') _RAM_TOTAL_MB=$(free -m | awk '/Mem:/{print $2}')
_RAM_USED_PCT=$(( ${_RAM_TOTAL_MB:-0} > 0 ? (${_RAM_TOTAL_MB:-0} - ${AVAIL_MB:-0}) * 100 / ${_RAM_TOTAL_MB:-1} : 0 )) _RAM_USED_PCT=$(( ${_RAM_TOTAL_MB:-0} > 0 ? (${_RAM_TOTAL_MB:-0} - ${AVAIL_MB:-0}) * 100 / ${_RAM_TOTAL_MB:-1} : 0 ))
@ -963,14 +975,14 @@ if [ -d "$PROJECTS_DIR" ]; then
# Load project config (overrides CODEBERG_REPO, PROJECT_REPO_ROOT, etc.) # Load project config (overrides CODEBERG_REPO, PROJECT_REPO_ROOT, etc.)
source "${FACTORY_ROOT}/lib/load-project.sh" "$project_toml" source "${FACTORY_ROOT}/lib/load-project.sh" "$project_toml"
check_project check_project || flog "check_project failed for ${project_toml} (per-project checks incomplete)"
done done
fi fi
if [ "$PROJECT_COUNT" -eq 0 ]; then if [ "$PROJECT_COUNT" -eq 0 ]; then
# Fallback: no project TOML files, use .env config (backwards compatible) # Fallback: no project TOML files, use .env config (backwards compatible)
flog "No projects/*.toml found, using .env defaults" flog "No projects/*.toml found, using .env defaults"
check_project check_project || flog "check_project failed with .env defaults (per-project checks incomplete)"
fi fi
# ############################################################################# # #############################################################################