9f5a6f9942
Add DISINTO_LOG_DIR to lib/env.sh: points to $HOME/data/logs inside the container (writable volume) and $FACTORY_ROOT on the host (existing behavior). Update all agent scripts to write logs, CI fix tracker, metrics, and vault locks to DISINTO_LOG_DIR instead of FACTORY_ROOT. This keeps the factory mount read-only while ensuring all writable state lands on the persistent data volume. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
237 lines
10 KiB
Bash
237 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# collect-metrics.sh — Collect factory metrics and write JSON for the dashboard
|
|
#
|
|
# Queries forge API for PR/issue stats across all managed projects,
|
|
# counts vault decisions, and checks CI pass rates. Writes a JSON snapshot
|
|
# to the live site directory so the dashboard can fetch it.
|
|
#
|
|
# Usage:
|
|
# bash site/collect-metrics.sh
|
|
#
|
|
# Cron: 0 */6 * * * cd /home/debian/dark-factory && bash site/collect-metrics.sh
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
FACTORY_ROOT="$(dirname "$SCRIPT_DIR")"
|
|
|
|
# shellcheck source=../lib/env.sh
|
|
source "$FACTORY_ROOT/lib/env.sh"
|
|
# shellcheck source=../lib/ci-helpers.sh
|
|
source "$FACTORY_ROOT/lib/ci-helpers.sh" 2>/dev/null || true
|
|
|
|
LOGFILE="${DISINTO_LOG_DIR}/site/collect-metrics.log"
|
|
log() {
|
|
printf '[%s] collect-metrics: %s\n' "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" "$*" >> "$LOGFILE"
|
|
}
|
|
|
|
# Output path: write to live site root if deployed, else to site/data/
|
|
SITE_ROOT="${DISINTO_SITE_ROOT:-/home/debian/disinto-site}"
|
|
if [ -d "$SITE_ROOT" ] || [ -L "$SITE_ROOT" ]; then
|
|
OUTPUT_DIR="$(readlink -f "$SITE_ROOT")/data"
|
|
else
|
|
OUTPUT_DIR="${SCRIPT_DIR}/data"
|
|
fi
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
NOW_ISO=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
WEEK_AGO=$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-7d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "")
|
|
MONTH_AGO=$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-30d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "")
|
|
|
|
# ── Per-project metrics ─────────────────────────────────────────────────────
|
|
|
|
collect_project_metrics() {
|
|
local project_toml="$1"
|
|
local repo repo_name
|
|
|
|
repo=$(grep '^repo ' "$project_toml" | head -1 | sed 's/.*= *"//;s/"//')
|
|
repo_name=$(grep '^name ' "$project_toml" | head -1 | sed 's/.*= *"//;s/"//')
|
|
local forge_url
|
|
forge_url=$(grep '^forge_url ' "$project_toml" | head -1 | sed 's/.*= *"//;s/"//') 2>/dev/null || true
|
|
forge_url="${forge_url:-${FORGE_URL:-http://localhost:3000}}"
|
|
local api_base="${forge_url}/api/v1/repos/${repo}"
|
|
|
|
# PRs merged (all time via state=closed + merged marker)
|
|
local prs_merged_week=0 prs_merged_month=0 prs_merged_total=0
|
|
local closed_prs
|
|
closed_prs=$(curl -sf -H "Authorization: token ${FORGE_TOKEN}" \
|
|
"${api_base}/pulls?state=closed&sort=updated&limit=50" 2>/dev/null || echo "[]")
|
|
|
|
prs_merged_total=$(printf '%s' "$closed_prs" | jq '[.[] | select(.merged)] | length' 2>/dev/null || echo 0)
|
|
|
|
if [ -n "$WEEK_AGO" ]; then
|
|
prs_merged_week=$(printf '%s' "$closed_prs" | \
|
|
jq --arg since "$WEEK_AGO" '[.[] | select(.merged and .merged_at >= $since)] | length' 2>/dev/null || echo 0)
|
|
fi
|
|
if [ -n "$MONTH_AGO" ]; then
|
|
prs_merged_month=$(printf '%s' "$closed_prs" | \
|
|
jq --arg since "$MONTH_AGO" '[.[] | select(.merged and .merged_at >= $since)] | length' 2>/dev/null || echo 0)
|
|
fi
|
|
|
|
# Issues closed
|
|
local issues_closed_week=0 issues_closed_month=0
|
|
local closed_issues
|
|
closed_issues=$(curl -sf -H "Authorization: token ${FORGE_TOKEN}" \
|
|
"${api_base}/issues?state=closed&sort=updated&type=issues&limit=50" 2>/dev/null || echo "[]")
|
|
|
|
if [ -n "$WEEK_AGO" ]; then
|
|
issues_closed_week=$(printf '%s' "$closed_issues" | \
|
|
jq --arg since "$WEEK_AGO" '[.[] | select(.closed_at >= $since)] | length' 2>/dev/null || echo 0)
|
|
fi
|
|
if [ -n "$MONTH_AGO" ]; then
|
|
issues_closed_month=$(printf '%s' "$closed_issues" | \
|
|
jq --arg since "$MONTH_AGO" '[.[] | select(.closed_at >= $since)] | length' 2>/dev/null || echo 0)
|
|
fi
|
|
|
|
local total_closed_header
|
|
total_closed_header=$(curl -sf -I -H "Authorization: token ${FORGE_TOKEN}" \
|
|
"${api_base}/issues?state=closed&type=issues&limit=1" 2>/dev/null | grep -i 'x-total-count' | tr -d '\r' | awk '{print $2}' || echo "0")
|
|
local issues_closed_total="${total_closed_header:-0}"
|
|
|
|
# Open issues by label
|
|
local backlog_count in_progress_count blocked_count
|
|
backlog_count=$(curl -sf -I -H "Authorization: token ${FORGE_TOKEN}" \
|
|
"${api_base}/issues?state=open&labels=backlog&type=issues&limit=1" 2>/dev/null | \
|
|
grep -i 'x-total-count' | tr -d '\r' | awk '{print $2}' || echo "0")
|
|
in_progress_count=$(curl -sf -H "Authorization: token ${FORGE_TOKEN}" \
|
|
"${api_base}/issues?state=open&labels=in-progress&type=issues&limit=50" 2>/dev/null | \
|
|
jq 'length' 2>/dev/null || echo 0)
|
|
blocked_count=$(curl -sf -H "Authorization: token ${FORGE_TOKEN}" \
|
|
"${api_base}/issues?state=open&labels=blocked&type=issues&limit=50" 2>/dev/null | \
|
|
jq 'length' 2>/dev/null || echo 0)
|
|
|
|
jq -nc \
|
|
--arg name "$repo_name" \
|
|
--arg repo "$repo" \
|
|
--argjson prs_week "$prs_merged_week" \
|
|
--argjson prs_month "$prs_merged_month" \
|
|
--argjson prs_total "${prs_merged_total:-0}" \
|
|
--argjson issues_week "$issues_closed_week" \
|
|
--argjson issues_month "$issues_closed_month" \
|
|
--argjson issues_total "${issues_closed_total:-0}" \
|
|
--argjson backlog "${backlog_count:-0}" \
|
|
--argjson in_progress "${in_progress_count:-0}" \
|
|
--argjson blocked "${blocked_count:-0}" \
|
|
'{
|
|
name: $name,
|
|
repo: $repo,
|
|
prs_merged: { week: $prs_week, month: $prs_month, total: $prs_total },
|
|
issues_closed: { week: $issues_week, month: $issues_month, total: $issues_total },
|
|
backlog: { queued: $backlog, in_progress: $in_progress, blocked: $blocked }
|
|
}'
|
|
}
|
|
|
|
# ── Vault decisions ─────────────────────────────────────────────────────────
|
|
|
|
collect_vault_metrics() {
|
|
local vault_dir="${FACTORY_ROOT}/vault"
|
|
local approved=0 rejected=0 escalated=0 pending=0 fired=0
|
|
|
|
[ -d "$vault_dir/fired" ] && fired=$(find "$vault_dir/fired" -name '*.json' 2>/dev/null | wc -l)
|
|
[ -d "$vault_dir/approved" ] && approved=$(find "$vault_dir/approved" -name '*.json' 2>/dev/null | wc -l)
|
|
[ -d "$vault_dir/rejected" ] && rejected=$(find "$vault_dir/rejected" -name '*.json' 2>/dev/null | wc -l)
|
|
[ -d "$vault_dir/pending" ] && {
|
|
pending=$(find "$vault_dir/pending" -name '*.json' 2>/dev/null | wc -l)
|
|
escalated=$(find "$vault_dir/pending" -name '*.json' -exec grep -l '"escalated"' {} + 2>/dev/null | wc -l)
|
|
pending=$((pending - escalated))
|
|
}
|
|
|
|
jq -nc \
|
|
--argjson approved "$((approved + fired))" \
|
|
--argjson escalated "$escalated" \
|
|
--argjson rejected "$rejected" \
|
|
--argjson pending "$pending" \
|
|
'{ approved: $approved, escalated: $escalated, rejected: $rejected, pending: $pending }'
|
|
}
|
|
|
|
# ── CI pass rate ────────────────────────────────────────────────────────────
|
|
|
|
collect_ci_metrics() {
|
|
local total=0 passed=0 failed=0 rate=0
|
|
|
|
# Query Woodpecker DB for last 30 days across all repos
|
|
local ci_stats
|
|
ci_stats=$(wpdb -A -c "
|
|
SELECT status, count(*)
|
|
FROM pipelines
|
|
WHERE finished > 0
|
|
AND to_timestamp(finished) > now() - interval '30 days'
|
|
GROUP BY status;" 2>/dev/null || echo "")
|
|
|
|
if [ -n "$ci_stats" ]; then
|
|
passed=$(echo "$ci_stats" | awk '$1 == "success" {print $3}' | tr -d ' ' || echo 0)
|
|
failed=$(echo "$ci_stats" | awk '$1 == "failure" {print $3}' | tr -d ' ' || echo 0)
|
|
passed=${passed:-0}
|
|
failed=${failed:-0}
|
|
total=$((passed + failed))
|
|
if [ "$total" -gt 0 ]; then
|
|
rate=$(( (passed * 100) / total ))
|
|
fi
|
|
fi
|
|
|
|
jq -nc \
|
|
--argjson total "$total" \
|
|
--argjson passed "$passed" \
|
|
--argjson failed "$failed" \
|
|
--argjson rate "$rate" \
|
|
'{ total: $total, passed: $passed, failed: $failed, pass_rate: $rate }'
|
|
}
|
|
|
|
# ── Agent activity ──────────────────────────────────────────────────────────
|
|
|
|
collect_agent_metrics() {
|
|
local active_sessions=0
|
|
active_sessions=$(tmux list-sessions 2>/dev/null | wc -l || echo 0)
|
|
|
|
local agents='[]'
|
|
local agent_name log_path age_min last_active
|
|
for log_entry in dev/dev-agent.log review/review.log gardener/gardener.log \
|
|
planner/planner.log predictor/predictor.log supervisor/supervisor.log \
|
|
action/action.log vault/vault.log; do
|
|
agent_name=$(basename "$(dirname "$log_entry")")
|
|
log_path="${FACTORY_ROOT}/${log_entry}"
|
|
if [ -f "$log_path" ]; then
|
|
age_min=$(( ($(date +%s) - $(stat -c %Y "$log_path" 2>/dev/null || echo 0)) / 60 ))
|
|
last_active=$(date -u -d "@$(stat -c %Y "$log_path" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "unknown")
|
|
agents=$(printf '%s' "$agents" | jq --arg n "$agent_name" --arg t "$last_active" --argjson a "$age_min" \
|
|
'. + [{ name: $n, last_active: $t, idle_minutes: $a }]')
|
|
fi
|
|
done
|
|
|
|
jq -nc --argjson sessions "$active_sessions" --argjson agents "$agents" \
|
|
'{ active_sessions: $sessions, agents: $agents }'
|
|
}
|
|
|
|
# ── Assemble ────────────────────────────────────────────────────────────────
|
|
|
|
log "Starting metrics collection"
|
|
|
|
PROJECTS_JSON="[]"
|
|
for toml in "$FACTORY_ROOT"/projects/*.toml; do
|
|
[ -f "$toml" ] || continue
|
|
log "Collecting metrics for $(basename "$toml")"
|
|
project_json=$(collect_project_metrics "$toml" 2>/dev/null || echo '{}')
|
|
PROJECTS_JSON=$(printf '%s\n%s' "$PROJECTS_JSON" "$project_json" | jq -s '.[0] + [.[1]]')
|
|
done
|
|
|
|
VAULT_JSON=$(collect_vault_metrics 2>/dev/null || echo '{}')
|
|
CI_JSON=$(collect_ci_metrics 2>/dev/null || echo '{}')
|
|
AGENTS_JSON=$(collect_agent_metrics 2>/dev/null || echo '{}')
|
|
|
|
# Build final output
|
|
jq -nc \
|
|
--arg ts "$NOW_ISO" \
|
|
--argjson projects "$PROJECTS_JSON" \
|
|
--argjson vault "$VAULT_JSON" \
|
|
--argjson ci "$CI_JSON" \
|
|
--argjson agents "$AGENTS_JSON" \
|
|
'{
|
|
generated_at: $ts,
|
|
projects: $projects,
|
|
vault: $vault,
|
|
ci: $ci,
|
|
agents: $agents
|
|
}' > "${OUTPUT_DIR}/metrics.json"
|
|
|
|
log "Metrics written to ${OUTPUT_DIR}/metrics.json"
|