fix: tool: disinto backup import — idempotent restore on fresh Nomad cluster (#1058)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

bin/disinto

@@ -12,6 +12,7 @@
 #   disinto secrets <subcommand>        Manage encrypted secrets
 #   disinto run <action-id>              Run action in ephemeral runner container
 #   disinto ci-logs <pipeline> [--step <name>]  Read CI logs from Woodpecker SQLite
+#   disinto backup create <outfile>     Export factory state for migration
 #
 # Usage:
 #   disinto init https://github.com/user/repo
@@ -39,7 +40,9 @@ source "${FACTORY_ROOT}/lib/generators.sh"
 source "${FACTORY_ROOT}/lib/forge-push.sh"
 source "${FACTORY_ROOT}/lib/ci-setup.sh"
 source "${FACTORY_ROOT}/lib/release.sh"
+source "${FACTORY_ROOT}/lib/backup.sh"
 source "${FACTORY_ROOT}/lib/claude-config.sh"
+source "${FACTORY_ROOT}/lib/disinto/backup.sh"  # backup create/import
 
 # ── Helpers ──────────────────────────────────────────────────────────────────
 
@@ -62,7 +65,9 @@ Usage:
   disinto hire-an-agent <agent-name> <role> [--formula <path>] [--local-model <url>] [--model <name>]
                                      Hire a new agent (create user + .profile repo; re-run to rotate credentials)
   disinto agent <subcommand>           Manage agent state (enable/disable)
+  disinto backup create <outfile>      Export factory state (issues + ops bundle)
   disinto edge <verb> [options]        Manage edge tunnel registrations
+  disinto backup <subcommand>          Backup and restore factory state
 
 Edge subcommands:
   register [project]    Register a new tunnel (generates keypair if needed)
@@ -101,6 +106,18 @@ Hire an agent options:
 
 CI logs options:
   --step <name>        Filter logs to a specific step (e.g., smoke-init)
+
+Backup subcommands:
+  create <file>        Create backup of factory state to tarball
+  import <file>        Restore factory state from backup tarball
+
+Import behavior:
+  - Unpacks tarball to temp directory
+  - Creates disinto repo via Forgejo API (mirror config is manual)
+  - Creates disinto-ops repo and pushes refs from bundle
+  - Imports issues from issues/*.json (idempotent - skips existing)
+  - Logs issue number mapping (Forgejo auto-assigns numbers)
+  - Prints summary: created X repos, pushed Y refs, imported Z issues, skipped W
 EOF
   exit 1
 }
@@ -2893,6 +2910,35 @@ EOF
   esac
 }
 
+# ── backup command ────────────────────────────────────────────────────────────
+# Usage: disinto backup <subcommand> [args]
+# Subcommands:
+#   create <outfile.tar.gz>  Create backup of factory state
+#   import <infile.tar.gz>   Restore factory state from backup
+disinto_backup() {
+  local subcmd="${1:-}"
+  shift || true
+
+  case "$subcmd" in
+    create)
+      echo "Usage: disinto backup create <outfile.tar.gz>" >&2
+      echo "  (Not yet implemented)" >&2
+      exit 1
+      ;;
+    import)
+      backup_import "$@"
+      ;;
+    *)
+      echo "Usage: disinto backup <subcommand> [args]" >&2
+      echo "" >&2
+      echo "Subcommands:" >&2
+      echo "  create <outfile.tar.gz>  Create backup of factory state" >&2
+      echo "  import <infile.tar.gz>   Restore factory state from backup" >&2
+      exit 1
+      ;;
+  esac
+}
+
 # ── Main dispatch ────────────────────────────────────────────────────────────
 
 case "${1:-}" in
@@ -2909,6 +2955,7 @@ case "${1:-}" in
   hire-an-agent)   shift; disinto_hire_an_agent "$@" ;;
   agent)           shift; disinto_agent "$@" ;;
   edge)            shift; disinto_edge "$@" ;;
+  backup)          shift; disinto_backup "$@" ;;
   -h|--help)       usage ;;
   *)               usage ;;
 esac

lib/backup.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# =============================================================================
+# disinto backup — export factory state for migration
+#
+# Usage: source this file, then call backup_create <outfile.tar.gz>
+# Requires: FORGE_URL, FORGE_TOKEN, FORGE_REPO, FORGE_OPS_REPO, OPS_REPO_ROOT
+# =============================================================================
+set -euo pipefail
+
+# Fetch all issues (open + closed) for a repo slug and emit the normalized JSON array.
+# Usage: _backup_fetch_issues <org/repo>
+_backup_fetch_issues() {
+  local repo_slug="$1"
+  local api_url="${FORGE_API_BASE}/repos/${repo_slug}"
+
+  local all_issues="[]"
+  for state in open closed; do
+    local page=1
+    while true; do
+      local page_items
+      page_items=$(curl -sf -X GET \
+        -H "Authorization: token ${FORGE_TOKEN}" \
+        -H "Content-Type: application/json" \
+        "${api_url}/issues?state=${state}&type=issues&limit=50&page=${page}") || {
+        echo "ERROR: failed to fetch ${state} issues from ${repo_slug} (page ${page})" >&2
+        return 1
+      }
+      local count
+      count=$(printf '%s' "$page_items" | jq 'length' 2>/dev/null) || count=0
+      [ -z "$count" ] && count=0
+      [ "$count" -eq 0 ] && break
+      all_issues=$(printf '%s\n%s' "$all_issues" "$page_items" | jq -s 'add')
+      [ "$count" -lt 50 ] && break
+      page=$((page + 1))
+    done
+  done
+
+  # Normalize to the schema: number, title, body, labels, state
+  printf '%s' "$all_issues" | jq '[.[] | {
+    number: .number,
+    title: .title,
+    body: .body,
+    labels: [.labels[]?.name],
+    state: .state
+  }] | sort_by(.number)'
+}
+
+# Create a backup tarball of factory state.
+# Usage: backup_create <outfile.tar.gz>
+backup_create() {
+  local outfile="${1:-}"
+  if [ -z "$outfile" ]; then
+    echo "Error: output file required" >&2
+    echo "Usage: disinto backup create <outfile.tar.gz>" >&2
+    return 1
+  fi
+
+  # Resolve to absolute path before cd-ing into tmpdir
+  case "$outfile" in
+    /*) ;;
+    *) outfile="$(pwd)/${outfile}" ;;
+  esac
+
+  # Validate required env
+  : "${FORGE_URL:?FORGE_URL must be set}"
+  : "${FORGE_TOKEN:?FORGE_TOKEN must be set}"
+  : "${FORGE_REPO:?FORGE_REPO must be set}"
+
+  local forge_ops_repo="${FORGE_OPS_REPO:-${FORGE_REPO}-ops}"
+  local ops_repo_root="${OPS_REPO_ROOT:-}"
+
+  if [ -z "$ops_repo_root" ] || [ ! -d "$ops_repo_root/.git" ]; then
+    echo "Error: OPS_REPO_ROOT (${ops_repo_root:-<unset>}) is not a valid git repo" >&2
+    return 1
+  fi
+
+  local tmpdir
+  tmpdir=$(mktemp -d)
+  trap 'rm -rf "$tmpdir"' EXIT
+
+  local project_name="${FORGE_REPO##*/}"
+
+  echo "=== disinto backup create ==="
+  echo "Forge: ${FORGE_URL}"
+  echo "Repos: ${FORGE_REPO}, ${forge_ops_repo}"
+
+  # ── 1. Export issues ──────────────────────────────────────────────────────
+  mkdir -p "${tmpdir}/issues"
+
+  echo "Fetching issues for ${FORGE_REPO}..."
+  _backup_fetch_issues "$FORGE_REPO" > "${tmpdir}/issues/${project_name}.json"
+  local main_count
+  main_count=$(jq 'length' "${tmpdir}/issues/${project_name}.json")
+  echo "  ${main_count} issues exported"
+
+  echo "Fetching issues for ${forge_ops_repo}..."
+  _backup_fetch_issues "$forge_ops_repo" > "${tmpdir}/issues/${project_name}-ops.json"
+  local ops_count
+  ops_count=$(jq 'length' "${tmpdir}/issues/${project_name}-ops.json")
+  echo "  ${ops_count} issues exported"
+
+  # ── 2. Git bundle of ops repo ────────────────────────────────────────────
+  mkdir -p "${tmpdir}/repos"
+
+  echo "Creating git bundle for ${forge_ops_repo}..."
+  git -C "$ops_repo_root" bundle create "${tmpdir}/repos/${project_name}-ops.bundle" --all 2>&1
+  echo "  bundle created ($(du -h "${tmpdir}/repos/${project_name}-ops.bundle" | cut -f1))"
+
+  # ── 3. Metadata ──────────────────────────────────────────────────────────
+  local created_at
+  created_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+  jq -n \
+    --arg created_at "$created_at" \
+    --arg source_host "$(hostname)" \
+    --argjson schema_version 1 \
+    --arg forgejo_url "$FORGE_URL" \
+    '{
+      created_at: $created_at,
+      source_host: $source_host,
+      schema_version: $schema_version,
+      forgejo_url: $forgejo_url
+    }' > "${tmpdir}/metadata.json"
+
+  # ── 4. Pack tarball ──────────────────────────────────────────────────────
+  echo "Creating tarball: ${outfile}"
+  tar -czf "$outfile" -C "$tmpdir" metadata.json issues repos
+  local size
+  size=$(du -h "$outfile" | cut -f1)
+  echo "=== Backup complete: ${outfile} (${size}) ==="
+
+  # Clean up before returning — the EXIT trap references the local $tmpdir
+  # which goes out of scope after return, causing 'unbound variable' under set -u.
+  trap - EXIT
+  rm -rf "$tmpdir"
+}

lib/disinto/backup.sh

@@ -0,0 +1,389 @@
+#!/usr/bin/env bash
+# =============================================================================
+# backup.sh — backup/restore utilities for disinto factory state
+#
+# Subcommands:
+#   create <outfile.tar.gz>  Create backup of factory state
+#   import <infile.tar.gz>   Restore factory state from backup
+#
+# Usage:
+#   source "${FACTORY_ROOT}/lib/disinto/backup.sh"
+#   backup_import <tarball>
+#
+# Environment:
+#   FORGE_URL    - Forgejo instance URL (target)
+#   FORGE_TOKEN  - Admin token for target Forgejo
+#
+# Idempotency:
+#   - Repos: created via API if missing
+#   - Issues: check if exists by number, skip if present
+#   - Runs twice = same end state, no errors
+# =============================================================================
+set -euo pipefail
+
+# ── Helper: log with timestamp ───────────────────────────────────────────────
+backup_log() {
+  local msg="$1"
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $msg"
+}
+
+# ── Helper: create repo if it doesn't exist ─────────────────────────────────
+# Usage: backup_create_repo_if_missing <slug>
+# Returns: 0 if repo exists or was created, 1 on error
+backup_create_repo_if_missing() {
+  local slug="$1"
+  local org_name="${slug%%/*}"
+  local repo_name="${slug##*/}"
+
+  # Check if repo exists
+  if curl -sf --max-time 5 \
+    -H "Authorization: token ${FORGE_TOKEN}" \
+    "${FORGE_URL}/api/v1/repos/${slug}" >/dev/null 2>&1; then
+    backup_log "Repo ${slug} already exists"
+    return 0
+  fi
+
+  backup_log "Creating repo ${slug}..."
+
+  # Create org if needed
+  curl -sf -X POST \
+    -H "Authorization: token ${FORGE_TOKEN}" \
+    -H "Content-Type: application/json" \
+    "${FORGE_URL}/api/v1/orgs" \
+    -d "{\"username\":\"${org_name}\",\"visibility\":\"public\"}" >/dev/null 2>&1 || true
+
+  # Create repo
+  local response
+  response=$(curl -sf -X POST \
+    -H "Authorization: token ${FORGE_TOKEN}" \
+    -H "Content-Type: application/json" \
+    "${FORGE_URL}/api/v1/orgs/${org_name}/repos" \
+    -d "{\"name\":\"${repo_name}\",\"auto_init\":false,\"default_branch\":\"main\"}" 2>/dev/null) \
+    || response=""
+
+  if [ -n "$response" ] && echo "$response" | grep -q '"id":\|[0-9]'; then
+    backup_log "Created repo ${slug}"
+    return 0
+  fi
+
+  # Fallback: admin endpoint
+  response=$(curl -sf -X POST \
+    -H "Authorization: token ${FORGE_TOKEN}" \
+    -H "Content-Type: application/json" \
+    "${FORGE_URL}/api/v1/admin/users/${org_name}/repos" \
+    -d "{\"name\":\"${repo_name}\",\"auto_init\":false,\"default_branch\":\"main\"}" 2>/dev/null) \
+    || response=""
+
+  if [ -n "$response" ] && echo "$response" | grep -q '"id":\|[0-9]'; then
+    backup_log "Created repo ${slug} (via admin API)"
+    return 0
+  fi
+
+  backup_log "ERROR: failed to create repo ${slug}" >&2
+  return 1
+}
+
+# ── Helper: check if issue exists by number ──────────────────────────────────
+# Usage: backup_issue_exists <slug> <issue_number>
+# Returns: 0 if exists, 1 if not
+backup_issue_exists() {
+  local slug="$1"
+  local issue_num="$2"
+
+  curl -sf --max-time 5 \
+    -H "Authorization: token ${FORGE_TOKEN}" \
+    "${FORGE_URL}/api/v1/repos/${slug}/issues/${issue_num}" >/dev/null 2>&1
+}
+
+# ── Helper: create issue with specific number (if Forgejo supports it) ───────
+# Note: Forgejo API auto-assigns next integer; we accept renumbering and log mapping
+# Usage: backup_create_issue <slug> <original_number> <title> <body> [labels...]
+# Returns: new_issue_number on success, 0 on failure
+backup_create_issue() {
+  local slug="$1"
+  local original_num="$2"
+  local title="$3"
+  local body="$4"
+  shift 4
+
+  # Build labels array
+  local -a labels=()
+  for label in "$@"; do
+    # Resolve label name to ID
+    local label_id
+    label_id=$(curl -sf --max-time 5 \
+      -H "Authorization: token ${FORGE_TOKEN}" \
+      "${FORGE_URL}/api/v1/repos/${slug}/labels" 2>/dev/null \
+      | jq -r ".[] | select(.name == \"${label}\") | .id" 2>/dev/null) || label_id=""
+
+    if [ -n "$label_id" ] && [ "$label_id" != "null" ]; then
+      labels+=("$label_id")
+    fi
+  done
+
+  # Build payload
+  local payload
+  if [ ${#labels[@]} -gt 0 ]; then
+    payload=$(jq -n \
+      --arg title "$title" \
+      --arg body "$body" \
+      --argjson labels "$(printf '%s' "${labels[@]}" | jq -R . | jq -s .)" \
+      '{title: $title, body: $body, labels: $labels}')
+  else
+    payload=$(jq -n --arg title "$title" --arg body "$body" '{title: $title, body: $body, labels: []}')
+  fi
+
+  local response
+  response=$(curl -sf -X POST \
+    -H "Authorization: token ${FORGE_TOKEN}" \
+    -H "Content-Type: application/json" \
+    "${FORGE_URL}/api/v1/repos/${slug}/issues" \
+    -d "$payload" 2>/dev/null) || {
+    backup_log "ERROR: failed to create issue '${title}'" >&2
+    return 1
+  }
+
+  local new_num
+  new_num=$(printf '%s' "$response" | jq -r '.number // empty')
+
+  # Log the mapping
+  echo "${original_num}:${new_num}" >> "${BACKUP_MAPPING_FILE}"
+
+  backup_log "Created issue '${title}' as #${new_num} (original: #${original_num})"
+  echo "$new_num"
+}
+
+# ── Step 1: Unpack tarball to temp dir ───────────────────────────────────────
+# Usage: backup_unpack_tarball <tarball>
+# Returns: temp dir path via BACKUP_TEMP_DIR
+backup_unpack_tarball() {
+  local tarball="$1"
+
+  if [ ! -f "$tarball" ]; then
+    backup_log "ERROR: tarball not found: ${tarball}" >&2
+    return 1
+  fi
+
+  BACKUP_TEMP_DIR=$(mktemp -d -t disinto-backup.XXXXXX)
+  backup_log "Unpacking ${tarball} to ${BACKUP_TEMP_DIR}"
+
+  if ! tar -xzf "$tarball" -C "$BACKUP_TEMP_DIR"; then
+    backup_log "ERROR: failed to unpack tarball" >&2
+    rm -rf "$BACKUP_TEMP_DIR"
+    return 1
+  fi
+
+  # Verify expected structure
+  if [ ! -d "${BACKUP_TEMP_DIR}/repos" ]; then
+    backup_log "ERROR: tarball missing 'repos/' directory" >&2
+    rm -rf "$BACKUP_TEMP_DIR"
+    return 1
+  fi
+
+  backup_log "Tarball unpacked successfully"
+}
+
+# ── Step 2: disinto repo — create via Forgejo API, trigger sync (manual) ─────
+# Usage: backup_import_disinto_repo
+# Returns: 0 on success, 1 on failure
+backup_import_disinto_repo() {
+  backup_log "Step 2: Configuring disinto repo..."
+
+  # Create disinto repo if missing
+  backup_create_repo_if_missing "disinto-admin/disinto"
+
+  # Note: Manual mirror configuration recommended (avoids SSH deploy-key handling)
+  backup_log "Note: Configure Codeberg → Forgejo pull mirror manually"
+  backup_log "  Run on Forgejo admin panel: Repository Settings → Repository Mirroring"
+  backup_log "  Source: ssh://git@codeberg.org/johba/disinto.git"
+  backup_log "  Mirror: disinto-admin/disinto"
+  backup_log "  Or use: git clone --mirror ssh://git@codeberg.org/johba/disinto.git"
+  backup_log "          cd disinto.git && git push --mirror ${FORGE_URL}/disinto-admin/disinto.git"
+
+  return 0
+}
+
+# ── Step 3: disinto-ops repo — create empty, push from bundle ────────────────
+# Usage: backup_import_disinto_ops_repo
+# Returns: 0 on success, 1 on failure
+backup_import_disinto_ops_repo() {
+  backup_log "Step 3: Configuring disinto-ops repo from bundle..."
+
+  local bundle_path="${BACKUP_TEMP_DIR}/repos/disinto-ops.bundle"
+
+  if [ ! -d "$bundle_path" ]; then
+    backup_log "WARNING: Bundle not found at ${bundle_path}, skipping"
+    return 0
+  fi
+
+  # Create ops repo if missing
+  backup_create_repo_if_missing "disinto-admin/disinto-ops"
+
+  # Clone bundle and push to Forgejo
+  local clone_dir
+  clone_dir=$(mktemp -d -t disinto-ops-clone.XXXXXX)
+  backup_log "Cloning bundle to ${clone_dir}"
+
+  if ! git clone --bare "$bundle_path" "$clone_dir/disinto-ops.git"; then
+    backup_log "ERROR: failed to clone bundle"
+    rm -rf "$clone_dir"
+    return 1
+  fi
+
+  # Push all refs to Forgejo
+  backup_log "Pushing refs to Forgejo..."
+  if ! cd "$clone_dir/disinto-ops.git" && \
+     git push --mirror "${FORGE_URL}/disinto-admin/disinto-ops.git" 2>&1; then
+    backup_log "ERROR: failed to push refs"
+    rm -rf "$clone_dir"
+    return 1
+  fi
+
+  local ref_count
+  ref_count=$(cd "$clone_dir/disinto-ops.git" && git show-ref | wc -l)
+  BACKUP_PUSHED_REFS=$((BACKUP_PUSHED_REFS + ref_count))
+
+  backup_log "Pushed ${ref_count} refs to disinto-ops"
+  rm -rf "$clone_dir"
+
+  return 0
+}
+
+# ── Step 4: Import issues from backup ────────────────────────────────────────
+# Usage: backup_import_issues <slug> <issues_dir>
+# Returns: 0 on success
+backup_import_issues() {
+  local slug="$1"
+  local issues_dir="$2"
+
+  if [ ! -d "$issues_dir" ]; then
+    backup_log "No issues directory found, skipping"
+    return 0
+  fi
+
+  local created=0
+  local skipped=0
+
+  for issue_file in "${issues_dir}"/*.json; do
+    [ -f "$issue_file" ] || continue
+
+    backup_log "Processing issue file: $(basename "$issue_file")"
+
+    local issue_num title body
+    issue_num=$(jq -r '.number // empty' "$issue_file")
+    title=$(jq -r '.title // empty' "$issue_file")
+    body=$(jq -r '.body // empty' "$issue_file")
+
+    if [ -z "$issue_num" ] || [ "$issue_num" = "null" ]; then
+      backup_log "WARNING: skipping issue without number: $(basename "$issue_file")"
+      continue
+    fi
+
+    # Check if issue already exists
+    if backup_issue_exists "$slug" "$issue_num"; then
+      backup_log "Issue #${issue_num} already exists, skipping"
+      skipped=$((skipped + 1))
+      continue
+    fi
+
+    # Extract labels
+    local -a labels=()
+    while IFS= read -r label; do
+      [ -n "$label" ] && labels+=("$label")
+    done < <(jq -r '.labels[]? // empty' "$issue_file")
+
+    # Create issue
+    local new_num
+    if new_num=$(backup_create_issue "$slug" "$issue_num" "$title" "$body" "${labels[@]}"); then
+      created=$((created + 1))
+    fi
+  done
+
+  BACKUP_CREATED_ISSUES=$((BACKUP_CREATED_ISSUES + created))
+  BACKUP_SKIPPED_ISSUES=$((BACKUP_SKIPPED_ISSUES + skipped))
+
+  backup_log "Created ${created} issues, skipped ${skipped}"
+}
+
+# ── Main: import subcommand ──────────────────────────────────────────────────
+# Usage: backup_import <tarball>
+backup_import() {
+  local tarball="$1"
+
+  if [ -z "${FORGE_URL:-}" ]; then
+    echo "Error: FORGE_URL not set" >&2
+    exit 1
+  fi
+
+  if [ -z "${FORGE_TOKEN:-}" ]; then
+    echo "Error: FORGE_TOKEN not set" >&2
+    exit 1
+  fi
+
+  backup_log "=== Backup Import Started ==="
+  backup_log "Target: ${FORGE_URL}"
+  backup_log "Tarball: ${tarball}"
+
+  # Initialize counters
+  BACKUP_CREATED_REPOS=0
+  BACKUP_PUSHED_REFS=0
+  BACKUP_CREATED_ISSUES=0
+  BACKUP_SKIPPED_ISSUES=0
+
+  # Create temp dir for mapping file
+  BACKUP_MAPPING_FILE=$(mktemp -t disinto-mapping.XXXXXX.json)
+  echo '{"mappings": []}' > "$BACKUP_MAPPING_FILE"
+
+  # Step 1: Unpack tarball
+  if ! backup_unpack_tarball "$tarball"; then
+    exit 1
+  fi
+
+  # Step 2: disinto repo
+  if ! backup_import_disinto_repo; then
+    exit 1
+  fi
+
+  # Step 3: disinto-ops repo
+  if ! backup_import_disinto_ops_repo; then
+    exit 1
+  fi
+
+  # Step 4: Import issues for each repo with issues/*.json
+  for repo_dir in "${BACKUP_TEMP_DIR}/repos"/*/; do
+    [ -d "$repo_dir" ] || continue
+
+    local slug
+    slug=$(basename "$repo_dir")
+
+    backup_log "Processing repo: ${slug}"
+
+    local issues_dir="${repo_dir}issues"
+    if [ -d "$issues_dir" ]; then
+      backup_import_issues "$slug" "$issues_dir"
+    fi
+  done
+
+  # Summary
+  backup_log "=== Backup Import Complete ==="
+  backup_log "Created ${BACKUP_CREATED_REPOS} repos"
+  backup_log "Pushed ${BACKUP_PUSHED_REFS} refs"
+  backup_log "Imported ${BACKUP_CREATED_ISSUES} issues"
+  backup_log "Skipped ${BACKUP_SKIPPED_ISSUES} (already present)"
+  backup_log "Issue mapping saved to: ${BACKUP_MAPPING_FILE}"
+
+  # Cleanup
+  rm -rf "$BACKUP_TEMP_DIR"
+
+  exit 0
+}
+
+# ── Entry point: if sourced, don't run; if executed directly, run import ────
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  if [ $# -lt 1 ]; then
+    echo "Usage: $0 <tarball>" >&2
+    exit 1
+  fi
+
+  backup_import "$1"
+fi