OpenKKnD/llm-configs-openkknd2/scripts/rotate-models-amanita.sh

#!/usr/bin/env bash
# rotate-models-amanita.sh — try coding models one-by-one until openkknd2 progresses.
#
# Run ON amanita (or: ssh amanita bash -s < this-script).
# Uses short Zusatzprompt; fresh OpenClaw session per model; progress gate on tree.
#
#   OPENKKND2_ROOT=~/src/openkknd2-am-coder \
#   LLM_CONFIGS=~/src/llm-configs \
#   bash rotate-models-amanita.sh
#
set -euo pipefail
export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH}"
export LLM_CONFIGS="${LLM_CONFIGS:-$HOME/src/llm-configs}"
ROOT="${OPENKKND2_ROOT:-$HOME/src/openkknd2-am-coder}"
LOG_DIR="${ROTATE_LOG_DIR:-$ROOT/rotate-logs}"
PROMPT_FILE="${PROMPT_FILE:-$LLM_CONFIGS/prompts/openkknd2-continue-short.md}"
MAX_TURNS="${ROTATE_MAX_TURNS:-12}"
AGENT="${AGENT_CODER:-$HOME/agent-coder}"
# Single-flight load policy: never run next model until previous fully stopped;
# unload other ollama models between slots; no parallel pulls.
SINGLE_FLIGHT="${ROTATE_SINGLE_FLIGHT:-1}"
mkdir -p "$LOG_DIR"
cd "$ROOT"

unload_other_ollama() {
  # Keep the model we are about to use; stop everything else loaded.
  local keep="${1:-}"
  command -v ollama >/dev/null 2>&1 || return 0
  local name
  while read -r name _; do
    [[ -z "${name:-}" || "$name" == "NAME" ]] && continue
    if [[ -n "$keep" && "$name" == *"$keep"* ]]; then
      continue
    fi
    ollama stop "$name" 2>/dev/null || true
  done < <(ollama ps 2>/dev/null | awk 'NR>1 {print $1}' || true)
  # Never start pulls mid-rotate
  pkill -TERM -f "ollama pull" 2>/dev/null || true
}

# agent-coder model flags in order (skip already-failed qwen first if SKIP_QWEN=1)
# Devstral = agentic coding specialist (Mistral + All Hands)
MODELS_DEFAULT=(
  "--devstral"
  "--coder"
  "--qwen314"
  "--qwen14"
  "--deepseek"
  "--qwen38"
  "--hermes"
)
# Optional: also try qwen3.6 last (heavy, already flaked once)
if [[ "${INCLUDE_QWEN36:-0}" == "1" ]]; then
  MODELS_DEFAULT+=("--qwen36")
fi

# shellcheck disable=SC2206
MODELS=( ${ROTATE_MODELS:-${MODELS_DEFAULT[*]}} )

ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }
stamp() { date -u +%Y%m%dT%H%M%SZ; }

code_stats() {
  python3 - <<'PY'
import os
from pathlib import Path
root = Path(os.environ.get("ROOT", "."))
skip = {".git", "build", "node_modules", "rotate-logs"}
exts = {".c", ".h", ".cpp", ".hpp", ".cc", ".py", ".sh", ".cmake"}
files, lines = 0, 0
names = []
for p in root.rglob("*"):
    if any(part in skip for part in p.parts):
        continue
    if not p.is_file():
        continue
    if p.suffix.lower() not in exts and p.name != "CMakeLists.txt":
        continue
    try:
        text = p.read_text(encoding="utf-8", errors="ignore")
    except Exception:
        continue
    n = text.count("\n") + (1 if text and not text.endswith("\n") else 0)
    files += 1
    lines += n
    names.append(str(p.relative_to(root)))
has_build = any("ok_build" in n for n in names)
has_ai = any("ok_ai" in n for n in names)
print(f"{files} {lines} {int(has_build)} {int(has_ai)}")
PY
}

ci_green() {
  if [[ -x "$ROOT/scripts/ci-headless.sh" ]]; then
    (cd "$ROOT" && ./scripts/ci-headless.sh) >/tmp/ok2-rotate-ci.log 2>&1 && return 0
    return 1
  fi
  return 1
}

progressed() {
  # args: before "files lines has_build has_ai"
  local b_files b_lines b_b b_a a_files a_lines a_b a_a
  read -r b_files b_lines b_b b_a <<<"$1"
  read -r a_files a_lines a_b a_a <<<"$2"
  # success if meaningful code growth OR new build/ai modules + still green
  if (( a_lines >= b_lines + 40 )); then return 0; fi
  if (( a_files > b_files && a_lines > b_lines )); then return 0; fi
  if (( a_b > b_b || a_a > b_a )) && (( a_lines > b_lines )); then return 0; fi
  return 1
}

if [[ ! -x "$AGENT" ]]; then
  echo "error: agent-coder missing at $AGENT" >&2
  exit 1
fi
if [[ ! -f "$PROMPT_FILE" ]]; then
  echo "error: prompt missing: $PROMPT_FILE" >&2
  exit 1
fi
PROMPT="$(cat "$PROMPT_FILE")"
SUMMARY="$LOG_DIR/rotate-$(stamp).md"
{
  echo "# openkknd2 model rotation"
  echo
  echo "- started: $(ts)"
  echo "- workdir: \`$ROOT\`"
  echo "- prompt: \`$PROMPT_FILE\`"
  echo "- max_turns: $MAX_TURNS"
  echo
  echo "| # | flag | session | before lines | after lines | progress | ci |"
  echo "|---|------|---------|--------------|-------------|----------|----|"
} >"$SUMMARY"

echo "=== model rotation start $(ts) ===" | tee -a "$LOG_DIR/rotate.out"
echo "ROOT=$ROOT models=${MODELS[*]}" | tee -a "$LOG_DIR/rotate.out"

i=0
for flag in "${MODELS[@]}"; do
  i=$((i + 1))
  sess="ok2-rot${i}-$(echo "$flag" | tr -cd 'A-Za-z0-9')"
  runlog="$LOG_DIR/run-${i}$(echo "$flag" | tr -cd 'A-Za-z0-9').log"
  echo | tee -a "$LOG_DIR/rotate.out"
  echo ">>> [$i/${#MODELS[@]}] model flag=$flag session=$sess $(ts)" | tee -a "$LOG_DIR/rotate.out"

  export ROOT
  before="$(code_stats)"
  read -r bf bl bb ba <<<"$before"
  echo "    before: files=$bf lines=$bl build=$bb ai=$ba" | tee -a "$LOG_DIR/rotate.out"

  # Stop any leftover registered agent for this tree (best effort)
  am-task stop all --reason "rotate: switch to $flag" 2>/dev/null || true
  sleep 2
  if [[ "$SINGLE_FLIGHT" == "1" ]]; then
    # Map flag → ollama name fragment for keep-list
    keep_frag=""
    case "$flag" in
      --devstral) keep_frag="devstral" ;;
      --coder) keep_frag="qwen3-coder" ;;
      --qwen36) keep_frag="qwen3.6" ;;
      --qwen314) keep_frag="qwen3:14b" ;;
      --qwen38) keep_frag="qwen3:8b" ;;
      --qwen14) keep_frag="qwen2.5-coder:14b" ;;
      --deepseek) keep_frag="deepseek" ;;
      --hermes) keep_frag="hermes" ;;
    esac
    echo "    single-flight: unload other ollama models (keep~$keep_frag)" | tee -a "$LOG_DIR/rotate.out"
    unload_other_ollama "$keep_frag"
    sleep 1
  fi

  set +e
  (
    cd "$ROOT"
    # shellcheck disable=SC2086
    "$AGENT" $flag -s "$sess" -A --max-turns "$MAX_TURNS" --detach "$PROMPT"
  ) >"$runlog" 2>&1
  launch_ec=$?
  set -e
  echo "    launch_ec=$launch_ec log=$runlog" | tee -a "$LOG_DIR/rotate.out"

  # Wait until no active am-task (or timeout)
  deadline=$(( $(date +%s) + ${ROTATE_WAIT_SECS:-3600} ))
  while (( $(date +%s) < deadline )); do
    active="$(ls "$HOME/.am-agent-tasks/active"/*.json 2>/dev/null | wc -l | tr -d ' ')"
    if [[ "${active:-0}" == "0" ]]; then
      break
    fi
    # still running
    sleep 15
  done
  # if still active after wait — stop and move on
  if ls "$HOME/.am-agent-tasks/active"/*.json >/dev/null 2>&1; then
    echo "    timeout waiting — stop all" | tee -a "$LOG_DIR/rotate.out"
    am-task stop all --reason "rotate: wait timeout after $flag" 2>/dev/null || true
    sleep 2
  fi

  export ROOT
  after="$(code_stats)"
  read -r af al ab aa <<<"$after"
  echo "    after:  files=$af lines=$al build=$ab ai=$aa" | tee -a "$LOG_DIR/rotate.out"

  ci="fail"
  if ci_green; then ci="green"; fi
  echo "    ci=$ci" | tee -a "$LOG_DIR/rotate.out"

  prog="no"
  if progressed "$before" "$after"; then prog="YES"; fi

  echo "| $i | \`$flag\` | \`$sess\` | $bl | $al | $prog | $ci |" >>"$SUMMARY"

  if [[ "$prog" == "YES" ]]; then
    {
      echo
      echo "## WINNER"
      echo
      echo "- flag: \`$flag\`"
      echo "- session: \`$sess\`"
      echo "- lines: $bl → $al"
      echo "- ci: $ci"
      echo "- finished: $(ts)"
    } >>"$SUMMARY"
    echo "=== PROGRESS with $flag — stop rotation ===" | tee -a "$LOG_DIR/rotate.out"
    echo "SUMMARY=$SUMMARY"
    exit 0
  fi
  echo "    no meaningful progress — next model" | tee -a "$LOG_DIR/rotate.out"
done

{
  echo
  echo "## RESULT"
  echo
  echo "No model produced meaningful code growth within max_turns=$MAX_TURNS."
  echo "- finished: $(ts)"
} >>"$SUMMARY"
echo "=== rotation exhausted without progress ===" | tee -a "$LOG_DIR/rotate.out"
echo "SUMMARY=$SUMMARY"
exit 1