breakpilot-compliance/scripts/qa/run_job.sh

#!/usr/bin/env bash
# ─────────────────────────────────────────────────────────────
# Robust job runner for QA scripts on Mac Mini
#
# Usage:
#   ./run_job.sh <script.py> [args...]    # start job
#   ./run_job.sh --status                 # show running jobs
#   ./run_job.sh --kill <script.py>       # kill a running job
#   ./run_job.sh --log <script.py>        # tail log
#
# Features:
#   - Loads .env automatically (COMPLIANCE_DATABASE_URL → DATABASE_URL)
#   - PID-file prevents duplicate runs
#   - Unbuffered Python output
#   - Structured log files in /tmp/qa_jobs/
# ─────────────────────────────────────────────────────────────
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
JOB_DIR="/tmp/qa_jobs"
mkdir -p "$JOB_DIR"

# ── Load .env ────────────────────────────────────────────────
load_env() {
    local envfile="$PROJECT_DIR/.env"
    if [[ -f "$envfile" ]]; then
        # Export all vars from .env
        set -a
        # shellcheck disable=SC1090
        source "$envfile"
        set +a
    fi
    # Map COMPLIANCE_DATABASE_URL → DATABASE_URL if needed
    if [[ -z "${DATABASE_URL:-}" && -n "${COMPLIANCE_DATABASE_URL:-}" ]]; then
        export DATABASE_URL="$COMPLIANCE_DATABASE_URL"
    fi
}

# ── Job name from script path ─────────────────────────────────
job_name() {
    basename "$1" .py
}

pid_file() {
    echo "$JOB_DIR/$(job_name "$1").pid"
}

log_file() {
    echo "$JOB_DIR/$(job_name "$1").log"
}

# ── Status ────────────────────────────────────────────────────
show_status() {
    echo "═══════════════════════════════════════════════════════"
    echo "QA Job Status ($(date '+%Y-%m-%d %H:%M:%S'))"
    echo "═══════════════════════════════════════════════════════"
    local found=0
    for pidfile in "$JOB_DIR"/*.pid; do
        [[ -f "$pidfile" ]] || continue
        found=1
        local name
        name=$(basename "$pidfile" .pid)
        local pid
        pid=$(cat "$pidfile")
        local logf="$JOB_DIR/$name.log"

        if kill -0 "$pid" 2>/dev/null; then
            local lines
            lines=$(wc -l < "$logf" 2>/dev/null || echo 0)
            local errors
            errors=$(grep -c "ERROR" "$logf" 2>/dev/null || echo 0)
            local last_line
            last_line=$(tail -1 "$logf" 2>/dev/null || echo "(empty)")
            echo "  ● $name (PID $pid) — RUNNING"
            echo "    Log: $logf ($lines lines, $errors errors)"
            echo "    Last: $last_line"
        else
            echo "  ○ $name (PID $pid) — STOPPED"
            echo "    Log: $logf"
            rm -f "$pidfile"
        fi
        echo ""
    done
    if [[ $found -eq 0 ]]; then
        echo "  No jobs running."
    fi
}

# ── Kill ──────────────────────────────────────────────────────
kill_job() {
    local script="$1"
    local pf
    pf=$(pid_file "$script")
    if [[ ! -f "$pf" ]]; then
        echo "No PID file for $(job_name "$script")"
        return 1
    fi
    local pid
    pid=$(cat "$pf")
    if kill -0 "$pid" 2>/dev/null; then
        kill "$pid"
        echo "Killed $(job_name "$script") (PID $pid)"
    else
        echo "Process $pid already stopped"
    fi
    rm -f "$pf"
}

# ── Tail log ──────────────────────────────────────────────────
tail_log() {
    local script="$1"
    local lf
    lf=$(log_file "$script")
    if [[ ! -f "$lf" ]]; then
        echo "No log file: $lf"
        return 1
    fi
    tail -50 "$lf"
}

# ── Start job ─────────────────────────────────────────────────
start_job() {
    local script="$1"
    shift
    local args=("$@")

    # Resolve script path
    local script_path="$script"
    if [[ ! -f "$script_path" ]]; then
        script_path="$SCRIPT_DIR/$script"
    fi
    if [[ ! -f "$script_path" ]]; then
        echo "ERROR: Script not found: $script"
        return 1
    fi

    local name
    name=$(job_name "$script")
    local pf
    pf=$(pid_file "$script")
    local lf
    lf=$(log_file "$script")

    # Check for already-running instance
    if [[ -f "$pf" ]]; then
        local existing_pid
        existing_pid=$(cat "$pf")
        if kill -0 "$existing_pid" 2>/dev/null; then
            echo "ERROR: $name already running (PID $existing_pid)"
            echo "Use: $0 --kill $script"
            return 1
        fi
        rm -f "$pf"
    fi

    # Load environment
    load_env

    # Verify required env vars
    if [[ -z "${DATABASE_URL:-}" ]]; then
        echo "ERROR: DATABASE_URL not set (checked .env)"
        return 1
    fi

    # Start
    echo "Starting $name..."
    echo "  Script: $script_path"
    echo "  Args:   ${args[*]:-none}"
    echo "  Log:    $lf"

    nohup python3 -u "$script_path" "${args[@]}" > "$lf" 2>&1 &
    local pid=$!
    echo "$pid" > "$pf"

    echo "  PID:    $pid"
    echo ""

    # Wait a moment and check it started OK
    sleep 3
    if ! kill -0 "$pid" 2>/dev/null; then
        echo "ERROR: Process died immediately. Log output:"
        cat "$lf"
        rm -f "$pf"
        return 1
    fi

    local lines
    lines=$(wc -l < "$lf" 2>/dev/null || echo 0)
    echo "Running OK ($lines log lines so far)"
    echo "Monitor with: $0 --status"
    echo "Tail log:     $0 --log $script"
}

# ── Main ──────────────────────────────────────────────────────
case "${1:-}" in
    --status|-s)
        show_status
        ;;
    --kill|-k)
        [[ -n "${2:-}" ]] || { echo "Usage: $0 --kill <script.py>"; exit 1; }
        kill_job "$2"
        ;;
    --log|-l)
        [[ -n "${2:-}" ]] || { echo "Usage: $0 --log <script.py>"; exit 1; }
        tail_log "$2"
        ;;
    --help|-h|"")
        echo "Usage:"
        echo "  $0 <script.py> [args...]  Start a QA job"
        echo "  $0 --status               Show running jobs"
        echo "  $0 --kill <script.py>     Kill a running job"
        echo "  $0 --log <script.py>      Tail job log"
        ;;
    *)
        start_job "$@"
        ;;
esac