feat: Control Library UI, dedup migration, QA tooling, docs
Some checks failed
CI/CD / go-lint (push) Has been skipped
CI/CD / python-lint (push) Has been skipped
CI/CD / nodejs-lint (push) Has been skipped
CI/CD / test-go-ai-compliance (push) Failing after 31s
CI/CD / test-python-backend-compliance (push) Successful in 1m35s
CI/CD / test-python-document-crawler (push) Successful in 20s
CI/CD / test-python-dsms-gateway (push) Successful in 17s
CI/CD / validate-canonical-controls (push) Successful in 10s
CI/CD / Deploy (push) Has been skipped
Some checks failed
CI/CD / go-lint (push) Has been skipped
CI/CD / python-lint (push) Has been skipped
CI/CD / nodejs-lint (push) Has been skipped
CI/CD / test-go-ai-compliance (push) Failing after 31s
CI/CD / test-python-backend-compliance (push) Successful in 1m35s
CI/CD / test-python-document-crawler (push) Successful in 20s
CI/CD / test-python-dsms-gateway (push) Successful in 17s
CI/CD / validate-canonical-controls (push) Successful in 10s
CI/CD / Deploy (push) Has been skipped
- Control Library: parent control display, ObligationTypeBadge, GenerationStrategyBadge variants, evidence string fallback - API: expose parent_control_uuid/id/title in canonical controls - Fix: DSFA SQLAlchemy 2.0 Row._mapping compatibility - Migration 074: control_parent_links + control_dedup_reviews tables - QA scripts: benchmark, gap analysis, OSCAL import, OWASP cleanup, phase5 normalize, phase74 gap fill, sync_db, run_job - Docs: dedup engine, RAG benchmark, lessons learned, pipeline docs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
218
scripts/qa/run_job.sh
Executable file
218
scripts/qa/run_job.sh
Executable file
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env bash
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Robust job runner for QA scripts on Mac Mini
|
||||
#
|
||||
# Usage:
|
||||
# ./run_job.sh <script.py> [args...] # start job
|
||||
# ./run_job.sh --status # show running jobs
|
||||
# ./run_job.sh --kill <script.py> # kill a running job
|
||||
# ./run_job.sh --log <script.py> # tail log
|
||||
#
|
||||
# Features:
|
||||
# - Loads .env automatically (COMPLIANCE_DATABASE_URL → DATABASE_URL)
|
||||
# - PID-file prevents duplicate runs
|
||||
# - Unbuffered Python output
|
||||
# - Structured log files in /tmp/qa_jobs/
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
JOB_DIR="/tmp/qa_jobs"
|
||||
mkdir -p "$JOB_DIR"
|
||||
|
||||
# ── Load .env ────────────────────────────────────────────────
|
||||
load_env() {
|
||||
local envfile="$PROJECT_DIR/.env"
|
||||
if [[ -f "$envfile" ]]; then
|
||||
# Export all vars from .env
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$envfile"
|
||||
set +a
|
||||
fi
|
||||
# Map COMPLIANCE_DATABASE_URL → DATABASE_URL if needed
|
||||
if [[ -z "${DATABASE_URL:-}" && -n "${COMPLIANCE_DATABASE_URL:-}" ]]; then
|
||||
export DATABASE_URL="$COMPLIANCE_DATABASE_URL"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Job name from script path ─────────────────────────────────
|
||||
job_name() {
|
||||
basename "$1" .py
|
||||
}
|
||||
|
||||
pid_file() {
|
||||
echo "$JOB_DIR/$(job_name "$1").pid"
|
||||
}
|
||||
|
||||
log_file() {
|
||||
echo "$JOB_DIR/$(job_name "$1").log"
|
||||
}
|
||||
|
||||
# ── Status ────────────────────────────────────────────────────
|
||||
show_status() {
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
echo "QA Job Status ($(date '+%Y-%m-%d %H:%M:%S'))"
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
local found=0
|
||||
for pidfile in "$JOB_DIR"/*.pid; do
|
||||
[[ -f "$pidfile" ]] || continue
|
||||
found=1
|
||||
local name
|
||||
name=$(basename "$pidfile" .pid)
|
||||
local pid
|
||||
pid=$(cat "$pidfile")
|
||||
local logf="$JOB_DIR/$name.log"
|
||||
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
local lines
|
||||
lines=$(wc -l < "$logf" 2>/dev/null || echo 0)
|
||||
local errors
|
||||
errors=$(grep -c "ERROR" "$logf" 2>/dev/null || echo 0)
|
||||
local last_line
|
||||
last_line=$(tail -1 "$logf" 2>/dev/null || echo "(empty)")
|
||||
echo " ● $name (PID $pid) — RUNNING"
|
||||
echo " Log: $logf ($lines lines, $errors errors)"
|
||||
echo " Last: $last_line"
|
||||
else
|
||||
echo " ○ $name (PID $pid) — STOPPED"
|
||||
echo " Log: $logf"
|
||||
rm -f "$pidfile"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
if [[ $found -eq 0 ]]; then
|
||||
echo " No jobs running."
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Kill ──────────────────────────────────────────────────────
|
||||
kill_job() {
|
||||
local script="$1"
|
||||
local pf
|
||||
pf=$(pid_file "$script")
|
||||
if [[ ! -f "$pf" ]]; then
|
||||
echo "No PID file for $(job_name "$script")"
|
||||
return 1
|
||||
fi
|
||||
local pid
|
||||
pid=$(cat "$pf")
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid"
|
||||
echo "Killed $(job_name "$script") (PID $pid)"
|
||||
else
|
||||
echo "Process $pid already stopped"
|
||||
fi
|
||||
rm -f "$pf"
|
||||
}
|
||||
|
||||
# ── Tail log ──────────────────────────────────────────────────
|
||||
tail_log() {
|
||||
local script="$1"
|
||||
local lf
|
||||
lf=$(log_file "$script")
|
||||
if [[ ! -f "$lf" ]]; then
|
||||
echo "No log file: $lf"
|
||||
return 1
|
||||
fi
|
||||
tail -50 "$lf"
|
||||
}
|
||||
|
||||
# ── Start job ─────────────────────────────────────────────────
|
||||
start_job() {
|
||||
local script="$1"
|
||||
shift
|
||||
local args=("$@")
|
||||
|
||||
# Resolve script path
|
||||
local script_path="$script"
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
script_path="$SCRIPT_DIR/$script"
|
||||
fi
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
echo "ERROR: Script not found: $script"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local name
|
||||
name=$(job_name "$script")
|
||||
local pf
|
||||
pf=$(pid_file "$script")
|
||||
local lf
|
||||
lf=$(log_file "$script")
|
||||
|
||||
# Check for already-running instance
|
||||
if [[ -f "$pf" ]]; then
|
||||
local existing_pid
|
||||
existing_pid=$(cat "$pf")
|
||||
if kill -0 "$existing_pid" 2>/dev/null; then
|
||||
echo "ERROR: $name already running (PID $existing_pid)"
|
||||
echo "Use: $0 --kill $script"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$pf"
|
||||
fi
|
||||
|
||||
# Load environment
|
||||
load_env
|
||||
|
||||
# Verify required env vars
|
||||
if [[ -z "${DATABASE_URL:-}" ]]; then
|
||||
echo "ERROR: DATABASE_URL not set (checked .env)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Start
|
||||
echo "Starting $name..."
|
||||
echo " Script: $script_path"
|
||||
echo " Args: ${args[*]:-none}"
|
||||
echo " Log: $lf"
|
||||
|
||||
nohup python3 -u "$script_path" "${args[@]}" > "$lf" 2>&1 &
|
||||
local pid=$!
|
||||
echo "$pid" > "$pf"
|
||||
|
||||
echo " PID: $pid"
|
||||
echo ""
|
||||
|
||||
# Wait a moment and check it started OK
|
||||
sleep 3
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
echo "ERROR: Process died immediately. Log output:"
|
||||
cat "$lf"
|
||||
rm -f "$pf"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local lines
|
||||
lines=$(wc -l < "$lf" 2>/dev/null || echo 0)
|
||||
echo "Running OK ($lines log lines so far)"
|
||||
echo "Monitor with: $0 --status"
|
||||
echo "Tail log: $0 --log $script"
|
||||
}
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────
|
||||
case "${1:-}" in
|
||||
--status|-s)
|
||||
show_status
|
||||
;;
|
||||
--kill|-k)
|
||||
[[ -n "${2:-}" ]] || { echo "Usage: $0 --kill <script.py>"; exit 1; }
|
||||
kill_job "$2"
|
||||
;;
|
||||
--log|-l)
|
||||
[[ -n "${2:-}" ]] || { echo "Usage: $0 --log <script.py>"; exit 1; }
|
||||
tail_log "$2"
|
||||
;;
|
||||
--help|-h|"")
|
||||
echo "Usage:"
|
||||
echo " $0 <script.py> [args...] Start a QA job"
|
||||
echo " $0 --status Show running jobs"
|
||||
echo " $0 --kill <script.py> Kill a running job"
|
||||
echo " $0 --log <script.py> Tail job log"
|
||||
;;
|
||||
*)
|
||||
start_job "$@"
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user