diff --git a/.claude/rules/loc-exceptions.txt b/.claude/rules/loc-exceptions.txt index 9653f85..74f314e 100644 --- a/.claude/rules/loc-exceptions.txt +++ b/.claude/rules/loc-exceptions.txt @@ -101,3 +101,11 @@ docs-src/control_generator_routes.py # splitting into multiple files awkward without sacrificing single-import ergonomics. consent-sdk/src/mobile/flutter/consent_sdk.dart consent-sdk/src/mobile/ios/ConsentManager.swift + +# --- admin-compliance: oversized component refactor backlog --- +# Phase 5+ target for splitting into smaller subcomponents per wizard step. +admin-compliance/components/sdk/ai-act/DecisionTreeWizard.tsx + +# --- ai-compliance-sdk: oversized handler refactor backlog --- +# Phase 5+ target for splitting handler groups into per-resource files. +ai-compliance-sdk/internal/api/handlers/tender_handlers.go diff --git a/.gitea/workflows/build-push-deploy.yml b/.gitea/workflows/build-push-deploy.yml index 69d211e..4d70c7a 100644 --- a/.gitea/workflows/build-push-deploy.yml +++ b/.gitea/workflows/build-push-deploy.yml @@ -1,5 +1,11 @@ # Build + push compliance service images to registry.meghsakha.com -# and trigger orca redeploy on every push to main that touches a service. +# and trigger orca redeploy after CI passes on main. +# +# This workflow is gated on the CI workflow completing successfully. +# It does not run independently — if CI fails, builds + deploy are skipped. +# Per-service builds are gated on detect-changes so only services with +# modified files are rebuilt; trigger-orca runs only if at least one build +# succeeded and none failed. # # Requires Gitea Actions secrets: # REGISTRY_USERNAME / REGISTRY_PASSWORD — registry.meghsakha.com credentials @@ -8,24 +14,68 @@ name: Build + Deploy on: - push: + workflow_run: + workflows: ["CI"] + types: [completed] branches: [main] - paths: - - 'admin-compliance/**' - - 'backend-compliance/**' - - 'ai-compliance-sdk/**' - - 'developer-portal/**' - - 'compliance-tts-service/**' - - 'document-crawler/**' - - 'dsms-gateway/**' - - 'dsms-node/**' jobs: - # ── per-service builds run in parallel ──────────────────────────────────── + # ── gate: only proceed if CI succeeded ──────────────────────────────────── + ci-passed: + runs-on: docker + container: alpine:3.20 + if: github.event.workflow_run.conclusion == 'success' + steps: + - name: CI passed, proceeding with build + deploy + run: echo "CI run ${{ github.event.workflow_run.id }} succeeded on ${{ github.event.workflow_run.head_branch }} @ ${{ github.event.workflow_run.head_sha }}" + + # ── detect which services changed since the last successful build ──────── + # Diff base = the last-build/main git tag, set by mark-last-build at the + # end of every successful run. Works across squash merges, multi-commit + # raw pushes, and force pushes (force pushes leave a stale tag → diff + # shows symmetric differences → safe over-rebuild). If the tag doesn't + # exist yet, scripts/detect-changes.sh falls back to rebuilding all. + detect-changes: + runs-on: docker + container: alpine:3.20 + needs: ci-passed + outputs: + admin: ${{ steps.diff.outputs.admin }} + backend: ${{ steps.diff.outputs.backend }} + sdk: ${{ steps.diff.outputs.sdk }} + portal: ${{ steps.diff.outputs.portal }} + tts: ${{ steps.diff.outputs.tts }} + crawler: ${{ steps.diff.outputs.crawler }} + dsms_gateway: ${{ steps.diff.outputs.dsms_gateway }} + dsms_node: ${{ steps.diff.outputs.dsms_node }} + steps: + - name: Checkout + run: | + apk add --no-cache git bash + git clone --depth 200 --branch ${GITHUB_REF_NAME} ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git . + git fetch --tags origin || true + - name: Resolve base SHA from last-build/main tag + run: | + BASE=$(git rev-parse --verify refs/tags/last-build/main 2>/dev/null || true) + echo "Base SHA: ${BASE:-}" + # Deepen if base isn't yet in the shallow clone. + if [ -n "$BASE" ] && ! git rev-parse --verify "${BASE}^{commit}" >/dev/null 2>&1; then + git fetch --unshallow origin 2>/dev/null \ + || git fetch --depth=10000 origin 2>/dev/null \ + || true + fi + echo "BASE_SHA=${BASE}" >> "$GITHUB_ENV" + - name: Detect changes + id: diff + run: bash scripts/detect-changes.sh + + # ── per-service builds run in parallel (only changed services) ──────────── build-admin-compliance: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.admin == 'true' steps: - name: Checkout run: | @@ -49,6 +99,8 @@ jobs: build-backend-compliance: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.backend == 'true' steps: - name: Checkout run: | @@ -72,6 +124,8 @@ jobs: build-ai-sdk: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.sdk == 'true' steps: - name: Checkout run: | @@ -95,6 +149,8 @@ jobs: build-developer-portal: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.portal == 'true' steps: - name: Checkout run: | @@ -118,6 +174,8 @@ jobs: build-tts: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.tts == 'true' steps: - name: Checkout run: | @@ -141,6 +199,8 @@ jobs: build-document-crawler: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.crawler == 'true' steps: - name: Checkout run: | @@ -164,6 +224,8 @@ jobs: build-dsms-gateway: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.dsms_gateway == 'true' steps: - name: Checkout run: | @@ -187,6 +249,8 @@ jobs: build-dsms-node: runs-on: docker container: docker:27-cli + needs: detect-changes + if: needs.detect-changes.outputs.dsms_node == 'true' steps: - name: Checkout run: | @@ -207,7 +271,52 @@ jobs: docker push registry.meghsakha.com/breakpilot/compliance-dsms-node:latest docker push registry.meghsakha.com/breakpilot/compliance-dsms-node:${SHORT_SHA} - # ── orca redeploy (only after all builds succeed) ───────────────────────── + # ── advance the last-build/main tag — the diff base for future runs ────── + # Runs when no build failed. Covers two cases: + # - at least one service was rebuilt → mark this SHA as the new baseline + # - all services were skipped (nothing changed) → still advance the tag + # so we don't keep re-evaluating the same skipped commits forever + # Skips if any build failed → tag stays put → next push retries those + # services from the previous known-good base. + mark-last-build: + runs-on: docker + container: alpine:3.20 + needs: + - build-admin-compliance + - build-backend-compliance + - build-ai-sdk + - build-developer-portal + - build-tts + - build-document-crawler + - build-dsms-gateway + - build-dsms-node + if: | + always() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + env: + GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + steps: + - name: Checkout + run: | + apk add --no-cache git + git clone --depth 1 --branch ${GITHUB_REF_NAME} ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git . + - name: Force-push last-build/main tag + run: | + set -e + SHA="${HEAD_SHA:-$(git rev-parse HEAD)}" + echo "Advancing last-build/main → ${SHA}" + git tag -f last-build/main "$SHA" + # Encode token into the push URL (no on-disk credential persistence). + PUSH_URL="${GITHUB_SERVER_URL/https:\/\//https:\/\/x-access-token:${GITEA_TOKEN}@}/${GITHUB_REPOSITORY}.git" + git push --force "$PUSH_URL" "refs/tags/last-build/main" + echo "Tag last-build/main now at ${SHA}" + + # ── orca redeploy — runs only if at least one build succeeded ───────────── + # `always()` lets this run when some builds are skipped (unchanged services). + # The contains() checks ensure we only redeploy when something actually built + # and no build failed. trigger-orca: runs-on: docker @@ -221,6 +330,11 @@ jobs: - build-document-crawler - build-dsms-gateway - build-dsms-node + if: | + always() && + contains(needs.*.result, 'success') && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') steps: - name: Checkout (for SHA) run: | diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index b091871..8de00a2 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -19,6 +19,49 @@ on: jobs: + # ── Change detection (always runs first) ───────────────────────────────── + # Diff base: + # PR → merge-base with the PR base branch + # push → last-build/main tag (set by build-push-deploy after a green build) + # Falls back to "rebuild all" when the base is missing or unreachable. + detect-changes: + runs-on: docker + container: alpine:3.20 + outputs: + admin: ${{ steps.diff.outputs.admin }} + backend: ${{ steps.diff.outputs.backend }} + sdk: ${{ steps.diff.outputs.sdk }} + portal: ${{ steps.diff.outputs.portal }} + tts: ${{ steps.diff.outputs.tts }} + crawler: ${{ steps.diff.outputs.crawler }} + dsms_gateway: ${{ steps.diff.outputs.dsms_gateway }} + dsms_node: ${{ steps.diff.outputs.dsms_node }} + any_python: ${{ steps.diff.outputs.any_python }} + any_node: ${{ steps.diff.outputs.any_node }} + any: ${{ steps.diff.outputs.any }} + steps: + - name: Checkout + run: | + apk add --no-cache git bash + git clone --depth 200 --branch ${GITHUB_REF_NAME} ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git . + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + git fetch --depth 200 origin "${GITHUB_BASE_REF}" || true + else + git fetch --tags origin || true + fi + - name: Resolve base SHA + run: | + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + BASE=$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD 2>/dev/null || true) + else + BASE=$(git rev-parse --verify refs/tags/last-build/main 2>/dev/null || true) + fi + echo "Base SHA: ${BASE:-}" + echo "BASE_SHA=${BASE}" >> "$GITHUB_ENV" + - name: Detect changes + id: diff + run: bash scripts/detect-changes.sh + # ── Branch naming convention (PR only) ────────────────────────────────── branch-name: runs-on: docker @@ -55,10 +98,12 @@ jobs: exit 1 fi - # ── LOC budget (always) ────────────────────────────────────────────────── + # ── LOC budget (only if files changed) ─────────────────────────────────── loc-budget: runs-on: docker container: alpine:3.20 + needs: detect-changes + if: needs.detect-changes.outputs.any == 'true' steps: - name: Checkout run: | @@ -86,10 +131,11 @@ jobs: --redact \ || { echo "::error::Secrets detected — remove them before merging."; exit 1; } - # ── Go lint + build (PR only) ──────────────────────────────────────────── + # ── Go lint + build (PR only, gated on ai-compliance-sdk changes) ──────── go-lint: runs-on: docker - if: github.event_name == 'pull_request' + needs: detect-changes + if: github.event_name == 'pull_request' && needs.detect-changes.outputs.sdk == 'true' container: golangci/golangci-lint:v1.62-alpine steps: - name: Checkout @@ -107,10 +153,11 @@ jobs: cd ai-compliance-sdk go build ./... - # ── Python lint + import check (PR only) ──────────────────────────────── + # ── Python lint + import check (PR only, gated on python service changes) ─ python-lint: runs-on: docker - if: github.event_name == 'pull_request' + needs: detect-changes + if: github.event_name == 'pull_request' && needs.detect-changes.outputs.any_python == 'true' container: python:3.12-slim steps: - name: Checkout @@ -137,10 +184,11 @@ jobs: python -c "import compliance; print('Import OK')" \ || { echo "::error::compliance package fails to import — missing import or syntax error."; exit 1; } - # ── Node.js lint + type-check (PR only) ───────────────────────────────── + # ── Node.js lint + type-check (PR only, gated on Next.js service changes) ─ nodejs-lint: runs-on: docker - if: github.event_name == 'pull_request' + needs: detect-changes + if: github.event_name == 'pull_request' && needs.detect-changes.outputs.any_node == 'true' container: node:20-alpine steps: - name: Checkout @@ -158,10 +206,12 @@ jobs: done exit $fail - # ── Node.js build — next build (PR + push to main) ─────────────────────── + # ── Node.js build — next build (gated on Next.js service changes) ─────── nodejs-build: runs-on: docker container: node:20-alpine + needs: detect-changes + if: needs.detect-changes.outputs.any_node == 'true' steps: - name: Checkout run: | @@ -244,10 +294,12 @@ jobs: - name: Vulnerability scan (fail on high+) run: grype sbom:sbom-out/sbom.cdx.json --fail-on high -q - # ── Tests (PR + push to main) ───────────────────────────────────────────── + # ── Tests (gated per service) ──────────────────────────────────────────── test-go: runs-on: docker container: golang:1.24-alpine + needs: detect-changes + if: needs.detect-changes.outputs.sdk == 'true' env: CGO_ENABLED: "0" steps: @@ -265,6 +317,8 @@ jobs: test-python-backend: runs-on: docker container: python:3.12-slim + needs: detect-changes + if: needs.detect-changes.outputs.backend == 'true' env: CI: "true" steps: @@ -284,6 +338,8 @@ jobs: test-python-document-crawler: runs-on: docker container: python:3.12-slim + needs: detect-changes + if: needs.detect-changes.outputs.crawler == 'true' env: CI: "true" steps: @@ -303,6 +359,8 @@ jobs: test-python-dsms-gateway: runs-on: docker container: python:3.12-slim + needs: detect-changes + if: needs.detect-changes.outputs.dsms_gateway == 'true' env: CI: "true" steps: diff --git a/scripts/check-loc.sh b/scripts/check-loc.sh index a303c79..546e417 100755 --- a/scripts/check-loc.sh +++ b/scripts/check-loc.sh @@ -54,6 +54,7 @@ is_excluded() { *.md|*.json|*.yaml|*.yml|*.lock|*.sum|*.mod|*.toml|*.cfg|*.ini) return 0 ;; *.html|*.html.j2|*.jinja|*.jinja2) return 0 ;; *.svg|*.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.woff|*.woff2|*.ttf) return 0 ;; + *.xls|*.xlsx|*.xlsm|*.docx|*.pptx|*.zip|*.tar|*.gz) return 0 ;; *.generated.*|*.gen.*|*_pb.go|*_pb2.py|*.pb.go) return 0 ;; esac return 1 diff --git a/scripts/detect-changes.sh b/scripts/detect-changes.sh new file mode 100755 index 0000000..d7f0d62 --- /dev/null +++ b/scripts/detect-changes.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Emit per-service + aggregate change flags for the CI / build workflows. +# +# Reads: +# BASE_SHA — diff base. Empty / unreachable → emit everything as true. +# HEAD_SHA — diff target. Defaults to HEAD. +# +# Writes key=value lines to $GITHUB_OUTPUT (defaults to /dev/stdout for local runs). +# +# Keys emitted: +# admin, backend, sdk, portal, tts, crawler, dsms_gateway, dsms_node +# any_python, any_node, any + +set -euo pipefail + +BASE_SHA="${BASE_SHA:-}" +HEAD_SHA="${HEAD_SHA:-HEAD}" +OUT="${GITHUB_OUTPUT:-/dev/stdout}" + +ALL_KEYS=(admin backend sdk portal tts crawler dsms_gateway dsms_node any_python any_node any) + +emit() { + echo "$1=$2" >> "$OUT" +} + +emit_all_true() { + reason=$1 + echo "→ rebuild all: $reason" + for k in "${ALL_KEYS[@]}"; do + emit "$k" true + done +} + +if [ -z "$BASE_SHA" ]; then + emit_all_true "no BASE_SHA provided" + exit 0 +fi + +if ! git rev-parse --verify "${BASE_SHA}^{commit}" >/dev/null 2>&1; then + emit_all_true "BASE_SHA ${BASE_SHA} unreachable" + exit 0 +fi + +changed=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" || true) +echo "Changed files since ${BASE_SHA}:" +echo "${changed:-(none)}" +echo "---" + +check() { + key=$1 + path=$2 + if echo "$changed" | grep -q "^${path}/"; then + emit "$key" true + echo " ${key} (${path}/): true" + else + emit "$key" false + echo " ${key} (${path}/): false" + fi +} + +check admin admin-compliance +check backend backend-compliance +check sdk ai-compliance-sdk +check portal developer-portal +check tts compliance-tts-service +check crawler document-crawler +check dsms_gateway dsms-gateway +check dsms_node dsms-node + +# Aggregate flags +if echo "$changed" | grep -qE "^(backend-compliance|document-crawler|dsms-gateway|compliance-tts-service)/"; then + emit any_python true + echo " any_python: true" +else + emit any_python false + echo " any_python: false" +fi + +if echo "$changed" | grep -qE "^(admin-compliance|developer-portal)/"; then + emit any_node true + echo " any_node: true" +else + emit any_node false + echo " any_node: false" +fi + +if [ -n "$changed" ]; then + emit any true + echo " any: true" +else + emit any false + echo " any: false" +fi