282 lines
14 KiB
YAML
282 lines
14 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
RUSTFLAGS: "-D warnings"
|
|
# Compile cache: sccache -> Hetzner S3 (breakpilot-sccache), runner-independent
|
|
# and persistent across CI runs (own key prefix). Reuses the shared cluster S3
|
|
# creds (same bucket as werkpilot). Requires repo secrets HETZNER_S3_ACCESS_KEY
|
|
# and HETZNER_S3_SECRET_KEY.
|
|
RUSTC_WRAPPER: /usr/local/bin/sccache
|
|
SCCACHE_BUCKET: breakpilot-sccache
|
|
SCCACHE_ENDPOINT: https://nbg1.your-objectstorage.com
|
|
SCCACHE_REGION: auto
|
|
SCCACHE_S3_USE_SSL: "true"
|
|
SCCACHE_S3_KEY_PREFIX: compliance-scanner
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
|
|
# compliance-agent depends on tramiton-core via git; use the system git so the
|
|
# credential rewrite below (see "Configure git auth ...") is honored on fetch.
|
|
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
|
|
# Throttle cargo so a ~670-crate concurrent download burst doesn't 429 the
|
|
# Kellnr mirror: fewer concurrent connections (HTTP/1.1) + more retries.
|
|
CARGO_NET_RETRY: "10"
|
|
CARGO_HTTP_MULTIPLEXING: "false"
|
|
|
|
# Cancel superseded PR runs, but NEVER cancel main-branch runs — those build and
|
|
# deploy per-service images, and cancelling one merge's deploy when the next
|
|
# merge lands leaves a service un-deployed (as happened between two back-to-back
|
|
# merges). So cancel-in-progress only for pull_request events.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
jobs:
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 1: Lint, audit, and test (single job to share cargo cache)
|
|
# ---------------------------------------------------------------------------
|
|
check:
|
|
name: Check
|
|
if: github.event_name == 'pull_request'
|
|
runs-on: docker
|
|
container:
|
|
image: rust:1.94-bookworm
|
|
steps:
|
|
- name: Checkout
|
|
run: |
|
|
git init
|
|
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
|
git fetch --depth=1 origin "${GITHUB_SHA}"
|
|
git checkout FETCH_HEAD
|
|
# Resolve crates.io deps through the self-hosted Kellnr mirror (cached,
|
|
# crates.io-independent). Git deps (tramiton-core) are unaffected — source
|
|
# replacement only applies to crates.io-sourced crates.
|
|
- name: Use Kellnr crates.io mirror
|
|
run: |
|
|
: "${CARGO_HOME:=/usr/local/cargo}"
|
|
mkdir -p "$CARGO_HOME"
|
|
{
|
|
echo '[source.crates-io]'
|
|
echo 'replace-with = "kellnr"'
|
|
echo '[registries.kellnr]'
|
|
echo 'index = "sparse+https://crates.meghsakha.com/api/v1/cratesio/"'
|
|
} >> "$CARGO_HOME/config.toml"
|
|
env:
|
|
RUSTC_WRAPPER: ""
|
|
- name: Install tools
|
|
run: |
|
|
rustup component add rustfmt clippy
|
|
curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz \
|
|
| tar xz --strip-components=1 -C /usr/local/bin/ sccache-v0.10.0-x86_64-unknown-linux-musl/sccache
|
|
chmod +x /usr/local/bin/sccache
|
|
cargo install cargo-audit --locked
|
|
env:
|
|
RUSTC_WRAPPER: ""
|
|
|
|
# compliance-agent has a git dependency on tramiton-core (a private repo on
|
|
# this Gitea instance). Rewrite its SSH URL to HTTPS + a PAT so the runner
|
|
# can fetch it. Requires the repo secret TRAMITON_FETCH_TOKEN (a Gitea PAT
|
|
# with read:repository, owned by a user with access to sharang/tramiton).
|
|
# (Honored on fetch because CARGO_NET_GIT_FETCH_WITH_CLI=true uses system git.)
|
|
- name: Configure git auth for private tramiton dependency
|
|
run: |
|
|
git config --global \
|
|
url."https://sharang:${{ secrets.TRAMITON_FETCH_TOKEN }}@gitea.meghsakha.com/".insteadOf \
|
|
"ssh://git@gitea.meghsakha.com:22222/"
|
|
env:
|
|
RUSTC_WRAPPER: ""
|
|
|
|
# Format (no compilation needed)
|
|
- name: Format
|
|
run: cargo fmt --all --check
|
|
env:
|
|
RUSTC_WRAPPER: ""
|
|
|
|
# Clippy (compiles once, sccache reuses across feature sets)
|
|
- name: Clippy (agent)
|
|
run: cargo clippy -p compliance-agent -- -D warnings
|
|
- name: Clippy (dashboard server)
|
|
run: cargo clippy -p compliance-dashboard --features server --no-default-features -- -D warnings
|
|
- name: Clippy (dashboard web)
|
|
run: cargo clippy -p compliance-dashboard --features web --no-default-features -- -D warnings
|
|
- name: Clippy (mcp)
|
|
run: cargo clippy -p compliance-mcp -- -D warnings
|
|
|
|
# Security audit
|
|
- name: Security Audit
|
|
run: cargo audit
|
|
env:
|
|
RUSTC_WRAPPER: ""
|
|
|
|
# Tests (reuses compilation artifacts from clippy)
|
|
- name: Tests (core + agent)
|
|
run: cargo test -p compliance-core -p compliance-agent --lib
|
|
- name: Tests (dashboard server)
|
|
run: cargo test -p compliance-dashboard --features server --no-default-features
|
|
- name: Tests (dashboard web)
|
|
run: cargo test -p compliance-dashboard --features web --no-default-features
|
|
|
|
- name: Show sccache stats
|
|
run: sccache --show-stats
|
|
if: always()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 2: Deploy (only on main, after checks pass)
|
|
# Each service only deploys when its relevant files changed.
|
|
# ---------------------------------------------------------------------------
|
|
detect-changes:
|
|
name: Detect Changes
|
|
runs-on: docker
|
|
if: github.ref == 'refs/heads/main'
|
|
container:
|
|
image: alpine:latest
|
|
outputs:
|
|
agent: ${{ steps.changes.outputs.agent }}
|
|
dashboard: ${{ steps.changes.outputs.dashboard }}
|
|
docs: ${{ steps.changes.outputs.docs }}
|
|
mcp: ${{ steps.changes.outputs.mcp }}
|
|
steps:
|
|
- name: Install git
|
|
run: apk add --no-cache git
|
|
- name: Checkout
|
|
run: |
|
|
git init
|
|
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
|
git fetch --depth=2 origin "${GITHUB_SHA}"
|
|
git checkout FETCH_HEAD
|
|
- name: Detect changed paths
|
|
id: changes
|
|
run: |
|
|
CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "")
|
|
echo "Changed files:"
|
|
echo "$CHANGED"
|
|
|
|
# Agent: core libs, agent code, agent Dockerfile
|
|
if echo "$CHANGED" | grep -qE '^(compliance-core/|compliance-agent/|compliance-graph/|compliance-dast/|Dockerfile\.agent|Cargo\.(toml|lock))'; then
|
|
echo "agent=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "agent=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# Dashboard: core libs, dashboard code, dashboard Dockerfile, assets
|
|
if echo "$CHANGED" | grep -qE '^(compliance-core/|compliance-dashboard/|Dockerfile\.dashboard|Dioxus\.toml|assets/|bin/|Cargo\.(toml|lock))'; then
|
|
echo "dashboard=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "dashboard=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# Docs: docs folder, docs Dockerfile
|
|
if echo "$CHANGED" | grep -qE '^(docs/|Dockerfile\.docs)'; then
|
|
echo "docs=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "docs=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# MCP: core libs, mcp code, mcp Dockerfile
|
|
if echo "$CHANGED" | grep -qE '^(compliance-core/|compliance-mcp/|Dockerfile\.mcp|Cargo\.(toml|lock))'; then
|
|
echo "mcp=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "mcp=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
deploy-agent:
|
|
name: Deploy Agent
|
|
runs-on: docker
|
|
needs: [detect-changes]
|
|
if: needs.detect-changes.outputs.agent == 'true'
|
|
container:
|
|
image: docker:27-cli
|
|
steps:
|
|
- name: Build, push and trigger orca redeploy
|
|
env:
|
|
# PAT for fetching the private tramiton-core git dependency during the
|
|
# image build (injected as a BuildKit secret, never baked into a layer).
|
|
TRAMITON_FETCH_TOKEN: ${{ secrets.TRAMITON_FETCH_TOKEN }}
|
|
run: |
|
|
apk add --no-cache git curl openssl
|
|
git init && git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
|
git fetch --depth=1 origin "${GITHUB_SHA}" && git checkout FETCH_HEAD
|
|
IMAGE=registry.meghsakha.com/compliance-agent
|
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login registry.meghsakha.com -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin
|
|
DOCKER_BUILDKIT=1 docker build --secret id=tramiton_token,env=TRAMITON_FETCH_TOKEN \
|
|
-f Dockerfile.agent -t "$IMAGE:latest" -t "$IMAGE:${GITHUB_SHA}" .
|
|
docker push "$IMAGE:latest" && docker push "$IMAGE:${GITHUB_SHA}"
|
|
PAYLOAD=$(printf '{"ref":"refs/heads/main","repository":{"full_name":"sharang/compliance-scanner-agent"},"head_commit":{"id":"%s","message":"deploy agent"}}' "${GITHUB_SHA}")
|
|
SIG=$(printf '%s' "$PAYLOAD" | openssl dgst -sha256 -hmac "${{ secrets.ORCA_WEBHOOK_SECRET }}" | awk '{print $2}')
|
|
RESP=$(curl -fsS -w "\nHTTP %{http_code}" -X POST "http://46.225.100.82:6880/api/v1/webhooks/github" -H "Content-Type: application/json" -H "X-Hub-Signature-256: sha256=$SIG" -d "$PAYLOAD"); echo "$RESP"
|
|
|
|
deploy-dashboard:
|
|
name: Deploy Dashboard
|
|
runs-on: docker
|
|
needs: [detect-changes]
|
|
if: needs.detect-changes.outputs.dashboard == 'true'
|
|
container:
|
|
image: docker:27-cli
|
|
steps:
|
|
- name: Build, push and trigger orca redeploy
|
|
env:
|
|
TRAMITON_FETCH_TOKEN: ${{ secrets.TRAMITON_FETCH_TOKEN }}
|
|
run: |
|
|
apk add --no-cache git curl openssl
|
|
git init && git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
|
git fetch --depth=1 origin "${GITHUB_SHA}" && git checkout FETCH_HEAD
|
|
IMAGE=registry.meghsakha.com/compliance-dashboard
|
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login registry.meghsakha.com -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin
|
|
DOCKER_BUILDKIT=1 docker build --secret id=tramiton_token,env=TRAMITON_FETCH_TOKEN \
|
|
-f Dockerfile.dashboard -t "$IMAGE:latest" -t "$IMAGE:${GITHUB_SHA}" .
|
|
docker push "$IMAGE:latest" && docker push "$IMAGE:${GITHUB_SHA}"
|
|
PAYLOAD=$(printf '{"ref":"refs/heads/main","repository":{"full_name":"sharang/compliance-scanner-agent"},"head_commit":{"id":"%s","message":"deploy dashboard"}}' "${GITHUB_SHA}")
|
|
SIG=$(printf '%s' "$PAYLOAD" | openssl dgst -sha256 -hmac "${{ secrets.ORCA_WEBHOOK_SECRET }}" | awk '{print $2}')
|
|
RESP=$(curl -fsS -w "\nHTTP %{http_code}" -X POST "http://46.225.100.82:6880/api/v1/webhooks/github" -H "Content-Type: application/json" -H "X-Hub-Signature-256: sha256=$SIG" -d "$PAYLOAD"); echo "$RESP"
|
|
|
|
deploy-docs:
|
|
name: Deploy Docs
|
|
runs-on: docker
|
|
needs: [detect-changes]
|
|
if: needs.detect-changes.outputs.docs == 'true'
|
|
container:
|
|
image: docker:27-cli
|
|
steps:
|
|
- name: Build, push and trigger orca redeploy
|
|
run: |
|
|
apk add --no-cache git curl openssl
|
|
git init && git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
|
git fetch --depth=1 origin "${GITHUB_SHA}" && git checkout FETCH_HEAD
|
|
IMAGE=registry.meghsakha.com/compliance-docs
|
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login registry.meghsakha.com -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin
|
|
docker build -f Dockerfile.docs -t "$IMAGE:latest" -t "$IMAGE:${GITHUB_SHA}" .
|
|
docker push "$IMAGE:latest" && docker push "$IMAGE:${GITHUB_SHA}"
|
|
PAYLOAD=$(printf '{"ref":"refs/heads/main","repository":{"full_name":"sharang/compliance-scanner-agent"},"head_commit":{"id":"%s","message":"deploy docs"}}' "${GITHUB_SHA}")
|
|
SIG=$(printf '%s' "$PAYLOAD" | openssl dgst -sha256 -hmac "${{ secrets.ORCA_WEBHOOK_SECRET }}" | awk '{print $2}')
|
|
RESP=$(curl -fsS -w "\nHTTP %{http_code}" -X POST "http://46.225.100.82:6880/api/v1/webhooks/github" -H "Content-Type: application/json" -H "X-Hub-Signature-256: sha256=$SIG" -d "$PAYLOAD"); echo "$RESP"
|
|
|
|
deploy-mcp:
|
|
name: Deploy MCP
|
|
runs-on: docker
|
|
needs: [detect-changes]
|
|
if: needs.detect-changes.outputs.mcp == 'true'
|
|
container:
|
|
image: docker:27-cli
|
|
steps:
|
|
- name: Build, push and trigger orca redeploy
|
|
env:
|
|
TRAMITON_FETCH_TOKEN: ${{ secrets.TRAMITON_FETCH_TOKEN }}
|
|
run: |
|
|
apk add --no-cache git curl openssl
|
|
git init && git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
|
git fetch --depth=1 origin "${GITHUB_SHA}" && git checkout FETCH_HEAD
|
|
IMAGE=registry.meghsakha.com/compliance-mcp
|
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login registry.meghsakha.com -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin
|
|
DOCKER_BUILDKIT=1 docker build --secret id=tramiton_token,env=TRAMITON_FETCH_TOKEN \
|
|
-f Dockerfile.mcp -t "$IMAGE:latest" -t "$IMAGE:${GITHUB_SHA}" .
|
|
docker push "$IMAGE:latest" && docker push "$IMAGE:${GITHUB_SHA}"
|
|
PAYLOAD=$(printf '{"ref":"refs/heads/main","repository":{"full_name":"sharang/compliance-scanner-agent"},"head_commit":{"id":"%s","message":"deploy mcp"}}' "${GITHUB_SHA}")
|
|
SIG=$(printf '%s' "$PAYLOAD" | openssl dgst -sha256 -hmac "${{ secrets.ORCA_WEBHOOK_SECRET }}" | awk '{print $2}')
|
|
RESP=$(curl -fsS -w "\nHTTP %{http_code}" -X POST "http://46.225.100.82:6880/api/v1/webhooks/github" -H "Content-Type: application/json" -H "X-Hub-Signature-256: sha256=$SIG" -d "$PAYLOAD"); echo "$RESP"
|