breakpilot-compliance/backend-compliance/tests/test_validate_controls.py

"""Tests for the CI/CD control validator script."""

import json
import subprocess
import sys
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parent.parent.parent
VALIDATOR = REPO_ROOT / "scripts" / "validate-controls.py"
CONTROLS_FILE = REPO_ROOT / "ai-compliance-sdk" / "policies" / "canonical_controls_v1.json"


class TestValidatorScript:
    """Integration tests for validate-controls.py."""

    def test_validator_passes_on_valid_controls(self):
        result = subprocess.run(
            [sys.executable, str(VALIDATOR)],
            capture_output=True, text=True, cwd=str(REPO_ROOT),
        )
        assert result.returncode == 0, f"Validator failed:\n{result.stdout}\n{result.stderr}"
        assert "ALL CHECKS PASSED" in result.stdout

    def test_validator_reports_control_count(self):
        result = subprocess.run(
            [sys.executable, str(VALIDATOR)],
            capture_output=True, text=True, cwd=str(REPO_ROOT),
        )
        assert "Controls:     10" in result.stdout
        assert "Open Anchors:" in result.stdout


class TestControlsJsonStructure:
    """Direct validation of the JSON file structure."""

    @pytest.fixture
    def controls_data(self):
        with open(CONTROLS_FILE) as f:
            return json.load(f)

    def test_top_level_keys(self, controls_data):
        assert "version" in controls_data
        assert "schema" in controls_data
        assert "framework" in controls_data
        assert "domains" in controls_data
        assert "controls" in controls_data

    def test_framework_metadata(self, controls_data):
        fw = controls_data["framework"]
        assert fw["id"] == "bp_security_v1"
        assert fw["version"] == "1.0"

    def test_all_controls_have_open_anchors(self, controls_data):
        for ctrl in controls_data["controls"]:
            anchors = ctrl.get("open_anchors", [])
            assert len(anchors) >= 1, (
                f"Control {ctrl['control_id']} has no open anchors"
            )

    def test_no_bsi_nomenclature_in_controls(self, controls_data):
        """Ensure no BSI-proprietary IDs leak into product-facing fields."""
        import re
        bsi_pattern = re.compile(r"O\.[A-Za-z]+_[0-9]+")
        for ctrl in controls_data["controls"]:
            for field in ["objective", "rationale", "title"]:
                text = ctrl.get(field, "")
                match = bsi_pattern.search(text)
                assert match is None, (
                    f"Control {ctrl['control_id']}.{field} contains BSI pattern: {match.group()}"
                )

    def test_control_id_format(self, controls_data):
        import re
        pattern = re.compile(r"^[A-Z]{2,6}-[0-9]{3}$")
        for ctrl in controls_data["controls"]:
            assert pattern.match(ctrl["control_id"]), (
                f"Invalid control_id format: {ctrl['control_id']}"
            )

    def test_valid_severities(self, controls_data):
        valid = {"low", "medium", "high", "critical"}
        for ctrl in controls_data["controls"]:
            assert ctrl["severity"] in valid, (
                f"Control {ctrl['control_id']} has invalid severity: {ctrl['severity']}"
            )

    def test_domains_referenced_by_controls(self, controls_data):
        domain_ids = {d["id"] for d in controls_data["domains"]}
        for ctrl in controls_data["controls"]:
            assert ctrl["domain"] in domain_ids, (
                f"Control {ctrl['control_id']} references unknown domain: {ctrl['domain']}"
            )

    def test_open_anchor_structure(self, controls_data):
        for ctrl in controls_data["controls"]:
            for i, anchor in enumerate(ctrl.get("open_anchors", [])):
                assert "framework" in anchor, (
                    f"Control {ctrl['control_id']}: anchor[{i}] missing 'framework'"
                )
                assert "ref" in anchor, (
                    f"Control {ctrl['control_id']}: anchor[{i}] missing 'ref'"
                )
                assert "url" in anchor, (
                    f"Control {ctrl['control_id']}: anchor[{i}] missing 'url'"
                )
                assert anchor["url"].startswith("https://"), (
                    f"Control {ctrl['control_id']}: anchor[{i}] URL not HTTPS"
                )

    def test_evidence_structure(self, controls_data):
        for ctrl in controls_data["controls"]:
            for i, ev in enumerate(ctrl.get("evidence", [])):
                assert "type" in ev, (
                    f"Control {ctrl['control_id']}: evidence[{i}] missing 'type'"
                )
                assert "description" in ev, (
                    f"Control {ctrl['control_id']}: evidence[{i}] missing 'description'"
                )

    def test_risk_scores_in_range(self, controls_data):
        for ctrl in controls_data["controls"]:
            if ctrl.get("risk_score") is not None:
                assert 0 <= ctrl["risk_score"] <= 10, (
                    f"Control {ctrl['control_id']}: risk_score {ctrl['risk_score']} out of range"
                )

    def test_total_controls_matches(self, controls_data):
        assert controls_data["total_controls"] == len(controls_data["controls"])

    def test_independent_taxonomy_no_tr_reference(self, controls_data):
        """Verify controls don't reference BSI TR documents in product text."""
        import re
        tr_pattern = re.compile(r"TR-03161|BSI-TR-")
        for ctrl in controls_data["controls"]:
            for field in ["objective", "rationale", "title"]:
                text = ctrl.get(field, "")
                match = tr_pattern.search(text)
                assert match is None, (
                    f"Control {ctrl['control_id']}.{field} references BSI TR: {match.group()}"
                )