Benjamin Admin
2400aa6a9e
New module consent-tester/services/cmp_llm_fallback.py:
- LLMCookieExtractor: single-endpoint adapter (Ollama OR OpenAI-compat)
- LLMCascade: tries Qwen (local Mac Mini Ollama) first; falls through to
OVH (managed 120B) when Qwen returns no usable strategy
- LLMCascade.from_env(): reads OLLAMA_URL/CMP_LLM_MODEL + OVH_LLM_URL/
OVH_LLM_KEY/OVH_LLM_MODEL from environment
- LLM returns JSON {strategy: url|selector|text, value: ...}
- Valkey-backed cache per netloc (cmp:hint:<netloc>, 7-day TTL) — next run
against the same domain skips the LLM entirely
dsi_discovery.py:
- Wired network_log collector (URL/status/content-type/size of every JSON
response on the page) — passed to LLM prompt as observation
- After Named CMP (Phase B) + Heuristic (Phase A) both fail AND DOM
< 300 words: invoke LLMCascade.analyze(...)
- _apply_llm_hint executes the LLM's strategy: refetch URL via Playwright
request context, query DOM selector, or use text directly
- Cache HIT path: apply cached hint, only fall back to LLM if cache is stale
docker-compose.yml:
- consent-tester gets env vars + cmp-data volume (for Phase E)
- All LLM endpoints configurable via env, sensible defaults
consent-tester/requirements.txt:
- redis>=5.0 (asyncio client, Valkey-compatible)
- httpx>=0.27
332 lines
11 KiB
YAML
332 lines
11 KiB
YAML
# =========================================================
|
|
# BreakPilot Compliance — Compliance SDK Platform
|
|
# =========================================================
|
|
# Voraussetzung: breakpilot-core muss laufen!
|
|
# Start: docker compose up -d
|
|
# =========================================================
|
|
|
|
networks:
|
|
breakpilot-network:
|
|
external: true
|
|
name: breakpilot-network
|
|
|
|
volumes:
|
|
dsms_data:
|
|
cmp-data: # consent-tester: CMP discovery log + auto-promoted modules
|
|
|
|
services:
|
|
|
|
# =========================================================
|
|
# CORE HEALTH CHECK — wartet auf Core-Infrastruktur
|
|
# =========================================================
|
|
core-health-check:
|
|
image: curlimages/curl:latest
|
|
container_name: bp-compliance-core-wait
|
|
command: >
|
|
sh -c "
|
|
echo 'Waiting for Core infrastructure...'
|
|
until curl -sf http://bp-core-health:8099/health; do
|
|
echo 'Core not ready, waiting 5s...'
|
|
sleep 5
|
|
done
|
|
echo 'Core is healthy!'
|
|
"
|
|
restart: "no"
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# =========================================================
|
|
# FRONTEND
|
|
# =========================================================
|
|
admin-compliance:
|
|
build:
|
|
context: ./admin-compliance
|
|
dockerfile: Dockerfile
|
|
args:
|
|
NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://macmini:8002}
|
|
NEXT_PUBLIC_SDK_URL: ${NEXT_PUBLIC_SDK_URL:-https://macmini:8093}
|
|
container_name: bp-compliance-admin
|
|
platform: linux/arm64
|
|
expose:
|
|
- "3000"
|
|
environment:
|
|
NODE_ENV: production
|
|
DATABASE_URL: ${COMPLIANCE_DATABASE_URL:-postgresql://breakpilot:breakpilot123@bp-core-postgres:5432/breakpilot_db}
|
|
BACKEND_URL: http://backend-compliance:8002
|
|
CONSENT_SERVICE_URL: http://bp-core-consent-service:8081
|
|
SDK_URL: http://ai-compliance-sdk:8090
|
|
OLLAMA_URL: ${OLLAMA_URL:-http://host.docker.internal:11434}
|
|
COMPLIANCE_LLM_MODEL: ${COMPLIANCE_LLM_MODEL:-qwen3.5:35b-a3b}
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
depends_on:
|
|
core-health-check:
|
|
condition: service_completed_successfully
|
|
backend-compliance:
|
|
condition: service_started
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
developer-portal:
|
|
build:
|
|
context: ./developer-portal
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-developer-portal
|
|
platform: linux/arm64
|
|
expose:
|
|
- "3000"
|
|
environment:
|
|
NODE_ENV: production
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# =========================================================
|
|
# BACKEND
|
|
# =========================================================
|
|
backend-compliance:
|
|
build:
|
|
context: ./backend-compliance
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-backend
|
|
platform: linux/arm64
|
|
expose:
|
|
- "8002"
|
|
environment:
|
|
PORT: 8002
|
|
DATABASE_URL: ${COMPLIANCE_DATABASE_URL:-postgresql://breakpilot:breakpilot123@bp-core-postgres:5432/breakpilot_db}
|
|
JWT_SECRET: ${JWT_SECRET:-your-super-secret-jwt-key-change-in-production}
|
|
ENVIRONMENT: ${ENVIRONMENT:-development}
|
|
CONSENT_SERVICE_URL: http://bp-core-consent-service:8081
|
|
VALKEY_URL: redis://bp-core-valkey:6379/0
|
|
SESSION_TTL_HOURS: ${SESSION_TTL_HOURS:-24}
|
|
COMPLIANCE_LLM_PROVIDER: ${COMPLIANCE_LLM_PROVIDER:-ollama}
|
|
SELF_HOSTED_LLM_URL: ${SELF_HOSTED_LLM_URL:-http://host.docker.internal:11434}
|
|
SELF_HOSTED_LLM_MODEL: ${SELF_HOSTED_LLM_MODEL:-qwen3.5:35b-a3b}
|
|
COMPLIANCE_LLM_MAX_TOKENS: ${COMPLIANCE_LLM_MAX_TOKENS:-4096}
|
|
COMPLIANCE_LLM_TEMPERATURE: ${COMPLIANCE_LLM_TEMPERATURE:-0.3}
|
|
COMPLIANCE_LLM_TIMEOUT: ${COMPLIANCE_LLM_TIMEOUT:-120}
|
|
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
|
DECOMPOSITION_LLM_MODEL: ${DECOMPOSITION_LLM_MODEL:-claude-haiku-4-5-20251001}
|
|
SMTP_HOST: ${SMTP_HOST:-bp-core-mailpit}
|
|
SMTP_PORT: ${SMTP_PORT:-1025}
|
|
SMTP_USERNAME: ${SMTP_USERNAME:-}
|
|
SMTP_PASSWORD: ${SMTP_PASSWORD:-}
|
|
SMTP_FROM_NAME: ${SMTP_FROM_NAME:-BreakPilot Compliance}
|
|
SMTP_FROM_ADDR: ${SMTP_FROM_ADDR:-compliance@breakpilot.app}
|
|
RAG_SERVICE_URL: http://bp-core-rag-service:8097
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
depends_on:
|
|
core-health-check:
|
|
condition: service_completed_successfully
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# =========================================================
|
|
# SDK SERVICES
|
|
# =========================================================
|
|
ai-compliance-sdk:
|
|
build:
|
|
context: ./ai-compliance-sdk
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-ai-sdk
|
|
platform: linux/arm64
|
|
environment:
|
|
PORT: 8090
|
|
ENVIRONMENT: ${ENVIRONMENT:-development}
|
|
DATABASE_URL: ${COMPLIANCE_DATABASE_URL:-postgresql://breakpilot:breakpilot123@bp-core-postgres:5432/breakpilot_db}
|
|
JWT_SECRET: ${JWT_SECRET:-your-super-secret-jwt-key-change-in-production}
|
|
LLM_PROVIDER: ${COMPLIANCE_LLM_PROVIDER:-ollama}
|
|
LLM_FALLBACK_PROVIDER: ${LLM_FALLBACK_PROVIDER:-}
|
|
OLLAMA_URL: ${OLLAMA_URL:-http://host.docker.internal:11434}
|
|
OLLAMA_DEFAULT_MODEL: ${OLLAMA_DEFAULT_MODEL:-qwen3.5:35b-a3b}
|
|
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
|
ANTHROPIC_DEFAULT_MODEL: ${ANTHROPIC_DEFAULT_MODEL:-claude-sonnet-4-5-20250929}
|
|
PII_REDACTION_ENABLED: ${PII_REDACTION_ENABLED:-true}
|
|
PII_REDACTION_LEVEL: ${PII_REDACTION_LEVEL:-standard}
|
|
AUDIT_RETENTION_DAYS: ${AUDIT_RETENTION_DAYS:-365}
|
|
AUDIT_LOG_PROMPTS: ${AUDIT_LOG_PROMPTS:-true}
|
|
ALLOWED_ORIGINS: "*"
|
|
TTS_SERVICE_URL: http://compliance-tts-service:8095
|
|
QDRANT_URL: ${QDRANT_URL:-https://qdrant-dev.breakpilot.ai}
|
|
QDRANT_API_KEY: ${QDRANT_API_KEY:-z9cKbT74vl1aKPD1QGIlKWfET47VH93u}
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
depends_on:
|
|
core-health-check:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8090/health"]
|
|
interval: 30s
|
|
timeout: 3s
|
|
start_period: 10s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# =========================================================
|
|
# =========================================================
|
|
# TTS SERVICE (Piper TTS + FFmpeg)
|
|
# =========================================================
|
|
compliance-tts-service:
|
|
build:
|
|
context: ./compliance-tts-service
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-tts
|
|
platform: linux/arm64
|
|
expose:
|
|
- "8095"
|
|
environment:
|
|
MINIO_ENDPOINT: nbg1.your-objectstorage.com
|
|
MINIO_ACCESS_KEY: T18RGFVXXG2ZHQ5404TP
|
|
MINIO_SECRET_KEY: KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss
|
|
MINIO_SECURE: "true"
|
|
PIPER_MODEL_PATH: /app/models/de_DE-thorsten-high.onnx
|
|
depends_on:
|
|
core-health-check:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8095/health')"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 60s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# DATA SOVEREIGNTY
|
|
# =========================================================
|
|
dsms-node:
|
|
build:
|
|
context: ./dsms-node
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-dsms-node
|
|
ports:
|
|
- "4001:4001"
|
|
- "5001:5001"
|
|
- "8085:8080"
|
|
volumes:
|
|
- dsms_data:/data/ipfs
|
|
environment:
|
|
IPFS_PROFILE: server
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "ipfs id"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 30s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
dsms-gateway:
|
|
build:
|
|
context: ./dsms-gateway
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-dsms-gateway
|
|
ports:
|
|
- "8082:8082"
|
|
environment:
|
|
IPFS_API_URL: http://dsms-node:5001
|
|
IPFS_GATEWAY_URL: http://dsms-node:8080
|
|
JWT_SECRET: ${JWT_SECRET:-your-super-secret-jwt-key-change-in-production}
|
|
depends_on:
|
|
dsms-node:
|
|
condition: service_healthy
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# =========================================================
|
|
# DOCUMENT CRAWLER & AUTO-ONBOARDING
|
|
# =========================================================
|
|
consent-tester:
|
|
build:
|
|
context: ./consent-tester
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-consent-tester
|
|
platform: linux/arm64
|
|
ports:
|
|
- "8094:8094"
|
|
mem_limit: 2g
|
|
depends_on:
|
|
core-health-check:
|
|
condition: service_completed_successfully
|
|
environment:
|
|
# LLM fallback for cookie-policy extraction (Phase C+D of cascade)
|
|
OLLAMA_URL: "${OLLAMA_URL:-http://bp-core-ollama:11434}"
|
|
CMP_LLM_MODEL: "${CMP_LLM_MODEL:-qwen3:30b-a3b}"
|
|
OVH_LLM_URL: "${OVH_LLM_URL:-}"
|
|
OVH_LLM_KEY: "${OVH_LLM_KEY:-}"
|
|
OVH_LLM_MODEL: "${OVH_LLM_MODEL:-}"
|
|
VALKEY_URL: "${VALKEY_URL:-redis://bp-core-valkey:6379}"
|
|
# CMP discovery log (Phase E auto-promote)
|
|
CMP_DISCOVERY_DB: "/data/cmp_discoveries.db"
|
|
volumes:
|
|
- cmp-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://127.0.0.1:8094/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 30s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
document-crawler:
|
|
build:
|
|
context: ./document-crawler
|
|
dockerfile: Dockerfile
|
|
container_name: bp-compliance-document-crawler
|
|
platform: linux/arm64
|
|
ports:
|
|
- "8098:8098"
|
|
environment:
|
|
PORT: 8098
|
|
DATABASE_URL: ${COMPLIANCE_DATABASE_URL:-postgresql://breakpilot:breakpilot123@bp-core-postgres:5432/breakpilot_db}
|
|
LLM_GATEWAY_URL: http://ai-compliance-sdk:8090
|
|
DSMS_GATEWAY_URL: http://dsms-gateway:8082
|
|
CRAWL_BASE_PATH: /data/crawl
|
|
MAX_FILE_SIZE_MB: 50
|
|
volumes:
|
|
- /tmp/breakpilot-crawl-data:/data/crawl:ro
|
|
depends_on:
|
|
core-health-check:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://127.0.0.1:8098/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 15s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|
|
|
|
# =========================================================
|
|
# DOCUMENTATION
|
|
# =========================================================
|
|
docs:
|
|
build:
|
|
context: .
|
|
dockerfile: docs-src/Dockerfile
|
|
container_name: bp-compliance-docs
|
|
profiles: [docs]
|
|
platform: linux/arm64
|
|
expose:
|
|
- "80"
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:80/"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
networks:
|
|
- breakpilot-network
|