diff --git a/.env.coolify.example b/.env.coolify.example new file mode 100644 index 0000000..6e4c910 --- /dev/null +++ b/.env.coolify.example @@ -0,0 +1,65 @@ +# ========================================================= +# BreakPilot Core — Coolify Environment Variables +# ========================================================= +# Copy these into Coolify's environment variable UI +# for the breakpilot-core Docker Compose resource. +# ========================================================= + +# --- External PostgreSQL (Coolify-managed) --- +POSTGRES_HOST= +POSTGRES_PORT=5432 +POSTGRES_USER=breakpilot +POSTGRES_PASSWORD=CHANGE_ME_STRONG_PASSWORD +POSTGRES_DB=breakpilot_db + +# --- Security --- +JWT_SECRET=CHANGE_ME_RANDOM_64_CHARS +JWT_REFRESH_SECRET=CHANGE_ME_ANOTHER_RANDOM_64_CHARS +INTERNAL_API_KEY=CHANGE_ME_INTERNAL_KEY + +# --- External S3 Storage --- +S3_ENDPOINT= +S3_ACCESS_KEY=CHANGE_ME_S3_ACCESS_KEY +S3_SECRET_KEY=CHANGE_ME_S3_SECRET_KEY +S3_BUCKET=breakpilot-rag +S3_SECURE=true + +# --- External Qdrant (Coolify-managed) --- +QDRANT_URL=http://:6333 +QDRANT_API_KEY= + +# --- SMTP (Real mail server) --- +SMTP_HOST=smtp.example.com +SMTP_PORT=587 +SMTP_USERNAME=noreply@breakpilot.ai +SMTP_PASSWORD=CHANGE_ME_SMTP_PASSWORD +SMTP_FROM_NAME=BreakPilot +SMTP_FROM_ADDR=noreply@breakpilot.ai + +# --- Session --- +SESSION_TTL_HOURS=24 + +# --- Frontend URLs (build args) --- +NEXT_PUBLIC_CORE_API_URL=https://api-core.breakpilot.ai +FRONTEND_URL=https://www.breakpilot.ai + +# --- Stripe (Billing) --- +STRIPE_SECRET_KEY= +STRIPE_WEBHOOK_SECRET= +STRIPE_PUBLISHABLE_KEY= +BILLING_SUCCESS_URL=https://www.breakpilot.ai/billing/success +BILLING_CANCEL_URL=https://www.breakpilot.ai/billing/cancel +TRIAL_PERIOD_DAYS=14 + +# --- Embedding Service --- +EMBEDDING_BACKEND=local +LOCAL_EMBEDDING_MODEL=BAAI/bge-m3 +LOCAL_RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2 +PDF_EXTRACTION_BACKEND=pymupdf +OPENAI_API_KEY= +COHERE_API_KEY= +LOG_LEVEL=INFO + +# --- Ollama (optional, for RAG embeddings) --- +OLLAMA_URL= +OLLAMA_EMBED_MODEL=bge-m3 diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 1af63e5..f7c69e7 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -140,117 +140,20 @@ jobs: python -m pytest tests/bqas/ -v --tb=short || true # ======================================== - # Build & Deploy auf Hetzner (nur main, kein PR) + # Deploy via Coolify (nur main, kein PR) # ======================================== - deploy-hetzner: + deploy-coolify: + name: Deploy runs-on: docker if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: - test-go-consent - container: docker:27-cli + container: + image: alpine:latest steps: - - name: Deploy + - name: Trigger Coolify deploy run: | - set -euo pipefail - DEPLOY_DIR="/opt/breakpilot-core" - COMPOSE_FILES="-f docker-compose.yml -f docker-compose.hetzner.yml" - COMMIT_SHA="${GITHUB_SHA:-unknown}" - SHORT_SHA="${COMMIT_SHA:0:8}" - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - - # Services die deployed werden - SERVICES="postgres valkey qdrant minio ollama mailpit embedding-service rag-service backend-core consent-service health-aggregator" - - echo "=== BreakPilot Core Deploy ===" - echo "Commit: ${SHORT_SHA}" - echo "Deploy Dir: ${DEPLOY_DIR}" - echo "Services: ${SERVICES}" - echo "" - - # 1. Repo auf dem Host erstellen/aktualisieren via Helper-Container - echo "=== Updating code on host ===" - docker run --rm \ - -v "${DEPLOY_DIR}:${DEPLOY_DIR}" \ - --entrypoint sh \ - alpine/git:latest \ - -c " - if [ ! -d '${DEPLOY_DIR}/.git' ]; then - echo 'Erstmaliges Klonen nach ${DEPLOY_DIR}...' - git clone '${REPO_URL}' '${DEPLOY_DIR}' - else - cd '${DEPLOY_DIR}' - git fetch origin main - git reset --hard origin/main - fi - " - echo "Code aktualisiert auf ${SHORT_SHA}" - - # 2. .env sicherstellen - docker run --rm -v "${DEPLOY_DIR}:${DEPLOY_DIR}" alpine \ - sh -c " - if [ ! -f '${DEPLOY_DIR}/.env' ]; then - echo 'WARNUNG: ${DEPLOY_DIR}/.env fehlt!' - echo 'Erstelle .env aus .env.example mit Defaults...' - if [ -f '${DEPLOY_DIR}/.env.example' ]; then - cp '${DEPLOY_DIR}/.env.example' '${DEPLOY_DIR}/.env' - echo '.env aus .env.example erstellt' - else - echo 'Kein .env.example gefunden — Services starten mit Defaults' - fi - else - echo '.env vorhanden' - fi - " - - # 3. Shared Network erstellen (falls noch nicht vorhanden) - docker network create breakpilot-network 2>/dev/null || true - - # 4. Build + Deploy via Helper-Container - echo "" - echo "=== Building + Deploying ===" - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "${DEPLOY_DIR}:${DEPLOY_DIR}" \ - -w "${DEPLOY_DIR}" \ - docker:27-cli \ - sh -c " - set -e - COMPOSE_FILES='-f docker-compose.yml -f docker-compose.hetzner.yml' - - echo '=== Building Docker Images ===' - docker compose \${COMPOSE_FILES} build --parallel \ - backend-core consent-service rag-service embedding-service health-aggregator - - echo '' - echo '=== Starting infrastructure ===' - docker compose \${COMPOSE_FILES} up -d postgres valkey qdrant minio mailpit - - echo 'Warte auf DB + Cache...' - sleep 10 - - echo '' - echo '=== Starting Ollama + pulling bge-m3 ===' - docker compose \${COMPOSE_FILES} up -d ollama - sleep 5 - - # bge-m3 Modell pullen (nur beim ersten Mal ~670MB) - echo 'Pulling bge-m3 model (falls noch nicht vorhanden)...' - docker exec bp-core-ollama ollama pull bge-m3 2>&1 || echo 'WARNUNG: bge-m3 pull fehlgeschlagen (wird spaeter nachgeholt)' - - echo '' - echo '=== Starting application services ===' - docker compose \${COMPOSE_FILES} up -d \ - embedding-service rag-service backend-core consent-service health-aggregator - - echo '' - echo '=== Health Checks ===' - sleep 15 - for svc in bp-core-postgres bp-core-valkey bp-core-qdrant bp-core-ollama bp-core-embedding-service bp-core-rag-service bp-core-backend bp-core-consent-service bp-core-health; do - STATUS=\$(docker inspect --format='{{.State.Status}}' \"\${svc}\" 2>/dev/null || echo 'not found') - echo \"\${svc}: \${STATUS}\" - done - " - - echo "" - echo "=== Deploy abgeschlossen: ${SHORT_SHA} ===" + apk add --no-cache curl + curl -sf "${{ secrets.COOLIFY_WEBHOOK }}" \ + -H "Authorization: Bearer ${{ secrets.COOLIFY_TOKEN }}" diff --git a/admin-core/Dockerfile b/admin-core/Dockerfile index 08ee6c8..ef77a22 100644 --- a/admin-core/Dockerfile +++ b/admin-core/Dockerfile @@ -18,6 +18,9 @@ ARG NEXT_PUBLIC_API_URL # Set environment variables for build ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL +# Ensure public directory exists +RUN mkdir -p public + # Build the application RUN npm run build @@ -30,8 +33,8 @@ WORKDIR /app ENV NODE_ENV=production # Create non-root user -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nextjs +RUN addgroup -S -g 1001 nodejs +RUN adduser -S -u 1001 -G nodejs nextjs # Copy built assets COPY --from=builder /app/public ./public diff --git a/backend-core/Dockerfile b/backend-core/Dockerfile index b638317..fa33d3f 100644 --- a/backend-core/Dockerfile +++ b/backend-core/Dockerfile @@ -43,11 +43,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Install DevSecOps tools (gitleaks, trivy, grype, syft) -ARG TARGETARCH=arm64 +ARG TARGETARCH RUN set -eux; \ + ARCH="${TARGETARCH:-$(dpkg --print-architecture)}"; \ # Gitleaks GITLEAKS_VERSION=8.21.2; \ - if [ "$TARGETARCH" = "arm64" ]; then GITLEAKS_ARCH=arm64; else GITLEAKS_ARCH=x64; fi; \ + if [ "$ARCH" = "arm64" ]; then GITLEAKS_ARCH=arm64; else GITLEAKS_ARCH=x64; fi; \ curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_${GITLEAKS_ARCH}.tar.gz" \ | tar xz -C /usr/local/bin gitleaks; \ # Trivy diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml new file mode 100644 index 0000000..32cc969 --- /dev/null +++ b/docker-compose.coolify.yml @@ -0,0 +1,165 @@ +# ========================================================= +# BreakPilot Core — Shared Infrastructure (Coolify) +# ========================================================= +# Deployed via Coolify. SSL termination handled by Traefik. +# External services (managed separately in Coolify): +# - PostgreSQL (PostGIS), Qdrant, S3-compatible storage +# Network: breakpilot-network (shared across all 3 repos) +# ========================================================= + +networks: + breakpilot-network: + name: breakpilot-network + driver: bridge + +volumes: + valkey_data: + embedding_models: + +services: + + # ========================================================= + # CACHE + # ========================================================= + valkey: + image: valkey/valkey:8-alpine + container_name: bp-core-valkey + volumes: + - valkey_data:/data + command: valkey-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + networks: + - breakpilot-network + + # ========================================================= + # SHARED SERVICES + # ========================================================= + consent-service: + build: + context: ./consent-service + dockerfile: Dockerfile + container_name: bp-core-consent-service + expose: + - "8081" + environment: + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT:-5432}/${POSTGRES_DB} + JWT_SECRET: ${JWT_SECRET} + JWT_REFRESH_SECRET: ${JWT_REFRESH_SECRET} + PORT: 8081 + ENVIRONMENT: production + ALLOWED_ORIGINS: "*" + VALKEY_URL: redis://valkey:6379/0 + SESSION_TTL_HOURS: ${SESSION_TTL_HOURS:-24} + SMTP_HOST: ${SMTP_HOST} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_USERNAME: ${SMTP_USERNAME} + SMTP_PASSWORD: ${SMTP_PASSWORD} + SMTP_FROM_NAME: ${SMTP_FROM_NAME:-BreakPilot} + SMTP_FROM_ADDR: ${SMTP_FROM_ADDR:-noreply@breakpilot.ai} + FRONTEND_URL: ${FRONTEND_URL:-https://www.breakpilot.ai} + depends_on: + valkey: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8081/health"] + interval: 30s + timeout: 10s + start_period: 15s + retries: 3 + restart: unless-stopped + networks: + - breakpilot-network + + # ========================================================= + # RAG & EMBEDDING SERVICES + # ========================================================= + rag-service: + build: + context: ./rag-service + dockerfile: Dockerfile + container_name: bp-core-rag-service + expose: + - "8097" + environment: + PORT: 8097 + QDRANT_URL: ${QDRANT_URL} + QDRANT_API_KEY: ${QDRANT_API_KEY:-} + MINIO_ENDPOINT: ${S3_ENDPOINT} + MINIO_ACCESS_KEY: ${S3_ACCESS_KEY} + MINIO_SECRET_KEY: ${S3_SECRET_KEY} + MINIO_BUCKET: ${S3_BUCKET:-breakpilot-rag} + MINIO_SECURE: ${S3_SECURE:-true} + EMBEDDING_SERVICE_URL: http://embedding-service:8087 + OLLAMA_URL: ${OLLAMA_URL:-} + OLLAMA_EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-bge-m3} + JWT_SECRET: ${JWT_SECRET} + ENVIRONMENT: production + depends_on: + embedding-service: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8097/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + restart: unless-stopped + networks: + - breakpilot-network + + embedding-service: + build: + context: ./embedding-service + dockerfile: Dockerfile + container_name: bp-core-embedding-service + volumes: + - embedding_models:/root/.cache/huggingface + environment: + EMBEDDING_BACKEND: ${EMBEDDING_BACKEND:-local} + LOCAL_EMBEDDING_MODEL: ${LOCAL_EMBEDDING_MODEL:-BAAI/bge-m3} + LOCAL_RERANKER_MODEL: ${LOCAL_RERANKER_MODEL:-cross-encoder/ms-marco-MiniLM-L-6-v2} + PDF_EXTRACTION_BACKEND: ${PDF_EXTRACTION_BACKEND:-pymupdf} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + COHERE_API_KEY: ${COHERE_API_KEY:-} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + deploy: + resources: + limits: + memory: 8G + healthcheck: + test: ["CMD", "python", "-c", "import httpx; r=httpx.get('http://127.0.0.1:8087/health'); r.raise_for_status()"] + interval: 30s + timeout: 10s + start_period: 120s + retries: 3 + restart: unless-stopped + networks: + - breakpilot-network + + # ========================================================= + # HEALTH AGGREGATOR + # ========================================================= + health-aggregator: + build: + context: ./scripts + dockerfile: Dockerfile.health + container_name: bp-core-health + expose: + - "8099" + environment: + PORT: 8099 + CHECK_SERVICES: "valkey:6379,consent-service:8081,rag-service:8097,embedding-service:8087" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8099/health"] + interval: 30s + timeout: 10s + retries: 3 + restart: unless-stopped + networks: + - breakpilot-network + diff --git a/rag-service/config.py b/rag-service/config.py index 951c04f..483308b 100644 --- a/rag-service/config.py +++ b/rag-service/config.py @@ -6,6 +6,7 @@ class Settings: # Qdrant QDRANT_URL: str = os.getenv("QDRANT_URL", "http://localhost:6333") + QDRANT_API_KEY: str = os.getenv("QDRANT_API_KEY", "") # MinIO MINIO_ENDPOINT: str = os.getenv("MINIO_ENDPOINT", "localhost:9000") diff --git a/rag-service/qdrant_client_wrapper.py b/rag-service/qdrant_client_wrapper.py index c4b27e4..daeafda 100644 --- a/rag-service/qdrant_client_wrapper.py +++ b/rag-service/qdrant_client_wrapper.py @@ -48,7 +48,11 @@ class QdrantClientWrapper: @property def client(self) -> QdrantClient: if self._client is None: - self._client = QdrantClient(url=settings.QDRANT_URL, timeout=30) + self._client = QdrantClient( + url=settings.QDRANT_URL, + api_key=settings.QDRANT_API_KEY or None, + timeout=30, + ) logger.info("Connected to Qdrant at %s", settings.QDRANT_URL) return self._client