feat: Add SDK Protection Middleware against systematic enumeration
Some checks failed
ci/woodpecker/push/integration Pipeline failed
ci/woodpecker/push/main Pipeline failed
CI/CD Pipeline / Docker Build & Push (push) Has been cancelled
CI/CD Pipeline / Linting (push) Has been cancelled
CI/CD Pipeline / Go Tests (push) Has been cancelled
CI/CD Pipeline / Python Tests (push) Has been cancelled
CI/CD Pipeline / Website Tests (push) Has been cancelled
CI/CD Pipeline / Security Scan (push) Has been cancelled
CI/CD Pipeline / Integration Tests (push) Has been cancelled
CI/CD Pipeline / Deploy to Staging (push) Has been cancelled
CI/CD Pipeline / Deploy to Production (push) Has been cancelled
CI/CD Pipeline / CI Summary (push) Has been cancelled
Security Scanning / Python Security Scan (push) Has been cancelled
Security Scanning / Node.js Security Scan (push) Has been cancelled
Security Scanning / Secret Scanning (push) Has been cancelled
Security Scanning / Dependency Vulnerability Scan (push) Has been cancelled
Security Scanning / Go Security Scan (push) Has been cancelled
Security Scanning / Docker Image Security (push) Has been cancelled
Security Scanning / Security Summary (push) Has been cancelled
Tests / Go Tests (push) Has been cancelled
Tests / Python Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Tests / Go Lint (push) Has been cancelled
Tests / Python Lint (push) Has been cancelled
Tests / Security Scan (push) Has been cancelled
Tests / All Checks Passed (push) Has been cancelled
Some checks failed
ci/woodpecker/push/integration Pipeline failed
ci/woodpecker/push/main Pipeline failed
CI/CD Pipeline / Docker Build & Push (push) Has been cancelled
CI/CD Pipeline / Linting (push) Has been cancelled
CI/CD Pipeline / Go Tests (push) Has been cancelled
CI/CD Pipeline / Python Tests (push) Has been cancelled
CI/CD Pipeline / Website Tests (push) Has been cancelled
CI/CD Pipeline / Security Scan (push) Has been cancelled
CI/CD Pipeline / Integration Tests (push) Has been cancelled
CI/CD Pipeline / Deploy to Staging (push) Has been cancelled
CI/CD Pipeline / Deploy to Production (push) Has been cancelled
CI/CD Pipeline / CI Summary (push) Has been cancelled
Security Scanning / Python Security Scan (push) Has been cancelled
Security Scanning / Node.js Security Scan (push) Has been cancelled
Security Scanning / Secret Scanning (push) Has been cancelled
Security Scanning / Dependency Vulnerability Scan (push) Has been cancelled
Security Scanning / Go Security Scan (push) Has been cancelled
Security Scanning / Docker Image Security (push) Has been cancelled
Security Scanning / Security Summary (push) Has been cancelled
Tests / Go Tests (push) Has been cancelled
Tests / Python Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Tests / Go Lint (push) Has been cancelled
Tests / Python Lint (push) Has been cancelled
Tests / Security Scan (push) Has been cancelled
Tests / All Checks Passed (push) Has been cancelled
Implements anomaly-score-based middleware to protect SDK/Compliance endpoints from systematic data harvesting. Includes 5 detection mechanisms (diversity, burst, sequential enumeration, unusual hours, multi-tenant), multi-window quota system, progressive throttling, HMAC watermarking, and graceful Valkey fallback. - backend/middleware/sdk_protection.py: Core middleware (~750 lines) - Admin API endpoints for score management and tier configuration - 14 new tests (all passing) - MkDocs documentation with clear explanations - Screen flow and middleware dashboard updates Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -102,6 +102,50 @@ class MiddlewareStatsResponse(BaseModel):
|
||||
top_ips: List[Dict[str, Any]]
|
||||
|
||||
|
||||
class SDKAnomalyScoreResponse(BaseModel):
|
||||
"""Response model for SDK anomaly score."""
|
||||
id: str
|
||||
user_id: str
|
||||
score: float
|
||||
throttle_level: int
|
||||
triggered_rules: List[str]
|
||||
endpoint_diversity_count: int
|
||||
request_count_1h: int
|
||||
snapshot_at: datetime
|
||||
|
||||
|
||||
class SDKProtectionStatsResponse(BaseModel):
|
||||
"""Response model for SDK protection statistics."""
|
||||
total_users_tracked: int
|
||||
users_level_0: int
|
||||
users_level_1: int
|
||||
users_level_2: int
|
||||
users_level_3: int
|
||||
avg_score: float
|
||||
max_score: float
|
||||
|
||||
|
||||
class SDKProtectionTierResponse(BaseModel):
|
||||
"""Response model for SDK protection tier."""
|
||||
tier_name: str
|
||||
quota_per_minute: int
|
||||
quota_per_hour: int
|
||||
quota_per_day: int
|
||||
quota_per_month: int
|
||||
diversity_threshold: int
|
||||
burst_threshold: int
|
||||
|
||||
|
||||
class SDKProtectionTierUpdateRequest(BaseModel):
|
||||
"""Request model for updating SDK protection tier."""
|
||||
quota_per_minute: Optional[int] = None
|
||||
quota_per_hour: Optional[int] = None
|
||||
quota_per_day: Optional[int] = None
|
||||
quota_per_month: Optional[int] = None
|
||||
diversity_threshold: Optional[int] = None
|
||||
burst_threshold: Optional[int] = None
|
||||
|
||||
|
||||
# ==============================================
|
||||
# Middleware Configuration Endpoints
|
||||
# ==============================================
|
||||
@@ -422,7 +466,7 @@ async def get_middleware_stats(
|
||||
pool = await get_db_pool()
|
||||
|
||||
stats = []
|
||||
middlewares = ["request_id", "security_headers", "cors", "rate_limiter", "pii_redactor", "input_gate"]
|
||||
middlewares = ["request_id", "security_headers", "cors", "rate_limiter", "pii_redactor", "input_gate", "sdk_protection"]
|
||||
|
||||
for mw in middlewares:
|
||||
# Get event counts
|
||||
@@ -533,3 +577,221 @@ async def get_middleware_stats_by_name(
|
||||
for r in top_ips
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ==============================================
|
||||
# SDK Protection Endpoints
|
||||
# ==============================================
|
||||
|
||||
|
||||
@router.get("/sdk-protection/scores", response_model=List[SDKAnomalyScoreResponse])
|
||||
async def list_sdk_anomaly_scores(
|
||||
min_score: float = Query(0, ge=0),
|
||||
limit: int = Query(50, le=500),
|
||||
session: Session = Depends(require_permission("settings:read")),
|
||||
):
|
||||
"""
|
||||
List current SDK anomaly scores.
|
||||
|
||||
Requires: settings:read permission
|
||||
"""
|
||||
pool = await get_db_pool()
|
||||
|
||||
rows = await pool.fetch("""
|
||||
SELECT DISTINCT ON (user_id)
|
||||
id, user_id, score, throttle_level, triggered_rules,
|
||||
endpoint_diversity_count, request_count_1h, snapshot_at
|
||||
FROM sdk_anomaly_scores
|
||||
WHERE score >= $1
|
||||
ORDER BY user_id, snapshot_at DESC
|
||||
LIMIT $2
|
||||
""", min_score, limit)
|
||||
|
||||
return [
|
||||
SDKAnomalyScoreResponse(
|
||||
id=str(row["id"]),
|
||||
user_id=row["user_id"],
|
||||
score=float(row["score"]),
|
||||
throttle_level=row["throttle_level"],
|
||||
triggered_rules=row["triggered_rules"] or [],
|
||||
endpoint_diversity_count=row["endpoint_diversity_count"] or 0,
|
||||
request_count_1h=row["request_count_1h"] or 0,
|
||||
snapshot_at=row["snapshot_at"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/sdk-protection/stats", response_model=SDKProtectionStatsResponse)
|
||||
async def get_sdk_protection_stats(
|
||||
session: Session = Depends(require_permission("settings:read")),
|
||||
):
|
||||
"""
|
||||
Get SDK protection statistics (users per throttle level).
|
||||
|
||||
Requires: settings:read permission
|
||||
"""
|
||||
pool = await get_db_pool()
|
||||
|
||||
row = await pool.fetchrow("""
|
||||
WITH latest_scores AS (
|
||||
SELECT DISTINCT ON (user_id)
|
||||
user_id, score, throttle_level
|
||||
FROM sdk_anomaly_scores
|
||||
WHERE snapshot_at > NOW() - INTERVAL '24 hours'
|
||||
ORDER BY user_id, snapshot_at DESC
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE throttle_level = 0) as level_0,
|
||||
COUNT(*) FILTER (WHERE throttle_level = 1) as level_1,
|
||||
COUNT(*) FILTER (WHERE throttle_level = 2) as level_2,
|
||||
COUNT(*) FILTER (WHERE throttle_level = 3) as level_3,
|
||||
COALESCE(AVG(score), 0) as avg_score,
|
||||
COALESCE(MAX(score), 0) as max_score
|
||||
FROM latest_scores
|
||||
""")
|
||||
|
||||
return SDKProtectionStatsResponse(
|
||||
total_users_tracked=row["total"] or 0,
|
||||
users_level_0=row["level_0"] or 0,
|
||||
users_level_1=row["level_1"] or 0,
|
||||
users_level_2=row["level_2"] or 0,
|
||||
users_level_3=row["level_3"] or 0,
|
||||
avg_score=float(row["avg_score"]),
|
||||
max_score=float(row["max_score"]),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sdk-protection/reset-score/{user_id}")
|
||||
async def reset_sdk_anomaly_score(
|
||||
user_id: str,
|
||||
session: Session = Depends(require_permission("settings:write")),
|
||||
):
|
||||
"""
|
||||
Reset anomaly score for a specific user.
|
||||
|
||||
Requires: settings:write permission
|
||||
"""
|
||||
pool = await get_db_pool()
|
||||
|
||||
# Insert a zero-score snapshot
|
||||
await pool.execute("""
|
||||
INSERT INTO sdk_anomaly_scores (user_id, score, throttle_level, triggered_rules,
|
||||
endpoint_diversity_count, request_count_1h)
|
||||
VALUES ($1, 0, 0, '[]'::jsonb, 0, 0)
|
||||
""", user_id)
|
||||
|
||||
# Clear Valkey score if available
|
||||
try:
|
||||
import redis.asyncio as r
|
||||
valkey_url = os.getenv("VALKEY_URL", "redis://localhost:6379")
|
||||
client = r.from_url(valkey_url, decode_responses=True, socket_timeout=1.0)
|
||||
await client.delete(f"sdk_protect:score:{user_id}")
|
||||
await client.aclose()
|
||||
except Exception:
|
||||
pass # Best-effort Valkey cleanup
|
||||
|
||||
# Log the event
|
||||
await pool.execute("""
|
||||
INSERT INTO middleware_events (middleware_name, event_type, user_id, details)
|
||||
VALUES ('sdk_protection', 'score_reset', $1, $2)
|
||||
""", session.user_id, {"target_user": user_id})
|
||||
|
||||
return {"message": f"Anomaly score reset for user {user_id}"}
|
||||
|
||||
|
||||
@router.get("/sdk-protection/tiers", response_model=List[SDKProtectionTierResponse])
|
||||
async def list_sdk_protection_tiers(
|
||||
session: Session = Depends(require_permission("settings:read")),
|
||||
):
|
||||
"""
|
||||
List SDK protection tier configurations.
|
||||
|
||||
Requires: settings:read permission
|
||||
"""
|
||||
pool = await get_db_pool()
|
||||
|
||||
rows = await pool.fetch("""
|
||||
SELECT tier_name, quota_per_minute, quota_per_hour, quota_per_day,
|
||||
quota_per_month, diversity_threshold, burst_threshold
|
||||
FROM sdk_protection_tiers
|
||||
ORDER BY quota_per_minute
|
||||
""")
|
||||
|
||||
return [
|
||||
SDKProtectionTierResponse(
|
||||
tier_name=row["tier_name"],
|
||||
quota_per_minute=row["quota_per_minute"],
|
||||
quota_per_hour=row["quota_per_hour"],
|
||||
quota_per_day=row["quota_per_day"],
|
||||
quota_per_month=row["quota_per_month"],
|
||||
diversity_threshold=row["diversity_threshold"],
|
||||
burst_threshold=row["burst_threshold"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
@router.put("/sdk-protection/tiers/{name}", response_model=SDKProtectionTierResponse)
|
||||
async def update_sdk_protection_tier(
|
||||
name: str,
|
||||
data: SDKProtectionTierUpdateRequest,
|
||||
session: Session = Depends(require_permission("settings:write")),
|
||||
):
|
||||
"""
|
||||
Update an SDK protection tier.
|
||||
|
||||
Requires: settings:write permission
|
||||
"""
|
||||
pool = await get_db_pool()
|
||||
|
||||
# Build update dynamically
|
||||
updates = []
|
||||
params = [name]
|
||||
param_idx = 2
|
||||
|
||||
for field_name in [
|
||||
"quota_per_minute", "quota_per_hour", "quota_per_day",
|
||||
"quota_per_month", "diversity_threshold", "burst_threshold",
|
||||
]:
|
||||
value = getattr(data, field_name)
|
||||
if value is not None:
|
||||
updates.append(f"{field_name} = ${param_idx}")
|
||||
params.append(value)
|
||||
param_idx += 1
|
||||
|
||||
if not updates:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(status_code=400, detail="No fields to update")
|
||||
|
||||
updates.append("updated_at = NOW()")
|
||||
query = f"""
|
||||
UPDATE sdk_protection_tiers
|
||||
SET {", ".join(updates)}
|
||||
WHERE tier_name = $1
|
||||
RETURNING tier_name, quota_per_minute, quota_per_hour, quota_per_day,
|
||||
quota_per_month, diversity_threshold, burst_threshold
|
||||
"""
|
||||
|
||||
row = await pool.fetchrow(query, *params)
|
||||
|
||||
if not row:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(status_code=404, detail=f"Tier '{name}' not found")
|
||||
|
||||
# Log the change
|
||||
await pool.execute("""
|
||||
INSERT INTO middleware_events (middleware_name, event_type, user_id, details)
|
||||
VALUES ('sdk_protection', 'tier_updated', $1, $2)
|
||||
""", session.user_id, {"tier": name, "changes": data.dict(exclude_none=True)})
|
||||
|
||||
return SDKProtectionTierResponse(
|
||||
tier_name=row["tier_name"],
|
||||
quota_per_minute=row["quota_per_minute"],
|
||||
quota_per_hour=row["quota_per_hour"],
|
||||
quota_per_day=row["quota_per_day"],
|
||||
quota_per_month=row["quota_per_month"],
|
||||
diversity_threshold=row["diversity_threshold"],
|
||||
burst_threshold=row["burst_threshold"],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user