fix: Pipeline-Skalierung — 6 Optimierungen für 80k+ Controls
1. control_generator: GeneratorResult.status Default "completed" → "running" (Bug) 2. control_generator: Anthropic API mit Phase-Timeouts + Retry bei Disconnect 3. control_generator: regulation_exclude Filter + Harmonization via Qdrant statt In-Memory 4. decomposition_pass: Enrich Pass Batch-UPDATEs (400k → ~400 DB-Calls) 5. decomposition_pass: Merge Pass single Query statt N+1 6. batch_dedup_runner: Cross-Group Dedup parallelisiert (asyncio.gather) 7. canonical_control_routes: Framework Controls API Pagination (limit/offset) 8. DB-Indizes: idx_oc_parent_release, idx_oc_trigger_null, idx_cc_framework Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -274,6 +274,8 @@ async def list_framework_controls(
|
||||
verification_method: Optional[str] = Query(None),
|
||||
category: Optional[str] = Query(None),
|
||||
target_audience: Optional[str] = Query(None),
|
||||
limit: Optional[int] = Query(None, ge=1, le=5000),
|
||||
offset: Optional[int] = Query(None, ge=0),
|
||||
):
|
||||
"""List controls belonging to a framework."""
|
||||
with SessionLocal() as db:
|
||||
@@ -309,6 +311,12 @@ async def list_framework_controls(
|
||||
params["ta"] = json.dumps([target_audience])
|
||||
|
||||
query += " ORDER BY control_id"
|
||||
if limit is not None:
|
||||
query += " LIMIT :lim"
|
||||
params["lim"] = limit
|
||||
if offset is not None:
|
||||
query += " OFFSET :off"
|
||||
params["off"] = offset
|
||||
rows = db.execute(text(query), params).fetchall()
|
||||
|
||||
return [_control_row(r) for r in rows]
|
||||
|
||||
@@ -55,6 +55,7 @@ class GenerateRequest(BaseModel):
|
||||
skip_web_search: bool = False
|
||||
dry_run: bool = False
|
||||
regulation_filter: Optional[List[str]] = None # Only process these regulation_code prefixes
|
||||
regulation_exclude: Optional[List[str]] = None # Skip these regulation_code prefixes
|
||||
skip_prefilter: bool = False # Skip local LLM pre-filter, send all chunks to API
|
||||
|
||||
|
||||
@@ -148,6 +149,7 @@ async def start_generation(req: GenerateRequest):
|
||||
skip_web_search=req.skip_web_search,
|
||||
dry_run=req.dry_run,
|
||||
regulation_filter=req.regulation_filter,
|
||||
regulation_exclude=req.regulation_exclude,
|
||||
skip_prefilter=req.skip_prefilter,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user