Add skip_prefilter option to control generator
Local LLM prefilter (llama3.2 3B) was incorrectly skipping annex chunks that contain concrete requirements. Added skip_prefilter flag to bypass the local pre-filter and send all chunks directly to Anthropic API. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -385,6 +385,7 @@ class GeneratorConfig(BaseModel):
|
||||
dry_run: bool = False
|
||||
existing_job_id: Optional[str] = None # If set, reuse this job instead of creating a new one
|
||||
regulation_filter: Optional[List[str]] = None # Only process chunks matching these regulation_code prefixes
|
||||
skip_prefilter: bool = False # If True, skip local LLM pre-filter (send all chunks to API)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1886,7 +1887,7 @@ Kategorien: {CATEGORY_LIST_STR}"""
|
||||
self._update_job(job_id, result)
|
||||
|
||||
# Stage 1.5: Local LLM pre-filter — skip chunks without requirements
|
||||
if not config.dry_run:
|
||||
if not config.dry_run and not config.skip_prefilter:
|
||||
is_relevant, prefilter_reason = await _prefilter_chunk(chunk.text)
|
||||
if not is_relevant:
|
||||
chunks_skipped_prefilter += 1
|
||||
|
||||
Reference in New Issue
Block a user