feat(F-16): chunk quality gate + section classification
- codex/quality.py: 3-signal filter (length / alpha-ratio / bib-score) + rule-based section classifier + run_quality_pass retroactive DB pass - codex/ingest.py: promote quality imports to module level; apply filter_chunks before embedding; store section column in chunks INSERT - codex/config.py: CHUNK_MIN_CHARS / CHUNK_MIN_ALPHA_RATIO / CHUNK_MAX_BIB_SCORE - infra/schema.sql: ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT - .env.example: document F-16 quality thresholds - codex/cli.py: quality run sub-command (scope by --paper-id or all papers) - tests/quality/: 35 new tests covering all quality functions - tests/ingest/: patch filter_chunks in source-path tests to isolate ingest Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -191,9 +191,7 @@ class Settings(BaseSettings):
|
||||
|
||||
synthesis_llm_url: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."
|
||||
),
|
||||
description=("Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."),
|
||||
)
|
||||
|
||||
synthesis_top_k: int = Field(
|
||||
@@ -256,6 +254,38 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# F-16 Chunk Quality Gate
|
||||
# ------------------------------------------------------------------
|
||||
chunk_min_chars: int = Field(
|
||||
default=60,
|
||||
gt=0,
|
||||
description=(
|
||||
"Minimum character count for a chunk to pass the quality gate. "
|
||||
"Chunks shorter than this value are discarded before embedding."
|
||||
),
|
||||
)
|
||||
|
||||
chunk_min_alpha_ratio: float = Field(
|
||||
default=0.40,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description=(
|
||||
"Minimum fraction of alphabetic characters in a chunk. "
|
||||
"Chunks with a lower ratio are treated as OCR artefacts and discarded."
|
||||
),
|
||||
)
|
||||
|
||||
chunk_max_bib_score: float = Field(
|
||||
default=0.70,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description=(
|
||||
"Maximum bibliography heuristic score before a chunk is rejected. "
|
||||
"Score is based on DOI density, 'et al.' and year-in-brackets patterns."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
|
||||
Reference in New Issue
Block a user