feat(F-16): chunk quality gate + section classification

3-signal quality filter (length/alpha-ratio/bib-score) + rule-based section classifier + retroactive CLI pass. 35 new tests, 287 total green.
This commit is contained in:
2026-06-15 01:22:44 +00:00
parent 1a9afb4433
commit 9647897173
9 changed files with 529 additions and 17 deletions

View File

@@ -191,9 +191,7 @@ class Settings(BaseSettings):
synthesis_llm_url: str | None = Field(
default=None,
description=(
"Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."
),
description=("Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."),
)
synthesis_top_k: int = Field(
@@ -256,6 +254,38 @@ class Settings(BaseSettings):
),
)
# ------------------------------------------------------------------
# F-16 Chunk Quality Gate
# ------------------------------------------------------------------
chunk_min_chars: int = Field(
default=60,
gt=0,
description=(
"Minimum character count for a chunk to pass the quality gate. "
"Chunks shorter than this value are discarded before embedding."
),
)
chunk_min_alpha_ratio: float = Field(
default=0.40,
ge=0.0,
le=1.0,
description=(
"Minimum fraction of alphabetic characters in a chunk. "
"Chunks with a lower ratio are treated as OCR artefacts and discarded."
),
)
chunk_max_bib_score: float = Field(
default=0.70,
ge=0.0,
le=1.0,
description=(
"Maximum bibliography heuristic score before a chunk is rejected. "
"Score is based on DOI density, 'et al.' and year-in-brackets patterns."
),
)
@lru_cache(maxsize=1)
def get_settings() -> Settings: