feat(F-16): chunk quality gate + section classification
- codex/quality.py: 3-signal filter (length / alpha-ratio / bib-score) + rule-based section classifier + run_quality_pass retroactive DB pass - codex/ingest.py: promote quality imports to module level; apply filter_chunks before embedding; store section column in chunks INSERT - codex/config.py: CHUNK_MIN_CHARS / CHUNK_MIN_ALPHA_RATIO / CHUNK_MAX_BIB_SCORE - infra/schema.sql: ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT - .env.example: document F-16 quality thresholds - codex/cli.py: quality run sub-command (scope by --paper-id or all papers) - tests/quality/: 35 new tests covering all quality functions - tests/ingest/: patch filter_chunks in source-path tests to isolate ingest Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -131,6 +131,7 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(tex_file))
|
||||
|
||||
@@ -178,6 +179,7 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
|
||||
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
||||
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user