diff --git a/.gitignore b/.gitignore index 9a69510..278f7af 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ build/ # Environment secrets — NEVER commit .env +.env.* +!.env.example # Type-checker and linter caches .mypy_cache/ diff --git a/codex/sources/semanticscholar.py b/codex/sources/semanticscholar.py index 90c383b..e82d75a 100644 --- a/codex/sources/semanticscholar.py +++ b/codex/sources/semanticscholar.py @@ -96,10 +96,7 @@ def fetch_references(paper_id: str) -> list[Citation]: context: str | None = contexts[0] if contexts else None cited_id: str = ( - external_ids.get("DOI") - or external_ids.get("ArXiv") - or cited_paper.get("paperId") - or "" + external_ids.get("DOI") or external_ids.get("ArXiv") or cited_paper.get("paperId") or "" ) if cited_id: citations.append(Citation(citing_id=paper_id, cited_id=cited_id, context=context)) diff --git a/codex/synthesis.py b/codex/synthesis.py index 2baba3f..dcc1586 100644 --- a/codex/synthesis.py +++ b/codex/synthesis.py @@ -365,11 +365,7 @@ def find_connections( continue seen_pairs.add(pair) # type: ignore[arg-type] - pair_chunks = [ - c - for c in chunks - if c.get("bibkey") in {bibkey, other} - ] + pair_chunks = [c for c in chunks if c.get("bibkey") in {bibkey, other}] prompt = _CONNECTION_PROMPT.format(chunks_block=_format_chunks(pair_chunks)) raw = _safe_llm_generate(llm, prompt, settings.synthesis_llm_model) if not raw.strip(): diff --git a/codex/wiki.py b/codex/wiki.py index 2efe15e..b47a98a 100644 --- a/codex/wiki.py +++ b/codex/wiki.py @@ -240,9 +240,36 @@ def _parse_claims(markdown: str) -> list[Claim]: _STOPWORDS: frozenset[str] = frozenset( { - "the", "a", "an", "in", "on", "of", "to", "is", "are", "was", "were", - "and", "or", "but", "for", "with", "this", "that", "it", "we", "they", - "be", "as", "at", "by", "from", "has", "have", "not", "which", + "the", + "a", + "an", + "in", + "on", + "of", + "to", + "is", + "are", + "was", + "were", + "and", + "or", + "but", + "for", + "with", + "this", + "that", + "it", + "we", + "they", + "be", + "as", + "at", + "by", + "from", + "has", + "have", + "not", + "which", } ) @@ -360,12 +387,8 @@ def _detect_conflicts( for bk2 in bibkeys[i + 1 :]: # Check if any chunk from bk1 contains a conflict keyword # and any chunk from bk2 also does — heuristic signal only - bk1_has_conflict = any( - _CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1] - ) - bk2_has_conflict = any( - _CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2] - ) + bk1_has_conflict = any(_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1]) + bk2_has_conflict = any(_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2]) if bk1_has_conflict and bk2_has_conflict: conflicts.append((slug, bk1, bk2)) diff --git a/docs/audit/AUDIT-2026-06-15-full-repo.md b/docs/audit/AUDIT-2026-06-15-full-repo.md new file mode 100644 index 0000000..f831084 --- /dev/null +++ b/docs/audit/AUDIT-2026-06-15-full-repo.md @@ -0,0 +1,616 @@ +# AUDIT — Full Repository (iterative) + +**Auditor:** Audit-Loop (Opus) +**Started:** 2026-06-15 +**Scope:** entire `main` tree, audited iteratively module-by-module. +**Companion doc:** [`AUDIT-2026-06-15-loop-1.md`](AUDIT-2026-06-15-loop-1.md) — +the F-15 / dev-loop-1 findings (S-1…U-4). This document continues the **same +finding-ID register** (so `C-4` here follows `C-3` there) and covers the rest of +the repo. + +> Severity: **HIGH** fix before relying · **MED** fix soon · **LOW** polish · +> **INFO** accepted/no-action. Findings only — remediation planned separately. + +## Coverage tracker + +| Part | Area | Status | +|------|------|--------| +| (loop-1) | F-15 graph, db/schema, ingest, discover, mcp, provenance-scan, cli(search+graph), config(graph) | ✅ documented | +| **A** | Grounding guard (`wiki.py` `_parse_claims`/`_run_grounding_guard`, quarantine) — shared by `synthesis.py` | ✅ documented | +| **B** | `sources/` (arxiv, openalex, semanticscholar) | ✅ documented | +| **C** | `parsing/` (grobid, nougat, mathpix, figures, tex) | ✅ documented | +| **D** | `synthesis.py` orchestration (find_connections/gaps/improvements, conjectures, write_leads) | ✅ documented | +| **E** | `wiki.py` rest (retrieve, compile_concept, cross-refs, index/log, embed) | ✅ documented | +| **F** | `embed.py`, `quality.py`, `models.py` | ✅ documented | +| **G** | `cli.py` rest (ingest/wiki/synthesis/provenance commands), `config.py` rest | ✅ documented | + +--- + +## Part A — Grounding guard (the "zero-fact-leak" core) + +The guard lives in `wiki.py` (`_parse_claims:216`, `_run_grounding_guard:299`) +and is imported by `synthesis.py:57-59`, so **one implementation gates both** +the wiki pages and the synthesis leads. A claim is "grounded" iff the **last +sentence** before its `[BibKey #loc]` citation contains a run of 5 consecutive +non-stopword tokens that appears as a substring of any chunk of the same bibkey. + +### C-4 — Zero-claim pages bypass quarantine and publish as "compiled" · **HIGH** +- **Evidence:** `wiki.py:730` — `if total_claims > 0 and grounding_rate < threshold:` + quarantines; the `else` (`:737-742`) **publishes to `wiki/` and records the + hash** as successfully compiled. +- **Bug:** when the LLM output has **no parseable citations** (`total_claims == 0`, + so `grounding_rate == 0.0`), the `total_claims > 0` guard is false → it takes + the *publish* branch. The quarantine gate is skipped exactly when grounding + evidence is **weakest** (zero citations). +- **Trigger paths:** (a) the LLM emits uncited prose; (b) **LLM outage** — the + module header guarantees "LLM unavailable → empty ConceptPage, no crash" + (`wiki.py:13-14`), so a transient Ollama outage writes near-empty stub pages to + `wiki/`, marks them compiled, and stores their hash. On the next + `changed_only` run the hash matches → the stub is **skipped forever** until a + forced recompile. A research wiki silently fills with empty/uncited pages. + +### C-5 — "Every claim is grounded" holds only for the last cited sentence · **MED** +- **Evidence:** `_CLAIM_RE` (`wiki.py:211-213`) captures `text = [^\[]+?` (the + span back to the previous `]`/start); `_last_sentence` (`:279-291`) then keeps + only the final sentence; `_run_grounding_guard` checks that sentence only. +- **Two holes in the guarantee:** + 1. **Paragraph passengers** — any sentence that is *not* the last one before a + citation is never grounding-checked and never `⚠`-marked. "Hallucination one. + Hallucination two. Grounded tail [Bib]." passes with only the tail verified. + 2. **Uncited text is invisible** — a sentence with no `[BibKey]` is not a Claim + at all (`_parse_claims` only yields cited spans), so it is never grounded + nor marked. The LLM can emit fully uncited assertions into a published page. +- **Impact:** the effective invariant is "*the last sentence immediately before + each citation has lexical overlap with the cited paper*" — materially weaker + than the N-02 "zero-fact-leak enforcement" framing. The module **header is + honest** ("Substring-Match MVP", `wiki.py:4,13`); the commit/ADR language is + not. Recommend aligning the claim or strengthening the guard (check every + sentence; flag uncited sentences). + +### C-6 — Substring match does not enforce the documented "5 consecutive tokens" · **LOW** +- **Evidence:** `wiki.py:347-349` — `gram = " ".join(content[i:i+5])`; + `if any(gram in src for src in sources)` where `src` is the space-joined + content-word string of the chunk. +- **Issue:** a *substring* test on the joined string lets the first/last gram + token match as a **prefix/suffix of a longer chunk word** (e.g. gram + `"cat dog …"` substring-matches inside `"scat dogma …"`). The docstring claims + matching "as 5 consecutive non-stopword **tokens**" — true sequence matching + would compare token lists with a sliding window (or add boundary sentinels). + Low real-world impact at 5-gram length, but the claimed invariant is not what + the code enforces. + +### R-1 — Tokenizer brittleness causes false-negative over-quarantine · **MED** +- **Evidence:** `_content_words` (`wiki.py:294-296`) lowercases and `.split()`s on + whitespace only — **no punctuation stripping**. `_last_sentence` splits on + `.!?` (`:276`). +- **Impact:** punctuation stays glued to tokens, so a faithful paraphrase grounds + only on *exact* surface form — `"volume,"` ≠ `"volume"`, `"L(gamma1)"` must + match character-for-character, and a trailing decimal like `"V = 1.5"` makes + `_last_sentence` return the fragment `"5"` (split at the `.`), which then has + `< 5` content words → **forced ungrounded** (`:339-344`). Math-heavy and terse + claims are systematically marked ungrounded even when verbatim, inflating the + ungrounded ratio and pushing real pages into quarantine (and into C-4's + blind spot when it drives `total_claims` interactions). The passing test + `test_grounding_guard_…` only succeeds because its fixture reproduces the chunk + text *exactly*. + +### R-2 — Reference-list filter can drop theorem-dense chunks · **LOW** +- **Evidence:** `_retrieve_chunks` (`wiki.py:188-199`) skips a chunk when + `>60 %` of its lines match `^\s*(\[\d+\]|[A-Z][a-z]+,?\s+[A-Z]\.)`. +- **Issue:** the second alternative (`[A-Z][a-z]+,?\s+[A-Z]\.`) also matches prose + like `"Theorem A."`, `"Lemma B."`, `"Section C."`. A chunk formatted as a list + of theorems/lemmas can exceed 60 % and be discarded as a "reference list" — + silently dropping exactly the mathematical content the wiki wants. + +### R-3 — Conflict detection has near-zero precision · **LOW** +- **Evidence:** `_detect_conflicts` (`wiki.py:367-…`) flags a pair `(bk1, bk2)` if + **each** has *any* chunk containing a hedging keyword + (`but|however|in contrast|…`, `_CONFLICT_KEYWORDS:361-364`). +- **Issue:** it never checks the keywords relate to a shared topic or to each + other. In a corpus where most papers contain "however"/"but" somewhere, this + flags ~O(n²) pairs as "conflicting" — noise surfaced in `CompileReport`. + Labeled "heuristic signal only", but as-is it is not actionable. + +**Part A net:** 1 HIGH (C-4), 2 MED (C-5, R-1), 3 LOW (C-6, R-2, R-3). The guard +is a reasonable lexical MVP, but (a) it has a concrete publish-bypass bug for +zero-citation output, and (b) the "zero-fact-leak" framing overstates what a +last-sentence substring check can guarantee. + +--- + +## Part B — `sources/` (external API clients) + +Solid foundations: all three clients set httpx timeouts; OpenAlex and S2 retry +429/5xx with exponential back-off (tenacity); S2 enforces a ≥1 req/s floor with a +monotonic clock under a lock. The findings are about **ID format consistency** +and a couple of dead/inert paths. + +### C-7 — `papers.id` is stored in OpenAlex *URL* form, breaking the bare-ID contract · **HIGH** +- **Evidence:** `openalex.py:94` — `_map_paper` sets + `id = data.get("doi") or data.get("id") or ""`. Real OpenAlex returns `doi` as + a **full URL** (`https://doi.org/10.x`) and `id` as `https://openalex.org/W…` + (the repo's own test confirms the URL form for `openalex_id`, + `test_openalex.py:92`). `ingest.py:82` takes `paper.id` from this unchanged and + upserts it as `papers.id` (`ingest.py:138`). +- **Why it's currently masked:** the test fixture `_SAMPLE_WORK["doi"]` is a + **bare** DOI `"10.1145/3592430"` (`test_openalex.py:18`) — not the URL form real + OpenAlex emits — and **no test asserts `paper.id`** at all. So the suite is + green while production stores `papers.id = "https://doi.org/10.1145/3592430"`. +- **Impact:** + 1. **Bare-ID lookups fail.** `ingest_all.sh` ingests `"10.1145/1964921.1964997"`, + but `papers.id` becomes the URL; `codex graph related 10.1145/…`, + `discover citing 10.1145/…`, search-by-id, etc. find nothing. + 2. **Second ID-mismatch axis (compounds C-1).** `cited_id` from the S2/GROBID + paths is a **bare** DOI/arXiv (`semanticscholar.py:98-99`, + `ingest.py:187`), so a bare-DOI reference can never equal a URL-form + `papers.id` — bare-DOI cross-citations stay "dangling" regardless of the + F-15 W-ID resolution (which only matches `papers.openalex_id`, the URL form). +- **Action:** confirm with `SELECT id FROM papers LIMIT 5` on the Jetson DB; if + URL-form, normalize `papers.id` to a canonical bare form at ingest (and the + `cited_id` writers to match), then add a representative fixture + `paper.id` + assertion (see T-3). + +### T-3 — OpenAlex test fixture is unrepresentative and skips `paper.id` · **MED** +- **Evidence:** `_SAMPLE_WORK["doi"] = "10.1145/3592430"` (bare) vs the real + `https://doi.org/…` URL form; `test_fetch_paper_*` assert title/year/authors/ + abstract/`openalex_id` but **never** `paper.id`. +- **Impact:** the id-mapping — the exact surface of C-7 — is untested, and the + fixture actively hides the URL-form behaviour. Any normalization fix needs this + closed first or it will regress invisibly. + +### R-5 — S2 `fetch_references` breaks on legacy arXiv IDs containing `/` · **LOW** +- **Evidence:** `semanticscholar.py:80` interpolates `paper_id` into the path + `…/paper/{paper_id}/references`. The ingest fallback passes `arXiv:math/0603097` + (`ingest.py:91`) — the embedded `/` yields a malformed path → 404 → + `fetch_references` silently returns `[]`. Legacy-arXiv papers that miss the + OpenAlex path get **zero references** with no error. + +### R-4 — `arxiv.fetch_source` no-op try/except + no retry · **LOW** +- **Evidence:** `arxiv.py:41-44` — `try: httpx.get(...) except httpx.RequestError: raise` + catches and immediately re-raises, doing nothing. Unlike OpenAlex/S2, the arXiv + client has **no** retry/back-off, so a transient network blip aborts ingest. + Minor resilience asymmetry. + +### C-9 — `semanticscholar.fetch_recommendations` is unreachable app code · **LOW** +- **Evidence:** referenced only by `tests/sources/test_semanticscholar.py`; no + `codex/` module or CLI calls it (`grep fetch_recommendations`). Tested but + never wired — dead feature surface (decide: wire a `discover recommend` + command, or drop). + +**Part B net:** 1 HIGH (C-7, ID-form contract — the big one; confirm on live DB), +1 MED (T-3, fixture masks it), 3 LOW (R-4, R-5, C-9). The retry/timeout/rate-limit +hygiene is genuinely good; the weakness is the **un-normalized, heterogeneous ID +space** (URL vs bare, W-ID vs DOI vs arXiv vs S2-hash) threaded through `papers.id` +and `citations.cited_id` — the same root cause behind C-1 and the F-15 ID dance. + +--- + +## Part C — `parsing/` (tex, grobid, nougat, mathpix, figures) + +> **Scope reality:** the live corpus is ingested entirely as **`.txt`** +> (`ingest_all.sh` maps every paper to a `*.txt` file), so the `.tex`-clean, +> GROBID, Nougat, Mathpix, and Figures paths are **latent** — only +> `tex.chunk_text` + `quality.*` run on the live data. The findings below are +> real but mostly fire only once `.tex`/`.pdf --rich` ingest is used. +> +> **Not a finding:** the broad `except Exception` in `mathpix.py:109,151` and +> `figures.py:125,152` are *justified* per-item graceful degradation +> (`fitz.open` fails → return empty; pix2tex/image fails → skip that item), not +> swallowed bugs. + +### C-10 — Formulas/figures key on the PDF *filename*, not `paper.id` · **MED** (latent, `.pdf --rich`) +- **Evidence:** `figures.py:119` and the mathpix extractor set + `paper_id = Path(pdf_path).stem`. `ingest.py:289,303` insert + `f.paper_id` / `fig.paper_id` **unchanged** into `formulas`/`figures`, whose + schema declares `paper_id TEXT REFERENCES papers(id)` (`schema.sql:97,115`). +- **Impact:** the filename stem (e.g. `springborn-2008-weighted-delaunay…`) will + not equal `papers.id` (an OpenAlex URL/bare id — see C-7), so the FK insert + **fails and aborts rich ingest**, or (without FK enforcement) writes orphaned + rows. Identity is derived from the *file*, not the *paper*. Fix: pass `paper.id` + into the extractors, or have ingest overwrite `f.paper_id = paper.id` before insert. + +### R-7 — `_clean_latex` misses `\(…\)` / `\[…\]` math + strips all math from chunks · **LOW** (`.tex` path) +- **Evidence:** `tex.py:21-56` strips `%comment`, `$…$`, `$$…$$`, + `equation`/`align` envs, `\cite/\label/\ref` — but **not** the `\(…\)` / `\[…\]` + math delimiters. Papers using those leave raw `\( … \)` markup in the chunk text. +- **Impact:** (a) leftover math markup pollutes embeddings; (b) *all* math is + removed from chunk text, so chunk-level semantic search can never match formula + content — it relies entirely on the separate formula FTS (F-09), which is itself + latent here. Acceptable as a separation-of-concerns choice, but worth stating. + +### R-6 — PDF-path external calls are unwrapped / un-retried · **LOW** (`.pdf` path) +- **Evidence:** `ingest.py:185` calls `extract_references()` outside any + `try`; GROBID (`grobid.py:54,133`) and the mathpix HTTP calls `raise_for_status` + with **no** retry/back-off (unlike OpenAlex/S2). A transient GROBID/Mathpix + 5xx aborts the whole PDF ingest. +- **INFO:** `grobid.py:56` uses stdlib `ET.fromstring` on the server's XML. + Fine for a trusted self-hosted GROBID; switch to `defusedxml` if it is ever + pointed at an untrusted endpoint. + +**Part C net:** 1 MED (C-10), 2 LOW (R-6, R-7), all **latent** for the current +`.txt` corpus. `nougat.py` was skimmed only (54 lines, `.pdf`-only wrapper) — no +deep review yet; flagged in the ledger. Parsing hygiene is otherwise good. + +> **Request (a) — second audit pass — is now complete:** §8 of the loop-1 doc +> (grounding guard ✓ Part A, `sources/` ✓ Part B, `parsing/` ✓ Part C) is closed. +> Remaining whole-repo parts D–G are the broader sweep. + +--- + +## Part D — `synthesis.py` orchestration + +The lead pipeline (connections → gaps → improvements → conjectures) reuses the +Part-A grounding guard for the three grounded kinds and degrades gracefully on +LLM/DB failure (`_safe_llm_generate`/`_list_paper_titles` swallow + log). The +conjecture invariants are genuinely solid (status hard-set `"unverified"`, +mandatory falsification path or the lead is dropped, written to `conjectures/` +only). Two real defects and a doc contradiction: + +### C-11 — Lead-ID collision silently overwrites synthesis output · **HIGH** +- **Evidence:** every stage restarts its counter at `seq = 1` + (`synthesis.py:341` connections, `:415` gaps, `:530` improvements, + `:634` conjectures) and ids are `L-0001, L-0002, …` (`_make_lead_id`). The CLI + concatenates the grounded kinds — `all_leads = connections + gaps + improvements` + (`cli.py:421`) — and `write_leads` routes **all** grounded kinds to the *same* + `grounded/.md` (`synthesis.py:783-796`), writing in list order. +- **Bug:** connection `L-0001`, gap `L-0001`, improvement `L-0001` all target + `grounded/L-0001.md`; later writes overwrite earlier → **improvements clobber + gaps clobber connections** at each number. Surviving files = + `max(#connections, #gaps, #improvements)`, **not** the sum. With 5/3/4 leads you + get 5 files, not 12; connections are lost first. Silent data loss of the + synthesis engine's core output, masked because "files do get written". +- **Fix direction:** namespace ids by kind (`L-C-0001` / `L-G-0001` / `L-I-0001`) + or use a single shared counter across stages. + +### R-8 — Default-topic gap detection is a false-positive generator · **MED** +- **Evidence:** `find_gaps` defaults `probe_topics` to paper **titles** when + `topics is None` (`synthesis.py:418-420`); a title query retrieves mostly its + *own* paper's chunks → `bibkeys_for_topic` ≈ 1, which is `< synthesis_gap_min_coverage` + (`:450-453`) → the topic is flagged as a gap and the LLM is asked to articulate + one. +- **Impact:** nearly every ingested paper becomes a spurious "Gap: '' + covered by only 1 bibkey(s)" candidate — the LLM invents a gap for material that + is actually well covered. The grounding guard catches some, but the default + signal is mostly noise. (User-supplied `--topics` avoid this.) + +### D-4 — `write_leads` vs `_update_index` docstrings contradict each other · **LOW** +- **Evidence:** `write_leads` (`synthesis.py:762-765`) claims it "maintains an + **append-only** `INDEX.md`: a previously-recorded id is kept on subsequent runs", + but `_update_index` (`:802-803`) states — and implements — a "**full rebuild**, + not append-only" by globbing the on-disk dirs. +- **Impact:** there is no append-only / id-retention logic; the index purely + reflects current disk state. The guarantee in the `write_leads` docstring is + false (and interacts with C-11: overwritten ids simply vanish from the index). + +**Part D net:** 1 HIGH (C-11, silent lead overwrite), 1 MED (R-8, gap noise), +1 LOW (D-4, doc contradiction). The conjecture path is exemplary; the grounded +path's persistence layer is where it breaks. + +--- + +## Part E — `wiki.py` rest (compile, cross-refs, state, index/log) + +Good: `yaml.safe_load`, deterministic `_chunk_hash` (sorted by chunk id) driving a +sound incremental-skip, append-only `log.md`, inline-code protection during +cross-ref injection. The issues are math-corruption, a dead step, and a marking +order bug. + +### R-9 — Cross-ref injection corrupts LaTeX math · **MED** +- **Evidence:** `_inject_cross_refs` (`wiki.py:426`) protects only inline-code + spans (`_INLINE_CODE_RE`); it then runs whole-word, case-insensitive title/alias + → `[[slug]]` replacement over everything else (`:429-438`). **Math spans + (`$…$`, `$$…$$`, `\(…\)`) are not protected.** +- **Impact:** the synthesis prompt explicitly asks for LaTeX `$ … $` + (`wiki.py:495`), so bodies contain math. A concept title/alias that appears + inside a formula (e.g. a concept named "Energy" inside `$E_{\text{Energy}}$`, + or a single-word alias) is rewritten to `[[slug]]`, corrupting the LaTeX. The + risk scales with how common the concept titles/aliases are as math tokens. + +### C-12 — `_try_embed_formulas` is a no-op that still costs a DB round-trip per concept · **LOW** +- **Evidence:** `wiki.py:623-637` opens a connection, queries + `information_schema.tables` for `formulas`, and returns — the actual embed is + `"(Future: …)"`. Called unconditionally per concept (`compile_concept:609`). +- **Impact:** step 4 of `compile_concept`'s docstring ("Embed formula chunks") + does nothing, and each concept compile pays an extra DB connection + query for + no effect. Placeholder masquerading as a feature. + +### C-13 — Ungrounded-claim marking can miss claims mutated by cross-ref injection · **LOW** +- **Evidence:** order in `compile_concept` — claims parsed from `raw_output` + (`:602-603`), then `raw_output` is rewritten by `_inject_cross_refs` (`:606`), + then `_render_page_markdown` (`:612`) re-finds the *original* `claim.text` by + regex in the **injected** body (`:544-552`). +- **Impact:** if an ungrounded claim's text contained a term that got replaced + with `[[slug]]`, the original text no longer matches → the `⚠` mark is silently + not applied. (Also note: wiki's marking lacks the `(?<!⚠ )` guard that + synthesis's `_mark_ungrounded` has — two divergent mark implementations.) + +### R-10 — Non-atomic compile-state write · **LOW** +- **Evidence:** `_save_compile_state` (`wiki.py:475-477`) writes + `.compile-state.json` in place. A crash mid-write corrupts it; `_load` + then catches `JSONDecodeError` → `{}` → every concept recompiles. Graceful but + loses incrementality. Write to a temp file + `os.replace` for atomicity. + +### R-11 — `load_concepts` raises `KeyError` on malformed YAML · **LOW** +- **Evidence:** `entry["slug"]` / `entry["title"]` (`wiki.py:135-136`) are + unguarded; a concepts.yaml entry missing a key crashes the whole compile with a + bare `KeyError` rather than a actionable message. + +> **Minor:** `compile_all` builds the index from `state.keys()` (`:745`), which +> accumulates slugs across runs — a concept removed from `concepts.yaml` lingers +> in `index.md`/state until manually pruned. Low. + +**Part E net:** 1 MED (R-9, math corruption), 4 LOW (C-12, C-13, R-10, R-11). The +incremental-compile machinery is sound; the rendering/cross-ref layer has the rough +edges. `OllamaClient.generate` (no retry, 120 s timeout, `raise` on error) feeds +directly into **C-4** (its exception → empty page → published), so C-4's blast +radius includes every transient Ollama hiccup. + +--- + +## Part F — `embed.py`, `quality.py`, `models.py` + +The best-built corner of the repo. `embed.py` L2-normalises correctly (cosine ⇒ +dot product), guards empty input and zero-norm rows, and caches a process +singleton. `quality.py` thresholds are all config-driven, `run_quality_pass` is +CLI-wired (`cli.py:513-517`) and atomic (single commit). `models.py` dataclasses +mirror the schema cleanly. Two dead/low-signal items and one corroboration: + +### C-14 — The sparse-embedding ("hybrid") path is dead code · **LOW** +- **Evidence:** `Embedder.encode_sparse` / `encode` / `_coerce_sparse` are called + nowhere in `codex/` (grep for `encode_sparse` / `.encode(` finds only + `str.encode` in `wiki.py:455`). Only `encode_dense` is wired (ingest, search, + wiki). There is no sparse column in the schema. +- **Impact:** the `embed.py` header and ADR-0002 advertise "**Hybrid dense + + sparse** embeddings via BGE-M3" as a headline, but the sparse half is + unreachable — the actual "hybrid" elsewhere is dense + Postgres FTS. Either wire + sparse retrieval or drop the capability + correct the docs. + +### R-12 — `classify_section` is low-signal as fed · **LOW** +- **Evidence:** `classify_section` matches section headers in the **first 200 + chars** (`quality.py:103-106`), but `tex.chunk_text` produces overlapping + *word-window* chunks that almost never begin at a header. So most chunks fall + through to `"body"` (or the DOI-density `"bibliography"` fallback). +- **Impact:** the `chunks.section` column is mostly `"body"` — limited value for + any section-aware retrieval that depends on it. Not wrong, just weak signal + given the chunker. + +### Corroboration of C-7 (not a new finding) +- `models.Paper.id` docstring (`models.py:20-21`) states the canonical id is "an + arXiv ID … **or a DOI (e.g. `10.1145/3592430`)**" — the **bare** form. Production + fills it from OpenAlex in **URL** form (C-7), so the code violates its own model + contract. This makes C-7 a documented-contract breach, not a judgement call. + +**Part F net:** 2 LOW (C-14, R-12) + a C-7 corroboration. These three modules are +clean; no correctness defects. + +--- + +## Part G — `cli.py` rest + `config.py` + +Most CLI commands are thin Typer wrappers delegating to already-audited modules +(`ingest`, `search`, `discover`, `wiki`, `synthesis`, `provenance`, `quality`, +`graph`). Settings are pydantic-validated with sensible bounds (`gt=0`, `0..1`). +Findings are config-coupling and output hygiene. + +### C-15 — `embedding_dim` is a footgun decoupled from the hardcoded schema · **MED** +- **Evidence:** `embedding_dim` is configurable (`config.py:67-74`, default 1024) + and `.env.example`/`schema.sql:7` invite other models (Qwen3 1024, **Jina v4 + 2048**). But `schema.sql` hardcodes `vector(1024)` on every embedding column + (`:25,38,102,119`), and `Embedder` derives the real output dim from the *model*, + using the `dim` setting only for empty-input zero-vectors (`embed.py:50,89`). +- **Impact:** set `EMBEDDING_DIM=2048` (or pick a non-1024 model) and inserts + fail on a pgvector dimension mismatch — silently for empty abstracts + (`(1,2048)` zero-vector into `vector(1024)`), structurally for real vectors. + Nothing couples the knob to the DDL; the setting reads as supported but is not. + +### R-13 — `export_bib` does no BibTeX escaping and forces `@article` · **LOW** +- **Evidence:** `provenance.py:174-181` interpolates `title`/`author`/`year` raw + into `@article{…}`; no escaping/brace-balancing of BibTeX specials + (`{ } \ % # $ _ &`). +- **Impact:** a title with an unbalanced brace or a stray special char produces a + malformed `.bib` entry that breaks Doxygen/BibTeX. Also every paper is emitted + as `@article` although the corpus includes theses/books/chapters + (`ingest_all.sh`) — bibliographically wrong (functional for `@cite` keys though). + +### S-4 — Secrets stored as plain `str`, not `SecretStr` · **LOW** +- **Evidence:** `database_url`, `mathpix_app_key`, `mcp_auth_token` + (`config.py:29,148,249`) are plain `str`. Any `repr(Settings())` or debug log of + the settings object prints the DB password / API key / bearer token in clear. +- **Impact:** log/exception-dump leakage surface (complements S-1). Use + `pydantic.SecretStr` for credential fields. + +### R-14 — `mcp_transport` enum not enforced · **LOW** +- **Evidence:** `config.py:229-236` documents "stdio or http" but no validator; + `main()` (`mcp_server.py:323`) treats anything non-`http` as stdio. A typo + (`htttp`) silently starts stdio instead of failing loudly. + +### D-5 — Inconsistent `validation_alias` on `nougat_url` only · **LOW** +- **Evidence:** `nougat_url` adds `AliasChoices("NOUGAT_URL", "nougat_url")` + (`config.py:47`) that no other field has — redundant under + `case_sensitive=False`, and the inconsistency invites confusion. + +> **Corroboration of C-7:** `cli.py:45` ingest does **no** id normalization and +> echoes `result.paper_id` (the URL-form id) straight back — no normalization +> layer exists at any tier (source → ingest → CLI). + +**Part G net:** 1 MED (C-15, dim/schema footgun), 4 LOW (R-13, S-4, R-14, D-5). +CLI/config are otherwise solid and well-documented. + +--- + +# Executive Summary — whole-repo audit complete (loop-1 + Parts A–G) + +**Tally: 45 findings — 8 HIGH, 11 MED, 26 LOW/INFO** (after live-DB verification: +**7 HIGH, 12 MED** — M-1 reclassified, see the verification section at the bottom). +Test suite 330 green, `ruff`/`mypy` clean. The codebase is well-structured and idiomatic; the defects +cluster into five root themes, not random scatter. + +### The 8 HIGH findings +| ID | Theme | One-liner | +|----|-------|-----------| +| S-1 | secrets | live DB password in un-ignored `.env.jetson-ingest` (mitigated this session) | +| M-1 | migration | `paper_identifiers` has no migration path; `schema.sql` not re-applyable, `apply_schema` never called | +| C-1 | identity | `discover.py` keeps the ID-mismatch bug F-15 fixed in `graph.py` (over-reports ingested papers as leads) | +| C-7 | identity | `papers.id` stored in OpenAlex **URL** form, violating the bare-ID contract (breaks lookups + cross-citation match) | +| C-4 | silent failure | zero-citation/LLM-outage wiki pages **bypass quarantine** and publish as "compiled" | +| C-11 | silent data loss | synthesis leads collide on `L-000N` across stages → improvements overwrite gaps overwrite connections | +| D-1 | doc integrity | ADR-F15 GO rests on spike numbers measured **before** the shipped resolution fix | +| T-1 | verification | the F-15 resolution SQL (the substance of the last 2 commits) has **zero** real-DB coverage | + +### Five root-cause themes +1. **Un-normalized identity (the dominant theme).** C-1, C-7, C-10, M-1, T-3, + and most of the F-15 ID dance are one problem: a paper's identity exists in + many un-reconciled forms (bare vs URL · DOI vs arXiv vs OpenAlex-W vs S2-hash + vs PDF-filename) with **no canonical normalization layer**. One normalization + boundary at ingest would retire ~5 HIGH/MED findings at once. +2. **"Green but unverified."** T-1/T-2/T-3 — the most error-prone code (resolution + SQL, migrations, id-mapping) is mocked or tested with unrepresentative fixtures, + so real bugs (C-4, C-7, C-11) sit behind 330 passing tests. +3. **Silent failure / data loss under degradation.** C-4 (publish empty), C-11 + (overwrite leads), R-1 (over-quarantine) corrupt *outputs* instead of erroring + — the worst failure mode for a research tool whose value is trust. +4. **Overstated guarantees.** D-1 (stale ADR), D-2/D-4 (doc↔code drift), C-5 + ("zero-fact-leak" ≫ a last-sentence substring check), C-14 ("hybrid" sparse is + dead) — the docs/commit-language claim more than the code delivers. +5. **Latent rich-path debt.** C-10, R-6, R-7 are real but dormant because the live + corpus is `.txt`; they detonate on the first `.pdf --rich` ingest. + +### Suggested remediation sequencing (themes, not a plan — planning deferred) +- **First:** S-1 (rotate credential — secret already protected), M-1 (migration + path), C-7+C-1 (one identity-normalization layer), T-1 (real-DB test) — these + unblock trust in the live corpus. +- **Then:** C-4 + C-11 (stop silent output corruption), D-1 (re-validate ADR-F15). +- **Then:** MED batch (R-1, R-8, R-9, C-15, C-10) and the LOW polish. + +### Coverage honesty +Every `codex/` module was read. After the follow-up pass **all are deep-reviewed** +(`nougat.py` included — see Runtime Validation). Runtime state that was initially +*inferred from code* (C-7 id-form, M-1 table existence, and the C-4/C-11 +behaviours) has since been **empirically confirmed** — see the *Live DB +Verification* and *Runtime Validation* sections below. + +--- + +# Live DB Verification — 2026-06-15 (read-only, via existing SSH tunnel) + +Ran read-only queries against the live Jetson corpus (`papers=36`, +`citations=811`). This **confirms the two identity HIGHs with hard numbers** and +**corrects an over-statement in M-1**. + +### C-7 — CONFIRMED ✅ (severity stands: HIGH) +`papers.id` form distribution: **`doi_url=35`, `bare_arxiv=1`, everything else 0** +(of 36). Samples: `https://doi.org/10.48550/arxiv.math/0603097`, +`https://doi.org/10.48550/arxiv.2310.17529`. Even arXiv papers are stored as their +arXiv-DOI **URL**, never the bare `math/0603097` / `2301.x` the schema, `models.Paper`, +and `ingest_all.sh` assume. The single bare id is the one paper OpenAlex 404'd +(fell to `ingest.py:94`). → bare-ID lookups fail for 35/36 papers. Confirmed. + +### C-1 — CONFIRMED ✅ with impact (severity stands: HIGH) +- `citations.cited_id` is uniformly **`oa_url=811`** (all OpenAlex URLs) — the live + corpus was ingested via the OpenAlex path, so the heterogeneous bare-DOI/S2 mix + theorised in Part B is **not present here** (it would appear only via S2/GROBID, + i.e. `.pdf`/fallback ingest). +- **`discover.py` wrongly flags 13 already-ingested papers as "dangling / not + ingested"** (their `cited_id` is an OpenAlex URL whose paper *is* in the corpus + via `openalex_id`). Meanwhile **`graph.py` correctly resolves 57 cross-citation + edges** via the `openalex_id` JOIN. The two "dangling" implementations + measurably disagree — exactly the C-1 divergence, now quantified. + +### M-1 — REFINED ⤵ (downgrade acute severity; structural finding stands) +`paper_identifiers` **exists on the live DB with 39 rows** — the catastrophic +"graph JOIN crashes with relation-does-not-exist" scenario **does not occur**; +the table was applied (manually or via DB re-create). The *structural* gap stands: +`apply_schema` is still non-idempotent and never invoked, so the **next** schema +change has no migration path — but the acute "live graph is broken" framing was +**over-stated**. Re-rank M-1 from HIGH → **MED** (latent migration debt, not an +active outage). + +### C-2 — PARTIALLY REFUTED ⤵ +`paper_identifiers` has **39 rows vs ~35 papers-with-openalex_id**, so ~4 papers +carry a diverging alias → the `pi` JOIN is **not** fully inert (it resolves a +handful of cases the `p` JOIN can't). Keep as LOW/INFO: the table earns its keep +for a few papers, though the cost/benefit at this corpus size is still marginal. + +### Net adjustment to the tally +HIGH **8 → 7** (M-1 → MED). C-7 and C-1 are now **confirmed**, not inferred — they +should lead remediation. The dominant root theme (**un-normalized identity**) is +empirically validated: 35/36 ids in the "wrong" form, 13 papers mis-classified by +the older code path, 57 edges rescued by the newer one. + +--- + +# Runtime Validation — 2026-06-15 (C-4 and C-11 reproduced end-to-end) + +The two HIGH findings flagged "code-certain, runtime-confirmable" were exercised +against the **real** code paths (injected fake LLM; no Ollama, no DB needed). + +### C-11 — CONFIRMED ✅ (lead-ID collision → silent loss; stays HIGH) +Reproduced the exact CLI assembly (`cli.py:421` `connections + gaps + +improvements`, each stage numbered from `L-0001`): **4 distinct leads** (2 +connection, 1 gap, 1 improvement) → `write_leads` produced **2 files**. +`grounded/L-0001.md` survived as the **improvement** ("IMPROVEMENT-one"); the +connection and gap `L-0001` were silently overwritten. **2 of 4 leads lost.** + +### C-4 — CONFIRMED ✅ (zero-claim page bypasses quarantine; stays HIGH) +Drove the real `compile_all` with a fake LLM returning confident **uncited** prose +(→ 0 parseable claims). With `wiki_min_grounding_rate = 0.5` and an actual +grounding rate of `0.0`, the page was **published to `wiki/` and marked +compiled** (`report.compiled=['test-concept']`, `report.quarantined=[]`) — not +quarantined to `wiki/draft/`. The `total_claims > 0` guard (`wiki.py:730`) is the +exact bypass; the bug fires precisely when grounding evidence is weakest. + +### `nougat.py` — now fully reviewed (no new finding) +Folds into **R-6** (PDF-path resilience). Detail: `pdf_to_markdown` retries **only** +`httpx.ConnectError` with `wait_fixed(0)` (no 5xx retry, no back-off) and is +unwrapped at `ingest.py:183` — a Nougat 5xx aborts PDF ingest. + +**Closure:** every `codex/` module is deep-reviewed; the two runtime-confirmable +HIGHs are empirically reproduced. The **audit / finding-identification pass is +complete**. Remediation planning is the open next phase. + +--- + +# Migration Incident — 2026-06-15 (M-1 confirmed live, new privilege dimension) + +While running the C-7 re-ingest migration (`infra/reingest_canonical_ids.sh`), +M-1 manifested exactly as predicted — and exposed a dimension the static audit +did **not** capture. + +**What happened.** The helper ran `TRUNCATE papers CASCADE` (corpus wiped to 0), +then `ingest_all.sh` failed on the first chunk insert: +`UndefinedColumn: column "section" of relation "chunks" does not exist`. The live +`chunks` table predates the F-16 `section` column, and nothing had ever applied +that DDL to the live DB (M-1: `apply_schema` is never called; `schema.sql` is not +re-applyable). The ingest code was ahead of the live schema. + +**New finding — privilege separation (extends M-1).** The fix +`ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT` then failed with +`InsufficientPrivilege: must be owner of table chunks`. The live ownership: + +| table | owner | +|-------|-------| +| papers, chunks, citations, code_links, formulas, figures | `postgres` | +| paper_identifiers | `researcher` (app user) | + +The app role `researcher` has DML + `TRUNCATE` but **not** DDL on the +postgres-owned core tables. So a `codex migrate` / `apply_schema` invoked with the +application's `DATABASE_URL` would fail on every `ALTER` / `CREATE INDEX` against +those tables. **Implication for the M-1 fix:** making `schema.sql` idempotent is +necessary but *not sufficient* — the migration must run under a privileged role +(owner/superuser), via a separate admin connection, not the least-privilege app +user. This is now part of M-1's remediation scope. + +**Consequences observed.** +- Corpus left wiped (papers = 0) until the owner adds the column and re-ingest is + re-run — a destructive-then-fail sequence. +- Helper hardened (`fix(infra)` on `fix/audit-wave-2`): a **preflight** verifies + `chunks.section` exists *before* the `TRUNCATE`, aborting early with the + owner-level `ALTER` to run, so a schema gap can no longer cost the corpus. + +**Unblock (owner action):** +`sudo -u postgres psql -d papers -c "ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;"` +then re-run `bash ingest_all.sh`. + +This incident is the strongest possible validation of M-1 and should anchor its +Wave-3 fix: **idempotent `schema.sql` + a `codex migrate` command that connects as +a privileged role**, plus a documented owner/grant model for the app user. diff --git a/docs/audit/AUDIT-2026-06-15-loop-1.md b/docs/audit/AUDIT-2026-06-15-loop-1.md new file mode 100644 index 0000000..1704b55 --- /dev/null +++ b/docs/audit/AUDIT-2026-06-15-loop-1.md @@ -0,0 +1,227 @@ +# AUDIT — Development Loop 1 (main branch) + +**Auditor:** Audit-Loop (Opus) +**Date:** 2026-06-15 +**Scope:** entire `main` branch (HEAD `5dfa6e4`) +**Method:** static review of source + schema + tests, branch-wide pattern scan +(`ruff`, `mypy`, defect-class greps), full test run (`330 passed`). + +> Status legend — **HIGH** = fix before relying on the feature · +> **MED** = fix soon, correctness/maintainability · **LOW** = polish · +> **INFO** = accepted risk / no action, documented for the record. + +This document **only records findings**. Remediation planning is a separate +step (next phase of the audit loop). + +--- + +## 0. Headline + +The most recent dev-loop work (**F-15 citation graph** + the follow-up fixes +`cited_id→papers.id` resolution and the `paper_identifiers` alias table) is +**functionally green but unverified where it matters**: the SQL that *is* the +fix has zero real-database coverage, the live-corpus validation in the ADR +predates the fix, and the same ID-mismatch bug the fix solves still lives in a +parallel code path (`discover.py`). Separately, a **live DB password sits in an +un-ignored file** and the new table has **no migration path onto the live DB**. + +Test suite: **330 passed**, `ruff check` clean, `ruff format --check` clean, +`mypy` clean on reviewed modules. + +--- + +## 1. Security + +### S-1 — Live DB credential in an un-ignored file · **HIGH** +- **Evidence:** `.env.jetson-ingest` contains + `DATABASE_URL=postgresql://researcher:05761b87…@localhost:5433/papers`. + `.gitignore` ignores only the literal `.env` (line: `# Environment secrets — NEVER commit` → `.env`). + `git check-ignore .env.jetson-ingest` → **NOT IGNORED**. +- **Impact:** one `git add -A && git commit` leaks a real Postgres password to + history. The file is currently untracked (`??`), so the leak has **not** + happened yet — this is a near-miss, not an incident. +- **Note:** the same gap exposes any future `.env.<profile>` file. + +### S-2 — Non-constant-time token comparison (MCP HTTP) · **LOW** +- **Evidence:** `codex/mcp_server.py:340` — `if auth != f"Bearer {token}":`. +- **Impact:** theoretical timing side-channel on the Bearer token. Low on a + trusted LAN, but trivially fixable with `secrets.compare_digest`. + +### S-3 — DNS-rebinding protection disabled · **INFO (accepted)** +- **Evidence:** `codex/mcp_server.py:35-37`, + `TransportSecuritySettings(enable_dns_rebinding_protection=False)` (commit `d29dcf2`). +- **Assessment:** documented and mitigated — HTTP transport requires a Bearer + token (`_TokenAuth`) and is firewalled to LAN IPs (UFW). A rebound browser + request lacks the token. Residual risk only if the firewall assumption fails. + No action; recorded so the trade-off stays visible. + +--- + +## 2. Correctness + +### C-1 — `discover.py` still has the ID-format-mismatch bug F-15 fixed · **HIGH** +- **Evidence:** `codex/discover.py:20` — + `WHERE cited_id NOT IN (SELECT id FROM papers)`; same pattern in + `cocited_papers` (`discover.py:52-61`), `citing_papers`, `cited_by`. +- **Root cause:** `citations.cited_id` is a **mix** of OpenAlex W-IDs (OpenAlex + API path, `ingest.py:229-231` ← `sources/openalex.py:155`) and DOIs/arXiv + (GROBID/S2 paths). `papers.id` is the canonical DOI/arXiv. An ingested paper + cited by its **OpenAlex ID** is *not* in `(SELECT id FROM papers)`, so it is + wrongly reported as a discovery lead. +- **Impact:** `codex discover leads`, `codex discover cocited`, and the MCP + `discover_leads` tool over-report — foundational papers the user has **already + ingested** show up as "cited but not ingested". This is exactly the + D-05-class issue the F-15 graph path resolves via the + `papers.openalex_id` JOIN (`graph.py:48-56`) — but the fix was **not + propagated** to `discover.py`. Two divergent "dangling" implementations now + disagree on what "ingested" means. + +### C-2 — `paper_identifiers` JOIN is effectively inert · **MED** +- **Evidence:** `graph.py:50-56` second LEFT JOIN; populated only at + `ingest.py:153-161`, which inserts **the same** `paper.openalex_id` that + `papers.openalex_id` already holds. +- **Impact:** because ingest never records an alias that differs from + `papers.openalex_id`, the `pi` JOIN can only resolve what the `p` JOIN already + resolves. It adds value **only** in the narrow case where a paper's + `openalex_id` changes across re-ingests (old alias retained via + `ON CONFLICT DO NOTHING`). The table + unique index + seed INSERT + migration + burden buy near-zero behaviour today — speculative generality. No test + exercises an alias that diverges from `papers.openalex_id`. + +### C-3 — MCP `wiki_read` returns the slug as its own "sources" · **MED** +- **Evidence:** `codex/mcp_server.py:138-149` — comment says "Collect cited + bibkeys", code sets `sources = [concept_slug]`. +- **Impact:** the `sources` field is a placeholder masquerading as data; any MCP + client trusting it to list cited bibkeys gets the concept's own slug. + +--- + +## 3. Documentation / Code Drift + +### D-1 — ADR-F15 validation predates the shipped graph · **HIGH (integrity)** +- **Evidence:** ADR-F15 (commit `a1a6f45`) lands **before** the resolution fixes + `664906f` (`cited_id→papers.id`) and `5dfa6e4` (`paper_identifiers`). +- **Stale claims:** + - "Spike Result": *478 dangling nodes*, specific PageRank scores, + *Bobenko/Springborn rank 7*, *5.7 % spread* — all measured on the + **pre-resolution** (bipartite DOI→OpenAlex) graph. Post-fix, in-KB cited + papers resolve to DOIs, so dangling count and PageRank distribution change. + - "Decision → Graph Construction": "inter-KB citations are structurally + invisible" — **now false**. + - "Known Limitation: ID-Format Mismatch": "cross-citations… not represented" + — **now resolved by the code**, contradicting the section. +- **Impact:** the **GO** decision rests on numbers the shipping code no longer + produces. Validation must be re-run on the post-fix graph (or the ADR marked + validation-pending). + +### D-2 — `graph_cite_boost_alpha` doc describes the *fixed* bug · **MED** +- **Evidence:** `config.py:296-300` describes + `Final score = dense_score * (1 + alpha * pagerank_score)` — a **multiply**. + Actual code (`cli.py:107`) divides distance: + `boosted = distance / (1.0 + alpha * pr_score)`. The ADR explicitly states the + multiply form was the "critical bug corrected during the F-15 review gate". +- **Impact:** the config docstring still documents the inverted (buggy) formula. + +### D-3 — `paper_identifiers` tagged "F-17" in schema, shipped as F-15 fix · **LOW** +- **Evidence:** `infra/schema.sql:129` comment "F-17 Paper identifier aliases", + but the table is introduced by the F-15 graph follow-up (`5dfa6e4`) and the + ADR-F15 work. Feature-tag inconsistency; ADR-F15 doesn't mention the table at all. + +--- + +## 4. Schema / Migration + +### M-1 — New table has no migration path onto the live DB · **HIGH (operational)** +- **Evidence chain:** + 1. `apply_schema` docstring (`db.py:46-49`) claims "all DDL statements use + CREATE … IF NOT EXISTS" — **false**: `papers`, `chunks`, `citations`, + `code_links` and their indexes use bare `CREATE TABLE/INDEX` + (`schema.sql:16,33,56,68,30,41,…`). Only F-09/F-16/F-17 DDL is idempotent. + 2. `apply_schema` is **never called** anywhere in the app — no `init`/`migrate` + CLI command (`grep apply_schema` → only `db.py` + a mocked unit test). + 3. `ingest_all.sh` does **not** apply the schema; it only runs `codex ingest`. +- **Impact:** re-running `infra/schema.sql` against the **existing** Jetson DB + fails on the first `CREATE TABLE papers` (already exists) and rolls back the + whole script in one implicit transaction — so the new `paper_identifiers` + table (and its seed INSERT) is **never created** on a DB that predates it. + `build_citation_graph` then fails with `relation "paper_identifiers" does not + exist` on the live corpus. The migration step is currently manual and undocumented. + +--- + +## 5. Test Integrity + +### T-1 — The F-15 resolution SQL has zero real coverage · **HIGH** +- **Evidence:** every test in `tests/graph/test_graph.py` and + `tests/graph/test_cli.py` builds the graph from a `MagicMock` conn + (`_make_conn`, `_make_conn_with_paper_ids`). `test_cross_citation_uses_doi_when_in_kb` + (`test_graph.py:83-99`) **pre-resolves** the rows in Python and asserts the + graph builds — it tests the mock, not the `COALESCE(p.id, pi.paper_id, …)` + + two LEFT JOINs that *are* the fix. +- **Impact:** the substance of the two most recent commits is unverified. 330 + green tests give false confidence about the resolution logic, the `p.id IS NULL` + correlated guard, and the `paper_identifiers` unique-index behaviour. Needs an + integration test against a real (or `pytest`-managed) Postgres. + +### T-2 — `apply_schema` idempotency untested · **MED** +- **Evidence:** `tests/scaffold/test_db.py:24-32` mocks the conn and only checks + `execute`/`commit` are called. The (false) idempotency claim in M-1 is never + exercised. + +--- + +## 6. Design / Usability + +### U-1 — `graph report` prints bare IDs, no titles · **LOW** +- `cli.py:582-588` emits `score <id>` and bare dangling IDs — a mix of DOIs and + `W…` OpenAlex IDs with no title/year. The ADR's readable author/title table is + not what the CLI produces; output is hard to interpret. + +### U-2 — `--cite-boost` can only reorder within the top-`limit` page · **LOW** +- `cli.py:84-94` applies `ORDER BY … LIMIT limit` in SQL **before** the boost + (`cli.py:96-109`). A high-PR paper just outside the semantic cut can never be + pulled in. With α=0.3 and PR≈0.002 the effect is ~0.06 % — effectively a + within-page tie-breaker. Consistent with ADR intent, but worth stating as a + limitation (the flag does far less than "weight results by PageRank" implies). + +### U-3 — `graph report --json` drops the small-corpus warning · **LOW** +- `cli.py:557-570` returns before the `graph_min_corpus_size` warning + (`cli.py:572-577`). JSON consumers get neither the warning nor a flag for it. + +### U-4 — `graph.find_co_cited` implemented + tested but not wired to a CLI · **LOW** +- `graph.find_co_cited` (`graph.py:142`) is unit-tested but exposed by no `graph` + sub-command (only the separate SQL-based `discover cocited` is wired). ADR-F15 + R-43's co-citation is only half-surfaced. + +--- + +## 7. State of Tree (INFO — no defect) + +- The uncommitted diff (`wiki.py`, `synthesis.py`, `semanticscholar.py`, + `tests/synthesis/test_cli.py`, `tests/wiki/test_compile.py`) is a **legitimate + `ruff format` normalization** to `line-length=100`. `ruff format --check` + reports all 25 files formatted *with* these changes applied, meaning the + **committed** code was non-compliant. These edits should be **committed**, not + reverted. (Not a finding; clarifies the working-tree churn.) + +--- + +## 8. Coverage of this audit (honesty ledger) + +**Deep-reviewed:** `graph.py`, `db.py`, `infra/schema.sql`, `ingest.py`, +`discover.py`, `provenance.py` (scan), `mcp_server.py`, `cli.py` (search + +graph), `config.py` (graph settings), F-15 tests, ADR-F15; branch-wide pattern +scan (SQL injection, bare except, eval/exec/subprocess, secrets, timeouts). + +**Not yet deep-reviewed (no findings asserted):** +- `synthesis.py` grounding guard / claim parser (`_run_grounding_guard`, + `_parse_claims`) — the "zero-fact-leak (N-02)" core. Prompts reviewed; guard + logic not yet traced. +- `wiki.py` grounding guard + conflict detection (only the format diff seen). +- `sources/` (arxiv, openalex, semanticscholar) beyond citation shapes. +- `parsing/` (grobid, nougat, mathpix, figures, tex) — error handling in + `mathpix.py`/`figures.py` uses broad `except Exception` without `noqa` + (`mathpix.py:109,151`, `figures.py:125,152`) — flagged for a later pass. +- `embed.py`, `quality.py`, `models.py`. + +A second pass should close these before the audit is declared complete. diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md new file mode 100644 index 0000000..f016a79 --- /dev/null +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -0,0 +1,160 @@ +# DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session) + +**Status:** baseline scan done 2026-06-15; four investigation items open (DQ-1…DQ-4). +**Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory +of the audit conversation — everything needed to continue is in this file. + +--- + +## Why this exists — the second audit axis + +The code/pipeline audit (`AUDIT-2026-06-15-loop-1.md`, `-full-repo.md`) verified the +**loader is correct**: IDs are canonical, the citation graph is consistent, the +ingest pipeline does what it claims. It did **not** ask whether the *information +in the database is actually good*. A correct loader can faithfully load thin, +incomplete, or unrepresentative data. This document is that second axis — a +**data-quality** assessment of the live corpus, independent of code correctness. + +--- + +## How to reach the data (self-contained) + +- **Live DB:** Jetson PostgreSQL via an SSH tunnel (open it first): + ``` + ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103 + ``` +- **Credentials:** `.env.jetson-ingest` (gitignored) holds `DATABASE_URL` + (`postgresql://researcher:…@localhost:5433/papers`). The `researcher` role is + DML-only (read freely; no DDL). +- **Run probes:** `set -a; source .env.jetson-ingest; set +a` then + `PYTHONPATH=. .venv/bin/python` with `psycopg` + `psycopg.rows.dict_row`. + Reusable domain helpers: `codex.quality` (`is_quality_chunk`, `_bib_score`, + `classify_section`), `codex.graph`, `codex.discover`. +- **Corpus snapshot (2026-06-15, post canonical-id migration):** + **29 papers, 1507 chunks, 590 citations.** + +--- + +## Baseline — what is already GOOD (measured, no action needed) + +- **Chunk content is clean.** Re-running the F-16 quality gate over all stored + chunks: **0 / 1507 fail** `is_quality_chunk` (none too-short, none low-alpha / + OCR-ish, none bibliography-like). The gate did its job on the `.txt` ingest. +- **Chunk sizing healthy.** chars: min 478 / median 2913 / max 4154; **0 exact + duplicate** chunk bodies. +- **`section` column is low-signal** (98 % `body`; 13 `bibliography`, 9 `theorem`, + 2 `intro`, 1 `proof`). This confirms code-audit **R-12** — word-window chunks + rarely start at a section header. Not a data defect; the column is just weak. + +--- + +## Findings (open investigation items) + +### DQ-1 — Citation coverage: 12 / 29 papers (41 %) have ZERO citations · **HIGH** +- **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph + comes from only ~17 papers. The whole F-15 layer (PageRank / coupling / + co-citation / discovery leads) therefore runs on ~59 % of the corpus. +- **Sharpening — it is mostly a source-coverage limit, not an ingest bug:** + **11 of the 12** zero-citation papers *do* have an `openalex_id`, i.e. ingest + called `openalex.fetch_citations(openalex_id)` and OpenAlex returned an **empty + `referenced_works`** list. Only `1911.00966` has no `openalex_id` (OpenAlex + 404 → arXiv/S2 fallback, which also yielded nothing). +- **Zero-citation ids:** `1005.2698`, `1505.01341`, `1911.00966` (no oa), + `2206.13461`, `2305.10988`, `2310.17529`, `2601.22903`, `math/0001176`, + `math/0503219`, `math/0603097`, `10.14279/depositonce-20357`, + `10.14279/depositonce-5415`. (Pattern: arXiv preprints + the two `depositonce` + theses — works OpenAlex indexes without parsed references.) +- **To investigate next:** + 1. Confirm it is OpenAlex coverage, not a `fetch_citations` bug: for 2–3 of the + ids, hit `GET https://api.openalex.org/works/<openalex_id>` and check whether + `referenced_works` is genuinely empty server-side. + 2. If genuinely empty: enrich via the **Semantic Scholar references** path + (`semanticscholar.fetch_references("arXiv:<id>")`) as a *supplement* for + OpenAlex-empty papers — S2 often has references where OpenAlex doesn't. Note + S2 cited_ids are bare DOI/arXiv, so they flow through the existing + `RESOLVED_CITATIONS_SQL` resolver (audit C-1) fine. + 3. Decide whether the F-15 `graph_min_corpus_size` warning should also flag *low + citing-paper coverage*, not just paper count. +- **Acceptance:** either (a) citation coverage materially improves after an S2 + supplement, or (b) documented as an inherent OpenAlex-coverage limit with the + graph caveated accordingly. + +### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **MED** +- **Measured:** + - **No abstract (3):** `10.1007/978-3-642-17413-1_7`, + `10.1007/s00454-019-00132-8`, `1911.00966`. These get a **zero-vector** + `abstract_emb`, so paper-level semantic search can't place them. + - **No bibkey / year (1):** `1911.00966` only — and its authors array is empty + too. This paper is **fully degraded** (OpenAlex 404 → `Paper(id, title="")` + fallback): no abstract, no bibkey, no year, empty authors, no citations. With + no bibkey it cannot be `@cite`d and is invisible to wiki grounding (which keys + on bibkey). +- **To investigate next:** + 1. `1911.00966` — confirm the arXiv id is correct and whether OpenAlex/S2 has it + under a different id (DOI?); if recoverable, re-ingest to populate metadata. + If not, decide: keep as a degraded node or drop. + 2. The 2 no-abstract DOIs — check if OpenAlex has an `abstract_inverted_index` + that the mapper missed, or if the abstract is genuinely absent upstream. +- **Acceptance:** every paper has at least a bibkey + non-zero abstract embedding, + or the exceptions are documented with rationale. + +### DQ-3 — Content fidelity (chunks vs source `.txt`): NOT YET VERIFIED · **MED** +- **Open question:** does the stored chunk set faithfully reconstruct each source + file, or did the chunker / `filter_chunks` silently drop material (e.g. an + abstract, a section, math-heavy passages)? The whole corpus was ingested from + `.txt` (`PAPERS_DIR=/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt`), so + the `.tex`/`.pdf` paths never ran. +- **To investigate next:** for a sample of ~5 papers, compare + `len("".join(stored chunks))` against `len(source.txt)` (coverage %), and + eyeball the first/last chunk vs the file head/tail. Flag papers where coverage + is low (content lost) — F-16 dropping >X % of a paper is a signal. +- **Acceptance:** sampled papers retain ≳ the expected fraction of source text; + no paper is silently gutted by the quality gate. + +### DQ-4 — Retrieval quality: NOT YET MEASURED · **MED** +- **Open question:** end-to-end, do real queries return *relevant* results? Clean + chunks + a working index don't guarantee useful retrieval. +- **To investigate next:** run a handful of domain queries through the actual + search path (`codex search paper "<q>"` and the chunk-level MCP `search`), e.g. + "discrete conformal map", "circle packing rigidity", "combinatorial Yamabe + flow", "discrete Laplace-Beltrami operator", and judge whether the top-5 hits + are on-topic and point at the right papers. Cross-check a couple against + `--cite-boost` to see if the boost helps or hurts on this corpus. +- **Acceptance:** a short relevance table (query → top-5 → on/off-topic) good + enough to trust the KB for lookups, or a list of failure modes to fix. + +--- + +## Priority for the next session +1. **DQ-1** (biggest lever — 41 % of the corpus is invisible to the graph; the S2 + supplement is concrete and reuses existing code). +2. **DQ-4** (cheap, high-information — tells you if the KB is actually usable). +3. **DQ-2**, then **DQ-3**. + +All four are *data* work (queries + maybe a re-ingest/enrichment), not code-audit +work — the code is already remediated (see the AUDIT-* docs and PRs #12–#14). + +--- + +## Appendix — reproduce the baseline scan + +```python +import os, psycopg, statistics +from collections import Counter, defaultdict +from psycopg.rows import dict_row +from codex.config import Settings +from codex.quality import is_quality_chunk, _bib_score + +s = Settings() +with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as c: + papers = c.execute("SELECT id, bibkey, title, abstract, year, authors, openalex_id FROM papers").fetchall() + chunks = c.execute("SELECT paper_id, content, section FROM chunks").fetchall() + cit = c.execute("SELECT citing_id, count(*) n FROM citations GROUP BY citing_id").fetchall() + +# zero-citation papers (DQ-1) +cited = {r["citing_id"] for r in cit} +zero = [p["id"] for p in papers if p["id"] not in cited] +# metadata gaps (DQ-2): p["abstract"] empty / p["bibkey"] None / not p["authors"] +# F-16 re-gate (baseline): [ch for ch in chunks if not is_quality_chunk(ch["content"], settings=s)] +# section dist (R-12): Counter(ch["section"] or "(null)" for ch in chunks) +``` diff --git a/ingest_all.sh b/ingest_all.sh new file mode 100755 index 0000000..7cb8586 --- /dev/null +++ b/ingest_all.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Batch ingest — writes directly to Jetson PostgreSQL via SSH tunnel. +# +# Prerequisites: +# ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103 +# uv sync (done once) +# +# Usage: bash ingest_all.sh 2>&1 | tee ingest_all.log +set -euo pipefail + +PAPERS_DIR="/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt" +CODEX_DIR="$(cd "$(dirname "$0")" && pwd)" +ENV_FILE="$CODEX_DIR/.env.jetson-ingest" +TUNNEL_PORT=5433 + +# ── 1. Verify SSH tunnel is active ─────────────────────────────────────────── +if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then + echo "ERROR: SSH tunnel not active on port $TUNNEL_PORT" + echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103" + exit 1 +fi +echo "✓ SSH tunnel active on port $TUNNEL_PORT" + +# ── 1b. Clear macOS UF_HIDDEN flag on editable-install .pth files ───────────── +# Python 3.13 skips .pth files with this flag set (new in 3.13); uv sets it. +# --no-sync below prevents uv from re-syncing (which would re-set the flag). +SP="$CODEX_DIR/.venv/lib/python3.13/site-packages" +for pth in "$SP"/_editable_impl_codex*.pth; do + [[ -f "$pth" ]] && chflags nohidden "$pth" 2>/dev/null || true +done +echo "✓ Cleared hidden flags on editable .pth files" + +# ── 2. Load env (DATABASE_URL → Jetson via tunnel) ─────────────────────────── +# pydantic-settings reads DATABASE_URL directly from the environment. +# DOTENV_PATH is not supported; export each var explicitly. +set -a; source "$ENV_FILE"; set +a +export DATABASE_URL GROBID_URL OLLAMA_BASE_URL EMBEDDING_MODEL EMBEDDING_DIM OPENALEX_MAILTO +echo "✓ DATABASE_URL: $DATABASE_URL" + +# ── 3. Paper ID → source file mapping ──────────────────────────────────────── +# Format: "PAPER_ID|filename.txt" +# SKIP: farkas-kra-1992-riemann-surfaces.txt (textbook, no OpenAlex entry) +declare -a PAPERS=( + # arXiv papers + "math/0603097|springborn-2008-weighted-delaunay-hyperideal.txt" + "1005.2698|bobenko-pinkall-springborn-2015-discrete-conformal-maps.txt" + "math/0306167|luo-2004-combinatorial-yamabe-flow.txt" + "math/0203250|bobenko-springborn-2004-circle-patterns.txt" + "math/0503219|bobenko-springborn-2007-discrete-laplace-beltrami.txt" + "2310.17529|bobenko-lutz-2025-non-euclidean-dce.txt" + "2305.10988|bobenko-lutz-2024-decorated-conformal-maps.txt" + "2206.13461|lutz-2023-canonical-tessellations.txt" + "2601.22903|bowers-bowers-lutz-2026-koebe-rigidity.txt" + "1911.00966|pinkall-springborn-2021-liouville.txt" + "1505.01341|born-bucking-springborn-2015-quasiconformal.txt" + "0906.1560|glickenstein-2011-discrete-conformal-variations.txt" + "math/0001176|rivin-schlenker-2000-schlafli-formula-arxiv-preprint.txt" + # DOI papers + "10.1007/s00454-019-00132-8|springborn-2020-ideal-hyperbolic-polyhedra.txt" + "10.1145/1964921.1964997|alexa-wardetzky-2011-discrete-laplacians-polygonal.txt" + "10.1111/cgf.13931|bunge-herholz-kazhdan-botsch-2020-polygon-laplacian.txt" + "10.1145/3450626.3459763|gillespie-springborn-crane-2021-discrete-conformal-equivalence.txt" + "10.1145/3306346.3323042|sharp-soliman-crane-2019-navigating-intrinsic-triangulations.txt" + "10.1145/3197517.3201367|soliman-slepcv-crane-2018-optimal-cone-singularities.txt" + "10.1145/3132705|sawhney-crane-2017-boundary-first-flattening.txt" + "10.1145/2767000|knoppel-crane-pinkall-schroder-2015-stripe-patterns.txt" + "10.5555/1070432.1070581|erickson-whittlesey-2005-greedy-homotopy-homology.txt" + "10.1145/1185657.1185665|desbrun-kanso-tong-2006-discrete-differential-forms.txt" + "10.1080/10586458.1993.10504266|pinkall-polthier-1993-computing-discrete-minimal-surfaces.txt" + "10.1007/s11040-021-09394-2|bobenko-bucking-2021-period-matrices.txt" + "10.14279/depositonce-20357|lutz-2024-thesis.txt" + "10.14279/depositonce-5415|sechelmann-2016-thesis.txt" + # Book chapters (txt contains full book; ID = cited chapter) + "10.1007/0-387-29555-0_13|precopa-molnar-eds-2006-non-euclidean-geometries-book.txt" + "10.1007/978-3-642-17413-1_7|bobenko-klein-eds-2011-computational-approach-riemann-surfaces-book.txt" +) + +# ── 4. Ingest loop ──────────────────────────────────────────────────────────── +OK=0; FAIL=0; SKIP=0 +FAILED_IDS=() + +cd "$CODEX_DIR" + +for entry in "${PAPERS[@]}"; do + PAPER_ID="${entry%%|*}" + FILENAME="${entry##*|}" + SOURCE="$PAPERS_DIR/$FILENAME" + + if [[ ! -f "$SOURCE" ]]; then + echo "SKIP (file missing): $FILENAME" + (( SKIP++ )) || true + continue + fi + + echo "" + echo "── Ingesting: $PAPER_ID" + echo " source: $FILENAME" + + if PYTHONPATH="$CODEX_DIR" "$CODEX_DIR/.venv/bin/codex" ingest "$PAPER_ID" --source "$SOURCE"; then + (( OK++ )) || true + else + echo "FAILED: $PAPER_ID" + FAILED_IDS+=("$PAPER_ID") + (( FAIL++ )) || true + fi +done + +# ── 5. Summary ──────────────────────────────────────────────────────────────── +echo "" +echo "════════════════════════════════════════" +echo "Ingest complete: $OK OK, $FAIL FAILED, $SKIP SKIPPED" +if [[ ${#FAILED_IDS[@]} -gt 0 ]]; then + echo "Failed IDs:" + for id in "${FAILED_IDS[@]}"; do echo " $id"; done +fi +echo "════════════════════════════════════════" diff --git a/tests/synthesis/test_cli.py b/tests/synthesis/test_cli.py index 7791d16..0d86145 100644 --- a/tests/synthesis/test_cli.py +++ b/tests/synthesis/test_cli.py @@ -55,9 +55,7 @@ def isolated_leads_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: monkeypatch.setattr("codex.cli.get_settings", lambda: settings, raising=False) monkeypatch.setattr("codex.synthesis.get_settings", lambda: settings) - monkeypatch.setattr( - "codex.config.get_settings", lambda: settings, raising=False - ) + monkeypatch.setattr("codex.config.get_settings", lambda: settings, raising=False) return leads_path diff --git a/tests/wiki/test_compile.py b/tests/wiki/test_compile.py index 02da74f..53ff8fc 100644 --- a/tests/wiki/test_compile.py +++ b/tests/wiki/test_compile.py @@ -201,9 +201,7 @@ def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored() This test verifies that a claim whose text = the last sentence grounds correctly. """ # The last sentence of the claim text is directly grounded in the fixture chunk. - grounded_sentence = ( - "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)" - ) + grounded_sentence = "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)" claim = Claim( text=grounded_sentence, bibkey="Springborn2008", @@ -523,9 +521,7 @@ def test_idempotent_no_rewrite_when_unchanged( call_count = 0 original_compile = compile_concept - def counting_compile( - concept: Concept, chunks: list[Any], **kwargs: Any - ) -> ConceptPage: + def counting_compile(concept: Concept, chunks: list[Any], **kwargs: Any) -> ConceptPage: nonlocal call_count call_count += 1 return original_compile(concept, chunks, **kwargs) @@ -569,9 +565,7 @@ def test_force_all_recompiles_even_when_unchanged( call_count = 0 original_compile = compile_concept - def counting_compile( - concept: Concept, chunks: list[Any], **kwargs: Any - ) -> ConceptPage: + def counting_compile(concept: Concept, chunks: list[Any], **kwargs: Any) -> ConceptPage: nonlocal call_count call_count += 1 return original_compile(concept, chunks, **kwargs)