b61d84d4bd
- codex/quality.py: 3-signal filter (length / alpha-ratio / bib-score) + rule-based section classifier + run_quality_pass retroactive DB pass - codex/ingest.py: promote quality imports to module level; apply filter_chunks before embedding; store section column in chunks INSERT - codex/config.py: CHUNK_MIN_CHARS / CHUNK_MIN_ALPHA_RATIO / CHUNK_MAX_BIB_SCORE - infra/schema.sql: ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT - .env.example: document F-16 quality thresholds - codex/cli.py: quality run sub-command (scope by --paper-id or all papers) - tests/quality/: 35 new tests covering all quality functions - tests/ingest/: patch filter_chunks in source-path tests to isolate ingest Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
35 lines
1.5 KiB
Plaintext
35 lines
1.5 KiB
Plaintext
# Copy this file to .env and fill in your values.
|
|
# Never commit .env to version control.
|
|
|
|
# PostgreSQL connection string (psycopg / asyncpg format)
|
|
# Example: postgresql://researcher:change_me@localhost:5432/papers
|
|
DATABASE_URL=postgresql://researcher:change_me@localhost:5432/papers
|
|
|
|
# GROBID service base URL (containerised — see infra/docker-compose.yml)
|
|
GROBID_URL=http://localhost:8070
|
|
|
|
# Ollama base URL for optional local LLM Q&A layer
|
|
OLLAMA_BASE_URL=http://localhost:11434
|
|
|
|
# Sentence-transformers model for dense embeddings.
|
|
# BGE-M3 (BAAI/bge-m3) produces 1024-dimensional vectors and supports
|
|
# dense + sparse (hybrid) retrieval. Change together with EMBEDDING_DIM.
|
|
EMBEDDING_MODEL=BAAI/bge-m3
|
|
|
|
# Dimension of the embedding vectors. Must match EMBEDDING_MODEL output.
|
|
# BGE-M3 = 1024 | Jina v4 = 2048 | Qwen3-Embedding-0.6B = 1024
|
|
EMBEDDING_DIM=1024
|
|
|
|
# E-mail address for the OpenAlex Polite Pool (faster rate limits).
|
|
# Required by OpenAlex ToS when making automated requests.
|
|
OPENALEX_MAILTO=you@example.com
|
|
|
|
# Nougat HTTP-Server (Jetson: http://192.168.178.103:8080 | lokal: http://localhost:8080)
|
|
# Used by `codex ingest <id> --rich` for full-PDF Mathpix Markdown output.
|
|
NOUGAT_URL=http://localhost:8080
|
|
|
|
# F-16 Chunk Quality Gate thresholds (all optional — defaults shown)
|
|
CHUNK_MIN_CHARS=60 # Discard chunks shorter than this many characters
|
|
CHUNK_MIN_ALPHA_RATIO=0.40 # Discard chunks with < 40% alphabetic characters
|
|
CHUNK_MAX_BIB_SCORE=0.70 # Discard chunks scoring above this bibliography threshold
|