Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
7 changes: 7 additions & 0 deletions .claude/agents/bench-analyzer.md
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,13 @@ Always parse JSONL with `bench-swe extract`. These files are too large for the

- Search results use XML-like tags: `<result:file filename="...">` and
`<result:chunk line-start="N" line-end="N" symbol="..." kind="..." score="N.NN">`.
BM25 related context carries `related-to="..." relation="..."` instead of a
score. A final `truncated="true"` attribute means the output budget
(`max_tokens`, default 4000 estimated tokens = 16,000 bytes) shortened the
snippet; line ranges still describe the whole source range. A trailing
`Output capped at max_tokens=...` footer reports shown, truncated and omitted
results. Treat truncated or omitted gold code as a budget effect, not a
ranking miss.
- Some tool_result entries may be permission denials (e.g., "Claude requested
permissions to use..."). Classify these as ERROR.

Expand Down
3 changes: 3 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,6 @@ internal/sqlitevec/sqlite-vec.[ch] -text -whitespace linguist-vendored
# core.autocrlf setting.
*.bat text eol=crlf
*.cmd text eol=crlf

# Preserve the BOM and CRLF bytes exercised by the retrieval line-number cases.
testdata/retrieval/corpora/unicode-bom/web/invoice_lines.ts -text
26 changes: 23 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,24 @@ jobs:
version: latest
args: --build-tags=fts5

bm25_e2e:
name: BM25 E2E
runs-on: ubuntu-latest
if: github.actor != 'release-please[bot]'
permissions:
contents: read
steps:
- uses: actions/checkout@v4

- uses: actions/setup-go@v5
with:
go-version: '1.26'
cache: true

# No embedding service is provisioned: BM25 must work offline.
- name: BM25 E2E tests
run: make e2e-bm25

semantic_e2e:
name: Semantic E2E
runs-on: ubuntu-latest
Expand Down Expand Up @@ -105,7 +123,7 @@ jobs:
'curl -sf http://localhost:11434/api/pull -d ''{"name":"all-minilm"}'''

- name: E2E tests
run: make e2e
run: make e2e-semantic
env:
OLLAMA_HOST: http://localhost:11434
LUMEN_EMBED_MODEL: all-minilm
Expand Down Expand Up @@ -252,20 +270,22 @@ jobs:
run: node scripts/test_run_spawn.js

# The repository ruleset requires the "E2E" check. This aggregator keeps
# that stable context while making semantic tests, fresh host installs,
# that stable context while making BM25 and semantic tests, fresh host installs,
# native artifacts, and every launcher platform transitively merge-gating.
e2e:
name: E2E
runs-on: ubuntu-latest
if: ${{ always() && github.actor != 'release-please[bot]' }}
needs: [semantic_e2e, build, scripts]
needs: [bm25_e2e, semantic_e2e, build, scripts]
steps:
- name: Require every distribution and runtime check
env:
BM25_E2E_RESULT: ${{ needs.bm25_e2e.result }}
SEMANTIC_E2E_RESULT: ${{ needs.semantic_e2e.result }}
BUILD_RESULT: ${{ needs.build.result }}
SCRIPTS_RESULT: ${{ needs.scripts.result }}
run: |
test "$BM25_E2E_RESULT" = success
test "$SEMANTIC_E2E_RESULT" = success
test "$BUILD_RESULT" = success
test "$SCRIPTS_RESULT" = success
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ lumen

bench-swe/bench-swe
bench-swe/overview.txt
bench-swe/node_modules/

**/.claude/worktrees
**/.claire
Expand All @@ -31,3 +32,6 @@ bench-swe/overview.txt
# scripts/ test-only npm install (cross-spawn for spawn-semantics test)
scripts/node_modules/
scripts/package-lock.json

# Locally generated analysis artifacts
/output/
6 changes: 4 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ runtime behavior for this repository outside the Claude plugin install path.

### Testing

- **Unit + integration**: `go test ./...`
- **E2E tests**: `go test -tags e2e ./...` (requires Ollama/LM Studio running)
- **Unit + integration**: `go test -tags fts5 ./...` (or `make test`; BM25
needs SQLite FTS5, which go-sqlite3 only compiles with the `fts5` tag)
- **E2E tests**: `go test -tags fts5,e2e ./...` (requires Ollama/LM Studio
running)
- **All tests must pass before commit**
- Coverage tracked but not enforced
- **TDD required**: All features and bug fixes must follow red/green TDD — write
Expand Down
31 changes: 28 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ GOFLAGS := -tags=$(GOTAGS)

XGORELEASER_IMAGE := oryd/xgoreleaser:1.26.0-2.14.1

.PHONY: build build-local build-bench-swe test e2e e2e-lang lint vet tidy clean format plugin-dev
.PHONY: build build-local build-bench-swe test eval e2e e2e-bm25 e2e-semantic e2e-semantic-catalog e2e-lang lint vet tidy clean format plugin-dev

build:
docker run --platform linux/amd64 --mount type=bind,source="$$(pwd)",target=/project \
Expand All @@ -20,11 +20,36 @@ build-bench-swe:
test:
CGO_ENABLED=1 $(GO) test $(GOFLAGS) ./...

# Offline retrieval catalog report (BM25, no embedding service).
eval:
CGO_ENABLED=1 LUMEN_EVAL_REPORT=$${LUMEN_EVAL_REPORT:-$(CURDIR)/bin/retrieval-bm25.md} \
$(GO) test $(GOFLAGS) -count=1 -run 'TestRetrievalCatalogBM25' ./internal/index

# All E2E suites. Semantic suites skip without Ollama + all-minilm.
e2e:
CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=10m -v -count=1 -run 'TestE2E_' ./...
CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=15m -v -count=1 -run 'TestE2E_' ./...

# BM25 executable suites; must pass with no embedding service.
e2e-bm25:
CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=15m -v -count=1 \
-run 'TestE2E_BM25|TestE2E_Catalog_BM25|TestLang_BM25SnapshotDiagnostics' .

# Semantic executable suites; a missing embedding service is a failure. Each
# test starts its own server, and one local Ollama serializes embeddings: on
# many-core machines the default parallelism queues requests past the
# failover timeouts and marks the backend unhealthy.
E2E_PARALLEL ?= 4

e2e-semantic:
LUMEN_E2E_SEMANTIC=require CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=20m -v -count=1 -parallel $(E2E_PARALLEL) -run 'TestE2E_' ./...

# Diagnostic: the retrieval catalog with real embeddings (slow).
e2e-semantic-catalog:
LUMEN_E2E_SEMANTIC=require LUMEN_E2E_SEMANTIC_CATALOG=1 CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e \
-timeout=120m -v -count=1 -run 'TestE2E_Catalog_Semantic' .

e2e-lang:
CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=30m -v -count=1 -run 'TestLang_' ./...
LUMEN_E2E_SEMANTIC=require CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=30m -v -count=1 -parallel $(E2E_PARALLEL) -run 'TestLang_' ./...

lint:
golangci-lint run
Expand Down
Loading
Loading