Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 57 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -251,21 +251,76 @@ jobs:
- name: Spawn-semantics regression guard
run: node scripts/test_run_spawn.js

worktree_e2e:
name: Worktree efficiency (${{ matrix.os }})
runs-on: ${{ matrix.os }}
if: github.actor != 'release-please[bot]'
permissions:
contents: read
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- uses: actions/checkout@v4

- uses: actions/setup-go@v5
with:
go-version: '1.26'
cache: true

# Offline: real Lumen binaries, Git worktrees, SessionStart children,
# and MCP stdio against a deterministic embedding server.
- name: Worktree integration suite
run: make e2e-worktrees

worktree_mutations:
name: Worktree mutation gate
runs-on: ubuntu-latest
if: github.actor != 'release-please[bot]'
permissions:
contents: read
timeout-minutes: 60
steps:
- uses: actions/checkout@v4

- uses: actions/setup-go@v5
with:
go-version: '1.26'
cache: true

- name: Curated production mutants
run: make test-worktree-mutations MUTATIONS_OUT="$RUNNER_TEMP/worktree-mutations"

- name: Upload mutation evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: worktree-mutations
path: ${{ runner.temp }}/worktree-mutations
if-no-files-found: ignore

# The repository ruleset requires the "E2E" check. This aggregator keeps
# that stable context while making semantic tests, fresh host installs,
# native artifacts, and every launcher platform transitively merge-gating.
# native artifacts, every launcher platform, the offline worktree suite, and
# its production mutation gate transitively merge-gating.
e2e:
name: E2E
runs-on: ubuntu-latest
if: ${{ always() && github.actor != 'release-please[bot]' }}
needs: [semantic_e2e, build, scripts]
needs: [semantic_e2e, build, scripts, worktree_e2e, worktree_mutations]
steps:
- name: Require every distribution and runtime check
env:
SEMANTIC_E2E_RESULT: ${{ needs.semantic_e2e.result }}
BUILD_RESULT: ${{ needs.build.result }}
SCRIPTS_RESULT: ${{ needs.scripts.result }}
WORKTREE_E2E_RESULT: ${{ needs.worktree_e2e.result }}
WORKTREE_MUTATIONS_RESULT: ${{ needs.worktree_mutations.result }}
run: |
test "$SEMANTIC_E2E_RESULT" = success
test "$BUILD_RESULT" = success
test "$SCRIPTS_RESULT" = success
test "$WORKTREE_E2E_RESULT" = success
test "$WORKTREE_MUTATIONS_RESULT" = success
3 changes: 3 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ make build # Cross-compile via Docker/xgoreleaser
make build-local # Build binary to bin/ (CGO_ENABLED=1)
make test # Run unit + integration tests
make e2e # Run E2E tests (requires Ollama/LM Studio)
make e2e-worktrees # Offline worktree efficiency suite (real binaries + Git)
make test-worktree-mutations # Curated production mutation gate for that suite
make lint # Run golangci-lint
make vet # Run go vet
make format # Format code & markdown
Expand Down Expand Up @@ -168,6 +170,7 @@ their repo-root counterparts. Install-specific entrypoints live in `.codex/`,
│ ├── chunker/ # Go AST parsing → chunks
│ ├── embedder/ # Ollama/LM Studio HTTP client
│ └── merkle/ # Change detection (SHA-256 tree)
├── e2e/worktrees/ # Offline worktree integration suite + mutation gate
└── testdata/ # Fixtures for E2E tests
```

Expand Down
11 changes: 10 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ GOFLAGS := -tags=$(GOTAGS)

XGORELEASER_IMAGE := oryd/xgoreleaser:1.26.0-2.14.1

.PHONY: build build-local build-bench-swe test e2e e2e-lang lint vet tidy clean format plugin-dev
.PHONY: build build-local build-bench-swe test e2e e2e-lang e2e-worktrees test-worktree-mutations lint vet tidy clean format plugin-dev

build:
docker run --platform linux/amd64 --mount type=bind,source="$$(pwd)",target=/project \
Expand All @@ -26,6 +26,15 @@ e2e:
e2e-lang:
CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=30m -v -count=1 -run 'TestLang_' ./...

# Offline: real Lumen binaries, Git worktrees, and MCP stdio against a
# deterministic embedding server. Requires git; no Ollama or LM Studio.
e2e-worktrees:
CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),worktree_e2e -timeout=20m -v -count=1 ./e2e/worktrees/

# Curated production mutants must each fail their designated assertion.
test-worktree-mutations:
CGO_ENABLED=1 $(GO) run ./e2e/worktrees/mutate $(if $(MUTATIONS_OUT),-out $(MUTATIONS_OUT))

lint:
golangci-lint run

Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,9 @@ Indexes with an indexer currently running are always kept.
**Git worktrees** are detected automatically. A new worktree attaches unchanged
path-and-content revisions directly from the repository collection and embeds
only missing chunk inputs. Removing an old worktree drops its memberships;
shared revisions and vectors remain until their final reference disappears.
shared revisions and vectors remain while referenced. Ordinary edits retain
recently unused revisions under a 256 MiB, seven-day logical cache policy;
explicit cleanup can reclaim them sooner.
Legacy per-worktree indexes migrate lazily, reusing unchanged float32 vectors
without contacting the embedding backend.

Expand Down
2 changes: 1 addition & 1 deletion cmd/clean.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func cleanIndex(reporter cleanReporter, name, hashDir string, days int, cutoff t
return false, store.CleanupStats{}, fmt.Errorf("clean shared collection %s: %w", name, sharedErr)
}
if sharedStats.ProjectsLeft > 0 {
reporter.Info(fmt.Sprintf("Cleaned %s: removed %d projects and %d vectors.", name, sharedStats.ProjectsRemoved, sharedStats.VectorsRemoved))
reporter.Info(fmt.Sprintf("Cleaned %s: removed %d projects, %d vectors, and %d orphan scan caches.", name, sharedStats.ProjectsRemoved, sharedStats.VectorsRemoved, sharedStats.ScanCachesRemoved))
return false, sharedStats, nil
}
// Empty collections have no future owner and can be removed as a
Expand Down
5 changes: 4 additions & 1 deletion cmd/embedder.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@ package cmd
import (
"github.com/ory/lumen/internal/config"
"github.com/ory/lumen/internal/embedder"
"path/filepath"
)

// newEmbedder creates a FailoverEmbedder from the given ConfigService.
func newEmbedder(cfg *config.ConfigService) *embedder.FailoverEmbedder {
return embedder.NewFailoverEmbedder(cfg)
emb := embedder.NewFailoverEmbedder(cfg)
emb.SetComputeBudget(filepath.Join(config.XDGDataDir(), "lumen", "compute"))
return emb
}
212 changes: 212 additions & 0 deletions cmd/experiment_input.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
// Copyright 2026 Aeneas Rekkas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.

package cmd

import (
"encoding/json"
"fmt"
"math"
"os"
"slices"

"github.com/ory/lumen/internal/embedder"
"github.com/spf13/cobra"
)

// The experiment accepts explicit relevance judgments rather than inventing
// synthetic quality claims from vector distances alone.
type inputDataset struct {
Chunks []struct {
Path string `json:"path"`
Body string `json:"body"`
} `json:"chunks"`
Queries []struct {
Text string `json:"text"`
RelevantPaths []string `json:"relevant_paths"`
} `json:"queries"`
}

type inputMetrics struct {
Mode string `json:"mode"`
UniqueInputs int `json:"unique_inputs"`
Recall float64 `json:"recall_at_k"`
MRR float64 `json:"mrr_at_k"`
}

func init() {
command := &cobra.Command{Use: "experiment-input <dataset.json>", Short: "Compare path-aware and body-only embeddings against labeled queries", Args: cobra.ExactArgs(1), RunE: runInputExperiment}
command.Flags().Int("k", 10, "number of retrieved chunks to evaluate")
command.Flags().String("model", "", "embedding model")
command.Flags().String("backend", "", "embedding backend")
rootCmd.AddCommand(command)
}

func runInputExperiment(cmd *cobra.Command, args []string) error {
data, err := os.ReadFile(args[0])
if err != nil {
return err
}
var dataset inputDataset
if err := json.Unmarshal(data, &dataset); err != nil {
return err
}
k, _ := cmd.Flags().GetInt("k")
if k < 1 || len(dataset.Chunks) == 0 || len(dataset.Queries) == 0 {
return fmt.Errorf("positive k, chunks, and labeled queries are required")
}
paths := map[string]bool{}
for _, chunk := range dataset.Chunks {
if chunk.Path == "" || chunk.Body == "" {
return fmt.Errorf("each chunk requires path and body")
}
paths[chunk.Path] = true
}
for _, query := range dataset.Queries {
if query.Text == "" || len(query.RelevantPaths) == 0 {
return fmt.Errorf("each query requires text and relevant_paths")
}
for _, path := range query.RelevantPaths {
if !paths[path] {
return fmt.Errorf("relevance label %q has no corpus chunk", path)
}
}
}
cfg, err := loadConfigWithFlags(cmd)
if err != nil {
return err
}
emb := newEmbedder(cfg)
model, dimensions := emb.ModelName(), emb.Dimensions()
ctx := embedder.BackgroundContext(cmd.Context())
encode := func(texts []string) ([][]float32, error) {
var result [][]float32
for start := 0; start < len(texts); start += 32 {
batch := texts[start:min(start+32, len(texts))]
vectors, err := emb.Embed(ctx, batch)
if err != nil {
return nil, err
}
if emb.ModelName() != model || emb.Dimensions() != dimensions {
return nil, fmt.Errorf("embedding profile changed during experiment; retry with a single model")
}
if len(vectors) != len(batch) {
return nil, fmt.Errorf("backend returned an unexpected vector count")
}
result = append(result, vectors...)
}
return result, nil
}
queries := make([]string, len(dataset.Queries))
for i, q := range dataset.Queries {
queries[i] = q.Text
}
queryVectors, err := encode(queries)
if err != nil {
return err
}
var report []inputMetrics
for _, mode := range []string{"path", "body"} {
positions := make([]int, len(dataset.Chunks))
unique := map[string]int{}
var inputs []string
for i, chunk := range dataset.Chunks {
text := chunk.Body
if mode == "path" {
text = "// " + chunk.Path + "\n" + text
}
position, ok := unique[text]
if !ok {
position = len(inputs)
unique[text] = position
inputs = append(inputs, text)
}
positions[i] = position
}
vectors, err := encode(inputs)
if err != nil {
return err
}
metrics := inputMetrics{Mode: mode, UniqueInputs: len(inputs)}
for qi, query := range dataset.Queries {
type match struct {
path string
score float64
ordinal int
}
matches := make([]match, len(dataset.Chunks))
for i, chunk := range dataset.Chunks {
score, err := cosineScore(queryVectors[qi], vectors[positions[i]])
if err != nil {
return err
}
matches[i] = match{chunk.Path, score, i}
}
slices.SortFunc(matches, func(a, b match) int {
if a.score > b.score {
return -1
}
if a.score < b.score {
return 1
}
if a.path < b.path {
return -1
}
if a.path > b.path {
return 1
}
return a.ordinal - b.ordinal
})
relevant := map[string]bool{}
for _, path := range query.RelevantPaths {
relevant[path] = true
}
found := map[string]bool{}
rr := 0.0
for rank, m := range matches[:min(k, len(matches))] {
if relevant[m.path] {
found[m.path] = true
if rr == 0 {
rr = 1 / float64(rank+1)
}
}
}
metrics.Recall += float64(len(found)) / float64(len(relevant))
metrics.MRR += rr
}
metrics.Recall /= float64(len(dataset.Queries))
metrics.MRR /= float64(len(dataset.Queries))
report = append(report, metrics)
}
encoder := json.NewEncoder(cmd.OutOrStdout())
encoder.SetIndent("", " ")
return encoder.Encode(struct {
Model string `json:"model"`
K int `json:"k"`
Results []inputMetrics `json:"results"`
}{model, k, report})
}

func cosineScore(a, b []float32) (float64, error) {
if len(a) == 0 || len(a) != len(b) {
return 0, fmt.Errorf("incompatible embedding dimensions")
}
var dot, aa, bb float64
for i, x := range a {
y := float64(b[i])
v := float64(x)
dot += v * y
aa += v * v
bb += y * y
}
if aa == 0 || bb == 0 {
return 0, fmt.Errorf("backend returned a zero vector")
}
score := dot / math.Sqrt(aa*bb)
if math.IsNaN(score) || math.IsInf(score, 0) {
return 0, fmt.Errorf("backend returned non-finite vectors")
}
return score, nil
}
Loading
Loading