diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 786529ce..862c241d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -251,21 +251,76 @@ jobs: - name: Spawn-semantics regression guard run: node scripts/test_run_spawn.js + worktree_e2e: + name: Worktree efficiency (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + if: github.actor != 'release-please[bot]' + permissions: + contents: read + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.26' + cache: true + + # Offline: real Lumen binaries, Git worktrees, SessionStart children, + # and MCP stdio against a deterministic embedding server. + - name: Worktree integration suite + run: make e2e-worktrees + + worktree_mutations: + name: Worktree mutation gate + runs-on: ubuntu-latest + if: github.actor != 'release-please[bot]' + permissions: + contents: read + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.26' + cache: true + + - name: Curated production mutants + run: make test-worktree-mutations MUTATIONS_OUT="$RUNNER_TEMP/worktree-mutations" + + - name: Upload mutation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: worktree-mutations + path: ${{ runner.temp }}/worktree-mutations + if-no-files-found: ignore + # The repository ruleset requires the "E2E" check. This aggregator keeps # that stable context while making semantic tests, fresh host installs, - # native artifacts, and every launcher platform transitively merge-gating. + # native artifacts, every launcher platform, the offline worktree suite, and + # its production mutation gate transitively merge-gating. e2e: name: E2E runs-on: ubuntu-latest if: ${{ always() && github.actor != 'release-please[bot]' }} - needs: [semantic_e2e, build, scripts] + needs: [semantic_e2e, build, scripts, worktree_e2e, worktree_mutations] steps: - name: Require every distribution and runtime check env: SEMANTIC_E2E_RESULT: ${{ needs.semantic_e2e.result }} BUILD_RESULT: ${{ needs.build.result }} SCRIPTS_RESULT: ${{ needs.scripts.result }} + WORKTREE_E2E_RESULT: ${{ needs.worktree_e2e.result }} + WORKTREE_MUTATIONS_RESULT: ${{ needs.worktree_mutations.result }} run: | test "$SEMANTIC_E2E_RESULT" = success test "$BUILD_RESULT" = success test "$SCRIPTS_RESULT" = success + test "$WORKTREE_E2E_RESULT" = success + test "$WORKTREE_MUTATIONS_RESULT" = success diff --git a/CLAUDE.md b/CLAUDE.md index 77d5d867..de7f4c9b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,6 +96,8 @@ make build # Cross-compile via Docker/xgoreleaser make build-local # Build binary to bin/ (CGO_ENABLED=1) make test # Run unit + integration tests make e2e # Run E2E tests (requires Ollama/LM Studio) +make e2e-worktrees # Offline worktree efficiency suite (real binaries + Git) +make test-worktree-mutations # Curated production mutation gate for that suite make lint # Run golangci-lint make vet # Run go vet make format # Format code & markdown @@ -168,6 +170,7 @@ their repo-root counterparts. Install-specific entrypoints live in `.codex/`, │ ├── chunker/ # Go AST parsing → chunks │ ├── embedder/ # Ollama/LM Studio HTTP client │ └── merkle/ # Change detection (SHA-256 tree) +├── e2e/worktrees/ # Offline worktree integration suite + mutation gate └── testdata/ # Fixtures for E2E tests ``` diff --git a/Makefile b/Makefile index 362c1473..3a2706e8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GOFLAGS := -tags=$(GOTAGS) XGORELEASER_IMAGE := oryd/xgoreleaser:1.26.0-2.14.1 -.PHONY: build build-local build-bench-swe test e2e e2e-lang lint vet tidy clean format plugin-dev +.PHONY: build build-local build-bench-swe test e2e e2e-lang e2e-worktrees test-worktree-mutations lint vet tidy clean format plugin-dev build: docker run --platform linux/amd64 --mount type=bind,source="$$(pwd)",target=/project \ @@ -26,6 +26,15 @@ e2e: e2e-lang: CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),e2e -timeout=30m -v -count=1 -run 'TestLang_' ./... +# Offline: real Lumen binaries, Git worktrees, and MCP stdio against a +# deterministic embedding server. Requires git; no Ollama or LM Studio. +e2e-worktrees: + CGO_ENABLED=1 $(GO) test -tags=$(GOTAGS),worktree_e2e -timeout=20m -v -count=1 ./e2e/worktrees/ + +# Curated production mutants must each fail their designated assertion. +test-worktree-mutations: + CGO_ENABLED=1 $(GO) run ./e2e/worktrees/mutate $(if $(MUTATIONS_OUT),-out $(MUTATIONS_OUT)) + lint: golangci-lint run diff --git a/README.md b/README.md index 8d33e76b..2b82433d 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,9 @@ Indexes with an indexer currently running are always kept. **Git worktrees** are detected automatically. A new worktree attaches unchanged path-and-content revisions directly from the repository collection and embeds only missing chunk inputs. Removing an old worktree drops its memberships; -shared revisions and vectors remain until their final reference disappears. +shared revisions and vectors remain while referenced. Ordinary edits retain +recently unused revisions under a 256 MiB, seven-day logical cache policy; +explicit cleanup can reclaim them sooner. Legacy per-worktree indexes migrate lazily, reusing unchanged float32 vectors without contacting the embedding backend. diff --git a/cmd/clean.go b/cmd/clean.go index 86753bae..418a7722 100644 --- a/cmd/clean.go +++ b/cmd/clean.go @@ -213,7 +213,7 @@ func cleanIndex(reporter cleanReporter, name, hashDir string, days int, cutoff t return false, store.CleanupStats{}, fmt.Errorf("clean shared collection %s: %w", name, sharedErr) } if sharedStats.ProjectsLeft > 0 { - reporter.Info(fmt.Sprintf("Cleaned %s: removed %d projects and %d vectors.", name, sharedStats.ProjectsRemoved, sharedStats.VectorsRemoved)) + reporter.Info(fmt.Sprintf("Cleaned %s: removed %d projects, %d vectors, and %d orphan scan caches.", name, sharedStats.ProjectsRemoved, sharedStats.VectorsRemoved, sharedStats.ScanCachesRemoved)) return false, sharedStats, nil } // Empty collections have no future owner and can be removed as a diff --git a/cmd/embedder.go b/cmd/embedder.go index d8a06210..cc2e98fe 100644 --- a/cmd/embedder.go +++ b/cmd/embedder.go @@ -17,9 +17,12 @@ package cmd import ( "github.com/ory/lumen/internal/config" "github.com/ory/lumen/internal/embedder" + "path/filepath" ) // newEmbedder creates a FailoverEmbedder from the given ConfigService. func newEmbedder(cfg *config.ConfigService) *embedder.FailoverEmbedder { - return embedder.NewFailoverEmbedder(cfg) + emb := embedder.NewFailoverEmbedder(cfg) + emb.SetComputeBudget(filepath.Join(config.XDGDataDir(), "lumen", "compute")) + return emb } diff --git a/cmd/experiment_input.go b/cmd/experiment_input.go new file mode 100644 index 00000000..bf286112 --- /dev/null +++ b/cmd/experiment_input.go @@ -0,0 +1,212 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "math" + "os" + "slices" + + "github.com/ory/lumen/internal/embedder" + "github.com/spf13/cobra" +) + +// The experiment accepts explicit relevance judgments rather than inventing +// synthetic quality claims from vector distances alone. +type inputDataset struct { + Chunks []struct { + Path string `json:"path"` + Body string `json:"body"` + } `json:"chunks"` + Queries []struct { + Text string `json:"text"` + RelevantPaths []string `json:"relevant_paths"` + } `json:"queries"` +} + +type inputMetrics struct { + Mode string `json:"mode"` + UniqueInputs int `json:"unique_inputs"` + Recall float64 `json:"recall_at_k"` + MRR float64 `json:"mrr_at_k"` +} + +func init() { + command := &cobra.Command{Use: "experiment-input ", Short: "Compare path-aware and body-only embeddings against labeled queries", Args: cobra.ExactArgs(1), RunE: runInputExperiment} + command.Flags().Int("k", 10, "number of retrieved chunks to evaluate") + command.Flags().String("model", "", "embedding model") + command.Flags().String("backend", "", "embedding backend") + rootCmd.AddCommand(command) +} + +func runInputExperiment(cmd *cobra.Command, args []string) error { + data, err := os.ReadFile(args[0]) + if err != nil { + return err + } + var dataset inputDataset + if err := json.Unmarshal(data, &dataset); err != nil { + return err + } + k, _ := cmd.Flags().GetInt("k") + if k < 1 || len(dataset.Chunks) == 0 || len(dataset.Queries) == 0 { + return fmt.Errorf("positive k, chunks, and labeled queries are required") + } + paths := map[string]bool{} + for _, chunk := range dataset.Chunks { + if chunk.Path == "" || chunk.Body == "" { + return fmt.Errorf("each chunk requires path and body") + } + paths[chunk.Path] = true + } + for _, query := range dataset.Queries { + if query.Text == "" || len(query.RelevantPaths) == 0 { + return fmt.Errorf("each query requires text and relevant_paths") + } + for _, path := range query.RelevantPaths { + if !paths[path] { + return fmt.Errorf("relevance label %q has no corpus chunk", path) + } + } + } + cfg, err := loadConfigWithFlags(cmd) + if err != nil { + return err + } + emb := newEmbedder(cfg) + model, dimensions := emb.ModelName(), emb.Dimensions() + ctx := embedder.BackgroundContext(cmd.Context()) + encode := func(texts []string) ([][]float32, error) { + var result [][]float32 + for start := 0; start < len(texts); start += 32 { + batch := texts[start:min(start+32, len(texts))] + vectors, err := emb.Embed(ctx, batch) + if err != nil { + return nil, err + } + if emb.ModelName() != model || emb.Dimensions() != dimensions { + return nil, fmt.Errorf("embedding profile changed during experiment; retry with a single model") + } + if len(vectors) != len(batch) { + return nil, fmt.Errorf("backend returned an unexpected vector count") + } + result = append(result, vectors...) + } + return result, nil + } + queries := make([]string, len(dataset.Queries)) + for i, q := range dataset.Queries { + queries[i] = q.Text + } + queryVectors, err := encode(queries) + if err != nil { + return err + } + var report []inputMetrics + for _, mode := range []string{"path", "body"} { + positions := make([]int, len(dataset.Chunks)) + unique := map[string]int{} + var inputs []string + for i, chunk := range dataset.Chunks { + text := chunk.Body + if mode == "path" { + text = "// " + chunk.Path + "\n" + text + } + position, ok := unique[text] + if !ok { + position = len(inputs) + unique[text] = position + inputs = append(inputs, text) + } + positions[i] = position + } + vectors, err := encode(inputs) + if err != nil { + return err + } + metrics := inputMetrics{Mode: mode, UniqueInputs: len(inputs)} + for qi, query := range dataset.Queries { + type match struct { + path string + score float64 + ordinal int + } + matches := make([]match, len(dataset.Chunks)) + for i, chunk := range dataset.Chunks { + score, err := cosineScore(queryVectors[qi], vectors[positions[i]]) + if err != nil { + return err + } + matches[i] = match{chunk.Path, score, i} + } + slices.SortFunc(matches, func(a, b match) int { + if a.score > b.score { + return -1 + } + if a.score < b.score { + return 1 + } + if a.path < b.path { + return -1 + } + if a.path > b.path { + return 1 + } + return a.ordinal - b.ordinal + }) + relevant := map[string]bool{} + for _, path := range query.RelevantPaths { + relevant[path] = true + } + found := map[string]bool{} + rr := 0.0 + for rank, m := range matches[:min(k, len(matches))] { + if relevant[m.path] { + found[m.path] = true + if rr == 0 { + rr = 1 / float64(rank+1) + } + } + } + metrics.Recall += float64(len(found)) / float64(len(relevant)) + metrics.MRR += rr + } + metrics.Recall /= float64(len(dataset.Queries)) + metrics.MRR /= float64(len(dataset.Queries)) + report = append(report, metrics) + } + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(struct { + Model string `json:"model"` + K int `json:"k"` + Results []inputMetrics `json:"results"` + }{model, k, report}) +} + +func cosineScore(a, b []float32) (float64, error) { + if len(a) == 0 || len(a) != len(b) { + return 0, fmt.Errorf("incompatible embedding dimensions") + } + var dot, aa, bb float64 + for i, x := range a { + y := float64(b[i]) + v := float64(x) + dot += v * y + aa += v * v + bb += y * y + } + if aa == 0 || bb == 0 { + return 0, fmt.Errorf("backend returned a zero vector") + } + score := dot / math.Sqrt(aa*bb) + if math.IsNaN(score) || math.IsInf(score, 0) { + return 0, fmt.Errorf("backend returned non-finite vectors") + } + return score, nil +} diff --git a/cmd/experiment_input_test.go b/cmd/experiment_input_test.go new file mode 100644 index 00000000..70cd435f --- /dev/null +++ b/cmd/experiment_input_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestInputExperimentMeasuresRetrievalAndReuse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/embed" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"models":[{"name":"experiment"}]}`)) + return + } + var request struct { + Input []string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + vectors := make([][]float32, len(request.Input)) + for i, text := range request.Input { + if strings.Contains(text, "cat") { + vectors[i] = []float32{1, 0} + } else { + vectors[i] = []float32{0, 1} + } + } + _ = json.NewEncoder(w).Encode(map[string]any{"embeddings": vectors}) + })) + defer server.Close() + command := flagsTestHarness(t, fmt.Sprintf("servers:\n - backend: ollama\n host: %s\n model: experiment\n dims: 2\n", server.URL)) + command.Flags().Int("k", 1, "") + command.SetContext(context.Background()) + t.Setenv("XDG_DATA_HOME", t.TempDir()) + dataset := filepath.Join(t.TempDir(), "dataset.json") + if err := os.WriteFile(dataset, []byte(`{"chunks":[{"path":"a.go","body":"cat"},{"path":"b.go","body":"cat"},{"path":"c.go","body":"dog"}],"queries":[{"text":"cat","relevant_paths":["a.go"]}]}`), 0600); err != nil { + t.Fatal(err) + } + var output bytes.Buffer + command.SetOut(&output) + if err := runInputExperiment(command, []string{dataset}); err != nil { + t.Fatal(err) + } + var report struct { + Results []inputMetrics `json:"results"` + } + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatal(err) + } + if len(report.Results) != 2 || report.Results[0].UniqueInputs != 3 || report.Results[1].UniqueInputs != 2 { + t.Fatalf("reuse metrics: %s", output.String()) + } + for _, result := range report.Results { + if result.Recall != 1 || result.MRR != 1 { + t.Fatalf("retrieval metrics: %+v", result) + } + } +} diff --git a/cmd/log.go b/cmd/log.go index 51dd93db..f2ed41bb 100644 --- a/cmd/log.go +++ b/cmd/log.go @@ -39,9 +39,10 @@ func newDebugLogger() (*slog.Logger, io.Closer) { return slog.New(slog.NewJSONHandler(os.Stderr, nil)), nil } + // Several Lumen processes append to one log; pid attributes each line. logger := slog.New(slog.NewJSONHandler(f, &slog.HandlerOptions{ Level: parseLogLevel(os.Getenv("LUMEN_LOG_LEVEL")), - })) + })).With("pid", os.Getpid()) return logger, f } diff --git a/cmd/stdio.go b/cmd/stdio.go index 30e02ee8..2500b886 100644 --- a/cmd/stdio.go +++ b/cmd/stdio.go @@ -501,7 +501,15 @@ func (ic *indexerCache) getOrCreate(projectPath string, preferredRoot string, mo // use preferredRoot if it already has an index — otherwise fall back to // findEffectiveRoot to avoid adopting the outer monorepo root. var effectiveRoot string - if preferredRoot != "" && !git.IsWorktree(projectPath) { + if root, err := git.RepoRoot(projectPath); err == nil && !pathCrossesSkipDir(root, projectPath) { + // Git projects are indexed at their worktree top level, exactly as + // `lumen index` and SessionStart do. The repository-scoped collection + // exists for every directory in the repository, so its presence cannot + // select a root: adopting a subdirectory (or an enclosing checkout) + // would create a separate membership with different relative paths and + // re-parse and re-embed content the worktree already has. + effectiveRoot = root + } else if preferredRoot != "" && !git.IsWorktree(projectPath) { clean := filepath.Clean(preferredRoot) // Only adopt the preferred root if an index already exists there. // Creating a brand-new index at cwd when path is a small subdirectory diff --git a/docs/INDEX_STORAGE.md b/docs/INDEX_STORAGE.md index 76477e0a..a0e038d9 100644 --- a/docs/INDEX_STORAGE.md +++ b/docs/INDEX_STORAGE.md @@ -52,13 +52,72 @@ the same vector. When a file changes, Lumen chunks the new revision, checks which exact inputs already exist, and sends only missing inputs to the embedding service. Unique -constraints make concurrent worktree indexing safe; two indexers may compute the -same embedding during a race, but only one stored vector remains. +constraints protect stored vectors. Before embedding, indexers claim exact input +hashes in SQLite with renewable two-minute leases. Concurrent worktrees reuse +completed results. Failed owners release their claims; crashed owners lose them +when the lease expires. A ten-minute, 64 MiB serialized-result cache bridges the +interval before all file revisions have been committed. Expired or excess results +are pruned when new results are published. Force-index bypasses this cache. + +Backend requests share one compute slot per user data directory, across +repositories and processes. The slot is admitted separately for every provider +HTTP request (at most 32 inputs), not for a whole indexing batch. Queries +announce priority using OS locks; background indexing yields before its next +HTTP request. Running requests are not preempted. +This coordination requires no standalone daemon. Separate `XDG_DATA_HOME` +settings intentionally have separate budgets. + +Search first materializes the requesting project's memberships and optional +literal path prefix, then scores each distinct eligible vector once. It sorts +matching chunks by distance and stable location keys in one SQL query. Other +worktrees and retained revisions do not contribute distance calculations. +This is an exact scan of eligible vectors, not an approximate nearest-neighbor +index; its cost still grows with the requesting worktree. + +## Incremental freshness + +A disposable JSON cache beside the collection stores worktree-specific content +hashes and file metadata. Ordinary checks walk directory entries and compare +size, mode, mtime, device/inode identity, and ctime. Unchanged files outside a +two-second timestamp race window reuse their hashes on macOS and Linux. Other +platforms conservatively read content until equivalent metadata support is added. + +In-process filesystem watchers invalidate changed paths. Metadata reconciliation +continues after restart and even when watches are unavailable, so correctness +does not rely on event delivery. Watch overflow, hourly reconciliation, and +`--force` trigger content hashing. Ignore rules are reevaluated each scan. +This avoids ordinary full-content reads; directory walking and Merkle root +construction remain proportional to file count. No Git object ID is substituted +for actual worktree content. + +## Comparing embedding inputs + +Path-aware embedding remains the default. Run an isolated quality experiment: -Search remains project-local. Lumen scans collection vectors adaptively, then -joins candidates through the requested project's memberships and optional path -filter. Results from another worktree are never returned merely because the -underlying storage is shared. +```bash +lumen experiment-input dataset.json --k 10 +``` + +The dataset contains already-chunked bodies and explicit relevance judgments: + +```json +{ + "chunks": [ + {"path": "auth/login.go", "body": "func Login() { /* ... */ }"}, + {"path": "legacy/login.go", "body": "func Login() { /* ... */ }"} + ], + "queries": [ + {"text": "login handler", "relevant_paths": ["auth/login.go"]} + ] +} +``` + +The command embeds both input formats using the configured backend and reports +unique-input counts, mean path recall at k, and mean reciprocal rank at k over +chunk-ranked results. Use representative queries, ambiguous bodies, and renamed +files; the tiny example above is only a format illustration. It does not modify +production indexes or add path ranking. No retrieval-quality improvement is +assumed until evaluated with real judgments. ## Vector storage @@ -92,7 +151,7 @@ The MCP `index_status` tool reports: | `indexed_files` | Current project | File memberships stored for this project | | `total_chunks` | Current project | Chunk references reachable by this project | | `unique_vectors` | Collection | Physical vectors stored once in the collection | -| `shared_references` | Collection | Chunk references beyond the unique-vector count | +| `shared_references` | Collection | Active chunk references beyond active unique vectors | | `deduplication_ratio` | Collection | `shared_references / (unique_vectors + shared_references)` | | `vector_storage` | Collection | `int8` or `float32` | | `database_bytes` | Collection | Allocated SQLite database pages | @@ -100,7 +159,7 @@ The MCP `index_status` tool reports: | `last_indexed_at` | Current project | Last completed indexing timestamp | | `stale` | Current project | Whether the source tree differs from the stored Merkle state | -For example, `10,000` unique vectors and `15,000` shared references means +When no unused vectors are retained, `10,000` unique vectors and `15,000` shared references means `25,000` total chunk references are represented by `10,000` physical vectors, for a deduplication ratio of 60%. @@ -126,8 +185,30 @@ the new index format: If legacy recovery cannot be completed, Lumen leaves the source database in place and embeds the missing inputs normally. +## Recently unused revisions + +Ordinary replacement retains unreferenced complete revisions for up to seven +days, under a 256 MiB logical byte budget per collection. Reattaching a revision +makes it active again. Eviction favors recently unused revisions and never +deletes a referenced revision or a vector still needed by another revision. +The budget conservatively counts vector bytes per revision plus chunk metadata; +SQLite page and WAL overhead are additional. Maintenance is lazy, during +replacement/removal; expiry does not run on a timer. The store API +`SetRevisionCachePolicy(bytes, ttl)` supports changing the policy or disabling it +with zero. There is currently no CLI/config option for this policy. + +`unique_vectors` includes retained physical vectors; `shared_references` counts +only active references. Consequently the reported deduplication ratio includes +retention overhead in its denominator. + ## Cleanup and disk reclamation +Cleanup also deletes the disposable scan sidecars +(`index.db.scan-.json`) of memberships that no longer exist, while it +holds the exclusive collection guard. In-progress temporary files, unknown +files, and per-project lock files are never removed: unlinking a lock file +could split flock protection across two inodes. + Run cleanup manually with: ```bash @@ -154,6 +235,37 @@ every 24 hours. The throttle stamp is stored at be nonzero until incremental vacuum can truncate pages, and it does not include WAL or filesystem allocation details. +## Storage policy in summary + +- Active memberships, revisions, and vectors are kept as long as any project + references them. +- Recently unused revisions: 256 MiB logical bytes and seven days per + collection by default. +- Serialized embedding-result rendezvous cache: 64 MiB and ten minutes. +- SQLite page, WAL, and scan-sidecar overhead are additional. There is no + absolute disk cap; the policies above bound logical retention only. + +## Verifying worktree efficiency + +Lumen writes structured diagnostics to `debug.log` in the data directory. Every +line carries the writing process's `pid`. + +- `scan completed`: `project`, `forced`, `files_total`, `files_hashed`, and + `files_reused` from the actual freshness scan, including unchanged scans. +- `index pass completed` (or `index pass failed`): `files_changed`, + `files_parsed` (chunker invocations), `revisions_attached` (files linked to an + existing revision without parsing), and `chunks_created`. + +`make e2e-worktrees` runs an offline suite that drives real Lumen binaries +(CLI, the SessionStart hook and its detached indexer, and the MCP stdio server) +against real Git worktrees sharing one data directory. A deterministic HTTP +server stands in for the embedding provider and records every input. The suite +asserts parsing, hashing, and embedding work; query priority; recovery; search +isolation; retention; cleanup; and physical size. +`make test-worktree-mutations` applies curated production-code mutants in a +disposable source copy. Each must fail its designated assertion, and the +restored source must pass again. + ## Reindexing versus wiping Use the narrowest operation that matches the problem: diff --git a/docs/research-agentic-retrieval-2026-10-11.html b/docs/research-agentic-retrieval-2026-10-11.html new file mode 100644 index 00000000..c7f21f74 --- /dev/null +++ b/docs/research-agentic-retrieval-2026-10-11.html @@ -0,0 +1,129 @@ + +Agentic retrieval: 25-agent research review — Lumen +
+

R&D RESEARCH REVIEW · 11 OCTOBER 2026 · LUMEN + GENERAL DOCUMENT RETRIEVAL

+

Retrieve the evidence an agent needs, within a measured budget

+

The strongest practical next step is query-aware hybrid retrieval with token-budgeted evidence selection. The most interesting research extension is to select complementary evidence bundles and adapt the next search action to the evidence still missing.

+
Decision: first improve Lumen's query-dependent ranking and global context budget; then compare lexical+dense fusion against that inexpensive baseline. Add structural expansion, reranking and stronger semantic representations only when held-out end-to-end gains justify their cost. This is an evidence-backed recommendation for experiments, not a verified universal winner.
+

Scope and method. Twenty-five separate AI subagents performed specialist investigations, including an independent local-code audit and skeptical review. The runtime permits three concurrent subagents, so the work ran in batches. The agents are not affiliated with a professional scientific organization. This review uses primary research papers, publication records and local source inspection; it is a broad research scan, not an exhaustive systematic review or independent reproduction of the papers' benchmarks. Publication searches include 2025–2026 work; recent preprints receive less weight than established results.

+

What was verified. The cited mechanisms and selected reported findings were checked against primary sources. The production dense-search path and existing index/chunking features were inspected. The local command below passed. It verifies a toy SQLite lexical/vector fusion query, not production integration, retrieval quality, latency superiority or the proposed complete system.

+
go test -tags fts5 ./internal/store -run '^TestHybridCTE_VecAndFTS5InCTE$' -count=1
+Result: PASS
+ +

Different failures need different retrieval operations

+ + + + + + + + + + + +
ProblemAppropriate responseKey limitation
Exact symbols, errors, paths and regexLiteral/symbol index; selective trigram prefilter; verify actual matchSemantic ranking must not silently relax an explicit literal requirement.
Typos and naming variantsBounded edit-distance search, identifier components, character gramsSpelling similarity does not resolve conceptual vocabulary mismatch.
Paraphrases and vague intentLexical plus dense or learned-sparse retrieval; selective rerankingTopical similarity is weaker than evidence of the requested behavior.
Several constraints or missing dependenciesFacet-aware selection; bounded linked evidence bundlesCoverage estimates can be wrong; multi-hop complementarity breaks simple diminishing-return assumptions.
Cross-file or cross-document reasoningTyped links, resolved references, bounded graph expansionSyntactic name matches are not proof of a call/dependency relationship.
Long documents and overview questionsHierarchy, contextual embeddings, selective summary indexesSummaries can omit precise facts; retrieve original spans for support.
Tables, scans, diagrams and numeric questionsLayout/visual retrieval and table-aware extraction; deterministic calculationsText-only ingestion can discard the needed relation before retrieval begins.
Versions, dates, negation and contradictionsExplicit scope predicates and competing evidence with provenanceSimilarity is not temporal validity, logical entailment or numeric correctness.
Large, changing corpora and repeated agent callsIncremental artifacts; measured ANN; snapshot-aware reuse; session deduplicationNew documents can invalidate a cached top-k even if every old hit is unchanged.
Missing or ambiguous evidencePreserve interpretations; bounded follow-up; expose searched scope and uncertaintyA top-k miss does not prove corpus-wide absence.
+ +

The recommended architecture

+
1 · SCOPEQuery + literals + revision2 · CANDIDATESExact / lexical / semantic3 · RELATIONSOptional bounded expansion4 · SELECTIONComplementary source spans5 · RETURNBudgeted evidence + handles6 · FEEDBACKUncovered facets or stop
+
    +
  1. Preserve exactness and scope. Keep original wording, explicit symbols and quoted text. Use revision, project and path membership to define eligible evidence. Fuzzy alternatives are labeled alternatives; they do not rewrite hard literal constraints.
  2. +
  3. Generate candidates cheaply. For exact lookups, prioritize direct matches. For conceptual questions, compare lexical+dense fusion against each branch alone. Use reciprocal rank fusion as a baseline, while ensuring explicit constraints are respected. Larger candidate pools cost ranking time, not necessarily reader context.
  4. +
  5. Rank for the actual question. Avoid unconditional penalties for tests or documents. A small reranker is optional and must demonstrate savings after including its inference cost.
  6. +
  7. Select sets of evidence. Prefer source units that address missing aspects rather than near-duplicates. Retain the decisive definition even if diversification appears attractive. Protect semantic units such as negations, operators, table headers, values and units.
  8. +
  9. Expand only when needed. Follow a bounded number of authoritative document links or resolved code references. Return a small implementation/caller/test bundle when it serves the question. Use limited graph radius and candidate budgets; do not construct an expensive global LLM knowledge graph by default.
  10. +
  11. Track what the agent has already received. Return source text with file/revision/span handles and offer expansion. Reuse retrieved evidence within the same snapshot, but do not suppress needed material merely because it appeared earlier in a compacted or different agent context.
  12. +
  13. Control additional work. Begin with simple rules and hard budgets. Learn a controller only after logging enough actual outcomes. Its own inference and bookkeeping must count in the budget.
  14. +
+

Illustration. For “where do retries stop when the caller cancels?”, returning several similarly named retry helpers may waste context. A better candidate evidence set could contain the retry loop, the caller's cancellation propagation, and a test of cancellation behavior. Their usefulness must be checked from source; proximity in a graph or high similarity alone does not prove the behavior.

+ +

The research hypothesis, separated from established components

+

Hypothesis: a snapshot-aware selector of complementary evidence bundles, coupled to a low-overhead search controller, will reduce agent rounds and total tokens at matched task success. Its ingredients draw from information retrieval, program analysis, submodular optimization, database optimization, incremental build systems and decision-theoretic metareasoning. This synthesis is a proposed Lumen experiment; scientific originality has not been established.

+
Evidence-selection surrogate:
+  F(S) = sum over query aspects a of w[a] * max over spans d in S of support[a,d]
+  subject to total returned tokens <= B
+
+Next-action heuristic:
+  estimated improvement in final task success
+  - lambda * additional latency
+  - mu * additional model/context cost
+

With fixed nonnegative weights and support scores, the coverage surrogate is monotone submodular. Real evidence utility can violate that assumption: two passages may be useful only together. Enumerating a small number of source-linked bundles is one experiment, but shared-span deduplication changes costs and does not automatically preserve a textbook approximation guarantee. Estimates of support and success gain require held-out calibration; the formula is not a correctness proof. Submodular knapsack · Selecting Computations

+

Most interesting individual algorithmic candidate: MUVERA's randomized reduction of multi-vector retrieval to fixed-dimensional candidate search. Test it if late interaction gives valuable quality improvements but its candidate stage becomes a bottleneck. This is distinct from recommending it as the first local implementation. MUVERA

+ +

Fit to the current Lumen implementation

+ + + + + + + +
Observed feature or gapConsequenceLocal source
Production shared search computes cosine distance for eligible unique vectors.Profile query embedding versus scanning before choosing ANN. The existing fusion test is not production hybrid retrieval.internal/store/shared.go:787
internal/store/hybrid_cte_test.go:27
AST/code, Markdown/MDX and YAML/JSON chunking already exist.Extend evidence hierarchy and unsupported formats instead of rebuilding chunkers.internal/chunker/languages.go:36
Embeddings already include filepath context and have incremental reuse.Do not credit the proposal with inventing these. Add new dependency/version keys if contextual encoding changes reuse semantics.internal/store/shared.go:360
internal/index/shared.go:198
Unconditional declaration boost and test penalty appear in result ranking.Test intent-conditioned ranking before adding complex models; “find the regression test” should not be penalized for asking for tests.cmd/stdio.go:1293
Summary, line limits and overlap merging already exist; a global evidence-token budget is absent.Extend the existing output contract with budgeted selection and expansion rather than duplicate those controls.cmd/stdio.go:54
cmd/stdio.go:720
cmd/stdio.go:1365
Benchmark reports can use best quality across runs and condition some comparisons on Lumen use.Report every assigned run and all-run quality distributions; keep usage-conditioned analysis secondary.bench-swe/internal/report/report.go:166
bench-swe/internal/report/report.go:459
+ +

All 25 research assignments

+

The decisions below are this review's engineering judgments. Research results on QA/passages, images or billion-vector collections do not automatically transfer to coding agents on local repositories.

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
#Workstream / dispositionFinding and primary sources
1Learned sparse retrieval
Pilot
SPLADE expands text into weighted vocabulary postings, allowing related wording to match through an inverted index. Preserve exact symbol postings. A promising semantic candidate branch; additional inference, index size, and out-of-domain code behavior need measurement.

SPLADE-v3 · Efficient SPLADE

2Late interaction and randomized sketches
Pilot
ColBERT retains token-level representations; PLAID prunes expensive comparisons. MUVERA sketches multi-vector similarity into fixed-dimensional encodings for candidate generation, followed by reranking. This is a particularly interesting algorithmic candidate for scale, but storage and local latency may exceed a compact single-vector baseline.

ColBERTv2 · MUVERA

3Associative graph memory
Pilot
HippoRAG uses Personalized PageRank to connect evidence across documents. Adapt it to source-backed document links and resolved code references, with bounded expansion from hybrid hits. GraphRAG community summaries serve corpus-wide synthesis; their original evaluation is not evidence of better precise code localization.

HippoRAG 2 · GraphRAG

4Hierarchy and contextual embeddings
Pilot
Late chunking encodes a contextual window before pooling individual passage vectors; it differs from late interaction. Test inexpensive heading/path context first. RAPTOR recursively summarizes clusters and can help global questions, but summary regeneration and noise are concerns for changing source code.

Late Chunking · RAPTOR

5Document and query expansion
Pilot
Filtered doc2query moves intent expansion to ingestion; synthetic questions must be filtered and stored separately from evidence. HyDE or Query2doc can rescue vague queries, but online generation adds latency and can anchor search to a wrong interpretation. Keep the original query as a retrieval branch.

Doc2Query-- · Query2doc

6Adaptive retrieval control
Build baseline
Adaptive-RAG and EfficientRAG support selective escalation and lightweight iteration. Start with rules and hard budgets; learn routing only after collecting outcome data. Score gaps and model confidence are features, not calibrated proof that evidence is sufficient.

Adaptive-RAG · EfficientRAG

7Set coverage and evidence selection
Priority experiment
Select complementary source units under a token budget instead of simply taking top-k. MMR is the cheap baseline. Weighted maximum facet coverage is a useful submodular surrogate; complete multi-hop chains can have increasing returns, so bounded evidence bundles need separate treatment. Naive marginal-gain-per-token greedy does not inherit the classical knapsack guarantee.

MMR · Submodular knapsack

8Metareasoning and value of computation
Priority experiment
Choose the next search action according to predicted improvement in the final task minus time and token costs. Metareasoning better describes within-query search than ordinary reward-collecting bandits. MBA-RAG supports adaptive strategy selection; calibration, logged-policy bias, and useful two-step searches remain concerns.

Selecting Computations · MBA-RAG

9Program analysis and dependency slicing
Pilot
RepoGraph supports source-structure-assisted repair; code property graphs supply richer dependency representations. Use semantic anchors, then small caller/callee/test neighborhoods. AST call syntax does not resolve targets by itself: use compiler/type information when available and label heuristic edges.

RepoGraph · Code Property Graphs

10Incremental builds and semantic caches
Build on existing
Reuse content artifacts by input and pipeline hashes, while keeping current paths and corpus membership separate. Contextual embeddings depend on their whole encoding window. A cached hit remaining unchanged is insufficient: a newly added document can outrank it. Result caches need corpus-generation validity or correct changed-partition reconciliation.

Build Systems a la Carte · Semantic Data Caching

11Adaptive dimensions and ANN
Scale-dependent
Matryoshka embeddings support small-prefix retrieval and full-vector reranking; ordinary embeddings cannot simply be truncated with the same expectation. Score-aware quantization preserves ranking-relevant information. Keep exact scans as the baseline for small corpora and measure index/update overhead before introducing ANN.

Matryoshka · Score-aware quantization

12Fuzzy lexical search and fusion
Build first
Trigram filtering plus exact verification speeds selective regex search; bounded edit-distance lookup handles misspelled identifiers. Index raw symbols, case/snake components and paths separately. Reciprocal rank fusion combines ranked channels but does not guarantee that exact matches remain first; explicit literal constraints need their own policy.

Indexed regular-expression search · Reciprocal Rank Fusion

13Context compression and progressive reads
Build carefully
RECOMP and LLMLingua-2 show that context reduction can save work. For source code and precise documents, prefer whole evidence spans with expandable handles. Protect negations, operators, values, units, and headers. Count compression and follow-up reads in total cost; token deletion is not a semantic-preservation guarantee.

RECOMP · LLMLingua-2

14Visual documents and tables
Specialist branch
ColPali/ColQwen retrieve page images using patch-level late interaction, retaining layout lost in text extraction. Use this selectively for figures, scans and tables. Retrieve the relevant table, then verify cells, units and dates and execute numeric predicates deterministically. Page or crop provenance is essential.

ColPali · Dense table retrieval

15Time, versions, negation and predicates
Build constraints
Separate hard metadata predicates from approximate relevance. Apply current revision and path filters before ranking; temporal validity is not synonymous with publication time. NevIR demonstrates negation failures. Numeric constraints require typed values and units; semantic negation cannot be reduced to absence of a keyword.

ACORN · NevIR

16Reasoning-trained retrieval
Pilot after baseline
ReasonIR learns difficult relevance distinctions. BRIGHT-Pro evaluates complementary evidence in agents: its 175-query agentic experiment shows that static retriever rankings and final answer quality can diverge. These results motivate evidence coverage and search-round measurements, not selecting the highest static score regardless of cost.

ReasonIR · BRIGHT-Pro / RTriever

17Generative document addressing
Defer replacement
Generating document IDs is an alternative to searching an explicit index. IncDSI and dynamic-corpus studies show that updates are possible, so blanket claims of impossible updates are wrong. For Lumen, borrowing semantic aliases while retaining an incrementally maintained source index is a lower-risk experiment than replacing the retrieval engine.

IncDSI · Generative retrieval on dynamic corpora

18RL-trained search policies
Defer training
Search-R1 learns interleaved retrieval but can search more and generate longer responses; accuracy rewards alone do not minimize cost. AnySearch explicitly studies variable budgets. CodeScout supplies an important terminal-only code-localization challenger. Test better routing before training a new agent.

Search-R1 · CodeScout

19Database query optimization
Priority experiment
Cascades suggests plan alternatives and cost limits; Eddies suggests adapting after observed results. Transfer these at retrieval-action boundaries. Lexical, dense and graph actions are not semantically equivalent database plans: a cheaper nonempty result cannot justify assuming no evidence was missed.

Cascades · Eddies

20Domain adaptation and hard negatives
Later optimization
GPL, InPars and CoRNStack support offline query synthesis, teacher distillation and code-specific negatives. This can amortize reasoning into a smaller retriever. Mine failures only after establishing a baseline; wrappers and related functions may be additional positives rather than negatives. Hold out repositories and time periods, and prevent clone leakage.

GPL · CoRNStack

21Evaluation and causal attribution
Required
Evaluate identical agents with competent iterative rg, lexical, dense, fixed hybrid and proposed extensions. Measure task success and total cost, evidence recall at fixed tokens, p95 latency, updates and cold indexing. Use held-out repositories, repeated paired trials and clustered uncertainty. Lumen's small published task set and LLM judging motivate broader testing; output tokens alone do not measure retrieved context.

BEIR · SWE-bench

22Ambiguity, conflicts and abstention
Build evidence fidelity
Keep competing interpretations and contradictory source spans separate. A similarity score or repeated agreement is not truth. CRAG supports corrective routing; conflict-aware RAG motivates preserving disagreements. Missing top-k results establish only failure to find evidence in that search, not nonexistence in the corpus.

CRAG · Conflicting Evidence / MADAM-RAG

23Independent Lumen fit audit
Verified in code
Existing features include filepath-prefixed embeddings, AST/Markdown/config chunks, incremental embedding reuse, summaries, max_lines and overlap merging. Clear gaps are production lexical fusion, a global token budget, query-conditioned ranking and broader document ingestion. Current code boosts declarations and penalizes tests regardless of query intent. Benchmark aggregation should retain all assigned runs and all quality outcomes.

MCP search, ranking and formatting · Benchmark aggregation

24Frontier: corpus-conditioned representations
Pilot
Contextual Document Embeddings conditions representations on neighboring documents and corpus context, analogous to collection-dependent IDF. This differs from late chunking and retains ordinary final vector search. Test encoding cost and sensitivity to repository changes. REFRAG's selective expansion is interesting but its actual embedding-to-decoder mechanism requires model integration; ordinary tool previews cannot inherit its claimed gains.

Contextual Document Embeddings · REFRAG

25Independent skeptical review
Applied
A strong terminal-only agent is a serious challenger; indexing complexity is not a prerequisite for good localization. Compare the proposed design against current Lumen plus better ranking and a shared evidence budget. Reject added components whose gains disappear after counting indexing, controller inference, follow-up reads or all assigned runs. No literature result or passing toy SQL test proves the combined proposal is best.

CodeScout · BRIGHT-Pro

+ +

Verification plan and rejection criteria

+

Incumbents. Compare a competent agent using iterative rg and file reads; current Lumen; current Lumen with query-aware ranking and a shared evidence budget; lexical-only; dense-only; lexical+dense fusion; and the proposed extensions. The simple improved-Lumen system is a mandatory challenger, not a deliberately weak baseline. Preserve the same agent, prompts, source snapshot and resource budget across comparisons.

+

Tasks. Pilot a held-out, human-reviewed set spanning exact identifiers, typos, paraphrases, symptoms without implementation names, cross-file chains, documents/configs, time/version constraints and no-answer cases. A planning range of 200–400 tasks is reasonable, but use pilot paired variance and a predeclared minimum meaningful effect to determine sample size. Include entire held-out repositories and query families; repeated runs do not become independent new tasks.

+

Primary outcome. Report executable task success for code changes and blinded supported-answer correctness for documents. Also report total spend divided by successful tasks, together with success rate and per-task paired differences. A cheaper run that fails is not a saving in successful work. Use task/repository-clustered uncertainty intervals and randomize paired run order.

+

Context and latency. Measure evidence recall and complete-chain recall at fixed token budgets; unique and repeated evidence tokens; cumulative model-input tokens including cache treatment; peak context; output tokens; tool rounds; cold and warm end-to-end p50/p95 latency; index size; peak memory; initial build and update costs; stale or wrong-revision citations. Include embedding, rewriting, reranking, controller, compression and subsequent expansion costs.

+
Amortized cost per task =
+  share of initial indexing and update cost
+  + all retrieval / embedding / reranking / control cost
+  + reader input, cached input and output cost
+  + retries and follow-up reads
+

Ablations. Independently test query-aware priors, global token budget, lexical fusion, reranking, graph bundles, session deduplication, query rewriting and adaptive stopping. Sweep candidate and context budgets instead of choosing favorable settings per method. Identify whether gains come from better retrieval, better packaging or a stronger agent. Evaluate both cold-start and repeated-use break-even points.

+

Freshness and leakage. Replay additions, modifications, deletions, renames, branch switches and encoder upgrades. Compare incremental results with fresh rebuilds. Exclude gold patches, future documentation and post-fix evidence; deduplicate clones across training/test partitions. Do not remove failed runs or runs where the agent did not choose Lumen. Report failures and all-run quality rather than the best repetition.

+
Reject an added component when its benefit disappears after total-cost accounting, when it introduces material exact-match or freshness regressions, or when its apparent advantage does not survive paired held-out evaluation. Controller overhead, wrong inferred aspects, distracting graph neighbors and context-selection mistakes are first-class failure modes.
+ +

Implementation order justified by the evidence

+
    +
  1. First: improve query-dependent ranking, establish a global evidence budget, and make all-run benchmark reporting reliable. These changes target concrete observed gaps.
  2. +
  3. Next: production lexical+dense candidate fusion with explicit literal/scope handling. Compare it against the first-stage incumbent before adding other machinery.
  4. +
  5. Then: complementary evidence selection and small authoritative relationship bundles; evaluate optional reranking separately.
  6. +
  7. Only after measured failures: test SPLADE, corpus-conditioned embeddings or ColBERT. Use MUVERA/ANN when profiling identifies a candidate-search bottleneck. Use document expansion or domain adaptation when query volume amortizes ingestion/training cost.
  8. +
  9. Specialized branches: visual/table retrieval for rich documents, hierarchy for global synthesis, and carefully budgeted multi-hop search for difficult cases. Keep expensive generative indexing and full agent RL as research alternatives rather than default dependencies.
  10. +
+

The best-supported decision today is an experimental roadmap with strong cheap challengers. The research supports the mechanisms and tradeoffs; a claim of lower cost or better task success for the complete Lumen design awaits the proposed comparison.

+
Prepared from 25 AI research assignments and primary-source checks. No external publication, messages or production code changes were made. The accompanying HTML is a research artifact. Literature claims remain authors' reported results unless explicitly described as locally verified.
+
diff --git a/e2e/worktrees/harness_test.go b/e2e/worktrees/harness_test.go new file mode 100644 index 00000000..044cf19c --- /dev/null +++ b/e2e/worktrees/harness_test.go @@ -0,0 +1,1113 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +//go:build worktree_e2e + +// Package worktrees proves Lumen's worktree indexing efficiency through its +// production entry points. Real Lumen binaries (CLI, SessionStart hook with its +// detached background indexer, and the MCP stdio server) run against real Git +// worktrees that share one isolated XDG data home. Only the remote embedding +// provider is replaced, by a deterministic HTTP server that records every +// request in a ledger. +// +// Assertion messages carry stable "[A:]" markers. The curated production +// mutation gate (see mutants/) requires a mutant to fail its designated test +// with its designated marker; any other failure does not count as a kill. +package worktrees + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/gofrs/flock" + "github.com/modelcontextprotocol/go-sdk/mcp" + + sqlitevec "github.com/ory/lumen/internal/sqlitevec" + "github.com/ory/lumen/internal/store" +) + +const testModel = "lumen-worktree-test-embed" + +// settleDelay exceeds the scanner's two-second timestamp race window, after +// which unchanged files are eligible for metadata-based hash reuse. +const settleDelay = 2100 * time.Millisecond + +var lumenBinary string + +func TestMain(m *testing.M) { + if runtime.GOOS == "windows" { + // Windows conservatively rehashes content on every scan; this suite + // asserts the macOS/Linux metadata reuse contract. + fmt.Println("skipping worktree integration suite on windows") + os.Exit(0) + } + if err := sqlitevec.Auto(); err != nil { + fmt.Fprintf(os.Stderr, "register sqlite-vec: %v\n", err) + os.Exit(1) + } + dir, err := os.MkdirTemp("", "lumen-worktree-e2e-bin-") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + root, err := moduleRoot() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + // Unique per run, so leftover detached children can be found and killed by + // executable path without touching any other Lumen installation. + lumenBinary = filepath.Join(dir, fmt.Sprintf("lumen-wt-%d-%d", os.Getpid(), time.Now().UnixNano())) + // Same flags as `make build-local`: CGO for sqlite-vec, fts5 for search. + build := exec.Command("go", "build", "-tags=fts5", "-o", lumenBinary, ".") + build.Dir = root + build.Env = append(os.Environ(), "CGO_ENABLED=1") + build.Stdout, build.Stderr = os.Stderr, os.Stderr + if err := build.Run(); err != nil { + fmt.Fprintf(os.Stderr, "build lumen from %s: %v\n", root, err) + os.Exit(1) + } + code := m.Run() + killStrayLumen() + _ = os.RemoveAll(dir) + os.Exit(code) +} + +func moduleRoot() (string, error) { + out, err := exec.Command("go", "env", "GOMOD").Output() + if err != nil { + return "", err + } + gomod := strings.TrimSpace(string(out)) + if gomod == "" || gomod == os.DevNull { + return "", errors.New("not inside a Go module") + } + return filepath.Dir(gomod), nil +} + +// killStrayLumen terminates detached children (e.g. SessionStart background +// indexers) still running from this run's unique binary. +func killStrayLumen() { + for _, pid := range lumenPIDs("") { + _ = syscall.Kill(pid, syscall.SIGKILL) + } +} + +// lumenPIDs lists processes running this run's binary whose command line +// contains every word of filter. +func lumenPIDs(filter string) []int { + out, err := exec.Command("ps", "-axo", "pid=,command=").Output() + if err != nil { + return nil + } + var pids []int + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[1] != lumenBinary { + continue + } + if filter != "" && !strings.Contains(strings.Join(fields[1:], " "), filter) { + continue + } + if pid, err := strconv.Atoi(fields[0]); err == nil && pid != os.Getpid() { + pids = append(pids, pid) + } + } + return pids +} + +// --------------------------------------------------------------------------- +// Deterministic embedding provider and request ledger +// --------------------------------------------------------------------------- + +type embedRequest struct { + Seq int + Endpoint string + Inputs []string + Status int + Query bool +} + +type gate struct { + match func([]string) bool + status int // nonzero: respond immediately with this HTTP status + arrived chan struct{} + release chan struct{} + once sync.Once + fired bool +} + +// Release lets a blocked request continue. Safe to call repeatedly. +func (g *gate) Release() { g.once.Do(func() { close(g.release) }) } + +type provider struct { + t *testing.T + srv *httptest.Server + dims int + mu sync.Mutex + reqs []embedRequest + query map[string]bool + gates []*gate + flight int + peak int +} + +func newProvider(t *testing.T, dims int) *provider { + p := &provider{t: t, dims: dims, query: map[string]bool{}} + p.srv = httptest.NewServer(http.HandlerFunc(p.serve)) + t.Cleanup(func() { + p.mu.Lock() + for _, g := range p.gates { + g.Release() + } + p.mu.Unlock() + p.srv.Close() + }) + return p +} + +func (p *provider) serve(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/tags": + _, _ = fmt.Fprintf(w, `{"models":[{"name":%q,"model":%q}]}`, testModel, testModel) + return + case r.Method == http.MethodGet && r.URL.Path == "/v1/models": + _, _ = fmt.Fprintf(w, `{"data":[{"id":%q}]}`, testModel) + return + case r.Method == http.MethodPost && (r.URL.Path == "/api/embed" || r.URL.Path == "/v1/embeddings"): + default: + w.WriteHeader(http.StatusNotFound) + return + } + var body struct { + Model string `json:"model"` + Input []string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Model != testModel { + w.WriteHeader(http.StatusBadRequest) + return + } + p.mu.Lock() + queries := 0 + for _, in := range body.Input { + if p.query[in] { + queries++ + } + } + entry := embedRequest{Seq: len(p.reqs), Endpoint: r.URL.Path, Inputs: body.Input, Status: http.StatusOK, Query: queries > 0} + if queries != 0 && queries != len(body.Input) { + p.t.Errorf("[A:query-batched-with-documents] request mixed %d queries into %d inputs", queries, len(body.Input)) + } + var g *gate + for _, candidate := range p.gates { + if !candidate.fired && candidate.match(body.Input) { + candidate.fired = true + g = candidate + break + } + } + if g != nil && g.status != 0 { + entry.Status = g.status + } + p.reqs = append(p.reqs, entry) + p.flight++ + p.peak = max(p.peak, p.flight) + p.mu.Unlock() + defer func() { + p.mu.Lock() + p.flight-- + p.mu.Unlock() + }() + + if g != nil { + close(g.arrived) + if g.status != 0 { + w.WriteHeader(g.status) + _, _ = fmt.Fprint(w, `{"error":"injected failure"}`) + return + } + select { + case <-g.release: + case <-r.Context().Done(): + return + } + } + vectors := make([][]float32, len(body.Input)) + for i, in := range body.Input { + vectors[i] = deterministicVector(in, p.dims) + } + if r.URL.Path == "/api/embed" { + _ = json.NewEncoder(w).Encode(map[string]any{"model": testModel, "embeddings": vectors}) + return + } + data := make([]map[string]any, len(vectors)) + for i, v := range vectors { + data[i] = map[string]any{"index": i, "embedding": v} + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": data}) +} + +func deterministicVector(input string, dims int) []float32 { + out := make([]float32, dims) + var norm float64 + for i := range out { + h := sha256.Sum256([]byte(fmt.Sprintf("%d\x00%s", i, input))) + v := float64(int32(binary.BigEndian.Uint32(h[:4]))) / math.MaxInt32 + out[i] = float32(v) + norm += v * v + } + norm = math.Sqrt(norm) + for i := range out { + out[i] = float32(float64(out[i]) / norm) + } + return out +} + +// registerQuery records a query string the harness is about to issue, so the +// ledger classifies requests by exact text rather than by request size. +func (p *provider) registerQuery(q string) { + p.mu.Lock() + defer p.mu.Unlock() + p.query[q] = true +} + +// block arms a gate that holds the next matching embedding request open. +func (p *provider) block(match func([]string) bool) *gate { + g := &gate{match: match, arrived: make(chan struct{}), release: make(chan struct{})} + p.mu.Lock() + p.gates = append(p.gates, g) + p.mu.Unlock() + return g +} + +// fail arms a gate that answers the next matching request with status. +func (p *provider) fail(match func([]string) bool, status int) *gate { + g := &gate{match: match, status: status, arrived: make(chan struct{}), release: make(chan struct{})} + p.mu.Lock() + p.gates = append(p.gates, g) + p.mu.Unlock() + return g +} + +func documentsMentioning(substr string) func([]string) bool { + return func(inputs []string) bool { + for _, in := range inputs { + if strings.HasPrefix(in, "// ") && strings.Contains(in, substr) { + return true + } + } + return false + } +} + +// mark returns a ledger position for later "since" queries. +func (p *provider) mark() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.reqs) +} + +func (p *provider) since(mark int) []embedRequest { + p.mu.Lock() + defer p.mu.Unlock() + return slices.Clone(p.reqs[mark:]) +} + +// documents returns successfully answered document inputs since mark, with +// multiplicity. +func (p *provider) documents(mark int) []string { + var docs []string + for _, r := range p.since(mark) { + if !r.Query && r.Status == http.StatusOK { + docs = append(docs, r.Inputs...) + } + } + return docs +} + +func (p *provider) queryRequests(mark int) int { + n := 0 + for _, r := range p.since(mark) { + if r.Query { + n++ + } + } + return n +} + +func (p *provider) peakInFlight() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.peak +} + +func multiplicity(inputs []string) map[string]int { + m := make(map[string]int, len(inputs)) + for _, in := range inputs { + m[in]++ + } + return m +} + +func requireEachOnce(t *testing.T, tag string, inputs []string) { + t.Helper() + for in, n := range multiplicity(inputs) { + if n != 1 { + t.Fatalf("[A:%s] document input embedded %d times: %q", tag, n, firstLine(in)) + } + } +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + " …" + } + return s +} + +// --------------------------------------------------------------------------- +// Isolated environment and production entry points +// --------------------------------------------------------------------------- + +type env struct { + t *testing.T + p *provider + dataHome string + home string + extra []string + backend string +} + +type envOption func(*env) + +func withEnv(kv ...string) envOption { return func(e *env) { e.extra = append(e.extra, kv...) } } + +func withBackend(backend string) envOption { return func(e *env) { e.backend = backend } } + +func withDims(dims int) envOption { return func(e *env) { e.p.dims = dims } } + +// newEnv creates one isolated XDG data home shared by every Lumen process the +// test starts, plus isolated config and Git settings. +func newEnv(t *testing.T, opts ...envOption) *env { + t.Helper() + t.Cleanup(killStrayLumen) + base := realDir(t, t.TempDir()) + e := &env{t: t, p: newProvider(t, 16), dataHome: filepath.Join(base, "data"), home: filepath.Join(base, "home"), backend: "ollama"} + for _, opt := range opts { + opt(e) + } + for _, dir := range []string{e.dataHome, e.home, filepath.Join(e.home, ".config")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + gitconfig := "[user]\n\tname = Lumen Test\n\temail = lumen@example.invalid\n[commit]\n\tgpgsign = false\n[tag]\n\tgpgsign = false\n[init]\n\tdefaultBranch = main\n" + if err := os.WriteFile(filepath.Join(e.home, ".gitconfig"), []byte(gitconfig), 0o644); err != nil { + t.Fatal(err) + } + // The MCP server performs daily maintenance on startup; a fresh stamp + // keeps it from racing assertions. Maintenance is tested explicitly. + if err := os.MkdirAll(filepath.Join(e.dataHome, "lumen"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.dataHome, "lumen", ".last-cleanup"), nil, 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if t.Failed() { + if data, err := os.ReadFile(e.debugLogPath()); err == nil { + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + t.Logf("debug.log tail:\n%s", strings.Join(lines[max(0, len(lines)-40):], "\n")) + } + } + }) + return e +} + +func (e *env) environ() []string { + host := "OLLAMA_HOST=" + e.p.srv.URL + if e.backend == "lmstudio" { + host = "LM_STUDIO_HOST=" + e.p.srv.URL + } + vars := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + e.home, + "XDG_DATA_HOME=" + e.dataHome, + "XDG_CONFIG_HOME=" + filepath.Join(e.home, ".config"), + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL=" + filepath.Join(e.home, ".gitconfig"), + "GIT_TERMINAL_PROMPT=0", + "LUMEN_BACKEND=" + e.backend, + "LUMEN_EMBED_MODEL=" + testModel, + "LUMEN_EMBED_DIMS=" + strconv.Itoa(e.p.dims), + "LUMEN_LOG_LEVEL=info", + host, + } + if tmp := os.Getenv("TMPDIR"); tmp != "" { + vars = append(vars, "TMPDIR="+tmp) + } + return append(vars, e.extra...) +} + +func (e *env) command(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, lumenBinary, args...) + cmd.Env = e.environ() + cmd.Dir = e.home + return cmd +} + +type result struct { + stdout, stderr string + err error + pid int +} + +// lumen runs one CLI invocation to completion. +func (e *env) lumen(args ...string) result { + e.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + cmd := e.command(ctx, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + r := result{stdout: stdout.String(), stderr: stderr.String(), err: err} + if cmd.Process != nil { + r.pid = cmd.Process.Pid + } + return r +} + +func (e *env) mustLumen(args ...string) result { + e.t.Helper() + r := e.lumen(args...) + if r.err != nil { + e.t.Fatalf("lumen %s: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), r.err, r.stdout, r.stderr) + } + return r +} + +type proc struct { + cmd *exec.Cmd + done chan struct{} + stdout bytes.Buffer + stderr bytes.Buffer + err error +} + +// start launches a CLI invocation without waiting for it. +func (e *env) start(args ...string) *proc { + e.t.Helper() + p := &proc{done: make(chan struct{})} + p.cmd = e.command(context.Background(), args...) + p.cmd.Stdout, p.cmd.Stderr = &p.stdout, &p.stderr + if err := p.cmd.Start(); err != nil { + e.t.Fatal(err) + } + go func() { p.err = p.cmd.Wait(); close(p.done) }() + e.t.Cleanup(func() { _ = p.cmd.Process.Kill(); <-p.done }) + return p +} + +func (p *proc) wait(t *testing.T) error { + t.Helper() + select { + case <-p.done: + return p.err + case <-time.After(3 * time.Minute): + t.Fatalf("process %v did not exit", p.cmd.Args) + return nil + } +} + +func (e *env) index(path string, extra ...string) result { + e.t.Helper() + return e.mustLumen(append(append([]string{"index"}, extra...), path)...) +} + +// sessionStart runs the real SessionStart hook, which may spawn a detached +// background `lumen index` child from the same binary. +func (e *env) sessionStart(cwd string) (string, int) { + e.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + cmd := e.command(ctx, "hook", "session-start", "lumen") + cmd.Stdin = strings.NewReader(fmt.Sprintf(`{"cwd":%q}`, cwd)) + out, err := cmd.Output() + if err != nil { + e.t.Fatalf("session-start: %v", err) + } + var decoded struct { + HookSpecificOutput struct { + AdditionalContext string `json:"additionalContext"` + } `json:"hookSpecificOutput"` + } + if err := json.Unmarshal(out, &decoded); err != nil { + e.t.Fatalf("decode hook output %q: %v", out, err) + } + return decoded.HookSpecificOutput.AdditionalContext, cmd.Process.Pid +} + +// waitNoIndexers waits until no `lumen index` process from this run remains. +func (e *env) waitNoIndexers() { + e.t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for len(lumenPIDs(" index ")) > 0 { + if time.Now().After(deadline) { + e.t.Fatalf("background indexers still running: %v", lumenPIDs(" index ")) + } + time.Sleep(20 * time.Millisecond) + } +} + +// --------------------------------------------------------------------------- +// MCP stdio sessions +// --------------------------------------------------------------------------- + +type mcpSession struct { + e *env + session *mcp.ClientSession + pid int +} + +func (e *env) mcp(extraEnv ...string) *mcpSession { + e.t.Helper() + cmd := e.command(context.Background(), "stdio") + cmd.Env = append(cmd.Env, extraEnv...) + client := mcp.NewClient(&mcp.Implementation{Name: "worktree-e2e", Version: "0.0.0"}, nil) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + session, err := client.Connect(ctx, &mcp.CommandTransport{Command: cmd}, nil) + if err != nil { + e.t.Fatalf("connect MCP: %v", err) + } + s := &mcpSession{e: e, session: session, pid: cmd.Process.Pid} + e.t.Cleanup(s.close) + return s +} + +func (s *mcpSession) close() { + if s.session != nil { + _ = s.session.Close() + s.session = nil + } +} + +// search issues semantic_search exactly as an agent host would and returns +// the result file names. min_score=-1 returns every eligible chunk up to limit. +func (s *mcpSession) search(query, path, cwd string) ([]string, string) { + s.e.t.Helper() + s.e.p.registerQuery(query) + ctx, cancel := contextWithTimeout(2 * time.Minute) + defer cancel() + res, err := s.session.CallTool(ctx, callParams(query, path, cwd)) + if err != nil { + s.e.t.Fatalf("semantic_search %q: %v", path, err) + } + var text strings.Builder + for _, c := range res.Content { + if tc, ok := c.(*mcp.TextContent); ok { + text.WriteString(tc.Text) + } + } + if res.IsError { + s.e.t.Fatalf("semantic_search %q returned error: %s", path, text.String()) + } + var files []string + for _, line := range strings.Split(text.String(), "\n") { + line = strings.TrimSpace(line) + if rest, ok := strings.CutPrefix(line, `= 0 { + files = append(files, rest[:end]) + } + } + } + sort.Strings(files) + return slices.Compact(files), text.String() +} + +func callParams(query, path, cwd string) *mcp.CallToolParams { + args := map[string]any{"query": query, "path": path, "limit": 200, "min_score": -1, "summary": true} + if cwd != "" { + args["cwd"] = cwd + } + return &mcp.CallToolParams{Name: "semantic_search", Arguments: args} +} + +func contextWithTimeout(d time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), d) +} + +// --------------------------------------------------------------------------- +// Production diagnostic events (debug.log) +// --------------------------------------------------------------------------- + +type event map[string]any + +func (ev event) msg() string { s, _ := ev["msg"].(string); return s } +func (ev event) str(k string) string { s, _ := ev[k].(string); return s } +func (ev event) num(k string) int { + f, _ := ev[k].(float64) + return int(f) +} +func (ev event) pid() int { return ev.num("pid") } + +func (e *env) debugLogPath() string { return filepath.Join(e.dataHome, "lumen", "debug.log") } + +// logMark returns the current debug.log offset. +func (e *env) logMark() int64 { + info, err := os.Stat(e.debugLogPath()) + if err != nil { + return 0 + } + return info.Size() +} + +func (e *env) events(since int64, msg string) []event { + e.t.Helper() + f, err := os.Open(e.debugLogPath()) + if err != nil { + return nil + } + defer func() { _ = f.Close() }() + if _, err := f.Seek(since, io.SeekStart); err != nil { + e.t.Fatal(err) + } + var out []event + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<24) + for sc.Scan() { + var ev event + if json.Unmarshal(sc.Bytes(), &ev) != nil { + continue // a line being appended concurrently + } + if msg == "" || ev.msg() == msg { + out = append(out, ev) + } + } + return out +} + +// work sums production work counters for project since a log mark. +type work struct { + scans, hashed, reused int + passes, parsed, attached int + failedPasses, forcedScans int +} + +func (e *env) work(since int64, project string) work { + e.t.Helper() + var w work + for _, ev := range e.events(since, "") { + if project != "" && ev.str("project") != project { + continue + } + switch ev.msg() { + case "scan completed": + w.scans++ + w.hashed += ev.num("files_hashed") + w.reused += ev.num("files_reused") + if b, _ := ev["forced"].(bool); b { + w.forcedScans++ + } + case "index pass completed": + w.passes++ + w.parsed += ev.num("files_parsed") + w.attached += ev.num("revisions_attached") + case "index pass failed": + w.failedPasses++ + w.parsed += ev.num("files_parsed") + w.attached += ev.num("revisions_attached") + } + } + return w +} + +func (e *env) waitEvent(since int64, timeout time.Duration, pred func(event) bool) event { + e.t.Helper() + deadline := time.Now().Add(timeout) + for { + for _, ev := range e.events(since, "") { + if pred(ev) { + return ev + } + } + if time.Now().After(deadline) { + e.t.Fatal("timed out waiting for debug.log event") + } + time.Sleep(20 * time.Millisecond) + } +} + +// waitEventTagged is waitEvent whose timeout is a designated assertion: a +// production event that never happens is the defect, not harness noise. +func (e *env) waitEventTagged(tag string, since int64, timeout time.Duration, pred func(event) bool) event { + e.t.Helper() + deadline := time.Now().Add(timeout) + for { + for _, ev := range e.events(since, "") { + if pred(ev) { + return ev + } + } + if time.Now().After(deadline) { + e.t.Fatalf("[A:%s] expected debug.log event never appeared", tag) + } + time.Sleep(20 * time.Millisecond) + } +} + +// --------------------------------------------------------------------------- +// Git fixtures +// --------------------------------------------------------------------------- + +func realDir(t *testing.T, dir string) string { + t.Helper() + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + return resolved +} + +func (e *env) git(dir string, args ...string) string { + e.t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = e.environ() + out, err := cmd.CombinedOutput() + if err != nil { + e.t.Fatalf("git %s in %s: %v\n%s", strings.Join(args, " "), dir, err, out) + } + return strings.TrimSpace(string(out)) +} + +// repo creates and commits a Git repository below the test workspace. +func (e *env) repo(name string, files map[string]string) string { + e.t.Helper() + dir := filepath.Join(filepath.Dir(e.dataHome), name) + if err := os.MkdirAll(dir, 0o755); err != nil { + e.t.Fatal(err) + } + e.git(dir, "init", "-q") + writeFiles(e.t, dir, files) + e.git(dir, "add", "-A") + e.git(dir, "commit", "-q", "-m", "initial") + return dir +} + +func (e *env) worktree(repo, path string, args ...string) string { + e.t.Helper() + e.git(repo, append(append([]string{"worktree", "add", "-q"}, args...), path)...) + return realDir(e.t, path) +} + +func writeFiles(t *testing.T, dir string, files map[string]string) { + t.Helper() + for rel, content := range files { + path := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// goFile renders a Go file whose functions each become one chunk. +func goFile(pkg string, funcs ...string) string { + var b strings.Builder + fmt.Fprintf(&b, "package %s\n", pkg) + for _, name := range funcs { + fmt.Fprintf(&b, "\n// %s computes a value unique to %s.\nfunc %s(input int) int {\n\ttotal := input\n\tfor i := 0; i < %d; i++ {\n\t\ttotal += i * %d\n\t}\n\treturn total // %s\n}\n", + name, name, name, len(name)+3, len(name)*7, name) + } + return b.String() +} + +func baseFiles() map[string]string { + return map[string]string{ + "alpha.go": goFile("fixture", "AlphaOne", "AlphaTwo", "AlphaThree"), + "beta.go": goFile("fixture", "BetaOne", "BetaTwo", "BetaThree"), + "pkg/gamma.go": goFile("pkg", "GammaOne", "GammaTwo", "GammaThree"), + "pkg/delta.go": goFile("pkg", "DeltaOne", "DeltaTwo"), + ".gitignore": "*.log\n", + } +} + +const baseFileCount = 4 // indexed source files in baseFiles + +// --------------------------------------------------------------------------- +// Read-only collection inspection +// --------------------------------------------------------------------------- + +// collections independently discovers collection databases on disk rather +// than recomputing Lumen's profile path. +func (e *env) collections() []string { + e.t.Helper() + matches, err := filepath.Glob(filepath.Join(e.dataHome, "lumen", "*", "index.db")) + if err != nil { + e.t.Fatal(err) + } + sort.Strings(matches) + return matches +} + +func (e *env) onlyCollection() string { + e.t.Helper() + dbs := e.collections() + if len(dbs) != 1 { + e.t.Fatalf("[A:one-collection] expected exactly one collection, found %d: %v", len(dbs), dbs) + } + return dbs[0] +} + +type snapshot struct { + projects map[string]int64 // path -> id + members map[string]map[string]int64 // project path -> relative path -> revision id + meta map[string]map[string]string + revisions int + retained int + retainedBytes int64 + vectors int + vecRows int + chunkDefs int + vectorIDs map[string]int64 // input hash -> vector id + orphanVectors int + orphanRevisions int + incomplete int + claimsPending int + claimsResults int + cacheBudget int64 +} + +func openReadOnly(t *testing.T, dbPath string) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite3", "file:"+dbPath+"?mode=ro&_busy_timeout=10000") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + return db +} + +func (e *env) inspect(dbPath string) snapshot { + e.t.Helper() + db := openReadOnly(e.t, dbPath) + defer func() { _ = db.Close() }() + q := func(dest any, query string, args ...any) { + e.t.Helper() + if err := db.QueryRow(query, args...).Scan(dest); err != nil { + e.t.Fatalf("inspect %q: %v", query, err) + } + } + s := snapshot{projects: map[string]int64{}, members: map[string]map[string]int64{}, meta: map[string]map[string]string{}, vectorIDs: map[string]int64{}} + rows, err := db.Query(`SELECT id, path FROM projects`) + if err != nil { + e.t.Fatal(err) + } + ids := map[int64]string{} + for rows.Next() { + var id int64 + var path string + if err := rows.Scan(&id, &path); err != nil { + e.t.Fatal(err) + } + s.projects[path], ids[id] = id, path + s.members[path], s.meta[path] = map[string]int64{}, map[string]string{} + } + _ = rows.Close() + rows, err = db.Query(`SELECT project_id, relative_path, file_revision_id FROM project_files`) + if err != nil { + e.t.Fatal(err) + } + for rows.Next() { + var pid, rid int64 + var rel string + if err := rows.Scan(&pid, &rel, &rid); err != nil { + e.t.Fatal(err) + } + s.members[ids[pid]][rel] = rid + } + _ = rows.Close() + rows, err = db.Query(`SELECT project_id, key, value FROM project_meta`) + if err != nil { + e.t.Fatal(err) + } + for rows.Next() { + var pid int64 + var k, v string + if err := rows.Scan(&pid, &k, &v); err != nil { + e.t.Fatal(err) + } + s.meta[ids[pid]][k] = v + } + _ = rows.Close() + rows, err = db.Query(`SELECT hex(input_hash), id FROM vector_keys`) + if err != nil { + e.t.Fatal(err) + } + for rows.Next() { + var h string + var id int64 + if err := rows.Scan(&h, &id); err != nil { + e.t.Fatal(err) + } + s.vectorIDs[h] = id + } + _ = rows.Close() + q(&s.revisions, `SELECT count(*) FROM file_revisions`) + q(&s.retained, `SELECT count(*) FROM revision_cache`) + q(&s.retainedBytes, `SELECT COALESCE(sum(bytes),0) FROM revision_cache`) + q(&s.vectors, `SELECT count(*) FROM vector_keys`) + q(&s.vecRows, `SELECT count(*) FROM vec_vectors`) + q(&s.chunkDefs, `SELECT count(*) FROM chunk_defs`) + q(&s.orphanVectors, `SELECT count(*) FROM vector_keys vk WHERE NOT EXISTS (SELECT 1 FROM chunk_defs cd WHERE cd.vector_id = vk.id)`) + q(&s.orphanRevisions, `SELECT count(*) FROM file_revisions fr WHERE NOT EXISTS (SELECT 1 FROM project_files pf WHERE pf.file_revision_id = fr.id) AND NOT EXISTS (SELECT 1 FROM revision_cache rc WHERE rc.revision_id = fr.id)`) + q(&s.incomplete, `SELECT count(*) FROM file_revisions WHERE complete = 0`) + q(&s.claimsPending, `SELECT count(*) FROM embedding_claims WHERE result IS NULL`) + q(&s.claimsResults, `SELECT count(*) FROM embedding_claims WHERE result IS NOT NULL`) + var budget sql.NullString + _ = db.QueryRow(`SELECT value FROM collection_meta WHERE key='revision_cache_bytes'`).Scan(&budget) + if budget.Valid { + s.cacheBudget, _ = strconv.ParseInt(budget.String, 10, 64) + } + return s +} + +// requireNoGarbage asserts that no durable data is unreachable. +func requireNoGarbage(t *testing.T, tag string, s snapshot) { + t.Helper() + if s.orphanVectors != 0 || s.orphanRevisions != 0 || s.vecRows != s.vectors || s.incomplete != 0 { + t.Fatalf("[A:%s] unreferenced durable data: orphanVectors=%d orphanRevisions=%d vecRows=%d vectors=%d incomplete=%d", + tag, s.orphanVectors, s.orphanRevisions, s.vecRows, s.vectors, s.incomplete) + } +} + +func revisionSet(members map[string]int64) map[int64]bool { + set := map[int64]bool{} + for _, id := range members { + set[id] = true + } + return set +} + +// fixture writes deliberately alter collection state for setup (policy, +// aging); they never insert memberships, revisions, or vectors. +func (e *env) fixtureExec(dbPath, query string, args ...any) int64 { + e.t.Helper() + db, err := sql.Open("sqlite3", dbPath+"?_busy_timeout=10000") + if err != nil { + e.t.Fatal(err) + } + defer func() { _ = db.Close() }() + res, err := db.Exec(query, args...) + if err != nil { + e.t.Fatalf("fixture %q: %v", query, err) + } + n, _ := res.RowsAffected() + return n +} + +// setRetention applies a revision-cache policy through the production store. +func (e *env) setRetention(dbPath, project string, bytes int64, ttl time.Duration) { + e.t.Helper() + s, err := store.NewCollection(dbPath, e.p.dims, "int8", project) + if err != nil { + e.t.Fatal(err) + } + defer func() { _ = s.Close() }() + if err := s.SetRevisionCachePolicy(bytes, ttl); err != nil { + e.t.Fatal(err) + } +} + +// expireRendezvous makes the serialized result cache unusable so it cannot +// mask the durable-vector reuse path. +func (e *env) expireRendezvous(dbPath string) { + e.fixtureExec(dbPath, `UPDATE embedding_claims SET expires_at = 1`) +} + +// scanFiles lists completed scan sidecars next to a collection. +func scanFiles(t *testing.T, dbPath string) []string { + t.Helper() + matches, err := filepath.Glob(dbPath + ".scan-*.json") + if err != nil { + t.Fatal(err) + } + return matches +} + +// physicalBytes sums DB, WAL and scan sidecars. Call only after every Lumen +// process and inspector has closed. +func physicalBytes(t *testing.T, dbPath string) int64 { + t.Helper() + var total int64 + for _, path := range append([]string{dbPath, dbPath + "-wal", dbPath + "-shm"}, scanFiles(t, dbPath)...) { + if info, err := os.Stat(path); err == nil { + total += info.Size() + } + } + return total +} + +// queryIntentHeld reports whether a query process holds the shared compute +// intent lock in this data home. +func (e *env) queryIntentHeld() bool { + lock := flock.New(filepath.Join(e.dataHome, "lumen", "compute", "query.lock")) + defer func() { _ = lock.Close() }() + ok, err := lock.TryLock() + if err != nil { + return false + } + if ok { + _ = lock.Unlock() + } + return !ok +} + +func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(10 * time.Millisecond) + } +} + +func waitGate(t *testing.T, g *gate, what string) { + t.Helper() + select { + case <-g.arrived: + case <-time.After(2 * time.Minute): + t.Fatalf("timed out waiting for %s", what) + } +} diff --git a/e2e/worktrees/mutants.json b/e2e/worktrees/mutants.json new file mode 100644 index 00000000..3020454f --- /dev/null +++ b/e2e/worktrees/mutants.json @@ -0,0 +1,146 @@ +[ + { + "id": "wrong-collection-identity", + "description": "Key the collection by worktree path instead of the shared Git common directory", + "file": "internal/config/config.go", + "find": "\tif commonDir, err := git.CommonDir(projectPath); err == nil {\n\t\tidentity = commonDir\n", + "replace": "\tif _, err := git.CommonDir(projectPath); err == nil {\n", + "test": "TestIdenticalWorktreesShareOneCollection", + "marker": "A:sibling-zero-parse" + }, + { + "id": "wrong-parent-root", + "description": "Let MCP adopt the cwd or a collection-file ancestor instead of the worktree root", + "file": "cmd/stdio.go", + "find": "\tif root, err := git.RepoRoot(projectPath); err == nil && !pathCrossesSkipDir(root, projectPath) {\n", + "replace": "\tif root, err := git.RepoRoot(projectPath); false && err == nil && !pathCrossesSkipDir(root, projectPath) {\n", + "test": "TestTopologiesKeepWorktreesIsolated", + "marker": "A:subdir-reuses-root" + }, + { + "id": "forced-full-hashing", + "description": "Never reuse metadata-validated hashes; read every file on every scan", + "file": "internal/merkle/cache.go", + "find": "\t\tif reusable {\n", + "replace": "\t\tif false && reusable {\n", + "test": "TestWarmSearchesAndRestartsReuseWork", + "marker": "A:warm-reuse-hashes" + }, + { + "id": "bypass-revision-attachment", + "description": "Skip AttachExistingFileRevision so siblings re-parse identical revisions", + "file": "internal/index/shared.go", + "find": "\t\tif !force {\n\t\t\tattached, err := idx.store.AttachExistingFileRevision(relativePath, contentHash)\n", + "replace": "\t\tif false {\n\t\t\tattached, err := idx.store.AttachExistingFileRevision(relativePath, contentHash)\n", + "test": "TestIdenticalWorktreesShareOneCollection", + "marker": "A:sibling-zero-parse" + }, + { + "id": "unchanged-files-as-changed", + "description": "Diff against an empty tree so every unchanged file is revisited", + "file": "internal/index/shared.go", + "find": "\t\tadded, removed, modified := merkle.Diff(&merkle.Tree{Files: oldHashes}, curTree)\n", + "replace": "\t\tadded, removed, modified := merkle.Diff(&merkle.Tree{Files: map[string]string{}}, curTree)\n", + "test": "TestEditsAndBranchSwitchesDoMinimalWork", + "marker": "A:edit-no-unchanged-work" + }, + { + "id": "disable-durable-vector-reuse", + "description": "Report every chunk input as missing from the durable vector table", + "file": "internal/store/shared.go", + "find": "\t\tif _, ok := present[hash]; !ok {\n", + "replace": "\t\tif _, ok := present[hash]; true || !ok {\n", + "test": "TestEditsAndBranchSwitchesDoMinimalWork", + "marker": "A:edit-embed-only-missing" + }, + { + "id": "bypass-embed-shared-claims", + "description": "Embed directly instead of claiming exact inputs through EmbedShared", + "file": "internal/index/shared.go", + "find": "\t\t\tembedded, err = idx.store.EmbedShared(ctx, batchTexts, idx.emb.Embed)\n", + "replace": "\t\t\tembedded, err = idx.emb.Embed(ctx, batchTexts)\n", + "test": "TestConcurrentProcessesComputeOnce", + "marker": "A:concurrent-cold-once" + }, + { + "id": "drop-search-membership-predicate", + "description": "Score chunks from every membership in the collection", + "file": "internal/store/shared.go", + "find": "\twhere := \"pf.project_id = ?\"\n", + "replace": "\twhere := \"(pf.project_id = ? OR 1)\"\n", + "test": "TestTopologiesKeepWorktreesIsolated", + "marker": "A:no-sibling-leak" + }, + { + "id": "disable-retention-prune", + "description": "Never evict retained revisions by TTL or byte budget", + "file": "internal/store/retention.go", + "find": "\t\tif ttl == 0 || now-unused >= ttl || unused > now || size > budget-retained {\n", + "replace": "\t\tif false && (ttl == 0 || now-unused >= ttl || unused > now || size > budget-retained) {\n", + "test": "TestRetentionAndCleanupBoundStorage", + "marker": "A:retention-budget" + }, + { + "id": "omit-orphan-vector-cleanup", + "description": "Keep vectors after the last revision referencing them is deleted", + "file": "internal/store/shared.go", + "find": "\t\t\tif used {\n", + "replace": "\t\t\tif used || true {\n", + "test": "TestRetentionAndCleanupBoundStorage", + "marker": "A:retention-gc" + }, + { + "id": "omit-scan-cache-cleanup", + "description": "Skip removal of scan sidecars whose membership was cleaned up", + "file": "internal/store/shared.go", + "find": "\tstats.ScanCachesRemoved, err = removeOrphanScanCaches(dbPath, paths)\n", + "replace": "\tstats.ScanCachesRemoved, err = len(paths)*0, nil\n", + "test": "TestRetentionAndCleanupBoundStorage", + "marker": "A:no-orphan-scan" + }, + { + "id": "disable-compute-coordination", + "description": "Admit every embedding request without the shared compute budget", + "file": "internal/embedder/budget.go", + "find": "\tif dir == \"\" {\n", + "replace": "\tif true || dir == \"\" {\n", + "test": "TestComputeSchedulingAcrossProcesses", + "marker": "A:single-flight" + }, + { + "id": "no-yield-between-http-batches", + "description": "Hold one budget admission across all provider requests of a call", + "file": "internal/embedder/failover.go", + "find": "\tif len(texts) <= embedBatchSize {\n", + "replace": "\tif true || len(texts) <= embedBatchSize {\n", + "test": "TestComputeSchedulingAcrossProcesses", + "marker": "A:query-before-next-batch" + }, + { + "id": "hook-no-spawn-when-no-index", + "description": "SessionStart does not spawn the background indexer for a project without an index", + "file": "cmd/hook.go", + "find": "\t\tbgIndexer(cwd)\n\t\tif donorPath := findDonor(cwd, modelName); donorPath != \"\" {\n", + "replace": "\t\t_ = bgIndexer\n\t\tif donorPath := findDonor(cwd, modelName); donorPath != \"\" {\n", + "test": "TestSessionStartWarmsThroughSpawnedIndexer", + "marker": "A:hook-warms" + }, + { + "id": "retention-bytes-omit-vectors", + "description": "Charge retained revisions only for metadata, not vector bytes", + "file": "internal/store/retention.go", + "find": "CASE (SELECT value FROM collection_meta WHERE key='vector_storage') WHEN 'float32' THEN 4 ELSE 1 END)", + "replace": "0)", + "test": "TestRetentionAndCleanupBoundStorage", + "marker": "A:retention-accounting" + }, + { + "id": "sweep-deletes-project-locks", + "description": "Orphan scan sweep also unlinks per-project flock files", + "file": "internal/store/scan.go", + "find": "\tfor _, entry := range entries {\n\t\tname := entry.Name()\n", + "replace": "\tfor _, entry := range entries {\n\t\tname := entry.Name()\n\t\tif strings.HasPrefix(name, filepath.Base(dbPath)+\".project-\") && strings.HasSuffix(name, \".lock\") {\n\t\t\t_ = os.Remove(filepath.Join(filepath.Dir(dbPath), name))\n\t\t}\n", + "test": "TestRetentionAndCleanupBoundStorage", + "marker": "A:locks-preserved" + } +] diff --git a/e2e/worktrees/mutate/main.go b/e2e/worktrees/mutate/main.go new file mode 100644 index 00000000..991aeab6 --- /dev/null +++ b/e2e/worktrees/mutate/main.go @@ -0,0 +1,352 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +// Command mutate is the curated production-mutation gate for the worktree +// integration suite. For every mutant in mutants.json it works in a disposable +// copy of the source tree: +// +// 1. baseline: every designated test passes on unmodified source; +// 2. applies one exact production-code edit that must still compile; +// 3. runs the designated test (which rebuilds the Lumen binary from the +// mutated copy) and requires it to fail with its designated assertion +// marker; +// 4. restores the file and requires the designated test to pass again. +// +// Compilation failures, timeouts, failures without the designated marker, +// and edits to test code all count as surviving mutants and fail the gate. +// Diffs, exact commands, and logs are kept in the output directory. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "time" +) + +type mutant struct { + ID string `json:"id"` + Description string `json:"description"` + File string `json:"file"` + Find string `json:"find"` + Replace string `json:"replace"` + Test string `json:"test"` + Marker string `json:"marker"` +} + +type outcome struct { + ID string `json:"id"` + Test string `json:"test"` + Marker string `json:"marker"` + Verdict string `json:"verdict"` // killed, survived, unevaluable + Reason string `json:"reason,omitempty"` + Command string `json:"command"` + Restored bool `json:"restored_pass"` + Duration string `json:"duration"` +} + +// Diagnostic counters are the observation channel. Mutating them would test +// the counters rather than production behavior, so such mutants are rejected. +var forbiddenInFind = []string{"FilesParsed", "RevisionsAttached", "FilesHashed", "FilesReused", "logIndexPass", "scan completed"} + +func main() { + manifest := flag.String("manifest", "e2e/worktrees/mutants.json", "mutant manifest relative to the module root") + only := flag.String("only", "", "comma-separated mutant IDs to run (default: all)") + out := flag.String("out", "", "artifact directory (default: a new temporary directory)") + timeout := flag.Duration("timeout", 10*time.Minute, "per designated test run") + keep := flag.Bool("keep", false, "keep the disposable source copy") + flag.Parse() + if err := run(*manifest, *only, *out, *timeout, *keep); err != nil { + fmt.Fprintln(os.Stderr, "mutation gate:", err) + os.Exit(1) + } +} + +func run(manifestPath, only, outDir string, timeout time.Duration, keep bool) error { + root, err := moduleRoot() + if err != nil { + return err + } + data, err := os.ReadFile(filepath.Join(root, manifestPath)) + if err != nil { + return err + } + var mutants []mutant + if err := json.Unmarshal(data, &mutants); err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + if only != "" { + ids := strings.Split(only, ",") + mutants = slices.DeleteFunc(mutants, func(m mutant) bool { return !slices.Contains(ids, m.ID) }) + } + if len(mutants) == 0 { + return errors.New("no mutants selected") + } + if outDir == "" { + if outDir, err = os.MkdirTemp("", "lumen-mutations-"); err != nil { + return err + } + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + return err + } + work, err := os.MkdirTemp("", "lumen-mutation-src-") + if err != nil { + return err + } + if keep { + fmt.Println("source copy:", work) + } else { + defer func() { _ = os.RemoveAll(work) }() + } + if err := copySource(root, work); err != nil { + return fmt.Errorf("copy source: %w", err) + } + fmt.Printf("artifacts: %s\n", outDir) + + var tests []string + for _, m := range mutants { + if !slices.Contains(tests, m.Test) { + tests = append(tests, m.Test) + } + } + for _, test := range tests { + fmt.Printf("baseline %s ... ", test) + cmd, output, err := runTest(work, test, timeout) + _ = os.WriteFile(filepath.Join(outDir, "baseline-"+test+".log"), []byte(cmd+"\n\n"+output), 0o644) + if err != nil { + fmt.Println("FAIL") + return fmt.Errorf("baseline %s is not green; see %s", test, outDir) + } + fmt.Println("ok") + } + + var results []outcome + for _, m := range mutants { + r := evaluate(work, outDir, m, timeout) + results = append(results, r) + fmt.Printf("%-34s %-11s restored=%-5v %s\n", m.ID, r.Verdict, r.Restored, r.Reason) + } + summary, _ := json.MarshalIndent(results, "", " ") + if err := os.WriteFile(filepath.Join(outDir, "summary.json"), summary, 0o644); err != nil { + return err + } + failed := 0 + for _, r := range results { + if r.Verdict != "killed" || !r.Restored { + failed++ + } + } + fmt.Printf("\n%d/%d mutants killed with designated assertions and restored green\n", len(results)-failed, len(results)) + if failed > 0 { + return fmt.Errorf("%d mutants survived, were unevaluable, or did not restore green", failed) + } + return nil +} + +func evaluate(work, outDir string, m mutant, timeout time.Duration) outcome { + start := time.Now() + r := outcome{ID: m.ID, Test: m.Test, Marker: m.Marker, Verdict: "unevaluable"} + defer func() { r.Duration = time.Since(start).Round(time.Second).String() }() + dir := filepath.Join(outDir, m.ID) + _ = os.MkdirAll(dir, 0o755) + if reason := validate(m); reason != "" { + r.Reason = reason + return r + } + path := filepath.Join(work, m.File) + original, err := os.ReadFile(path) + if err != nil { + r.Reason = err.Error() + return r + } + if n := strings.Count(string(original), m.Find); n != 1 { + r.Reason = fmt.Sprintf("find text occurs %d times in %s", n, m.File) + return r + } + mutated := strings.Replace(string(original), m.Find, m.Replace, 1) + if err := os.WriteFile(path, []byte(mutated), 0o644); err != nil { + r.Reason = err.Error() + return r + } + restore := func() { _ = os.WriteFile(path, original, 0o644) } + _ = os.WriteFile(filepath.Join(dir, "mutant.diff"), []byte(unifiedDiff(m.File, string(original), mutated)), 0o644) + + build := exec.Command("go", "build", "-tags=fts5", "-o", os.DevNull, ".") + build.Dir, build.Env = work, append(os.Environ(), "CGO_ENABLED=1") + if output, err := build.CombinedOutput(); err != nil { + restore() + _ = os.WriteFile(filepath.Join(dir, "build.log"), output, 0o644) + r.Reason = "mutant does not compile" + return r + } + + cmd, output, err := runTest(work, m.Test, timeout) + r.Command = cmd + _ = os.WriteFile(filepath.Join(dir, "command.txt"), []byte(cmd+"\n"), 0o644) + _ = os.WriteFile(filepath.Join(dir, "mutated.log"), []byte(output), 0o644) + restore() + switch { + case err == nil: + r.Verdict, r.Reason = "survived", "designated test passed" + case errors.Is(err, context.DeadlineExceeded) || strings.Contains(output, "panic: test timed out"): + r.Reason = "designated test timed out" + case strings.Contains(output, "[build failed]") || strings.Contains(output, "build lumen from"): + r.Reason = "test or binary build failed" + case !strings.Contains(output, "--- FAIL: "+m.Test): + r.Reason = "designated test did not report a failure" + case !strings.Contains(output, "["+m.Marker+"]"): + r.Verdict, r.Reason = "survived", "failed without designated marker "+m.Marker+": "+firstFailure(output) + default: + r.Verdict, r.Reason = "killed", firstFailure(output) + } + _ = os.WriteFile(filepath.Join(dir, "designated-failure.txt"), []byte(r.Reason+"\n"), 0o644) + + _, restored, err := runTest(work, m.Test, timeout) + _ = os.WriteFile(filepath.Join(dir, "restored.log"), []byte(restored), 0o644) + r.Restored = err == nil + return r +} + +func validate(m mutant) string { + switch { + case m.ID == "" || m.File == "" || m.Find == "" || m.Test == "" || m.Marker == "": + return "incomplete manifest entry" + case m.Find == m.Replace: + return "replacement does not change the source" + case strings.HasSuffix(m.File, "_test.go") || strings.HasPrefix(filepath.ToSlash(m.File), "e2e/"): + return "mutants must change production code, not tests" + case !strings.HasSuffix(m.File, ".go"): + return "mutants must change Go production code" + } + for _, word := range forbiddenInFind { + if strings.Contains(m.Find, word) || strings.Contains(m.Replace, word) { + return "mutants must not alter diagnostic counters (" + word + ")" + } + } + return "" +} + +func runTest(dir, test string, timeout time.Duration) (string, string, error) { + args := []string{"test", "-tags=fts5,worktree_e2e", "-count=1", "-v", "-timeout=" + timeout.String(), "-run", "^" + test + "$", "./e2e/worktrees/"} + ctx, cancel := context.WithTimeout(context.Background(), timeout+time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "go", args...) + cmd.Dir, cmd.Env = dir, append(os.Environ(), "CGO_ENABLED=1") + var buf bytes.Buffer + cmd.Stdout, cmd.Stderr = &buf, &buf + err := cmd.Run() + if ctx.Err() != nil { + err = ctx.Err() + } + return "(cd " + dir + " && CGO_ENABLED=1 go " + strings.Join(args, " ") + ")", buf.String(), err +} + +func firstFailure(output string) string { + for line := range strings.SplitSeq(output, "\n") { + if strings.Contains(line, "[A:") { + return strings.TrimSpace(line) + } + } + for line := range strings.SplitSeq(output, "\n") { + if strings.Contains(line, "--- FAIL") { + return strings.TrimSpace(line) + } + } + return "" +} + +func unifiedDiff(name, before, after string) string { + dir, err := os.MkdirTemp("", "lumen-mutant-diff-") + if err != nil { + return "" + } + defer func() { _ = os.RemoveAll(dir) }() + a, b := filepath.Join(dir, "a"), filepath.Join(dir, "b") + _ = os.WriteFile(a, []byte(before), 0o644) + _ = os.WriteFile(b, []byte(after), 0o644) + out, _ := exec.Command("git", "diff", "--no-index", "--no-color", "--src-prefix=a/"+name+"#", "--dst-prefix=b/"+name+"#", a, b).Output() + return string(out) +} + +func moduleRoot() (string, error) { + out, err := exec.Command("go", "env", "GOMOD").Output() + if err != nil { + return "", err + } + gomod := strings.TrimSpace(string(out)) + if gomod == "" || gomod == os.DevNull { + return "", errors.New("run inside the lumen module") + } + return filepath.Dir(gomod), nil +} + +// copySource copies tracked and untracked, non-ignored files, so uncommitted +// work is evaluated exactly as it is in the checkout. +func copySource(root, dst string) error { + cmd := exec.Command("git", "ls-files", "-z", "--cached", "--others", "--exclude-standard") + cmd.Dir = root + out, err := cmd.Output() + if err != nil { + return err + } + for rel := range strings.SplitSeq(strings.TrimRight(string(out), "\x00"), "\x00") { + src := filepath.Join(root, rel) + info, err := os.Lstat(src) + if os.IsNotExist(err) { + continue // deleted in the working tree + } + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + link, err := os.Readlink(src) + if err != nil { + return err + } + if err := os.Symlink(link, target); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + continue + } + if err := copyFile(src, target, info.Mode().Perm()); err != nil { + return err + } + } + return nil +} + +func copyFile(src, dst string, perm os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} diff --git a/e2e/worktrees/scenarios_test.go b/e2e/worktrees/scenarios_test.go new file mode 100644 index 00000000..d7e41b0c --- /dev/null +++ b/e2e/worktrees/scenarios_test.go @@ -0,0 +1,989 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +//go:build worktree_e2e + +package worktrees + +import ( + "crypto/sha256" + "fmt" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +// Scenario 1: identical linked worktrees share one repository/profile +// collection. New siblings hash their files once but parse and embed nothing. +func TestIdenticalWorktreesShareOneCollection(t *testing.T) { + for _, backend := range []string{"ollama", "lmstudio"} { + t.Run(backend, func(t *testing.T) { + e := newEnv(t, withBackend(backend)) + main := e.repo("main repo", baseFiles()) + sibling := e.worktree(main, filepath.Join(filepath.Dir(main), "sibling wt"), "-b", "sibling") + third := e.worktree(main, filepath.Join(filepath.Dir(main), "third"), "-b", "third") + + // Positive control: a cold index parses and embeds everything. + m, l := e.p.mark(), e.logMark() + e.index(main) + cold, coldDocs := e.work(l, main), e.p.documents(m) + if cold.hashed != baseFileCount || cold.parsed != baseFileCount || len(coldDocs) == 0 { + t.Fatalf("[A:cold-control] cold index hashed=%d parsed=%d docs=%d", cold.hashed, cold.parsed, len(coldDocs)) + } + requireEachOnce(t, "cold-each-once", coldDocs) + + m, l = e.p.mark(), e.logMark() + e.index(sibling) + w := e.work(l, sibling) + if w.hashed != baseFileCount { + t.Fatalf("[A:sibling-initial-hash] new sibling hashed %d files, want %d", w.hashed, baseFileCount) + } + if w.parsed != 0 { + t.Fatalf("[A:sibling-zero-parse] identical sibling parsed %d files", w.parsed) + } + if w.attached != baseFileCount { + t.Fatalf("[A:sibling-attached] identical sibling attached %d revisions, want %d", w.attached, baseFileCount) + } + if docs := e.p.documents(m); len(docs) != 0 { + t.Fatalf("[A:sibling-zero-embed] identical sibling embedded %d inputs", len(docs)) + } + + // The MCP server is the path agent hosts use for a new worktree. + s := e.mcp() + m, l = e.p.mark(), e.logMark() + files, _ := s.search("how are totals computed", third, third) + w = e.work(l, third) + if w.parsed != 0 || len(e.p.documents(m)) != 0 { + t.Fatalf("[A:sibling-zero-parse] MCP sibling parsed=%d embedded=%d", w.parsed, len(e.p.documents(m))) + } + if q := e.p.queryRequests(m); q != 1 { + t.Fatalf("[A:query-count] MCP search issued %d query requests, want 1", q) + } + if !slices.Equal(files, []string{"alpha.go", "beta.go", "pkg/delta.go", "pkg/gamma.go"}) { + t.Fatalf("[A:sibling-results] MCP sibling results = %v", files) + } + s.close() + + snap := e.inspect(e.onlyCollection()) + if len(snap.projects) != 3 { + t.Fatalf("[A:memberships] projects = %v", snap.projects) + } + want := revisionSet(snap.members[main]) + for _, project := range []string{main, sibling, third} { + members, ok := snap.members[project] + if !ok || len(members) != baseFileCount { + t.Fatalf("[A:memberships] %s has %d memberships", project, len(members)) + } + if got := revisionSet(members); !mapsEqual(got, want) { + t.Fatalf("[A:shared-revisions] %s revisions %v differ from %v", project, got, want) + } + } + if distinct := len(multiplicity(coldDocs)); snap.vectors != distinct || snap.chunkDefs != distinct { + t.Fatalf("[A:shared-vectors] vectors=%d chunkDefs=%d, want %d", snap.vectors, snap.chunkDefs, distinct) + } + requireNoGarbage(t, "no-garbage", snap) + }) + } +} + +// Scenario 2: warm searches inside the freshness TTL do not scan; after TTL +// expiry or a restart, stable files reuse hashes and nothing is parsed or +// embedded. Revision and vector IDs remain stable. +func TestWarmSearchesAndRestartsReuseWork(t *testing.T) { + const ttl = 4 * time.Second + e := newEnv(t, withEnv("LUMEN_FRESHNESS_TTL="+ttl.String())) + main := e.repo("warm", baseFiles()) + time.Sleep(settleDelay) + e.index(main) + indexedAt := time.Now() + db := e.onlyCollection() + before := e.inspect(db) + + s := e.mcp() + m, l := e.p.mark(), e.logMark() + s.search("first warm query", main, main) + s.search("second warm query", main, main) + if time.Since(indexedAt) < ttl { + if w := e.work(l, main); w.scans != 0 { + t.Fatalf("[A:ttl-no-scan] searches inside the freshness TTL scanned %d times", w.scans) + } + } else { + t.Logf("SKIPPED [A:ttl-no-scan]: searches finished %s after indexing, beyond the %s TTL", time.Since(indexedAt), ttl) + } + if q, d := e.p.queryRequests(m), len(e.p.documents(m)); q != 2 || d != 0 { + t.Fatalf("[A:warm-zero-embed] warm searches: queries=%d documents=%d", q, d) + } + + time.Sleep(ttl + 200*time.Millisecond) + m, l = e.p.mark(), e.logMark() + s.search("after ttl query", main, main) + w := e.work(l, main) + if w.scans != 1 || w.hashed != 0 || w.reused != baseFileCount { + t.Fatalf("[A:warm-reuse-hashes] after TTL: scans=%d hashed=%d reused=%d", w.scans, w.hashed, w.reused) + } + if w.parsed != 0 || w.passes != 0 || len(e.p.documents(m)) != 0 { + t.Fatalf("[A:warm-zero-embed] after TTL: parsed=%d passes=%d documents=%d", w.parsed, w.passes, len(e.p.documents(m))) + } + s.close() + + // Restart: a new server process loads the scan sidecar. + s = e.mcp() + m, l = e.p.mark(), e.logMark() + s.search("after restart query", main, main) + w = e.work(l, main) + if w.scans != 1 || w.hashed != 0 || w.reused != baseFileCount { + t.Fatalf("[A:warm-reuse-hashes] after restart: scans=%d hashed=%d reused=%d", w.scans, w.hashed, w.reused) + } + if w.parsed != 0 || len(e.p.documents(m)) != 0 { + t.Fatalf("[A:warm-zero-embed] after restart: parsed=%d documents=%d", w.parsed, len(e.p.documents(m))) + } + s.close() + + l = e.logMark() + r := e.index(main) + if !strings.Contains(r.stdout, "already up to date") { + t.Fatalf("[A:warm-zero-embed] CLI restart reindexed:\n%s", r.stdout) + } + if w := e.work(l, main); w.hashed != 0 || w.reused != baseFileCount { + t.Fatalf("[A:warm-reuse-hashes] CLI restart hashed=%d reused=%d", w.hashed, w.reused) + } + after := e.inspect(db) + if !mapsEqual(after.members[main], before.members[main]) || !mapsEqual(after.vectorIDs, before.vectorIDs) { + t.Fatalf("[A:stable-ids] revision or vector IDs changed across warm restarts") + } +} + +// Scenario 3: a one-file edit parses only that file and embeds only its +// genuinely missing inputs; switching branches back to a retained revision +// needs no parsing or embedding. +func TestEditsAndBranchSwitchesDoMinimalWork(t *testing.T) { + e := newEnv(t) + main := e.repo("edits", baseFiles()) + e.index(main) + db := e.onlyCollection() + cold := multiplicity(e.p.documents(0)) + betaChunks := 0 + for in := range cold { + if strings.HasPrefix(in, "// beta.go\n") { + betaChunks++ + } + } + if betaChunks < 3 { + t.Fatalf("[A:cold-control] beta.go produced %d chunks", betaChunks) + } + time.Sleep(settleDelay) + + e.git(main, "checkout", "-q", "-b", "feature") + editFile(t, filepath.Join(main, "beta.go"), "return total // BetaTwo\n", "return total * 2 // BetaTwo edited\n") + // Rendezvous results must not mask the durable vector reuse path. + e.expireRendezvous(db) + m, l := e.p.mark(), e.logMark() + e.index(main) + w := e.work(l, main) + if w.hashed != 1 || w.reused != baseFileCount-1 { + t.Fatalf("[A:edit-hash-one] one-file edit hashed=%d reused=%d", w.hashed, w.reused) + } + if w.parsed != 1 { + t.Fatalf("[A:edit-parse-one] one-file edit parsed %d files", w.parsed) + } + if w.attached != 0 { + t.Fatalf("[A:edit-no-unchanged-work] one-file edit revisited %d unchanged files", w.attached) + } + docs := e.p.documents(m) + requireEachOnce(t, "edit-each-once", docs) + if len(docs) == 0 { + t.Fatalf("[A:edit-embed-only-missing] edited chunk was not embedded") + } + for _, in := range docs { + if !strings.HasPrefix(in, "// beta.go\n") || !strings.Contains(in, "BetaTwo edited") || cold[in] != 0 { + t.Fatalf("[A:edit-embed-only-missing] embedded an input that was not missing: %q", firstLine(in)) + } + } + if len(docs) >= betaChunks { + t.Fatalf("[A:edit-embed-only-missing] embedded %d of beta.go's %d chunks", len(docs), betaChunks) + } + e.git(main, "commit", "-q", "-am", "edit beta") + + for _, branch := range []string{"main", "feature"} { + e.expireRendezvous(db) + e.git(main, "checkout", "-q", branch) + m, l = e.p.mark(), e.logMark() + e.index(main) + w = e.work(l, main) + if w.parsed != 0 { + t.Fatalf("[A:switch-no-parse] switching to %s parsed %d files", branch, w.parsed) + } + if w.attached != 1 { + t.Fatalf("[A:switch-attach] switching to %s attached %d revisions, want 1", branch, w.attached) + } + if docs := e.p.documents(m); len(docs) != 0 { + t.Fatalf("[A:switch-no-embed] switching to %s embedded %d inputs", branch, len(docs)) + } + } + snap := e.inspect(db) + if snap.retained != 1 || snap.revisions != baseFileCount+1 { + t.Fatalf("[A:switch-retained] retained=%d revisions=%d", snap.retained, snap.revisions) + } + requireNoGarbage(t, "no-garbage", snap) +} + +// Scenario 4: uncommitted edits, additions, deletions, renames, ignore-rule +// changes, and same-size edits with restored mtimes across restarts. +func TestFilesystemChangesAreDetected(t *testing.T) { + e := newEnv(t) + main := e.repo("filesystem", baseFiles()) + e.index(main) + db := e.onlyCollection() + members := func() []string { + var paths []string + for rel := range e.inspect(db).members[main] { + paths = append(paths, rel) + } + slices.Sort(paths) + return paths + } + + writeFiles(t, main, map[string]string{"epsilon.go": goFile("fixture", "EpsilonOne")}) + l := e.logMark() + e.index(main) + if w := e.work(l, main); w.parsed != 1 || !slices.Contains(members(), "epsilon.go") { + t.Fatalf("[A:fs-addition] untracked addition parsed=%d members=%v", w.parsed, members()) + } + + if err := os.Remove(filepath.Join(main, "alpha.go")); err != nil { + t.Fatal(err) + } + m, l := e.p.mark(), e.logMark() + e.index(main) + if w := e.work(l, main); w.parsed != 0 || len(e.p.documents(m)) != 0 || slices.Contains(members(), "alpha.go") { + t.Fatalf("[A:fs-deletion] deletion parsed=%d members=%v", w.parsed, members()) + } + + e.git(main, "mv", "beta.go", "beta_renamed.go") + m, l = e.p.mark(), e.logMark() + e.index(main) + w := e.work(l, main) + docs := e.p.documents(m) + if w.parsed != 1 || len(docs) < 3 || slices.Contains(members(), "beta.go") || !slices.Contains(members(), "beta_renamed.go") { + t.Fatalf("[A:fs-rename] rename parsed=%d docs=%d members=%v", w.parsed, len(docs), members()) + } + for _, in := range docs { + // Embedding inputs are filepath-aware, so a rename embeds new inputs. + if !strings.HasPrefix(in, "// beta_renamed.go\n") { + t.Fatalf("[A:fs-rename] unexpected input %q", firstLine(in)) + } + } + + writeFiles(t, main, map[string]string{".gitignore": "*.log\npkg/\n"}) + m, l = e.p.mark(), e.logMark() + e.index(main) + if got := members(); slices.Contains(got, "pkg/gamma.go") || slices.Contains(got, "pkg/delta.go") { + t.Fatalf("[A:fs-ignore] ignored files remain members: %v", got) + } + if w := e.work(l, main); w.parsed != 0 || len(e.p.documents(m)) != 0 { + t.Fatalf("[A:fs-ignore] ignoring files parsed=%d embedded=%d", w.parsed, len(e.p.documents(m))) + } + writeFiles(t, main, map[string]string{".gitignore": "*.log\n"}) + m, l = e.p.mark(), e.logMark() + e.index(main) + if w := e.work(l, main); w.parsed != 0 || w.attached != 2 || len(e.p.documents(m)) != 0 || !slices.Contains(members(), "pkg/gamma.go") { + t.Fatalf("[A:fs-ignore] un-ignoring parsed=%d attached=%d members=%v", w.parsed, w.attached, members()) + } + + // Same-size edit with the original mtime, observed by a new process after + // the timestamp race window: only identity/ctime reveal the change. + time.Sleep(settleDelay) + e.index(main) + gamma := filepath.Join(main, "pkg", "gamma.go") + info, err := os.Stat(gamma) + if err != nil { + t.Fatal(err) + } + editFile(t, gamma, "total += i * 56", "total += i * 65") + if after, _ := os.Stat(gamma); after.Size() != info.Size() { + t.Fatal("fixture edit changed the file size") + } + if err := os.Chtimes(gamma, info.ModTime(), info.ModTime()); err != nil { + t.Fatal(err) + } + time.Sleep(settleDelay) + l = e.logMark() + e.index(main) + if w := e.work(l, main); w.parsed != 1 || w.hashed != 1 { + t.Fatalf("[A:fs-same-size-edit] same-size edit with restored mtime: hashed=%d parsed=%d", w.hashed, w.parsed) + } + + s := e.mcp() + files, _ := s.search("filesystem state", main, main) + if want := []string{"beta_renamed.go", "epsilon.go", "pkg/delta.go", "pkg/gamma.go"}; !slices.Equal(files, want) { + t.Fatalf("[A:fs-search] results %v, want %v", files, want) + } + requireNoGarbage(t, "no-garbage", e.inspect(db)) +} + +// Scenario 5: external and nested worktrees, detached HEAD, subdirectory +// working directories, paths with spaces, and symlink aliases. Separate clones +// and profiles remain isolated. +func TestTopologiesKeepWorktreesIsolated(t *testing.T) { + e := newEnv(t) + main := e.repo("topology main", baseFiles()) + parent := filepath.Dir(main) + nested := e.worktree(main, filepath.Join(main, ".worktrees", "nested"), "-b", "nested") + writeFiles(t, nested, map[string]string{"nested_only.go": goFile("fixture", "NestedOnly")}) + external := e.worktree(main, filepath.Join(parent, "external wt"), "-b", "external") + writeFiles(t, external, map[string]string{"external_only.go": goFile("fixture", "ExternalOnly")}) + detached := e.worktree(main, filepath.Join(parent, "detached"), "--detach") + alias := filepath.Join(parent, "alias") + if err := os.Symlink(main, alias); err != nil { + t.Fatal(err) + } + base := []string{"alpha.go", "beta.go", "pkg/delta.go", "pkg/gamma.go"} + + e.index(main) + db := e.onlyCollection() + for rel := range e.inspect(db).members[main] { + if strings.HasPrefix(rel, ".worktrees/") { + t.Fatalf("[A:main-excludes-nested] main project indexed nested worktree file %s", rel) + } + } + + s := e.mcp() + pkg := filepath.Join(main, "pkg") + m, l := e.p.mark(), e.logMark() + files, _ := s.search("subdirectory query", pkg, pkg) + if w := e.work(l, ""); w.parsed != 0 || len(e.p.documents(m)) != 0 { + t.Fatalf("[A:subdir-reuses-root] subdirectory cwd parsed=%d embedded=%d", w.parsed, len(e.p.documents(m))) + } + if !slices.Equal(files, []string{"pkg/delta.go", "pkg/gamma.go"}) && !slices.Equal(files, []string{"delta.go", "gamma.go"}) { + t.Fatalf("[A:subdir-results] subdirectory results = %v", files) + } + + m, l = e.p.mark(), e.logMark() + files, _ = s.search("nested worktree query", nested, main) + if w := e.work(l, ""); w.parsed != 1 || w.attached != baseFileCount || !allFrom(e.p.documents(m), "nested_only.go") { + t.Fatalf("[A:nested-own-root] nested worktree parsed=%d attached=%d documents=%d", w.parsed, w.attached, len(e.p.documents(m))) + } + if want := append([]string{"nested_only.go"}, base...); !sameSet(files, want) { + t.Fatalf("[A:nested-own-root] nested results = %v, want %v", files, want) + } + + files, _ = s.search("main isolation query", main, main) + if !sameSet(files, base) { + t.Fatalf("[A:no-sibling-leak] main results leak sibling content: %v", files) + } + + m, l = e.p.mark(), e.logMark() + files, _ = s.search("external worktree query", external, external) + if !sameSet(files, append([]string{"external_only.go"}, base...)) { + t.Fatalf("[A:no-sibling-leak] external results = %v", files) + } + if w := e.work(l, external); w.parsed != 1 || !allFrom(e.p.documents(m), "external_only.go") { + t.Fatalf("[A:external-results] external parsed %d files, want 1; documents=%d", w.parsed, len(e.p.documents(m))) + } + + m, l = e.p.mark(), e.logMark() + files, _ = s.search("detached query", detached, detached) + if w := e.work(l, detached); w.parsed != 0 || len(e.p.documents(m)) != 0 || !sameSet(files, base) { + t.Fatalf("[A:detached-shares] detached HEAD parsed=%d embedded=%d results=%v", w.parsed, len(e.p.documents(m)), files) + } + s.close() + + // A fresh server reached through a symlink alias reuses the canonical + // worktree: no new membership, parsing, or embedding. + s = e.mcp() + m, l = e.p.mark(), e.logMark() + files, _ = s.search("alias query", alias, alias) + if w := e.work(l, ""); w.parsed != 0 || len(e.p.documents(m)) != 0 || !sameSet(files, base) { + t.Fatalf("[A:alias-no-rework] alias parsed=%d embedded=%d results=%v", w.parsed, len(e.p.documents(m)), files) + } + s.close() + snap := e.inspect(db) + want := map[string]bool{main: true, nested: true, external: true, detached: true} + for project := range snap.projects { + if !want[project] { + t.Fatalf("[A:canonical-roots] unexpected membership %s (all: %v)", project, keys(snap.projects)) + } + } + + // Separate clones and profiles never share a collection. + clone := filepath.Join(parent, "clone") + e.git(parent, "clone", "-q", main, clone) + l = e.logMark() + e.index(clone) + if w := e.work(l, clone); w.parsed != baseFileCount { + t.Fatalf("[A:clone-isolated] clone parsed %d files, want %d", w.parsed, baseFileCount) + } + if n := len(e.collections()); n != 2 { + t.Fatalf("[A:clone-isolated] %d collections after indexing a clone, want 2", n) + } + e.extra = append(e.extra, "LUMEN_MAX_CHUNK_TOKENS=256") + e.index(main) + if n := len(e.collections()); n != 3 { + t.Fatalf("[A:profile-isolated] %d collections after a profile change, want 3", n) + } + if got := e.inspect(db); !mapsEqual(got.members[main], snap.members[main]) { + t.Fatalf("[A:profile-isolated] profile change modified the original collection") + } +} + +// Scenario 6a: independent cold processes compute each unique document input +// once, and same-worktree startup avoids duplicate passes. +func TestConcurrentProcessesComputeOnce(t *testing.T) { + e := newEnv(t) + main := e.repo("concurrent", baseFiles()) + wt1 := e.worktree(main, filepath.Join(filepath.Dir(main), "c1"), "-b", "c1") + wt2 := e.worktree(main, filepath.Join(filepath.Dir(main), "c2"), "-b", "c2") + + g := e.p.block(documentsMentioning("")) + m, l := e.p.mark(), e.logMark() + a, b := e.start("index", wt1), e.start("index", wt2) + waitGate(t, g, "first document request") + for _, wt := range []string{wt1, wt2} { + e.waitEvent(l, time.Minute, func(ev event) bool { return ev.msg() == "scan completed" && ev.str("project") == wt }) + } + // Both processes have scanned; let the second reach its claims while the + // first request is still held open. + time.Sleep(500 * time.Millisecond) + g.Release() + if err := a.wait(t); err != nil { + t.Fatalf("index %s: %v\n%s", wt1, err, a.stderr.String()) + } + if err := b.wait(t); err != nil { + t.Fatalf("index %s: %v\n%s", wt2, err, b.stderr.String()) + } + docs := e.p.documents(m) + requireEachOnce(t, "concurrent-cold-once", docs) + snap := e.inspect(e.onlyCollection()) + if snap.vectors != len(docs) || !mapsEqual(revisionSet(snap.members[wt1]), revisionSet(snap.members[wt2])) || len(snap.members[wt1]) != baseFileCount { + t.Fatalf("[A:concurrent-cold-once] vectors=%d docs=%d members=%d/%d", snap.vectors, len(docs), len(snap.members[wt1]), len(snap.members[wt2])) + } + if peak := e.p.peakInFlight(); peak != 1 { + t.Fatalf("[A:single-flight] peak in-flight embedding requests = %d", peak) + } + + // Same worktree: CLI, SessionStart background child, and MCP all start + // while the first indexer holds its request open. + same := e.repo("same", map[string]string{"same_a.go": goFile("same", "SameOne", "SameTwo"), "same_b.go": goFile("same", "SameThree")}) + g = e.p.block(documentsMentioning("// same_")) + m, l = e.p.mark(), e.logMark() + first := e.start("index", same) + waitGate(t, g, "same-worktree document request") + second := e.lumen("index", same) + if second.err != nil || !strings.Contains(second.stderr, "already running") { + t.Fatalf("[A:same-worktree-skip] second CLI indexer was not skipped: err=%v\n%s", second.err, second.stderr) + } + _, hookPID := e.sessionStart(same) + ev := e.waitEvent(l, time.Minute, func(ev event) bool { + return ev.msg() == "index skipped: another indexer is already running" && ev.str("project") == same + }) + if ev.pid() == hookPID || ev.pid() == first.cmd.Process.Pid { + t.Fatalf("[A:same-worktree-skip] skip was not reported by the spawned hook child") + } + s := e.mcp() + _, text := s.search("same worktree query", same, same) + if !strings.Contains(text, "being updated in the background") { + t.Fatalf("[A:same-worktree-skip] MCP did not defer to the running indexer:\n%s", text) + } + g.Release() + if err := first.wait(t); err != nil { + t.Fatalf("index %s: %v", same, err) + } + e.waitNoIndexers() + docs = e.p.documents(m) + requireEachOnce(t, "same-worktree-once", docs) + if w := e.work(l, same); w.passes != 1 || w.parsed != 2 { + t.Fatalf("[A:same-worktree-one-pass] passes=%d parsed=%d", w.passes, w.parsed) + } +} + +// Scenario 6b: the real SessionStart hook warms the index through the binary +// it spawns, and a following search in the session does no indexing work. +func TestSessionStartWarmsThroughSpawnedIndexer(t *testing.T) { + e := newEnv(t) + main := e.repo("hook", baseFiles()) + m, l := e.p.mark(), e.logMark() + text, hookPID := e.sessionStart(main) + if !strings.Contains(text, "indexing in background") { + t.Fatalf("[A:hook-warms] unexpected hook context: %s", text) + } + ev := e.waitEventTagged("hook-warms", l, 2*time.Minute, func(ev event) bool { + return ev.msg() == "index pass completed" && ev.str("project") == main + }) + if ev.pid() == hookPID || ev.pid() == 0 { + t.Fatalf("[A:hook-warms] index pass ran in pid %d, not a spawned child of hook %d", ev.pid(), hookPID) + } + e.waitNoIndexers() + if w := e.work(l, main); w.parsed != baseFileCount || len(e.p.documents(m)) == 0 { + t.Fatalf("[A:hook-warms] background child parsed=%d embedded=%d", w.parsed, len(e.p.documents(m))) + } + if root := e.inspect(e.onlyCollection()).meta[main]["root_hash"]; root == "" { + t.Fatalf("[A:hook-warms] background child did not complete the index") + } + + s := e.mcp() + m, l = e.p.mark(), e.logMark() + files, _ := s.search("hook warmed query", main, main) + if w := e.work(l, main); w.scans != 0 || w.parsed != 0 || len(e.p.documents(m)) != 0 { + t.Fatalf("[A:hook-warm-search] first search after warmup scans=%d parsed=%d embedded=%d", w.scans, w.parsed, len(e.p.documents(m))) + } + if len(files) != baseFileCount { + t.Fatalf("[A:hook-warm-search] results = %v", files) + } + + l = e.logMark() + text, _ = e.sessionStart(main) + if !strings.Contains(text, "Lumen index ready") { + t.Fatalf("[A:hook-ready] unexpected hook context: %s", text) + } + time.Sleep(500 * time.Millisecond) + e.waitNoIndexers() + if w := e.work(l, main); w.scans != 0 || w.passes != 0 { + t.Fatalf("[A:hook-ready] ready session start triggered scans=%d passes=%d", w.scans, w.passes) + } +} + +// Scenario 7: one embedding HTTP request at a time per data home across +// repositories and processes; queries precede the next background request; +// terminated, failed, and expired owners recover without false completion. +func TestComputeSchedulingAcrossProcesses(t *testing.T) { + e := newEnv(t) + names := make([]string, 80) + for i := range names { + names[i] = fmt.Sprintf("BigFunction%02d", i) + } + big := e.repo("big", map[string]string{"big.go": goFile("big", names...)}) + other := e.repo("other", map[string]string{"other.go": goFile("other", "OtherOne", "OtherTwo")}) + small := e.repo("small", baseFiles()) + e.index(small) + s := e.mcp("LUMEN_FRESHNESS_TTL=10m") + s.search("warm the search session", small, small) + + m, l := e.p.mark(), e.logMark() + g := e.p.block(documentsMentioning("// big.go\n")) + a := e.start("index", big) + waitGate(t, g, "first big.go request") + // A different repository indexed by a different process in the same data + // home must not overlap the held request. + c := e.start("index", other) + e.waitEvent(l, time.Minute, func(ev event) bool { return ev.msg() == "scan completed" && ev.str("project") == other }) + time.Sleep(500 * time.Millisecond) + if peak := e.p.peakInFlight(); peak != 1 { + t.Fatalf("[A:single-flight] another process's request overlapped the held one: peak in-flight = %d", peak) + } + + const query = "which function handles the priority query" + queryDone := make(chan error, 1) + go func() { queryDone <- s.trySearch(query, small, small) }() + waitFor(t, 30*time.Second, "query intent", e.queryIntentHeld) + g.Release() + if err := <-queryDone; err != nil { + t.Fatal(err) + } + if err := a.wait(t); err != nil { + t.Fatalf("index big: %v\n%s", err, a.stderr.String()) + } + if err := c.wait(t); err != nil { + t.Fatalf("index other: %v\n%s", err, c.stderr.String()) + } + if peak := e.p.peakInFlight(); peak != 1 { + t.Fatalf("[A:single-flight] peak in-flight embedding requests = %d", peak) + } + reqs := e.p.since(m) + bigRequests := 0 + for _, r := range reqs { + if !r.Query && strings.HasPrefix(r.Inputs[0], "// big.go\n") { + bigRequests++ + if len(r.Inputs) > 32 { + t.Fatalf("[A:request-size] provider request carried %d inputs", len(r.Inputs)) + } + } + } + if bigRequests < 3 { + t.Fatalf("[A:multi-request-control] big.go used %d HTTP requests; the scenario needs more than 32 inputs", bigRequests) + } + if len(reqs) < 2 || !reqs[1].Query || reqs[1].Inputs[0] != query { + order := make([]string, len(reqs)) + for i, r := range reqs { + order[i] = firstLine(r.Inputs[0]) + } + t.Fatalf("[A:query-before-next-batch] query was not served before the next background request: %v", order) + } + + // Terminating the owner of a running request releases capacity, and the + // interrupted index never receives a completed root marker. + deadNames := make([]string, 40) + for i := range deadNames { + deadNames[i] = fmt.Sprintf("DeadFunction%02d", i) + } + dead := e.repo("dead", map[string]string{"dead.go": goFile("dead", deadNames...)}) + g = e.p.block(documentsMentioning("// dead.go\n")) + d := e.start("index", dead) + waitGate(t, g, "dead.go request") + _ = d.cmd.Process.Kill() + _ = d.wait(t) + started := time.Now() + if err := s.trySearch("query after the owner died", small, small); err != nil { + t.Fatalf("[A:killed-owner-releases-capacity] %v", err) + } + if elapsed := time.Since(started); elapsed > 10*time.Second { + t.Fatalf("[A:killed-owner-releases-capacity] query waited %s after the owner died", elapsed) + } + deadDB := e.collectionFor(dead) + snap := e.inspect(deadDB) + if snap.meta[dead]["root_hash"] != "" || snap.claimsPending == 0 { + t.Fatalf("[A:no-false-completion] killed index: root_hash=%q pending claims=%d", snap.meta[dead]["root_hash"], snap.claimsPending) + } + // Simulate the two-minute lease passing; the next indexer recovers. + e.fixtureExec(deadDB, `UPDATE embedding_claims SET expires_at = 1 WHERE result IS NULL`) + m = e.p.mark() + e.index(dead) + if snap = e.inspect(deadDB); snap.meta[dead]["root_hash"] == "" || snap.claimsPending != 0 { + t.Fatalf("[A:expired-claims-recover] recovery: root_hash=%q pending=%d", snap.meta[dead]["root_hash"], snap.claimsPending) + } + requireEachOnce(t, "expired-claims-recover", e.p.documents(m)) + + // A failed request releases its claims immediately. + failing := e.repo("failing", map[string]string{"failing.go": goFile("failing", "FailOne", "FailTwo")}) + e.p.fail(documentsMentioning("// failing.go\n"), http.StatusBadRequest) + if r := e.lumen("index", failing); r.err == nil { + t.Fatalf("[A:failure-control] index succeeded despite an injected provider failure") + } + failDB := e.collectionFor(failing) + if snap = e.inspect(failDB); snap.meta[failing]["root_hash"] != "" || snap.claimsPending != 0 { + t.Fatalf("[A:failed-claims-released] failed index: root_hash=%q pending=%d", snap.meta[failing]["root_hash"], snap.claimsPending) + } + e.index(failing) + if snap = e.inspect(failDB); snap.meta[failing]["root_hash"] == "" { + t.Fatalf("[A:failed-claims-released] retry did not complete") + } + if peak := e.p.peakInFlight(); peak != 1 { + t.Fatalf("[A:single-flight] peak in-flight embedding requests = %d", peak) + } +} + +// Scenario 8: edits and disposable worktree churn stay within the retention +// policy, active data is protected, and cleanup leaves no garbage or orphan +// scan sidecars. Zero-policy and TTL expiry run through real maintenance. +func TestRetentionAndCleanupBoundStorage(t *testing.T) { + // Realistic dimensions make vector bytes dominate per-chunk metadata, so + // the accounting oracle below cannot be satisfied by metadata alone. + e := newEnv(t, withDims(768)) + main := e.repo("retention", baseFiles()) + e.index(main) + db := e.onlyCollection() + + edit := func(dir string, n int) { + t.Helper() + writeFiles(t, dir, map[string]string{"beta.go": goFile("fixture", "BetaOne", "BetaTwo", fmt.Sprintf("BetaEdit%03d", n))}) + } + edit(main, 0) + e.index(main) + snap := e.inspect(db) + if snap.retained != 1 || snap.retainedBytes <= 0 { + t.Fatalf("[A:retention-control] first replacement retained=%d bytes=%d", snap.retained, snap.retainedBytes) + } + // Independent oracle: every retained revision is charged at least one + // int8 vector (dims bytes) per chunk it owns. + if under := e.countRows(db, `SELECT count(*) FROM revision_cache rc WHERE rc.bytes < ? * (SELECT count(*) FROM chunk_defs cd WHERE cd.file_revision_id = rc.revision_id)`, e.p.dims); under != 0 { + t.Fatalf("[A:retention-accounting] %d retained revisions are charged less than their vectors", under) + } + budget := snap.retainedBytes*2 + snap.retainedBytes/2 + e.setRetention(db, main, budget, time.Hour) + + var physicalHigh int64 + for i := 1; i <= 6; i++ { + edit(main, i) + e.index(main) + if i%2 == 0 { + // Disposable worktree: index, diverge, then delete it. + wt := e.worktree(main, filepath.Join(filepath.Dir(main), fmt.Sprintf("disposable-%d", i)), "-b", fmt.Sprintf("disposable-%d", i)) + e.index(wt) + writeFiles(t, wt, map[string]string{"alpha.go": goFile("fixture", "AlphaOne", fmt.Sprintf("Disposable%03d", i))}) + e.index(wt) + e.git(main, "worktree", "remove", "--force", wt) + } + physicalHigh = max(physicalHigh, physicalBytes(t, db)) + snap = e.inspect(db) + if snap.retainedBytes > budget || snap.retained > 2 { + t.Fatalf("[A:retention-budget] step %d retained %d revisions, %d bytes > budget %d", i, snap.retained, snap.retainedBytes, budget) + } + active := map[int64]bool{} + for _, members := range snap.members { + for _, id := range members { + active[id] = true + } + } + if snap.revisions != len(active)+snap.retained { + t.Fatalf("[A:retention-gc] step %d revisions=%d active=%d retained=%d", i, snap.revisions, len(active), snap.retained) + } + requireNoGarbage(t, "retention-gc", snap) + } + + projectLocks, _ := filepath.Glob(db + ".project-*.lock") + if len(projectLocks) == 0 { + t.Fatalf("[A:locks-control] no per-project lock files exist before clean") + } + e.mustLumen("clean") + snap = e.inspect(db) + if len(snap.projects) != 1 || snap.projects[main] == 0 { + t.Fatalf("[A:cleanup-memberships] projects after clean: %v", keys(snap.projects)) + } + requireNoGarbage(t, "cleanup-no-garbage", snap) + if scans := scanFiles(t, db); len(scans) != 1 || filepath.Base(scans[0]) != fmt.Sprintf("index.db.scan-%x.json", sha256.Sum256([]byte(main))) { + t.Fatalf("[A:no-orphan-scan] scan sidecars after clean: %v", scans) + } + for _, lock := range append([]string{db + ".lock"}, projectLocks...) { + if _, err := os.Stat(lock); err != nil { + t.Fatalf("[A:locks-preserved] cleanup removed %s: %v", lock, err) + } + } + + // Active content is intact: searching does no indexing work. + s := e.mcp() + m, l := e.p.mark(), e.logMark() + files, _ := s.search("retention survivors", main, main) + if w := e.work(l, main); w.parsed != 0 || len(e.p.documents(m)) != 0 || len(files) != baseFileCount { + t.Fatalf("[A:active-protected] parsed=%d embedded=%d results=%v", w.parsed, len(e.p.documents(m)), files) + } + s.close() + + // Zero policy retains nothing after the next replacement. + e.setRetention(db, main, 0, 0) + edit(main, 100) + e.index(main) + if snap = e.inspect(db); snap.retained != 0 || snap.revisions != baseFileCount { + t.Fatalf("[A:zero-policy] retained=%d revisions=%d", snap.retained, snap.revisions) + } + requireNoGarbage(t, "zero-policy", snap) + + // TTL expiry through ordinary replacement maintenance. + e.setRetention(db, main, 64<<20, time.Hour) + edit(main, 101) + e.index(main) + if snap = e.inspect(db); snap.retained != 1 { + t.Fatalf("[A:ttl-expiry] retained=%d after one replacement", snap.retained) + } + e.fixtureExec(db, `UPDATE revision_cache SET unused_at = unused_at - 7200`) + edit(main, 102) + e.index(main) + if snap = e.inspect(db); snap.retained != 1 || snap.revisions != baseFileCount+1 { + t.Fatalf("[A:ttl-expiry] expired revision survived: retained=%d revisions=%d", snap.retained, snap.revisions) + } + requireNoGarbage(t, "ttl-expiry", snap) + t.Logf("physical high-water during churn: %d bytes; logical retention budget %d bytes", physicalHigh, budget) +} + +// The serialized embedding-result rendezvous cache expires and is bounded by +// its 64 MiB budget independently of revision retention. +func TestRendezvousCacheExpiryAndByteBudget(t *testing.T) { + e := newEnv(t) + main := e.repo("rendezvous", baseFiles()) + e.index(main) + db := e.onlyCollection() + if snap := e.inspect(db); snap.claimsResults == 0 { + t.Fatalf("[A:rendezvous-control] cold index published no rendezvous results") + } + edit := func(n int) { + writeFiles(t, main, map[string]string{"beta.go": goFile("fixture", "BetaOne", fmt.Sprintf("Rendezvous%03d", n))}) + e.index(main) + } + + e.fixtureExec(db, `UPDATE embedding_claims SET expires_at = 1000 WHERE result IS NOT NULL`) + edit(1) + if n := e.countRows(db, `SELECT count(*) FROM embedding_claims WHERE expires_at <= ?`, time.Now().UnixMilli()); n != 0 { + t.Fatalf("[A:rendezvous-expiry] %d expired results survived publication", n) + } + + planted := sha256.Sum256([]byte("oversized rendezvous fixture")) + e.fixtureExec(db, `INSERT INTO embedding_claims(input_hash, owner, expires_at, result) VALUES (?, 'fixture', ?, zeroblob(?))`, + planted[:], time.Now().Add(5*time.Minute).UnixMilli(), 65<<20) + edit(2) + if n := e.countRows(db, `SELECT count(*) FROM embedding_claims WHERE owner = 'fixture'`); n != 0 { + t.Fatalf("[A:rendezvous-byte-budget] oversized result survived the 64 MiB budget") + } + if n := e.countRows(db, `SELECT COALESCE(sum(length(result)),0) FROM embedding_claims`); n > 64<<20 { + t.Fatalf("[A:rendezvous-byte-budget] rendezvous cache holds %d bytes", n) + } + if snap := e.inspect(db); snap.claimsResults == 0 { + t.Fatalf("[A:rendezvous-byte-budget] newest results were pruned too") + } +} + +// Physical storage for a vector-heavy fixture: five identical worktrees stay +// within twice a complete single-worktree collection. +func TestPhysicalStorageForFiveIdenticalWorktrees(t *testing.T) { + files := map[string]string{} + for f := range 6 { + names := make([]string, 25) + for i := range names { + names[i] = fmt.Sprintf("Heavy%dFunction%02d", f, i) + } + files[fmt.Sprintf("heavy%d.go", f)] = goFile("heavy", names...) + } + + single := newEnv(t, withDims(768)) + repo := single.repo("physical", files) + single.index(repo) + singleDB := single.onlyCollection() + baseline := physicalBytes(t, singleDB) + one := single.inspect(singleDB) + + five := newEnv(t, withDims(768)) + repo = five.repo("physical", files) + roots := []string{repo} + for i := range 4 { + roots = append(roots, five.worktree(repo, filepath.Join(filepath.Dir(repo), fmt.Sprintf("physical-%d", i)), "-b", fmt.Sprintf("p%d", i))) + } + var high int64 + var fiveDB string + for _, root := range roots { + five.index(root) + fiveDB = five.onlyCollection() + high = max(high, physicalBytes(t, fiveDB)) + } + total := physicalBytes(t, fiveDB) + all := five.inspect(fiveDB) + if all.vectors != one.vectors || all.revisions != one.revisions || len(all.projects) != 5 { + t.Fatalf("[A:five-unique-data] vectors %d vs %d, revisions %d vs %d, projects %d", all.vectors, one.vectors, all.revisions, one.revisions, len(all.projects)) + } + if total > 2*baseline { + t.Fatalf("[A:five-physical-bound] five worktrees use %d bytes, more than twice the single-worktree %d", total, baseline) + } + t.Logf("single worktree %d bytes; five worktrees %d bytes (%.2fx); high-water %d bytes", baseline, total, float64(total)/float64(baseline), high) +} + +// Scenario 9: force does full parsing and embedding for the selected worktree +// while sibling memberships, IDs, and results are preserved. +func TestForceReindexPreservesSiblings(t *testing.T) { + e := newEnv(t) + main := e.repo("force", baseFiles()) + wt := e.worktree(main, filepath.Join(filepath.Dir(main), "force-wt"), "-b", "force") + e.index(main) + e.index(wt) + db := e.onlyCollection() + before := e.inspect(db) + + m, l := e.p.mark(), e.logMark() + e.index(wt, "--force") + w := e.work(l, wt) + if w.forcedScans != 1 || w.hashed != baseFileCount { + t.Fatalf("[A:force-rehash] forced scans=%d hashed=%d", w.forcedScans, w.hashed) + } + if w.parsed != baseFileCount { + t.Fatalf("[A:force-parse] force parsed %d files, want %d", w.parsed, baseFileCount) + } + docs := e.p.documents(m) + requireEachOnce(t, "force-embed-all", docs) + if len(docs) != before.vectors { + t.Fatalf("[A:force-embed-all] force embedded %d inputs, want %d", len(docs), before.vectors) + } + after := e.inspect(db) + if !mapsEqual(after.members[main], before.members[main]) || !mapsEqual(after.vectorIDs, before.vectorIDs) { + t.Fatalf("[A:force-preserves-siblings] sibling memberships or vector IDs changed") + } + requireNoGarbage(t, "no-garbage", after) + s := e.mcp() + m, l = e.p.mark(), e.logMark() + files, _ := s.search("force sibling query", main, main) + if w := e.work(l, main); w.parsed != 0 || len(e.p.documents(m)) != 0 || len(files) != baseFileCount { + t.Fatalf("[A:force-preserves-siblings] sibling search parsed=%d embedded=%d results=%v", w.parsed, len(e.p.documents(m)), files) + } +} + +// --------------------------------------------------------------------------- + +func editFile(t *testing.T, path, old, replacement string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), old) { + t.Fatalf("fixture %s does not contain %q", path, old) + } + if err := os.WriteFile(path, []byte(strings.Replace(string(data), old, replacement, 1)), 0o644); err != nil { + t.Fatal(err) + } +} + +// allFrom reports whether docs is non-empty and every input belongs to file. +func allFrom(docs []string, file string) bool { + for _, in := range docs { + if !strings.HasPrefix(in, "// "+file+"\n") { + return false + } + } + return len(docs) > 0 +} + +func mapsEqual[K comparable, V comparable](a, b map[K]V) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || w != v { + return false + } + } + return true +} + +func sameSet(a, b []string) bool { + a, b = slices.Clone(a), slices.Clone(b) + slices.Sort(a) + slices.Sort(b) + return slices.Equal(slices.Compact(a), slices.Compact(b)) +} + +func keys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} + +// collectionFor finds the collection holding a membership for project. +func (e *env) collectionFor(project string) string { + e.t.Helper() + for _, db := range e.collections() { + if _, ok := e.inspect(db).projects[project]; ok { + return db + } + } + e.t.Fatalf("no collection has a membership for %s", project) + return "" +} + +func (e *env) countRows(dbPath, query string, args ...any) int64 { + e.t.Helper() + db := openReadOnly(e.t, dbPath) + defer func() { _ = db.Close() }() + var n int64 + if err := db.QueryRow(query, args...).Scan(&n); err != nil { + e.t.Fatal(err) + } + return n +} + +// trySearch is search without t.Fatal, for use off the test goroutine. +func (s *mcpSession) trySearch(query, path, cwd string) error { + s.e.p.registerQuery(query) + ctx, cancel := contextWithTimeout(2 * time.Minute) + defer cancel() + res, err := s.session.CallTool(ctx, callParams(query, path, cwd)) + if err != nil { + return err + } + if res.IsError { + return fmt.Errorf("semantic_search returned an error result") + } + return nil +} diff --git a/internal/embedder/budget.go b/internal/embedder/budget.go new file mode 100644 index 00000000..aa0ce999 --- /dev/null +++ b/internal/embedder/budget.go @@ -0,0 +1,99 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package embedder + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/gofrs/flock" +) + +type backgroundKey struct{} + +// BackgroundContext marks indexing batches as lower priority than queries. +func BackgroundContext(ctx context.Context) context.Context { + return context.WithValue(ctx, backgroundKey{}, true) +} + +// acquireBudget serializes backend requests across processes for this user. +// Query intent is a shared OS lock, so process death cannot strand priority. +// Background callers recheck intent after acquiring capacity to avoid jumping +// ahead of queries queued behind an already-running request. +func acquireBudget(ctx context.Context, dir string) (func(), error) { + if dir == "" { + return func() {}, nil + } + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + capacity := flock.New(filepath.Join(dir, "compute.lock")) + intent := flock.New(filepath.Join(dir, "query.lock")) + closeAll := func() { _ = capacity.Close(); _ = intent.Close() } + background, _ := ctx.Value(backgroundKey{}).(bool) + if !background { + ok, err := intent.TryRLockContext(ctx, 10*time.Millisecond) + if err != nil || !ok { + closeAll() + if err == nil { + err = ctx.Err() + } + return nil, err + } + ok, err = capacity.TryLockContext(ctx, 10*time.Millisecond) + if err != nil || !ok { + closeAll() + if err == nil { + err = ctx.Err() + } + return nil, err + } + return closeAll, nil + } + for { + if err := ctx.Err(); err != nil { + closeAll() + return nil, err + } + available, err := intent.TryLock() + if err != nil { + closeAll() + return nil, err + } + if available { + _ = intent.Unlock() + acquired, err := capacity.TryLock() + if err != nil { + closeAll() + return nil, err + } + if acquired { + available, err = intent.TryLock() + if err != nil { + closeAll() + return nil, err + } + if available { + _ = intent.Unlock() + return closeAll, nil + } + _ = capacity.Unlock() + } + } + select { + case <-ctx.Done(): + closeAll() + return nil, ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } +} + +// SetComputeBudget enables a shared on-disk compute budget for this embedder. +// Configure it before use. An empty path disables coordination for tests. +func (f *FailoverEmbedder) SetComputeBudget(dir string) { f.budgetDir = dir } diff --git a/internal/embedder/budget_test.go b/internal/embedder/budget_test.go new file mode 100644 index 00000000..fd936513 --- /dev/null +++ b/internal/embedder/budget_test.go @@ -0,0 +1,186 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package embedder + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/gofrs/flock" + + "github.com/ory/lumen/internal/config" +) + +func TestBudgetHonorsQueryIntentAndCancellation(t *testing.T) { + dir := t.TempDir() + intent := flock.New(filepath.Join(dir, "query.lock")) + if ok, err := intent.TryRLock(); err != nil || !ok { + t.Fatalf("intent: %v %v", ok, err) + } + defer func() { _ = intent.Close() }() + ctx, cancel := context.WithTimeout(BackgroundContext(context.Background()), 40*time.Millisecond) + defer cancel() + if release, err := acquireBudget(ctx, dir); err == nil { + release() + t.Fatal("background request jumped ahead of query") + } + if err := intent.Unlock(); err != nil { + t.Fatal(err) + } + release, err := acquireBudget(context.Background(), dir) + if err != nil { + t.Fatal(err) + } + ctx2, cancel2 := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel2() + if other, err := acquireBudget(ctx2, dir); err == nil { + other() + t.Fatal("budget admitted overlapping requests") + } + release() + next, err := acquireBudget(BackgroundContext(context.Background()), dir) + if err != nil { + t.Fatal(err) + } + next() +} + +// TestQueryPrecedesNextDocumentHTTPRequest proves the compute budget is admitted +// per backend HTTP request: a query that announces intent while the first +// 32-input document request is running must be served before the next one. +func TestQueryPrecedesNextDocumentHTTPRequest(t *testing.T) { + for _, backend := range []string{config.BackendOllama, config.BackendLMStudio} { + t.Run(backend, func(t *testing.T) { + const query = "where is the query handled" + arrived := make(chan struct{}) + release := make(chan struct{}) + var mu sync.Mutex + var order []string + var inFlight, maxInFlight int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/tags": + _, _ = fmt.Fprint(w, `{"models":[{"name":"test"}]}`) + return + case "/v1/models": + _, _ = fmt.Fprint(w, `{"data":[{"id":"test"}]}`) + return + } + var body struct { + Input []string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + w.WriteHeader(400) + return + } + mu.Lock() + inFlight++ + maxInFlight = max(maxInFlight, inFlight) + label := fmt.Sprintf("doc:%d", len(body.Input)) + if len(body.Input) == 1 && body.Input[0] == query { + label = "query" + } + order = append(order, label) + first := len(order) == 1 + mu.Unlock() + if first { + close(arrived) + <-release + } + mu.Lock() + inFlight-- + mu.Unlock() + vectors := make([][]float32, len(body.Input)) + for i := range vectors { + vectors[i] = []float32{1, 0, 0} + } + if r.URL.Path == "/api/embed" { + _ = json.NewEncoder(w).Encode(map[string]any{"embeddings": vectors}) + return + } + data := make([]map[string]any, len(vectors)) + for i, v := range vectors { + data[i] = map[string]any{"index": i, "embedding": v} + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": data}) + })) + defer srv.Close() + + cfg := testConfigService(t, config.ServerConfig{Backend: backend, Host: srv.URL, Model: "test", Dims: 3}) + budgetDir := t.TempDir() + indexer := NewFailoverEmbedder(cfg) + indexer.SetComputeBudget(budgetDir) + searcher := NewFailoverEmbedder(cfg) + searcher.SetComputeBudget(budgetDir) + + docs := make([]string, 40) + for i := range docs { + docs[i] = fmt.Sprintf("// f.go\nfunc F%d() {}", i) + } + docDone := make(chan error, 1) + go func() { + _, err := indexer.Embed(BackgroundContext(context.Background()), docs) + docDone <- err + }() + select { + case <-arrived: + case <-time.After(10 * time.Second): + t.Fatal("first document request never arrived") + } + queryDone := make(chan error, 1) + go func() { + _, err := searcher.Embed(context.Background(), []string{query}) + queryDone <- err + }() + // Wait until the query holds its shared intent lock. + intent := flock.New(filepath.Join(budgetDir, "query.lock")) + deadline := time.Now().Add(10 * time.Second) + for { + ok, err := intent.TryLock() + if err != nil { + t.Fatal(err) + } + if !ok { + break + } + _ = intent.Unlock() + if time.Now().After(deadline) { + t.Fatal("query never announced intent") + } + time.Sleep(5 * time.Millisecond) + } + _ = intent.Close() + close(release) + for name, done := range map[string]chan error{"documents": docDone, "query": queryDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("%s: %v", name, err) + } + case <-time.After(10 * time.Second): + t.Fatalf("%s embed did not finish", name) + } + } + mu.Lock() + defer mu.Unlock() + want := []string{"doc:32", "query", "doc:8"} + if !slices.Equal(order, want) { + t.Fatalf("HTTP request order = %v, want %v", order, want) + } + if maxInFlight != 1 { + t.Fatalf("max in-flight embedding requests = %d, want 1", maxInFlight) + } + }) + } +} diff --git a/internal/embedder/failover.go b/internal/embedder/failover.go index 26e48c0d..1234290f 100644 --- a/internal/embedder/failover.go +++ b/internal/embedder/failover.go @@ -33,6 +33,7 @@ const reprobeInterval = 30 * time.Second // When the ConfigService's server list changes (e.g. via hot reload), the // embedder re-initializes on the next Embed call. type FailoverEmbedder struct { + budgetDir string cfg *config.ConfigService logger *slog.Logger mu sync.Mutex @@ -115,7 +116,34 @@ func (f *FailoverEmbedder) ModelName() string { // Embed generates embeddings for the given texts. On first call it probes // all servers for health and selects the first healthy one. On transient // errors (5xx, network) it fails over to the next healthy server. +// +// Texts are submitted in provider-sized requests, and the shared compute +// budget is admitted separately for each one. A query that announces intent +// while a background request is running is therefore served before the next +// background request instead of after the whole call. func (f *FailoverEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) { + if len(texts) <= embedBatchSize { + return f.embedRequest(ctx, texts) + } + vectors := make([][]float32, 0, len(texts)) + for start := 0; start < len(texts); start += embedBatchSize { + batch, err := f.embedRequest(ctx, texts[start:min(start+embedBatchSize, len(texts))]) + if err != nil { + return nil, fmt.Errorf("embedding batch starting at %d: %w", start, err) + } + vectors = append(vectors, batch...) + } + return vectors, nil +} + +// embedRequest embeds at most embedBatchSize texts, which each provider sends +// as a single HTTP request, while holding one compute budget admission. +func (f *FailoverEmbedder) embedRequest(ctx context.Context, texts []string) ([][]float32, error) { + release, err := acquireBudget(ctx, f.budgetDir) + if err != nil { + return nil, err + } + defer release() f.mu.Lock() f.maybeReprobe(ctx, true) f.mu.Unlock() diff --git a/internal/index/freshness.go b/internal/index/freshness.go new file mode 100644 index 00000000..66405163 --- /dev/null +++ b/internal/index/freshness.go @@ -0,0 +1,61 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package index + +import ( + "path/filepath" + + "github.com/ory/lumen/internal/merkle" + "github.com/ory/lumen/internal/store" +) + +func (idx *Indexer) buildTree(projectDir string, force bool) (*merkle.Tree, error) { + root, err := filepath.Abs(projectDir) + if err != nil { + return nil, err + } + value, _ := idx.scanCaches.LoadOrStore(root, &merkle.Cache{}) + cachePath := "" + if idx.dsn != "" && idx.dsn != ":memory:" { + cachePath = store.ScanCachePath(idx.dsn, root) + } + tree, err := value.(*merkle.Cache).Build(root, makeSkip(root), cachePath, force) + if err == nil && idx.logger != nil { + idx.logger.Info("scan completed", + "project", root, + "forced", force, + "files_total", len(tree.Files), + "files_hashed", tree.FilesHashed, + "files_reused", tree.FilesReused, + "root_hash", tree.RootHash, + ) + } + return tree, err +} + +// logIndexPass records the work one indexing pass actually performed. It is +// emitted for failed passes too, so interrupted work remains observable. +func (idx *Indexer) logIndexPass(projectDir string, force bool, rootHash string, stats Stats, err error) { + if idx.logger == nil { + return + } + args := []any{ + "project", projectDir, + "forced", force, + "files_total", stats.TotalFiles, + "files_changed", stats.FilesChanged, + "files_removed", stats.FilesRemoved, + "files_parsed", stats.FilesParsed, + "revisions_attached", stats.RevisionsAttached, + "chunks_created", stats.ChunksCreated, + "root_hash", rootHash, + } + if err != nil { + idx.logger.Warn("index pass failed", append(args, "error", err.Error())...) + return + } + idx.logger.Info("index pass completed", args...) +} diff --git a/internal/index/index.go b/internal/index/index.go index db258862..4dc7590b 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -62,6 +62,14 @@ type Stats struct { FilesModified int FilesRemoved int + // Work actually performed. FilesHashed and FilesReused come from the + // freshness scan; FilesParsed counts chunker invocations; RevisionsAttached + // counts files linked to an existing complete revision without parsing. + FilesHashed int + FilesReused int + FilesParsed int + RevisionsAttached int + // Reason explains why reindexing was triggered. Reason string @@ -88,6 +96,7 @@ type StatusInfo struct { // Indexer orchestrates chunking, embedding, and storage for a code index. type Indexer struct { + scanCaches sync.Map // absolute worktree root -> metadata/hash cache mu sync.Mutex projectMu sync.RWMutex store *store.Store @@ -165,6 +174,7 @@ func (idx *Indexer) rebuildStore(projectPath string) error { // Close closes the underlying store. func (idx *Indexer) Close() error { + idx.scanCaches.Range(func(_, value any) bool { _ = value.(*merkle.Cache).Close(); return true }) idx.mu.Lock() defer idx.mu.Unlock() idx.projectMu.Lock() @@ -189,8 +199,8 @@ func makeSkip(projectDir string) merkle.SkipFunc { // Index indexes the project at projectDir. If force is true, all files are // re-indexed regardless of whether they have changed. func (idx *Indexer) Index(ctx context.Context, projectDir string, force bool, progress ProgressFunc) (Stats, error) { - // Build tree outside the lock: it is read-only and can be slow for large projects. - curTree, err := merkle.BuildTree(projectDir, makeSkip(projectDir)) + // Scan outside the index lock; the cache serializes per-worktree checks. + curTree, err := idx.buildTree(projectDir, force) if err != nil { return Stats{}, fmt.Errorf("build merkle tree: %w", err) } @@ -265,8 +275,8 @@ func (idx *Indexer) Index(ctx context.Context, projectDir string, force bool, pr // EnsureFresh checks if the index is stale and re-indexes if needed. // Returns whether a re-index occurred, the stats, and any error. func (idx *Indexer) EnsureFresh(ctx context.Context, projectDir string, progress ProgressFunc) (bool, Stats, error) { - // Build tree outside the lock: it is read-only and can be slow for large projects. - curTree, err := merkle.BuildTree(projectDir, makeSkip(projectDir)) + // Scan outside the index lock; the cache serializes per-worktree checks. + curTree, err := idx.buildTree(projectDir, false) if err != nil { return false, Stats{}, fmt.Errorf("build merkle tree: %w", err) } @@ -335,6 +345,7 @@ func (idx *Indexer) EnsureFresh(ctx context.Context, projectDir string, progress // merkle tree, so callers that already have one (e.g. EnsureFresh) do not need // to build it again. func (idx *Indexer) indexWithTree(ctx context.Context, projectDir, oldRootHash string, force bool, curTree *merkle.Tree, progress ProgressFunc) (stats Stats, retErr error) { + ctx = embedder.BackgroundContext(ctx) defer func() { if retErr != nil { _ = idx.store.SetMeta(store.MetaLastIndexError, retErr.Error()) @@ -347,6 +358,7 @@ func (idx *Indexer) indexWithTree(ctx context.Context, projectDir, oldRootHash s } stats.TotalFiles = len(curTree.Files) + stats.FilesHashed, stats.FilesReused = curTree.FilesHashed, curTree.FilesReused // Load stored hashes once — used for extension purge in both paths, // and for deletion diff in both paths. @@ -509,6 +521,7 @@ func (idx *Indexer) indexWithTree(ctx context.Context, projectDir, oldRootHash s return stats, fmt.Errorf("upsert file placeholder %s: %w", relPath, err) } + stats.FilesParsed++ chunks, err := idx.chunker.Chunk(relPath, content) if err != nil { if idx.logger != nil { @@ -610,7 +623,7 @@ func (idx *Indexer) LastIndexedAt(projectDir string) (time.Time, bool) { // IsFresh does not acquire the indexer mutex; it reads through the store's // read-only connection (SQLite WAL isolation). func (idx *Indexer) IsFresh(projectDir string) (bool, error) { - curTree, err := merkle.BuildTree(projectDir, makeSkip(projectDir)) + curTree, err := idx.buildTree(projectDir, false) if err != nil { return false, fmt.Errorf("build merkle tree: %w", err) } diff --git a/internal/index/shared.go b/internal/index/shared.go index 9c42e925..08687d5e 100644 --- a/internal/index/shared.go +++ b/internal/index/shared.go @@ -27,8 +27,9 @@ import ( // indexSharedWithTree indexes one project membership in a repository-scoped // content-addressed collection. Existing path+content revisions and exact // embedding inputs are reused without calling the embedding backend. -func (idx *Indexer) indexSharedWithTree(ctx context.Context, projectDir, _ string, force bool, curTree *merkle.Tree, progress ProgressFunc) (Stats, error) { - stats := Stats{TotalFiles: len(curTree.Files)} +func (idx *Indexer) indexSharedWithTree(ctx context.Context, projectDir, _ string, force bool, curTree *merkle.Tree, progress ProgressFunc) (stats Stats, retErr error) { + stats = Stats{TotalFiles: len(curTree.Files), FilesHashed: curTree.FilesHashed, FilesReused: curTree.FilesReused} + defer func() { idx.logIndexPass(projectDir, force, curTree.RootHash, stats, retErr) }() oldHashes, err := idx.store.GetFileHashes() if err != nil { return stats, fmt.Errorf("get project file hashes: %w", err) @@ -159,7 +160,13 @@ func (idx *Indexer) indexSharedWithTree(ctx context.Context, projectDir, _ strin if len(batchTexts) == 0 { return drainReady() } - embedded, err := idx.emb.Embed(ctx, batchTexts) + var embedded [][]float32 + var err error + if force { + embedded, err = idx.emb.Embed(ctx, batchTexts) + } else { + embedded, err = idx.store.EmbedShared(ctx, batchTexts, idx.emb.Embed) + } if err != nil { return fmt.Errorf("embed shared batch: %w", err) } @@ -194,6 +201,7 @@ func (idx *Indexer) indexSharedWithTree(ctx context.Context, projectDir, _ strin } if attached { stats.IndexedFiles++ + stats.RevisionsAttached++ continue } } @@ -206,6 +214,9 @@ func (idx *Indexer) indexSharedWithTree(ctx context.Context, projectDir, _ strin } return stats, fmt.Errorf("read file %s: %w", relativePath, err) } + if fmt.Sprintf("%x", sha256.Sum256(content)) != contentHash { + return stats, fmt.Errorf("file changed after freshness scan: %s", relativePath) + } if isBinaryContent(content) { if err := idx.store.DeleteFileChunks(relativePath); err != nil { return stats, fmt.Errorf("remove binary file %s: %w", relativePath, err) @@ -213,6 +224,7 @@ func (idx *Indexer) indexSharedWithTree(ctx context.Context, projectDir, _ strin continue } + stats.FilesParsed++ chunks, err := idx.chunker.Chunk(relativePath, content) if err != nil { if idx.logger != nil { diff --git a/internal/index/shared_batch_test.go b/internal/index/shared_batch_test.go index 8c237238..e6711b59 100644 --- a/internal/index/shared_batch_test.go +++ b/internal/index/shared_batch_test.go @@ -140,6 +140,10 @@ func TestSharedIndexRetriesWhenPreviouslySharedVectorVanishes(t *testing.T) { t.Fatal(err) } defer func() { _ = idx.Close() }() + if err := idx.store.SetRevisionCachePolicy(0, 0); err != nil { + t.Fatal(err) + } + idx.chunker = fixedChunker{chunks: []chunker.Chunk{chunkA, chunkB}} if _, err := idx.store.StoreFileRevision("main.go", "old", []chunker.Chunk{chunkA}, map[int][]float32{0: {1, 0, 0, 0}}); err != nil { t.Fatal(err) diff --git a/internal/index/work_stats_test.go b/internal/index/work_stats_test.go new file mode 100644 index 00000000..2b5cff8b --- /dev/null +++ b/internal/index/work_stats_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package index + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "path/filepath" + "testing" +) + +func TestIndexerReportsActualScanParseAndAttachWork(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + for _, dir := range []string{first, second} { + writeGoFile(t, dir, "a.go", "package p\n\nfunc A() {}\n") + writeGoFile(t, dir, "b.go", "package p\n\nfunc B() {}\n") + } + var logs bytes.Buffer + idx, err := NewIndexerForProject(filepath.Join(t.TempDir(), "index.db"), &mockEmbedder{dims: 4, model: "m"}, 0, "int8", first) + if err != nil { + t.Fatal(err) + } + defer func() { _ = idx.Close() }() + idx.SetLogger(slog.New(slog.NewJSONHandler(&logs, nil))) + + check := func(name string, got Stats, hashed, parsed, attached int) { + t.Helper() + if got.FilesHashed != hashed || got.FilesParsed != parsed || got.RevisionsAttached != attached { + t.Fatalf("%s: hashed=%d parsed=%d attached=%d, want %d/%d/%d", name, got.FilesHashed, got.FilesParsed, got.RevisionsAttached, hashed, parsed, attached) + } + } + stats, err := idx.Index(context.Background(), first, false, nil) + if err != nil { + t.Fatal(err) + } + check("cold", stats, 2, 2, 0) + stats, err = idx.Index(context.Background(), second, false, nil) + if err != nil { + t.Fatal(err) + } + check("identical sibling", stats, 2, 0, 2) + writeGoFile(t, first, "a.go", "package p\n\nfunc A() { _ = 1 }\n") + stats, err = idx.Index(context.Background(), first, false, nil) + if err != nil { + t.Fatal(err) + } + if stats.FilesParsed != 1 || stats.RevisionsAttached != 0 || stats.FilesHashed+stats.FilesReused != 2 { + t.Fatalf("one-file edit: %+v", stats) + } + + events := map[string]int{} + for _, line := range bytes.Split(bytes.TrimSpace(logs.Bytes()), []byte("\n")) { + var entry struct { + Msg string `json:"msg"` + } + if err := json.Unmarshal(line, &entry); err != nil { + t.Fatal(err) + } + events[entry.Msg]++ + } + if events["scan completed"] != 3 || events["index pass completed"] != 3 { + t.Fatalf("diagnostic events = %v", events) + } +} diff --git a/internal/merkle/cache.go b/internal/merkle/cache.go new file mode 100644 index 00000000..880b4fe1 --- /dev/null +++ b/internal/merkle/cache.go @@ -0,0 +1,132 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package merkle + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +// Cache retains hashes only while the file's identity and metadata agree. +// It still walks directory entries, so additions, removals, and ignore-rule +// changes are discovered without relying on event delivery. +type Cache struct { + watcher *fsnotify.Watcher + watched map[string]bool + closed bool + mu sync.Mutex + Files map[string]cachedFile `json:"files"` + Reconciled time.Time `json:"reconciled"` +} + +type cachedFile struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + Modified int64 `json:"modified"` + Mode uint32 `json:"mode"` + Identity string `json:"identity"` + Changed int64 `json:"changed"` +} + +// Build reconciles metadata on every call and content at least hourly. force +// bypasses all cached hashes. Cache files are disposable and atomically replaced. +func (c *Cache) Build(root string, skip SkipFunc, cachePath string, force bool) (*Tree, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.Files == nil && cachePath != "" { + if data, err := os.ReadFile(cachePath); err == nil { + var saved struct { + Files map[string]cachedFile `json:"files"` + Reconciled time.Time `json:"reconciled"` + } + if json.Unmarshal(data, &saved) == nil { + c.Files, c.Reconciled = saved.Files, saved.Reconciled + } + } + } + now := time.Now() + reconcile := force || c.Reconciled.IsZero() || now.Sub(c.Reconciled) >= time.Hour || now.Before(c.Reconciled) + if skip == nil { + skip = DefaultSkip + } + c.watchLocked(root) + paths, err := collectFilePaths(root, skip, c.watchDirectoryLocked) + if err != nil { + return nil, err + } + next := make(map[string]cachedFile, len(paths)) + tree := &Tree{Files: make(map[string]string, len(paths))} + for _, rel := range paths { + path := filepath.Join(root, rel) + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !before.Mode().IsRegular() || before.Size() > maxFileSize { + continue + } + stamp := stampFile(before) + old, ok := c.Files[rel] + // Recently modified files are deliberately rehashed: coarse timestamps + // can hide a second write in the same timestamp tick. + reusable := ok && len(old.Hash) == 64 && !reconcile && stamp.Identity != "" && stamp.Changed < now.Add(-2*time.Second).UnixNano() && before.ModTime().Before(now.Add(-2*time.Second)) && old.Size == stamp.Size && old.Modified == stamp.Modified && old.Mode == stamp.Mode && old.Identity == stamp.Identity + if reusable { + stamp.Hash = old.Hash + tree.FilesReused++ + } else { + data, err := os.ReadFile(path) + if os.IsPermission(err) { + continue + } + if err != nil { + return nil, err + } + after, err := os.Lstat(path) + if err != nil { + return nil, err + } + if stampFile(after) != stamp { + return nil, fmt.Errorf("file changed while hashing: %s", rel) + } + stamp.Hash = fmt.Sprintf("%x", sha256.Sum256(data)) + tree.FilesHashed++ + } + next[rel] = stamp + tree.Files[rel] = stamp.Hash + } + c.Files = next + if reconcile { + c.Reconciled = now + } + tree.RootHash = buildDirHash(tree.Files) + if cachePath != "" { + // A failed cache write affects performance only, never index validity. + if data, err := json.Marshal(c); err == nil { + if f, err := os.CreateTemp(filepath.Dir(cachePath), ".scan-*"); err == nil { + name := f.Name() + _, writeErr := f.Write(data) + closeErr := f.Close() + if writeErr == nil && closeErr == nil { + _ = os.Rename(name, cachePath) + } + _ = os.Remove(name) + } + } + } + return tree, nil +} + +func stampFile(info os.FileInfo) cachedFile { + identity, changed := fileIdentity(info) + return cachedFile{Size: info.Size(), Modified: info.ModTime().UnixNano(), Mode: uint32(info.Mode()), Identity: identity, Changed: changed} +} diff --git a/internal/merkle/cache_test.go b/internal/merkle/cache_test.go new file mode 100644 index 00000000..43d681a3 --- /dev/null +++ b/internal/merkle/cache_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package merkle + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestCacheReconcilesEditsAndRestarts(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "a.go") + cachePath := filepath.Join(t.TempDir(), "scan.json") + if err := os.WriteFile(path, []byte("package a"), 0600); err != nil { + t.Fatal(err) + } + oldTime := time.Now().Add(-time.Hour) + if err := os.Chtimes(path, oldTime, oldTime); err != nil { + t.Fatal(err) + } + c := &Cache{closed: true} + first, err := c.Build(root, nil, cachePath, false) + if err != nil { + t.Fatal(err) + } + // Reopen the persisted cache, then overwrite with equal-length content and + // restore mtime. ctime/identity must prevent a false freshness result. + c = &Cache{closed: true} + if err := os.WriteFile(path, []byte("package b"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, oldTime, oldTime); err != nil { + t.Fatal(err) + } + second, err := c.Build(root, nil, cachePath, false) + if err != nil { + t.Fatal(err) + } + if first.RootHash == second.RootHash { + t.Fatal("same-size, restored-mtime edit was missed") + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "b.go"), []byte("package b"), 0600); err != nil { + t.Fatal(err) + } + third, err := c.Build(root, nil, cachePath, false) + if err != nil { + t.Fatal(err) + } + if _, ok := third.Files["a.go"]; ok || len(third.Files) != 1 { + t.Fatalf("stale membership: %v", third.Files) + } + full, err := BuildTree(root, nil) + if err != nil { + t.Fatal(err) + } + if full.RootHash != third.RootHash { + t.Fatal("incremental and full trees differ") + } +} + +func TestCacheForceAndExpiryRehash(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "a.go") + if err := os.WriteFile(path, []byte("package a"), 0600); err != nil { + t.Fatal(err) + } + oldTime := time.Now().Add(-time.Hour) + if err := os.Chtimes(path, oldTime, oldTime); err != nil { + t.Fatal(err) + } + c := &Cache{closed: true} + want, err := c.Build(root, nil, "", false) + if err != nil { + t.Fatal(err) + } + for _, force := range []bool{true, false} { + entry := c.Files["a.go"] + entry.Hash = "stale" + c.Files["a.go"] = entry + if !force { + c.Reconciled = time.Now().Add(-2 * time.Hour) + } + got, err := c.Build(root, nil, "", force) + if err != nil { + t.Fatal(err) + } + if got.RootHash != want.RootHash { + t.Fatal("reconciliation reused stale hash") + } + } +} + +func TestCacheReusesStableHashes(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a.go"), []byte("package a"), 0600); err != nil { + t.Fatal(err) + } + c := &Cache{closed: true} + first, err := c.Build(root, nil, "", false) + if err != nil { + t.Fatal(err) + } + if c.Files["a.go"].Identity == "" { + t.Skip("platform conservatively hashes all files") + } + // Let both mtime and ctime leave the racy timestamp window. + time.Sleep(2100 * time.Millisecond) + second, err := c.Build(root, nil, "", false) + if err != nil { + t.Fatal(err) + } + if first.RootHash != second.RootHash || second.FilesHashed != 0 || second.FilesReused != 1 { + t.Fatalf("unchanged file was not reused: %+v", second) + } +} + +func TestCacheWatcherInvalidatesChangedPath(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "a.go") + if err := os.WriteFile(path, []byte("package a"), 0600); err != nil { + t.Fatal(err) + } + c := &Cache{} + defer func() { _ = c.Close() }() + if _, err := c.Build(root, nil, "", false); err != nil { + t.Fatal(err) + } + if c.watcher == nil { + t.Skip("filesystem watches unavailable") + } + if err := os.WriteFile(path, []byte("package b"), 0600); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + c.mu.Lock() + _, present := c.Files["a.go"] + c.mu.Unlock() + if !present { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("watcher did not invalidate the changed path") +} diff --git a/internal/merkle/merkle.go b/internal/merkle/merkle.go index c4e0e9d2..add6a799 100644 --- a/internal/merkle/merkle.go +++ b/internal/merkle/merkle.go @@ -28,8 +28,10 @@ import ( // Tree holds the Merkle tree state for a project directory. type Tree struct { - RootHash string // SHA-256 of the root directory - Files map[string]string // relative path -> content SHA-256 hash + FilesHashed int // files read during this scan + FilesReused int // hashes reused after metadata validation + RootHash string // SHA-256 of the root directory + Files map[string]string // relative path -> content SHA-256 hash } // SkipFunc returns true for paths that should be skipped during tree building. @@ -44,7 +46,6 @@ func DefaultSkip(relPath string, isDir bool) bool { return !strings.HasSuffix(base, ".go") } - const merkleWorkers = 8 const maxFileSize = 10 * 1024 * 1024 // 10 MB @@ -57,7 +58,7 @@ func BuildTree(rootDir string, skip SkipFunc) (*Tree, error) { skip = DefaultSkip } - relPaths, err := collectFilePaths(rootDir, skip) + relPaths, err := collectFilePaths(rootDir, skip, nil) if err != nil { return nil, err } @@ -67,11 +68,12 @@ func BuildTree(rootDir string, skip SkipFunc) (*Tree, error) { return nil, err } + tree.FilesHashed = len(tree.Files) tree.RootHash = buildDirHash(tree.Files) return tree, nil } -func collectFilePaths(rootDir string, skip SkipFunc) ([]string, error) { +func collectFilePaths(rootDir string, skip SkipFunc, visitDir func(string)) ([]string, error) { var relPaths []string err := filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -85,12 +87,18 @@ func collectFilePaths(rootDir string, skip SkipFunc) ([]string, error) { } rel, _ := filepath.Rel(rootDir, path) if rel == "." { + if visitDir != nil { + visitDir(path) + } return nil } if d.IsDir() { if skip(rel, true) { return filepath.SkipDir } + if visitDir != nil { + visitDir(path) + } return nil } if d.Type()&os.ModeSymlink != 0 { diff --git a/internal/merkle/merkle_test.go b/internal/merkle/merkle_test.go index 104a374a..26082e8d 100644 --- a/internal/merkle/merkle_test.go +++ b/internal/merkle/merkle_test.go @@ -174,7 +174,7 @@ func TestCollectFilePaths_SkipsSymlinks(t *testing.T) { t.Skip("symlinks not supported:", err) } - paths, err := collectFilePaths(dir, DefaultSkip) + paths, err := collectFilePaths(dir, DefaultSkip, nil) if err != nil { t.Fatal(err) } @@ -197,7 +197,7 @@ func TestCollectFilePaths_SkipsLargeFiles(t *testing.T) { t.Fatal(err) } - paths, err := collectFilePaths(dir, DefaultSkip) + paths, err := collectFilePaths(dir, DefaultSkip, nil) if err != nil { t.Fatal(err) } diff --git a/internal/merkle/stat_darwin.go b/internal/merkle/stat_darwin.go new file mode 100644 index 00000000..be30959c --- /dev/null +++ b/internal/merkle/stat_darwin.go @@ -0,0 +1,20 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package merkle + +import ( + "fmt" + "os" + "syscall" +) + +func fileIdentity(info os.FileInfo) (string, int64) { + s, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", 0 + } + return fmt.Sprintf("%d:%d:%d:%d", s.Dev, s.Ino, s.Ctimespec.Sec, s.Ctimespec.Nsec), int64(s.Ctimespec.Sec)*1e9 + int64(s.Ctimespec.Nsec) +} diff --git a/internal/merkle/stat_linux.go b/internal/merkle/stat_linux.go new file mode 100644 index 00000000..090dfdb4 --- /dev/null +++ b/internal/merkle/stat_linux.go @@ -0,0 +1,20 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package merkle + +import ( + "fmt" + "os" + "syscall" +) + +func fileIdentity(info os.FileInfo) (string, int64) { + s, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", 0 + } + return fmt.Sprintf("%d:%d:%d:%d", s.Dev, s.Ino, s.Ctim.Sec, s.Ctim.Nsec), int64(s.Ctim.Sec)*1e9 + int64(s.Ctim.Nsec) +} diff --git a/internal/merkle/stat_other.go b/internal/merkle/stat_other.go new file mode 100644 index 00000000..0145f255 --- /dev/null +++ b/internal/merkle/stat_other.go @@ -0,0 +1,13 @@ +//go:build !darwin && !linux + +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package merkle + +import "os" + +// Unknown stat formats conservatively disable metadata-only reuse. +func fileIdentity(os.FileInfo) (string, int64) { return "", 0 } diff --git a/internal/merkle/watch.go b/internal/merkle/watch.go new file mode 100644 index 00000000..cfefa406 --- /dev/null +++ b/internal/merkle/watch.go @@ -0,0 +1,88 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package merkle + +import ( + "os" + "path/filepath" + "strings" + "time" + + "github.com/fsnotify/fsnotify" +) + +// watchLocked is best effort. Metadata reconciliation remains active even +// when the OS cannot watch a directory or events overflow. +func (c *Cache) watchLocked(root string) { + if c.closed { + return + } + if c.watcher == nil { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return + } + c.watcher = watcher + c.watched = map[string]bool{} + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + rel, err := filepath.Rel(root, event.Name) + if err != nil { + continue + } + c.mu.Lock() + delete(c.Files, rel) + if c.watched[event.Name] { + for path := range c.Files { + if strings.HasPrefix(path, rel+string(os.PathSeparator)) { + delete(c.Files, path) + } + } + } + if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) { + for path := range c.watched { + if path == event.Name || strings.HasPrefix(path, event.Name+string(os.PathSeparator)) { + delete(c.watched, path) + } + } + } + c.mu.Unlock() + case _, ok := <-watcher.Errors: + if !ok { + return + } + c.mu.Lock() + c.Reconciled = time.Time{} + c.mu.Unlock() + } + } + }() + } +} + +func (c *Cache) watchDirectoryLocked(path string) { + if c.watcher != nil && !c.closed && !c.watched[path] { + if c.watcher.Add(path) == nil { + c.watched[path] = true + } + } +} + +// Close releases OS watches. Future builds still use metadata reconciliation. +func (c *Cache) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + c.closed = true + if c.watcher != nil { + return c.watcher.Close() + } + return nil +} diff --git a/internal/store/claims.go b/internal/store/claims.go new file mode 100644 index 00000000..2d243999 --- /dev/null +++ b/internal/store/claims.go @@ -0,0 +1,160 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package store + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/json" + "fmt" + "time" +) + +const embeddingLease = 2 * time.Minute + +// EmbedShared claims exact inputs before computing them. Completed results are +// handed to concurrent worktrees through a short-lived, bounded rendezvous +// cache; durable vector ownership still belongs to file revisions. +func (s *Store) EmbedShared(ctx context.Context, texts []string, compute func(context.Context, []string) ([][]float32, error)) ([][]float32, error) { + owner := rand.Text() + defer func() { _, _ = s.db.Exec(`DELETE FROM embedding_claims WHERE owner=? AND result IS NULL`, owner) }() + output := make([][]float32, len(texts)) + positions := map[[32]byte][]int{} + for i, text := range texts { + h := sha256.Sum256([]byte(text)) + positions[h] = append(positions[h], i) + } + for len(positions) > 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + var hashes [][32]byte + var inputs []string + for hash, indices := range positions { + now := time.Now().UnixMilli() + res, err := s.db.ExecContext(ctx, `INSERT INTO embedding_claims(input_hash,owner,expires_at) VALUES (?,?,?) + ON CONFLICT(input_hash) DO UPDATE SET owner=excluded.owner,expires_at=excluded.expires_at,result=NULL + WHERE embedding_claims.expires_at<=?`, hash[:], owner, now+embeddingLease.Milliseconds(), now) + if err != nil { + return nil, err + } + claimed, err := res.RowsAffected() + if err != nil { + return nil, err + } + if claimed > 0 { + hashes = append(hashes, hash) + inputs = append(inputs, texts[indices[0]]) + continue + } + var blob []byte + err = s.reader().QueryRowContext(ctx, `SELECT result FROM embedding_claims WHERE input_hash=? AND result IS NOT NULL AND expires_at>?`, hash[:], now).Scan(&blob) + if err == sql.ErrNoRows { + continue + } + if err != nil { + return nil, err + } + var vector []float32 + if err := json.Unmarshal(blob, &vector); err != nil { + return nil, fmt.Errorf("decode shared embedding: %w", err) + } + if len(vector) != s.dimensions { + return nil, fmt.Errorf("shared embedding dimensions mismatch") + } + for _, i := range indices { + output[i] = vector + } + delete(positions, hash) + } + if len(inputs) > 0 { + vectors, err := s.computeClaimed(ctx, owner, inputs, compute) + if err != nil { + return nil, err + } + if len(vectors) != len(inputs) { + return nil, fmt.Errorf("embed returned %d vectors for %d inputs", len(vectors), len(inputs)) + } + for i, hash := range hashes { + if len(vectors[i]) != s.dimensions { + return nil, fmt.Errorf("embedding dimensions mismatch") + } + blob, err := json.Marshal(vectors[i]) + if err != nil { + return nil, err + } + now := time.Now().UnixMilli() + res, err := s.db.ExecContext(ctx, `UPDATE embedding_claims SET result=?,expires_at=? WHERE input_hash=? AND owner=? AND expires_at>?`, blob, now+(10*time.Minute).Milliseconds(), hash[:], owner, now) + if err != nil { + return nil, err + } + n, err := res.RowsAffected() + if err != nil { + return nil, err + } + if n != 1 { + return nil, fmt.Errorf("embedding lease expired before publication") + } + for _, position := range positions[hash] { + output[position] = vectors[i] + } + delete(positions, hash) + } + if _, err := s.db.ExecContext(ctx, `DELETE FROM embedding_claims WHERE expires_at<=? OR input_hash IN ( + SELECT input_hash FROM (SELECT input_hash, sum(length(result)) OVER (ORDER BY expires_at DESC,input_hash) AS bytes + FROM embedding_claims WHERE result IS NOT NULL) WHERE bytes>?)`, time.Now().UnixMilli(), 64<<20); err != nil { + return nil, err + } + } else if len(positions) > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } + } + return output, nil +} + +func (s *Store) computeClaimed(ctx context.Context, owner string, inputs []string, compute func(context.Context, []string) ([][]float32, error)) ([][]float32, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan struct{}) + stopped := make(chan error, 1) + go func() { + ticker := time.NewTicker(embeddingLease / 4) + defer ticker.Stop() + for { + select { + case <-done: + stopped <- nil + return + case <-ctx.Done(): + stopped <- ctx.Err() + return + case <-ticker.C: + now := time.Now().UnixMilli() + if _, err := s.db.ExecContext(ctx, `UPDATE embedding_claims SET expires_at=? WHERE owner=? AND result IS NULL AND expires_at>?`, now+embeddingLease.Milliseconds(), owner, now); err != nil { + cancel() + stopped <- err + return + } + } + } + }() + vectors, err := compute(ctx, inputs) + close(done) + renewalErr := <-stopped + if err != nil { + return nil, err + } + if renewalErr != nil { + return nil, renewalErr + } + return vectors, nil +} diff --git a/internal/store/claims_test.go b/internal/store/claims_test.go new file mode 100644 index 00000000..6dd5ba81 --- /dev/null +++ b/internal/store/claims_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package store + +import ( + "context" + "crypto/sha256" + "errors" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +func TestEmbeddingClaimsShareConcurrentWork(t *testing.T) { + path := filepath.Join(t.TempDir(), "index.db") + a, err := NewCollection(path, 4, "int8", "/a") + if err != nil { + t.Fatal(err) + } + defer func() { _ = a.Close() }() + b, err := NewCollection(path, 4, "int8", "/b") + if err != nil { + t.Fatal(err) + } + defer func() { _ = b.Close() }() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var calls atomic.Int32 + started := make(chan struct{}) + unblock := make(chan struct{}) + compute := func(ctx context.Context, texts []string) ([][]float32, error) { + if calls.Add(1) == 1 { + close(started) + } + select { + case <-unblock: + case <-ctx.Done(): + return nil, ctx.Err() + } + return [][]float32{{1, 0, 0, 0}}, nil + } + done := make(chan error, 2) + run := func(s *Store) { + vectors, err := s.EmbedShared(ctx, []string{"same", "same"}, compute) + if err == nil && (len(vectors) != 2 || vectors[1][0] != 1) { + err = errors.New("incorrect duplicate results") + } + done <- err + } + go run(a) + <-started + go run(b) + close(unblock) + for range 2 { + if err := <-done; err != nil { + t.Fatal(err) + } + } + if calls.Load() != 1 { + t.Fatalf("computed %d times", calls.Load()) + } +} + +func TestEmbeddingClaimsRecoverAndCancel(t *testing.T) { + s, err := NewCollection(":memory:", 4, "float32", "/a") + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + hash := sha256.Sum256([]byte("input")) + if _, err := s.db.Exec(`INSERT INTO embedding_claims(input_hash,owner,expires_at) VALUES (?,?,?)`, hash[:], "dead", time.Now().Add(-time.Second).UnixMilli()); err != nil { + t.Fatal(err) + } + failure := errors.New("backend failed") + _, err = s.EmbedShared(context.Background(), []string{"input"}, func(context.Context, []string) ([][]float32, error) { return nil, failure }) + if !errors.Is(err, failure) { + t.Fatal(err) + } + var count int + if err := s.db.QueryRow(`SELECT count(*) FROM embedding_claims`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatal("failed owner stranded its claim") + } + if _, err := s.db.Exec(`INSERT INTO embedding_claims(input_hash,owner,expires_at) VALUES (?,?,?)`, hash[:], "busy", time.Now().Add(time.Minute).UnixMilli()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + _, err = s.EmbedShared(ctx, []string{"input"}, func(context.Context, []string) ([][]float32, error) { + t.Fatal("computed another owner's input") + return nil, nil + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cancellation: %v", err) + } +} diff --git a/internal/store/retention.go b/internal/store/retention.go new file mode 100644 index 00000000..46716c55 --- /dev/null +++ b/internal/store/retention.go @@ -0,0 +1,109 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package store + +import ( + "database/sql" + "fmt" + "strconv" + "time" +) + +// SetRevisionCachePolicy sets a collection-wide logical byte budget and TTL. +// The budget counts vectors conservatively once per retained revision, plus +// chunk metadata; SQLite page/WAL overhead is not included. Zero disables it. +func (s *Store) SetRevisionCachePolicy(bytes int64, ttl time.Duration) error { + if bytes < 0 || ttl < 0 { + return fmt.Errorf("cache budget and TTL must be nonnegative") + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + for key, value := range map[string]int64{"revision_cache_bytes": bytes, "revision_cache_seconds": int64(ttl / time.Second)} { + if _, err := tx.Exec(`INSERT INTO collection_meta(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, key, strconv.FormatInt(value, 10)); err != nil { + return err + } + } + if err := pruneRevisionCacheTx(tx); err != nil { + return err + } + return tx.Commit() +} + +func gcRevisionsTx(tx *sql.Tx, revisionIDs ...int64) error { + for _, id := range revisionIDs { + if id == 0 { + continue + } + _, err := tx.Exec(`INSERT OR IGNORE INTO revision_cache(revision_id, unused_at, bytes) + SELECT fr.id, ?, 128 + length(fr.relative_path) + COALESCE(( + SELECT sum(96 + length(cd.chunk_key) + length(cd.symbol) + length(cd.kind) + + CAST((SELECT value FROM collection_meta WHERE key='vec_dimensions') AS INTEGER) * + CASE (SELECT value FROM collection_meta WHERE key='vector_storage') WHEN 'float32' THEN 4 ELSE 1 END) + FROM chunk_defs cd WHERE cd.file_revision_id=fr.id),0) + FROM file_revisions fr WHERE fr.id=? AND fr.complete=1 + AND NOT EXISTS(SELECT 1 FROM project_files WHERE file_revision_id=fr.id)`, time.Now().Unix(), id) + if err != nil { + return err + } + var retained bool + if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM revision_cache WHERE revision_id=?)`, id).Scan(&retained); err != nil { + return err + } + if !retained { + if err := deleteRevisionsTx(tx, id); err != nil { + return err + } + } + } + return pruneRevisionCacheTx(tx) +} + +func pruneRevisionCacheTx(tx *sql.Tx) error { + budget, ttl := int64(256<<20), int64((7*24*time.Hour)/time.Second) + for key, target := range map[string]*int64{"revision_cache_bytes": &budget, "revision_cache_seconds": &ttl} { + var value string + err := tx.QueryRow(`SELECT value FROM collection_meta WHERE key=?`, key).Scan(&value) + if err == sql.ErrNoRows { + continue + } + if err != nil { + return err + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed < 0 { + return fmt.Errorf("invalid %s: %q", key, value) + } + *target = parsed + } + rows, err := tx.Query(`SELECT revision_id, unused_at, bytes FROM revision_cache ORDER BY unused_at DESC, revision_id DESC`) + if err != nil { + return err + } + var remove []int64 + var retained int64 + now := time.Now().Unix() + for rows.Next() { + var id, unused, size int64 + if err := rows.Scan(&id, &unused, &size); err != nil { + _ = rows.Close() + return err + } + if ttl == 0 || now-unused >= ttl || unused > now || size > budget-retained { + remove = append(remove, id) + } else { + retained += size + } + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return err + } + return deleteRevisionsTx(tx, remove...) +} diff --git a/internal/store/retention_test.go b/internal/store/retention_test.go new file mode 100644 index 00000000..68292394 --- /dev/null +++ b/internal/store/retention_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package store + +import ( + "testing" + "time" + + "github.com/ory/lumen/internal/chunker" +) + +func TestRevisionCacheReuseAndBound(t *testing.T) { + s, err := NewCollection(":memory:", 4, "int8", "/project") + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + for _, version := range []string{"a", "b"} { + c := chunker.Chunk{ID: version, FilePath: "a.go", Content: version} + if _, err := s.StoreFileRevision("a.go", version, []chunker.Chunk{c}, map[int][]float32{0: {1, 0, 0, 0}}); err != nil { + t.Fatal(err) + } + } + attached, err := s.AttachExistingFileRevision("a.go", "a") + if err != nil || !attached { + t.Fatalf("switch-back reuse: %v, %v", attached, err) + } + if err := s.SetRevisionCachePolicy(1, time.Hour); err != nil { + t.Fatal(err) + } + attached, err = s.AttachExistingFileRevision("a.go", "b") + if err != nil || attached { + t.Fatalf("over-budget revision retained: %v, %v", attached, err) + } + stats, err := s.CollectionStats() + if err != nil { + t.Fatal(err) + } + if stats.UniqueVectors != 1 || stats.ProjectFiles != 1 { + t.Fatalf("eviction damaged active membership: %+v", stats) + } +} + +func TestRevisionCacheExpires(t *testing.T) { + s, err := NewCollection(":memory:", 4, "float32", "/project") + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + if _, err := s.StoreFileRevision("a.go", "a", nil, nil); err != nil { + t.Fatal(err) + } + if err := s.DeleteFileChunks("a.go"); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`UPDATE revision_cache SET unused_at=?`, time.Now().Add(-8*24*time.Hour).Unix()); err != nil { + t.Fatal(err) + } + if err := s.SetRevisionCachePolicy(256<<20, 7*24*time.Hour); err != nil { + t.Fatal(err) + } + attached, err := s.AttachExistingFileRevision("a.go", "a") + if err != nil || attached { + t.Fatalf("expired revision retained: %v %v", attached, err) + } +} diff --git a/internal/store/scan.go b/internal/store/scan.go new file mode 100644 index 00000000..2fe3bd8a --- /dev/null +++ b/internal/store/scan.go @@ -0,0 +1,65 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package store + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + scanCacheInfix = ".scan-" + scanCacheSuffix = ".json" +) + +// ScanCachePath returns the disposable freshness-scan sidecar for one +// worktree root of the collection at dbPath. +func ScanCachePath(dbPath, projectRoot string) string { + return fmt.Sprintf("%s%s%x%s", dbPath, scanCacheInfix, sha256.Sum256([]byte(projectRoot)), scanCacheSuffix) +} + +// removeOrphanScanCaches deletes completed scan sidecars whose worktree no +// longer has a membership. Only exact ".scan-.json" names are +// considered: in-progress temporary files, lock files (which must keep their +// inode for flock), and unknown files are left alone. Callers must hold the +// exclusive collection guard. +func removeOrphanScanCaches(dbPath string, projectPaths []string) (int, error) { + surviving := make(map[string]bool, len(projectPaths)) + for _, path := range projectPaths { + surviving[filepath.Base(ScanCachePath(dbPath, path))] = true + } + entries, err := os.ReadDir(filepath.Dir(dbPath)) + if err != nil { + return 0, err + } + prefix := filepath.Base(dbPath) + scanCacheInfix + var removed int + var errs []error + for _, entry := range entries { + name := entry.Name() + if !entry.Type().IsRegular() || surviving[name] || !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, scanCacheSuffix) { + continue + } + digest := strings.TrimSuffix(strings.TrimPrefix(name, prefix), scanCacheSuffix) + if len(digest) != 2*sha256.Size || strings.ToLower(digest) != digest { + continue + } + if _, err := hex.DecodeString(digest); err != nil { + continue + } + if err := os.Remove(filepath.Join(filepath.Dir(dbPath), name)); err != nil && !os.IsNotExist(err) { + errs = append(errs, err) + continue + } + removed++ + } + return removed, errors.Join(errs...) +} diff --git a/internal/store/scan_cleanup_test.go b/internal/store/scan_cleanup_test.go new file mode 100644 index 00000000..4170ed57 --- /dev/null +++ b/internal/store/scan_cleanup_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +package store + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ory/lumen/internal/chunker" +) + +func TestCleanupCollectionAtRemovesOnlyOrphanScanCaches(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "index.db") + active, stale, vanished := t.TempDir(), t.TempDir(), filepath.Join(t.TempDir(), "gone") + s, err := NewCollection(dbPath, 4, "int8", active) + if err != nil { + t.Fatal(err) + } + chunk := chunker.Chunk{ID: "a", FilePath: "a.go", Symbol: "A", Kind: "function", StartLine: 1, EndLine: 1, Content: "func A() {}"} + if _, err := s.StoreFileRevision("a.go", "aa", []chunker.Chunk{chunk}, map[int][]float32{0: {1, 0, 0, 0}}); err != nil { + t.Fatal(err) + } + if err := s.UseProject(stale); err != nil { + t.Fatal(err) + } + if attached, err := s.AttachExistingFileRevision("a.go", "aa"); err != nil || !attached { + t.Fatalf("attach: %v %v", attached, err) + } + old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339) + if _, err := s.db.Exec(`UPDATE projects SET last_accessed_at = ? WHERE path = ?`, old, stale); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + + scan := func(project string) string { + return fmt.Sprintf("%s.scan-%x.json", dbPath, sha256.Sum256([]byte(project))) + } + keep := []string{ + scan(active), + filepath.Join(dir, ".scan-123456"), // in-progress atomic write + dbPath + ".lock", // collection guard + dbPath + ".project-0123456789abcdef.lock", // per-project guard + dbPath + ".scan-NOTHEX.json", // unknown naming + dbPath + ".scan-" + fmt.Sprintf("%x", [8]byte{}) + ".json", // wrong length + filepath.Join(dir, "notes.txt"), + } + remove := []string{scan(stale), scan(vanished)} + for _, path := range append(append([]string{}, keep...), remove...) { + if err := os.WriteFile(path, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + } + + stats, shared, err := CleanupCollectionAt(dbPath, time.Now().Add(-30*24*time.Hour)) + if err != nil || !shared { + t.Fatalf("cleanup: shared=%v err=%v", shared, err) + } + if stats.ProjectsRemoved != 1 || stats.ProjectsLeft != 1 { + t.Fatalf("unexpected cleanup stats: %+v", stats) + } + for _, path := range keep { + if _, err := os.Stat(path); err != nil { + t.Errorf("cleanup removed %s: %v", filepath.Base(path), err) + } + } + for _, path := range remove { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("orphan scan cache %s survived cleanup: %v", filepath.Base(path), err) + } + } +} diff --git a/internal/store/shared.go b/internal/store/shared.go index 43fe856f..c9fd21e3 100644 --- a/internal/store/shared.go +++ b/internal/store/shared.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/mattn/go-sqlite3" "github.com/ory/lumen/internal/chunker" sqlite_vec "github.com/ory/lumen/internal/sqlitevec" ) @@ -48,10 +49,11 @@ type CollectionStats struct { // CleanupStats reports a shared collection garbage-collection pass. type CleanupStats struct { - ProjectsRemoved int - VectorsRemoved int - BytesReclaimed int64 - ProjectsLeft int + ProjectsRemoved int + VectorsRemoved int + BytesReclaimed int64 + ProjectsLeft int + ScanCachesRemoved int } func openCollection(dsn string, dimensions int, vectorStorage string) (*Store, error) { @@ -69,7 +71,19 @@ func openCollection(dsn string, dimensions int, vectorStorage string) (*Store, e "PRAGMA cache_size=-64000", "PRAGMA temp_store=MEMORY", } { - if _, err := db.Exec(pragma); err != nil { + var pragmaErr error + deadline := time.Now().Add(5 * time.Second) + for { + _, pragmaErr = db.Exec(pragma) + var sqliteErr sqlite3.Error + if !errors.As(pragmaErr, &sqliteErr) || (sqliteErr.Code != sqlite3.ErrBusy && sqliteErr.Code != sqlite3.ErrLocked) || time.Now().After(deadline) { + break + } + // Switching a newly-created database to WAL can return BUSY without + // invoking SQLite's busy handler while another opener changes the mode. + time.Sleep(10 * time.Millisecond) + } + if err := pragmaErr; err != nil { _ = db.Close() return nil, fmt.Errorf("exec %q: %w", pragma, err) } @@ -133,6 +147,12 @@ func openCollection(dsn string, dimensions int, vectorStorage string) (*Store, e func createCollectionSchema(db *sql.DB, dimensions int, vectorStorage string) error { stmts := []string{ + `CREATE TABLE IF NOT EXISTS embedding_claims ( + input_hash BLOB PRIMARY KEY, + owner TEXT NOT NULL, + expires_at INTEGER NOT NULL, + result BLOB + ) WITHOUT ROWID`, `CREATE TABLE IF NOT EXISTS collection_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -163,6 +183,11 @@ func createCollectionSchema(db *sql.DB, dimensions int, vectorStorage string) er PRIMARY KEY(project_id, relative_path) ) WITHOUT ROWID`, `CREATE INDEX IF NOT EXISTS idx_project_files_revision ON project_files(file_revision_id)`, + `CREATE TABLE IF NOT EXISTS revision_cache ( + revision_id INTEGER PRIMARY KEY REFERENCES file_revisions(id) ON DELETE CASCADE, + unused_at INTEGER NOT NULL, + bytes INTEGER NOT NULL + )`, `CREATE TABLE IF NOT EXISTS vector_keys ( id INTEGER PRIMARY KEY, input_hash BLOB NOT NULL UNIQUE @@ -523,6 +548,9 @@ func (s *Store) StoreFileRevision(relativePath, contentHash string, chunks []chu } func replaceProjectFileTx(tx *sql.Tx, projectID int64, path string, revisionID int64) (int64, error) { + if _, err := tx.Exec(`DELETE FROM revision_cache WHERE revision_id = ?`, revisionID); err != nil { + return 0, err + } var displaced int64 err := tx.QueryRow(`SELECT file_revision_id FROM project_files WHERE project_id = ? AND relative_path = ?`, projectID, path).Scan(&displaced) if err != nil && err != sql.ErrNoRows { @@ -648,7 +676,7 @@ func (s *Store) removeProjectFile(path string) error { return tx.Commit() } -func gcRevisionsTx(tx *sql.Tx, revisionIDs ...int64) error { +func deleteRevisionsTx(tx *sql.Tx, revisionIDs ...int64) error { seen := make(map[int64]struct{}, len(revisionIDs)) for _, revisionID := range revisionIDs { if revisionID == 0 { @@ -764,75 +792,58 @@ func (s *Store) searchShared(ctx context.Context, queryVec []float32, limit int, if err != nil { return nil, fmt.Errorf("serialize query: %w", err) } - var totalVectors int - if err := s.reader().QueryRow(`SELECT count(*) FROM vector_keys`).Scan(&totalVectors); err != nil { - return nil, err - } - if totalVectors == 0 { - return nil, nil + // Materialize project membership before touching vectors. Each eligible + // vector is scored once, even when several chunks reference it. This also + // preserves deterministic chunk-level tie breaking without global KNN + // overfetching (which can otherwise search the collection repeatedly). + where := "pf.project_id = ?" + args := []any{s.projectID} + if pathPrefix != "" { + where += " AND (pf.relative_path = ? OR substr(pf.relative_path, 1, length(?) + 1) = ? || '/')" + args = append(args, pathPrefix, pathPrefix, pathPrefix) } - candidates := min(totalVectors, max(limit, 32)) - for { - results, boundaryDistance, err := s.searchSharedCandidates(ctx, blob, limit, candidates, maxDistance, pathPrefix) - if err != nil { - return nil, err - } - if candidates >= totalVectors || (len(results) >= limit && results[len(results)-1].Distance < boundaryDistance) { - return results, nil - } - candidates = min(totalVectors, candidates*2) + queryExpression := "?" + vectorExpression := "v.embedding" + if s.vectorStorage == "int8" { + queryExpression = "vec_int8(?)" + vectorExpression = "vec_int8(v.embedding)" } -} - -func (s *Store) searchSharedCandidates(ctx context.Context, blob []byte, limit, candidates int, maxDistance float64, pathPrefix string) ([]SearchResult, float64, error) { - where := []string{"pf.project_id = ?"} - args := []any{blob, candidates, s.projectID} + args = append(args, blob) + distanceWhere := "" if maxDistance > 0 { - where = append(where, "knn.distance < ?") + distanceWhere = "WHERE scored.distance < ?" args = append(args, maxDistance) } - if pathPrefix != "" { - where = append(where, "(pf.relative_path = ? OR pf.relative_path LIKE ? || '/%')") - args = append(args, pathPrefix, pathPrefix) - } args = append(args, limit) - matchExpression := "?" - if s.vectorStorage == "int8" { - matchExpression = "vec_int8(?)" - } - query := `WITH knn AS ( - SELECT vector_id, distance FROM vec_vectors - WHERE embedding MATCH ` + matchExpression + ` AND k = ? + query := `WITH eligible AS MATERIALIZED ( + SELECT pf.relative_path, cd.* + FROM project_files pf + JOIN chunk_defs cd ON cd.file_revision_id = pf.file_revision_id + WHERE ` + where + ` + ), scored AS MATERIALIZED ( + SELECT ids.vector_id, vec_distance_cosine(` + vectorExpression + `, ` + queryExpression + `) AS distance + FROM (SELECT DISTINCT vector_id FROM eligible) ids + CROSS JOIN vec_vectors v ON v.vector_id = ids.vector_id ) - SELECT pf.relative_path, cd.symbol, cd.kind, cd.start_line, cd.end_line, knn.distance, - max(knn.distance) OVER () - FROM knn - JOIN chunk_defs cd ON cd.vector_id = knn.vector_id - JOIN project_files pf ON pf.file_revision_id = cd.file_revision_id - WHERE ` + strings.Join(where, " AND ") + ` - ORDER BY knn.distance, pf.relative_path, cd.start_line, cd.end_line, cd.symbol, cd.id + SELECT e.relative_path, e.symbol, e.kind, e.start_line, e.end_line, scored.distance + FROM eligible e JOIN scored ON scored.vector_id = e.vector_id + ` + distanceWhere + ` + ORDER BY scored.distance, e.relative_path, e.start_line, e.end_line, e.symbol, e.id LIMIT ?` rows, err := s.reader().QueryContext(ctx, query, args...) if err != nil { - return nil, 0, fmt.Errorf("shared search query: %w", err) + return nil, fmt.Errorf("shared search query: %w", err) } defer func() { _ = rows.Close() }() var results []SearchResult - var boundaryDistance float64 for rows.Next() { var result SearchResult - if err := rows.Scan(&result.FilePath, &result.Symbol, &result.Kind, &result.StartLine, &result.EndLine, &result.Distance, &boundaryDistance); err != nil { - return nil, 0, err + if err := rows.Scan(&result.FilePath, &result.Symbol, &result.Kind, &result.StartLine, &result.EndLine, &result.Distance); err != nil { + return nil, err } results = append(results, result) } - if err := rows.Err(); err != nil { - return nil, 0, err - } - if err := rows.Close(); err != nil { - return nil, 0, err - } - return results, boundaryDistance, nil + return results, rows.Err() } func (s *Store) sharedStats() (StoreStats, error) { @@ -863,7 +874,11 @@ func (s *Store) CollectionStats() (CollectionStats, error) { if err != nil { return stats, err } - stats.SharedReferences = stats.ChunkReferences - stats.UniqueVectors + var activeVectors int + if err := s.reader().QueryRow(`SELECT count(DISTINCT cd.vector_id) FROM chunk_defs cd JOIN project_files pf ON pf.file_revision_id=cd.file_revision_id`).Scan(&activeVectors); err != nil { + return stats, err + } + stats.SharedReferences = stats.ChunkReferences - activeVectors stats.VectorStorage = s.vectorStorage var pageCount, pageSize, freelist int64 if err := s.reader().QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil { @@ -1015,5 +1030,30 @@ func CleanupCollectionAt(dbPath string, cutoff time.Time) (CleanupStats, bool, e } defer func() { _ = s.Close() }() stats, err := s.CleanupStaleProjects(cutoff) - return stats, true, err + if err != nil { + return stats, true, err + } + // Sweeping after every pass also retries sidecars orphaned by an earlier + // partial failure. + rows, err := s.reader().Query(`SELECT path FROM projects`) + if err != nil { + return stats, true, err + } + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + _ = rows.Close() + return stats, true, err + } + paths = append(paths, path) + } + if err := errors.Join(rows.Err(), rows.Close()); err != nil { + return stats, true, err + } + stats.ScanCachesRemoved, err = removeOrphanScanCaches(dbPath, paths) + if err != nil { + return stats, true, fmt.Errorf("remove orphan scan caches: %w", err) + } + return stats, true, nil } diff --git a/internal/store/shared_test.go b/internal/store/shared_test.go index 1a8a40e7..8820b4bb 100644 --- a/internal/store/shared_test.go +++ b/internal/store/shared_test.go @@ -20,7 +20,7 @@ import ( "github.com/ory/lumen/internal/chunker" ) -func TestSharedReplacementGarbageCollectsDisplacedRevision(t *testing.T) { +func TestSharedReplacementRetainsDisplacedRevision(t *testing.T) { s, err := NewCollection(":memory:", 4, "int8", t.TempDir()) if err != nil { t.Fatal(err) @@ -38,8 +38,8 @@ func TestSharedReplacementGarbageCollectsDisplacedRevision(t *testing.T) { if err != nil { t.Fatal(err) } - if stats.UniqueVectors != 1 || stats.ChunkReferences != 1 { - t.Fatalf("displaced revision was not collected: %+v", stats) + if stats.UniqueVectors != 2 || stats.ChunkReferences != 1 { + t.Fatalf("displaced revision was not retained: %+v", stats) } } @@ -96,6 +96,9 @@ func TestSharedCollectionReusesRevisionsAndVectors(t *testing.T) { t.Fatal(err) } defer func() { _ = s.Close() }() + if err := s.SetRevisionCachePolicy(0, 0); err != nil { + t.Fatal(err) + } chunks := []chunker.Chunk{ {ID: "a", FilePath: "main.go", Symbol: "A", Kind: "function", StartLine: 1, EndLine: 2, Content: "func A() {}"}, @@ -216,7 +219,7 @@ func TestSharedCollectionValidatesStorageProfile(t *testing.T) { } } -func TestSharedSearchAdaptivelyExpandsSparseProjectCandidates(t *testing.T) { +func TestSharedSearchScopesSparseProjectCandidates(t *testing.T) { s, err := NewCollection(":memory:", 4, "int8", "/project-a") if err != nil { t.Fatal(err) @@ -246,13 +249,13 @@ func TestSharedSearchAdaptivelyExpandsSparseProjectCandidates(t *testing.T) { } // The nearest 80 global vectors belong to project A. Project B's result - // is only found after candidate doubling exhausts that dense prefix. + // must be found without allowing that dense prefix to consume its limit. results, err := s.Search(context.Background(), []float32{1, 0, 0, 0}, 1, 0, "nested") if err != nil { t.Fatal(err) } if len(results) != 1 || results[0].Symbol != "Target" { - t.Fatalf("adaptive sparse-project search failed: %+v", results) + t.Fatalf("scoped sparse-project search failed: %+v", results) } } @@ -538,3 +541,30 @@ func TestSharedInt8StorageAtMostTwentyPercentOfSeparateFloat32(t *testing.T) { t.Fatalf("shared int8 size ratio = %.3f (%d/%d), want <= 0.20", ratio, sharedInfo.Size(), separateBytes) } } + +func TestSharedSearchLiteralPrefixAndDeterministicTies(t *testing.T) { + for _, storage := range []string{"int8", "float32"} { + t.Run(storage, func(t *testing.T) { + s, err := NewCollection(":memory:", 4, storage, "/project") + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + for _, path := range []string{"pkg_/z.go", "pkg_/a.go", "pkgX/wrong.go"} { + chunk := chunker.Chunk{ID: path, FilePath: path, Content: path, StartLine: 1, EndLine: 1} + if _, err := s.StoreFileRevision(path, path, []chunker.Chunk{chunk}, map[int][]float32{0: {1, 0, 0, 0}}); err != nil { + t.Fatal(err) + } + } + for range 3 { + results, err := s.Search(context.Background(), []float32{1, 0, 0, 0}, 1, 0, "pkg_") + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].FilePath != "pkg_/a.go" { + t.Fatalf("prefix/tie result: %+v", results) + } + } + }) + } +}