Skip to content
22 changes: 16 additions & 6 deletions cmd/gortex/daemon_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -315,15 +315,17 @@ type warmupTimings struct {
// parallel reconcile pool. Any incomplete result fails closed to the existing
// repo/global pipeline; an ordinary warm delta stays file-scoped end to end.
type warmupDeltaFrontier struct {
mu sync.Mutex
exact bool
plans map[string]indexer.DerivedInvalidationPlan
mu sync.Mutex
exact bool
plans map[string]indexer.DerivedInvalidationPlan
crossRepoFiles map[string]struct{}
}

func newWarmupDeltaFrontier() *warmupDeltaFrontier {
return &warmupDeltaFrontier{
exact: true,
plans: make(map[string]indexer.DerivedInvalidationPlan),
exact: true,
plans: make(map[string]indexer.DerivedInvalidationPlan),
crossRepoFiles: make(map[string]struct{}),
}
}

Expand All @@ -343,13 +345,21 @@ func (frontier *warmupDeltaFrontier) record(result *indexer.IndexResult) {
plan := frontier.plans[result.RepoPrefix]
plan.Merge(result.DerivedInvalidation)
frontier.plans[result.RepoPrefix] = plan
for _, file := range result.CrossRepoResolveFiles {
if file != "" {
frontier.crossRepoFiles[file] = struct{}{}
}
}
}

func (frontier *warmupDeltaFrontier) snapshot(changedRepos int, scopeUnknown bool) (map[string]indexer.DerivedInvalidationPlan, []string, bool) {
frontier.mu.Lock()
defer frontier.mu.Unlock()
plans := make(map[string]indexer.DerivedInvalidationPlan, len(frontier.plans))
fileSet := make(map[string]struct{})
fileSet := make(map[string]struct{}, len(frontier.crossRepoFiles))
for file := range frontier.crossRepoFiles {
fileSet[file] = struct{}{}
}
for prefix, plan := range frontier.plans {
plans[prefix] = plan
for _, file := range plan.Files {
Expand Down
128 changes: 128 additions & 0 deletions cmd/gortex/daemon_state_warm_npm_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package main

import (
"context"
"os"
"path/filepath"
"strconv"
"testing"
"time"

"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/config"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
"github.com/zzet/gortex/internal/indexer"
"github.com/zzet/gortex/internal/parser"
"github.com/zzet/gortex/internal/parser/languages"
"github.com/zzet/gortex/internal/search"
"go.uber.org/zap"
)

func TestWarmupDeltaFrontierTrackedNpmOfflineEdit(t *testing.T) {
for _, providerFirst := range []bool{true, false} {
t.Run(strconv.FormatBool(providerFirst), func(t *testing.T) {
dir := t.TempDir()
provider, consumer := filepath.Join(dir, "provider"), filepath.Join(dir, "consumer")
write := func(rel, body string) {
t.Helper()
p := filepath.Join(dir, rel)
require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
require.NoError(t, os.WriteFile(p, []byte(body), 0o644))
}
write("provider/package.json", `{"name":"p","exports":"./src/index.ts"}`)
write("provider/src/index.ts", "export function f(): number { return 42; }\n")
for _, n := range []string{"a", "b", "c", "d"} {
write("provider/src/"+n+".ts", "export function h"+n+"(): number { return 1; }\n")
}
write("consumer/package.json", `{"name":"app","dependencies":{"p":"^1"}}`)
write("consumer/src/app.ts", "import { f } from 'p';\nexport function run(): number { return f(); }\n")
entries := []config.RepoEntry{{Path: provider, Name: "provider", Workspace: "shared"}, {Path: consumer, Name: "consumer", Workspace: "shared"}}
if !providerFirst {
entries[0], entries[1] = entries[1], entries[0]
}
cfgPath := filepath.Join(t.TempDir(), "config.yaml")
gc := &config.GlobalConfig{Repos: entries}
gc.SetConfigPath(cfgPath)
require.NoError(t, gc.Save())
cm, err := config.NewConfigManager(cfgPath)
require.NoError(t, err)
db := filepath.Join(t.TempDir(), "graph.sqlite")
sqlite, err := store_sqlite.Open(db)
require.NoError(t, err)
t.Cleanup(func() { _ = sqlite.Close() })
var g graph.Store = sqlite
newIndexer := func() *indexer.MultiIndexer {
reg := parser.NewRegistry()
reg.Register(languages.NewTypeScriptExtractor())
return indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop())
}
mi := newIndexer()
t.Cleanup(func() { _ = mi.Close(context.Background()) })
for _, entry := range entries {
_, err := mi.TrackRepoCtx(context.Background(), entry)
require.NoError(t, err)
}
require.NoError(t, mi.RunPreEnrichResolve(context.Background(), nil, nil))
check := func(stage string) {
t.Helper()
var imports, calls []string
for _, e := range g.AllEdges() {
if e.FilePath != "consumer/src/app.ts" {
continue
}
switch e.Kind {
case graph.EdgeImports:
imports = append(imports, e.To)
require.True(t, e.CrossRepo, stage)
case graph.EdgeCalls:
calls = append(calls, e.To)
require.True(t, e.CrossRepo, stage)
}
}
require.ElementsMatch(t, []string{"provider/src/index.ts", "provider/src/index.ts::f"}, imports, stage)
require.Equal(t, []string{"provider/src/index.ts::f"}, calls, stage)
}
check("cold")
mtimes := map[string]map[string]int64{}
for _, entry := range entries {
mtimes[entry.Name] = mi.FileMtimes(entry.Name)
}
require.NoError(t, mi.Close(context.Background()))
require.NoError(t, sqlite.Close())
// Offline provider edit.
p := filepath.Join(provider, "src/index.ts")
require.NoError(t, os.WriteFile(p, []byte("export function f(v = 0): number { return 43 + v; }\n"), 0o644))
future := time.Now().Add(5 * time.Second)
require.NoError(t, os.Chtimes(p, future, future))
sqlite, err = store_sqlite.Open(db)
require.NoError(t, err)
g = sqlite
warm := newIndexer()
mi = warm
warm.BeginParallelBatch()
frontier := newWarmupDeltaFrontier()
require.NoError(t, warm.RunRepositoryTopologyBatch(context.Background(), func(ctx context.Context) error {
for _, entry := range entries {
res, err := warm.ReconcileRepoCtx(ctx, entry, mtimes[entry.Name])
require.NoError(t, err)
if res != nil {
require.False(t, res.FullRetrack)
if entry.Name == "provider" {
require.Equal(t, 1, res.StaleFileCount)
require.Equal(t, []string{"provider/src/index.ts"}, res.DerivedInvalidation.Files)
}
frontier.record(res)
}
}
return nil
}))
plans, files, exact := frontier.snapshot(2, false)
require.True(t, exact)
require.Equal(t, []string{"provider/src/index.ts"}, plans["provider"].Files)
require.NoError(t, warm.RunPreEnrichResolveFiles(context.Background(), files, nil))
warm.EndBatch()
check("after exact warm resolve")
})
}
}
5 changes: 4 additions & 1 deletion docs/multi-repo.md
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,9 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h
## How it works

- **Qualified node IDs** — IDs are always `<repo_prefix>/<path>::<Symbol>` (e.g., `frontend/src/app.ts::App`), including for a workspace tracking a single repo: a lone repo is simply the first tracked repo. Tools still accept an unqualified path (`src/app.ts`) when exactly one repo is tracked, since there is then nothing to be ambiguous about.
- **Cross-repo edges** — the resolver links symbols across repo boundaries with same-repo preference. Cross-repo edges carry a `cross_repo: true` flag.
- **Cross-repo edges** — the resolver links symbols across repo boundaries with same-repo preference. Cross-repo edges carry a `cross_repo: true` flag. JS/TS imports can link a declared npm dependency to a tracked repository whose `package.json` claims that package name, including npm and pnpm workspace members. A provider in the caller's workspace wins over eligible foreign providers. Multiple manifest claims in the winning tier keep the import external, even if only one entry is indexed.
- Entry lookup respects `exports` keys, wildcard subpaths and `null` blocks. It probes the selected `import`/`default` target first, then a `types` condition when that file is not indexed. Without `exports`, it tries `types`, `main` and directory indexes. Unsupported-condition subpaths never guess a directory target.
- Bare calls use the caller file's named import binding, including its local alias, not another same-named function anywhere in the provider. A per-binding import links to a symbol marked public by the extractor; otherwise it links only to the entry file and creates no call binding. Linked imports retain their specifier, survive provider eviction as external imports, and are rechecked by whole-graph resolution.
- **Limits:** Build-output-only entries, such as `main` and `types` pointing into excluded `dist/`, stay external unless an indexed target is declared. Gortex does not infer `src/` from `dist/`. The importing repository's own packages, `npm:` aliases and workspace paths that escape the repository are not linked by this pass. Sibling dependencies declared with `file:`, `link:` or `portal:` are not supported. Default-import bare calls have no exact binding evidence yet and stay unresolved. Existing linked imports from older builds need the consumer file reindexed to acquire the specifier and alias metadata. Namespace member calls, CommonJS `require` calls and package re-exports are not linked by this npm import pass. Export targets without `./` are accepted leniently, as in workspace resolution, even though Node rejects them. Call binding is file-scoped: a parameter or local callable with the imported name can bind to the import.
- **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository.
- **Shared repos** — the same repo can appear in multiple projects with different reference tags. It's indexed once and shared across projects.
2 changes: 1 addition & 1 deletion internal/indexer/affected_by_bound_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ func TestDeletionShapedStagesCarryNoInEdges(t *testing.T) {
stages := []*incrementalBatchStage{{graphPath: restubDefPath, priorNodes: prior}}
view := loadIncrementalPriorView(st, stages)

carried := restubIncomingRefsFromView(st, stages, view)
carried, _ := restubIncomingRefsFromView(st, stages, view)
require.Empty(t, carried,
"a deletion stage must park every in-edge; a carried edge has no re-state path here")
assert.Equal(t, 1, st.restubWrites("Foo"),
Expand Down
2 changes: 2 additions & 0 deletions internal/indexer/file_meta.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,8 @@ func persistFileMetaRows(target graph.Store, repoPrefix string, rows []graph.Fil
const reparsePendingEnrichmentBatchSize = 256

type reparsePendingEnrichmentBatch struct {
// npmConsumerFiles are unchanged importers parked by this mutation.
npmConsumerFiles []string
byFile map[string]bool
deferResolverCatchup bool
deferredAffectedFiles map[string]struct{}
Expand Down
12 changes: 8 additions & 4 deletions internal/indexer/incremental_batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -1092,7 +1092,10 @@ func (idx *Indexer) commitStructuralIncrementalBatch(
markerBatch.recordDeferredPriorBindings(stagePriorBindings(idx.graph, stages, view))
}
idx.applyLap("prior_bindings")
carried := restubIncomingRefsFromView(idx.graph, stages, view)
carried, npmConsumers := restubIncomingRefsFromView(idx.graph, stages, view)
if markerBatch != nil {
markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...)
}
idx.applyLap("restub")
// The capability state of the files, read before the eviction below
// deletes it; the carried set says which incoming accesses_field rows
Expand Down Expand Up @@ -1349,9 +1352,10 @@ func restubIncomingRefsFromView(
g graph.Store,
stages []*incrementalBatchStage,
view incrementalPriorView,
) []*graph.Edge {
) ([]*graph.Edge, []string) {
evicted := structuralPriorIDs(stages)
reparsed := reparsedPaths(stages)
npmConsumers := restubTrackedNpmImportsFromView(g, evicted, view.inByNode)
var reindexes []graph.EdgeReindex
var carried []*graph.Edge
carriedSeen := make(map[*graph.Edge]struct{})
Expand Down Expand Up @@ -1505,7 +1509,7 @@ func restubIncomingRefsFromView(
if len(reindexes) > 0 {
g.ReindexEdges(reindexes)
}
return carried
return carried, npmConsumers
}

// dataflowSourceSite is a reference's site: the file that records it and its
Expand Down Expand Up @@ -2483,7 +2487,7 @@ func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInva
// carry therefore means a deletion stage stopped being conservative,
// which would destroy those edges with no compile error to show for
// it — so the assumption is checked, not just documented.
if carried := restubIncomingRefsFromView(idx.graph, stages, view); len(carried) > 0 {
if carried, _ := restubIncomingRefsFromView(idx.graph, stages, view); len(carried) > 0 {
idx.logger.Error("indexer: deletion restub carried in-edges with no re-state path",
zap.String("repo", idx.repoPrefix),
zap.Int("carried", len(carried)),
Expand Down
2 changes: 1 addition & 1 deletion internal/indexer/incremental_restub_gate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func runRestubStage(
stages := []*incrementalBatchStage{stage}
view := loadIncrementalPriorView(st, stages)

carried := restubIncomingRefsFromView(st, stages, view)
carried, _ := restubIncomingRefsFromView(st, stages, view)

evictFilesBatched(st, []string{restubDefPath})
st.AddBatch(fresh, append(append([]*graph.Edge{}, freshEdges...), carried...))
Expand Down
5 changes: 4 additions & 1 deletion internal/indexer/incremental_watcher_batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,9 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode(
batch.dropDeferredPriorBindings()
}
crossRepoFiles := appendUniqueSorted(nil, files...)
if batch != nil {
crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.npmConsumerFiles...)
}
if result != nil {
crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...)
}
Expand Down Expand Up @@ -585,7 +588,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode(
}
crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...)
}
if needed || len(semanticFiles) > 0 {
if needed || len(semanticFiles) > 0 || batch != nil && len(batch.npmConsumerFiles) > 0 {
if idx != nil {
idx.observeIncrementalCatchup("cross_repo", crossRepoFiles)
}
Expand Down
20 changes: 15 additions & 5 deletions internal/indexer/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ type IndexResult struct {
// incremental reconcile. A zero plan proves that no graph-wide derived
// pass is required; callers must not infer work merely from stale count.
DerivedInvalidation DerivedInvalidationPlan `json:"derived_invalidation,omitempty"`
// CrossRepoResolveFiles contains parked consumers outside this repository's
// derived frontier. Batch callers must include them in cross-repo resolution.
CrossRepoResolveFiles []string `json:"cross_repo_resolve_files,omitempty"`

// mutationErr carries a point-operation compatibility error through the
// resolver/semantic/derived tail. It is intentionally unexported and never
Expand Down Expand Up @@ -3409,6 +3412,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index
if sqlite, ok := diskTarget.(*store_sqlite.Store); ok && sqlite.SupportsShadowReplacementRetirement() {
var err error
shadowReplacementStarted = true
restubTrackedNpmRepoIncoming(diskTarget, idx.RepoPrefix())
evictedNodes, evictedEdges, err = sqlite.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix())
if err != nil {
retErr = fmt.Errorf("indexer: retire shadow replacement rows: %w", err)
Expand Down Expand Up @@ -5111,7 +5115,10 @@ func (idx *Indexer) indexFile(
// batch replacement by deleting its prior documents explicitly;
// canonical contracts follow the store's owner-retention decision.
idx.deleteSymbolFTS(oldFTSNodeIDs)
idx.restubIncomingRefs(graphPath)
npmConsumers := idx.restubIncomingRefs(graphPath)
if markerBatch != nil {
markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...)
}
idx.graph.EvictFile(graphPath)
}

Expand Down Expand Up @@ -5866,10 +5873,10 @@ func (idx *Indexer) reresolveFileScopedRaw(filePath string) error {
// agnostic: GetInEdges + ReindexEdges are the same Store primitives the
// resolver uses, so this behaves identically on the in-memory and disk
// stores.
func (idx *Indexer) restubIncomingRefs(graphPath string) {
func (idx *Indexer) restubIncomingRefs(graphPath string) []string {
nodes := idx.graph.GetFileNodes(graphPath)
if len(nodes) == 0 {
return
return nil
}
evicted := make(map[string]struct{}, len(nodes))
refIDs := make([]string, 0, len(nodes))
Expand All @@ -5878,17 +5885,18 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) {
continue
}
evicted[n.ID] = struct{}{}
if n.Name != "" && graph.IsReferenceableSymbol(n.Kind) {
if n.Kind == graph.KindFile || (n.Name != "" && graph.IsReferenceableSymbol(n.Kind)) {
refIDs = append(refIDs, n.ID)
}
}
if len(refIDs) == 0 {
return
return nil
}
// Fetch the whole changed file's incoming adjacency in one bounded store
// operation. On SQLite this is chunked under the parameter limit; more
// importantly, it avoids one transaction/query per definition in the file.
inEdges := idx.graph.GetInEdgesByNodeIDs(refIDs)
npmConsumers := restubTrackedNpmImportsFromView(idx.graph, evicted, inEdges)
var batch []graph.EdgeReindex
for _, n := range nodes {
if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) {
Expand Down Expand Up @@ -5919,6 +5927,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) {
if len(batch) > 0 {
idx.graph.ReindexEdges(batch)
}
return npmConsumers
}

// embeddingDimsOrDefault returns the embedder's reported vector width,
Expand Down Expand Up @@ -7310,6 +7319,7 @@ func (idx *Indexer) incrementalReindexPathsMode(
CountsCached: countsCached,
capabilityPrior: idx.takeCapabilityPrior(),
}
result.CrossRepoResolveFiles = appendUniqueSorted(nil, markerBatch.npmConsumerFiles...)
for _, path := range failedFiles {
result.Errors = append(result.Errors, IndexError{FilePath: path, Error: idx.fileIndexFailureError(path).Error()})
}
Expand Down
13 changes: 9 additions & 4 deletions internal/indexer/multi.go
Original file line number Diff line number Diff line change
Expand Up @@ -2580,8 +2580,7 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) {
oldIdx.Close()
}

// TODO: After re-indexing, run CrossRepoResolver.ResolveForRepo(repoPrefix)
// to update cross-repo edges. This will be implemented in Task 7.1.
mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(repoPrefix))

mi.ReconcileContractEdges()

Expand Down Expand Up @@ -3146,6 +3145,7 @@ func (mi *MultiIndexer) trackPreparedRepoSourceCtx(
mi.indexers[prefix] = idx
mi.mu.Unlock()
installed = true
mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix))

// Add to global config.
entry.Path = absPath
Expand Down Expand Up @@ -3438,8 +3438,13 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE
prefix: result.DerivedInvalidation,
})
}
} else if result.FullRetrack && !batchMode.deferGlobalPasses {
mi.ReconcileContractEdges()
} else if result.FullRetrack {
if !batchMode.deferResolve {
mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix))
}
if !batchMode.deferGlobalPasses {
mi.ReconcileContractEdges()
}
}

publishTiming.complete(nil)
Expand Down
Loading
Loading