diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 16b792e..42ff98c 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -509,7 +509,7 @@ func Run(configPath string, force bool, logger *slog.Logger) error { var wg sync.WaitGroup var progress atomic.Int64 - totalDirs := len(extractUniqueDirs(walkResult.Files)) + totalDirs := len(currentDirs) totalItems := len(walkResult.Files) + totalDirs // Create directory tracker — directories are analyzed automatically diff --git a/internal/store/qdrant.go b/internal/store/qdrant.go index 2ba799f..06084fa 100644 --- a/internal/store/qdrant.go +++ b/internal/store/qdrant.go @@ -39,6 +39,7 @@ func (q *QdrantStore) EnsureCollection() error { if err != nil { return fmt.Errorf("check collection: %w", err) } + io.Copy(io.Discard, resp.Body) resp.Body.Close() if resp.StatusCode == http.StatusOK { @@ -159,28 +160,22 @@ func (q *QdrantStore) GetAllFilePoints() ([]*Point, error) { q.logger.Debug("GetAllFilePoints") start := time.Now() - body := map[string]any{ - "filter": map[string]any{ - "must": []map[string]any{ - { - "key": "type", - "match": map[string]any{ - "value": "file", - }, + filter := map[string]any{ + "must": []map[string]any{ + { + "key": "type", + "match": map[string]any{ + "value": "file", }, }, }, - "limit": 1000, - "with_payload": true, - "with_vector": false, } - var result qdrantScrollResponse - if err := q.postJSON("/collections/"+q.collection+"/points/scroll", body, &result); err != nil { + points, err := q.scrollAll(filter) + if err != nil { return nil, err } - points := parsePoints(result.Result.Points) q.logger.Debug("GetAllFilePoints completed", "count", len(points), "duration", time.Since(start), @@ -193,28 +188,22 @@ func (q *QdrantStore) GetAllDirPoints() ([]*Point, error) { q.logger.Debug("GetAllDirPoints") start := time.Now() - body := map[string]any{ - "filter": map[string]any{ - "must": []map[string]any{ - { - "key": "type", - "match": map[string]any{ - "value": "directory", - }, + filter := map[string]any{ + "must": []map[string]any{ + { + "key": "type", + "match": map[string]any{ + "value": "directory", }, }, }, - "limit": 1000, - "with_payload": true, - "with_vector": false, } - var result qdrantScrollResponse - if err := q.postJSON("/collections/"+q.collection+"/points/scroll", body, &result); err != nil { + points, err := q.scrollAll(filter) + if err != nil { return nil, err } - points := parsePoints(result.Result.Points) q.logger.Debug("GetAllDirPoints completed", "count", len(points), "duration", time.Since(start), @@ -307,6 +296,40 @@ func (q *QdrantStore) Search(vector []float32, limit int) ([]*SearchResult, erro return results, nil } +// scrollAll paginates through all points matching the given filter. +func (q *QdrantStore) scrollAll(filter map[string]any) ([]*Point, error) { + const scrollLimit = 1000 + + var allPoints []*Point + var offset *string + + for { + body := map[string]any{ + "filter": filter, + "limit": scrollLimit, + "with_payload": true, + "with_vector": false, + } + if offset != nil { + body["offset"] = *offset + } + + var result qdrantScrollResponse + if err := q.postJSON("/collections/"+q.collection+"/points/scroll", body, &result); err != nil { + return nil, err + } + + allPoints = append(allPoints, parsePoints(result.Result.Points)...) + + if result.Result.NextPageOffset == nil { + break + } + offset = result.Result.NextPageOffset + } + + return allPoints, nil +} + // --- HTTP helpers --- func (q *QdrantStore) put(path string, body any) error { @@ -428,7 +451,8 @@ func (q *QdrantStore) postExpectOK(path string, body any) error { type qdrantScrollResponse struct { Result struct { - Points []qdrantPoint `json:"points"` + Points []qdrantPoint `json:"points"` + NextPageOffset *string `json:"next_page_offset"` } `json:"result"` } diff --git a/internal/store/qdrant_test.go b/internal/store/qdrant_test.go index fec657e..db7c466 100644 --- a/internal/store/qdrant_test.go +++ b/internal/store/qdrant_test.go @@ -558,3 +558,61 @@ func TestInterfaceCompliance(t *testing.T) { // Compile-time check that QdrantStore implements Store var _ Store = (*QdrantStore)(nil) } + +func TestGetAllFilePoints_Pagination(t *testing.T) { + callCount := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req map[string]any + json.Unmarshal(body, &req) + + callCount++ + var resp string + if callCount == 1 { + // First page: has next_page_offset + if _, hasOffset := req["offset"]; hasOffset { + t.Error("first request should not have offset") + } + resp = `{ + "result": { + "points": [ + {"id": "uuid-1", "payload": {"file_path": "a.go", "type": "file", "summary": "file a"}}, + {"id": "uuid-2", "payload": {"file_path": "b.go", "type": "file", "summary": "file b"}} + ], + "next_page_offset": "uuid-2" + } + }` + } else { + // Second page: no next_page_offset + if req["offset"] != "uuid-2" { + t.Errorf("expected offset uuid-2, got %v", req["offset"]) + } + resp = `{ + "result": { + "points": [ + {"id": "uuid-3", "payload": {"file_path": "c.go", "type": "file", "summary": "file c"}} + ] + } + }` + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(resp)) + })) + defer srv.Close() + + s := NewQdrantStore(srv.URL, "vedcode_", "test", 3072, noopLogger) + points, err := s.GetAllFilePoints() + if err != nil { + t.Fatalf("GetAllFilePoints failed: %v", err) + } + + if callCount != 2 { + t.Errorf("expected 2 scroll requests, got %d", callCount) + } + if len(points) != 3 { + t.Fatalf("expected 3 points across pages, got %d", len(points)) + } + if points[2].FilePath != "c.go" { + t.Errorf("expected third point file_path c.go, got %s", points[2].FilePath) + } +}