reimplement collection

Makefile

@@ -5,7 +5,7 @@ FLAGS = -ldflags "\
 "
 
 test:
-	go test -cover ./...
+	go test ./...
 
 run:
 	STATICS=statics/www/ go run $(FLAGS) ./cmd/inceptiondb/...

api/apicollectionv1/0_build.go

@@ -19,8 +19,6 @@ func BuildV1Collection(v1 *box.R, s service.Servicer) *box.R {
 		WithActions(
 			box.Get(getCollection),
 			box.ActionPost(insert),
-			box.ActionPost(insertStream),     // todo: experimental!!
-			box.ActionPost(insertFullduplex), // todo: experimental!!
 			box.ActionPost(find),
 			box.ActionPost(remove),
 			box.ActionPost(patch),

api/apicollectionv1/0_traverse.go

@@ -3,14 +3,16 @@ package apicollectionv1
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/SierraSoftworks/connor"
+	"github.com/buger/jsonparser"
 
-	"github.com/fulldump/inceptiondb/collection"
+	"github.com/fulldump/inceptiondb/collectionv4"
 	"github.com/fulldump/inceptiondb/utils"
 )
 
-func traverse(requestBody []byte, col *collection.Collection, f func(row *collection.Row) bool) error {
+func traverse(requestBody []byte, col *collectionv4.Collection, f func(id int64, payload []byte) bool) error {
 
 	options := &struct {
 		Index  *string
@@ -28,18 +30,97 @@ func traverse(requestBody []byte, col *collection.Collection, f func(row *collec
 		return err
 	}
 
-	hasFilter := options.Filter != nil && len(options.Filter) > 0
+	hasFilter := len(options.Filter) > 0
+
+	// Add simple equality filter check
+	isSimple := true
+	if hasFilter {
+		for k, v := range options.Filter {
+			if strings.HasPrefix(k, "$") || strings.Contains(k, ".") {
+				isSimple = false
+				break
+			}
+			switch v.(type) {
+			case string, float64, bool, nil:
+				// supported
+			default:
+				isSimple = false
+				break
+			}
+		}
+	}
 
 	skip := options.Skip
 	limit := options.Limit
-	iterator := func(r *collection.Row) bool {
+	iterator := func(id int64, payload []byte) bool {
 		if limit == 0 {
 			return false
 		}
 
 		if hasFilter {
+			// Fast path for simple equality queries
+			if isSimple {
+				match := true
+				for k, expected := range options.Filter {
+					val, dataType, _, err := jsonparser.Get(payload, k)
+					if err != nil {
+						if expected != nil {
+							match = false
+							break
+						}
+						continue
+					}
+
+					switch exp := expected.(type) {
+					case string:
+						if dataType != jsonparser.String {
+							match = false
+							break
+						}
+						parsedStr, err := jsonparser.ParseString(val)
+						if err != nil || parsedStr != exp {
+							match = false
+						}
+					case float64:
+						if dataType != jsonparser.Number {
+							match = false
+							break
+						}
+						parsedNum, err := jsonparser.ParseFloat(val)
+						if err != nil || parsedNum != exp {
+							match = false
+						}
+					case bool:
+						if dataType != jsonparser.Boolean {
+							match = false
+							break
+						}
+						parsedBool, err := jsonparser.ParseBoolean(val)
+						if err != nil || parsedBool != exp {
+							match = false
+						}
+					case nil:
+						if dataType != jsonparser.Null {
+							match = false
+						}
+					default:
+						match = false
+					}
+
+					if !match {
+						break
+					}
+				}
+
+				if !match {
+					return true
+				}
+				goto evaluate
+			}
+
+			// Slow path via json.Unmarshal and connor
 			rowData := map[string]interface{}{}
-			json.Unmarshal(r.Payload, &rowData) // todo: handle error here?
+			json.Unmarshal(payload, &rowData) // todo: handle error here?
 
 			match, err := connor.Match(options.Filter, rowData)
 			if err != nil {
@@ -52,12 +133,13 @@ func traverse(requestBody []byte, col *collection.Collection, f func(row *collec
 			}
 		}
 
+	evaluate:
 		if skip > 0 {
 			skip--
 			return true
 		}
 		limit--
-		return f(r)
+		return f(id, payload)
 	}
 
 	// Fullscan
@@ -66,24 +148,20 @@ func traverse(requestBody []byte, col *collection.Collection, f func(row *collec
 		return nil
 	}
 
-	index, exists := col.Indexes[*options.Index]
+	indexes := col.ListIndexes()
+	index, exists := indexes[*options.Index]
 	if !exists {
-		return fmt.Errorf("index '%s' not found, available indexes %v", *options.Index, utils.GetKeys(col.Indexes))
+		return fmt.Errorf("index '%s' not found, available indexes %v", *options.Index, utils.GetKeys(indexes))
 	}
 
-	index.Traverse(requestBody, iterator)
-
-	return nil
+	_ = index
+	return col.TraverseIndex(*options.Index, requestBody, iterator)
 }
 
-func traverseFullscan(col *collection.Collection, f func(row *collection.Row) bool) error {
-
-	for _, row := range col.Rows {
-		next := f(row)
-		if !next {
-			break
-		}
-	}
+func traverseFullscan(col *collectionv4.Collection, f func(id int64, payload []byte) bool) error {
+	col.TraverseRecords(func(id int64, payload []byte) bool {
+		return f(id, payload)
+	})
 
 	return nil
 }

api/apicollectionv1/createCollection.go

@@ -40,7 +40,7 @@ func createCollection(ctx context.Context, w http.ResponseWriter, input *createC
 	w.WriteHeader(http.StatusCreated)
 	return &CollectionResponse{
 		Name:     input.Name,
-		Total:    len(collection.Rows),
-		Defaults: collection.Defaults,
+		Total:    int(collection.Count()),
+		Defaults: collection.Defaults(),
 	}, nil
 }

api/apicollectionv1/createIndex.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/fulldump/box"
 
-	"github.com/fulldump/inceptiondb/collection"
+	"github.com/fulldump/inceptiondb/collectionv4"
 	"github.com/fulldump/inceptiondb/service"
 )
 
@@ -59,11 +59,15 @@ func createIndex(ctx context.Context, r *http.Request) (*listIndexesItem, error)
 
 	switch input.Type {
 	case "map":
-		options = &collection.IndexMapOptions{}
+		options = &collectionv4.IndexMapOptions{}
 	case "btree":
-		options = &collection.IndexBTreeOptions{}
+		options = &collectionv4.IndexBTreeOptions{}
+	case "fts":
+		options = &collectionv4.IndexFTSOptions{}
+	case "pk":
+		options = &collectionv4.IndexPKOptions{}
 	default:
-		return nil, fmt.Errorf("unexpected type '%s' instead of [map|btree]", input.Type)
+		return nil, fmt.Errorf("unexpected type '%s' instead of [map|btree|fts|pk]", input.Type)
 	}
 
 	err = json.Unmarshal(requestBody, &options)

api/apicollectionv1/dropIndex.go

@@ -2,6 +2,7 @@ package apicollectionv1
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 
 	"github.com/fulldump/box"
@@ -32,9 +33,15 @@ func dropIndex(ctx context.Context, w http.ResponseWriter, input *dropIndexReque
 		return err // todo: handle/wrap this properly
 	}
 
+	indexes := col.ListIndexes()
+	_, exists := indexes[input.Name]
+	if !exists {
+		w.WriteHeader(http.StatusBadRequest)
+		return fmt.Errorf("index '%s' not found", input.Name)
+	}
+
 	err = col.DropIndex(input.Name)
 	if err != nil {
-		w.WriteHeader(http.StatusBadRequest)
 		return err
 	}
 

api/apicollectionv1/find.go

@@ -7,8 +7,6 @@ import (
 	"net/http"
 
 	"github.com/fulldump/box"
-
-	"github.com/fulldump/inceptiondb/collection"
 )
 
 func find(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
@@ -33,8 +31,9 @@ func find(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
 		return err // todo: handle/wrap this properly
 	}
 
-	return traverse(requestBody, col, func(row *collection.Row) bool {
-		w.Write(row.Payload)
+	return traverse(requestBody, col, func(id int64, payload []byte) bool {
+		_ = id
+		w.Write(payload)
 		w.Write([]byte("\n"))
 		return true
 	})

api/apicollectionv1/getCollection.go

@@ -22,10 +22,11 @@ func getCollection(ctx context.Context) (*CollectionResponse, error) {
 		return nil, err
 	}
 
+	indexes := collection.ListIndexes()
 	return &CollectionResponse{
 		Name:     collectionName,
-		Total:    len(collection.Rows),
-		Indexes:  len(collection.Indexes),
-		Defaults: collection.Defaults,
+		Total:    int(collection.Count()),
+		Indexes:  len(indexes),
+		Defaults: collection.Defaults(),
 	}, nil
 }