feat: Add support for views with Lens transforms (sourcenetwork#2311)

## Relevant issue(s)

Resolves sourcenetwork#2070

## Description

Adds support for views with Lens transforms.

.gitignore

@@ -17,6 +17,12 @@ tests/lenses/rust_wasm32_remove/pkg
 tests/lenses/rust_wasm32_copy/Cargo.lock
 tests/lenses/rust_wasm32_copy/target
 tests/lenses/rust_wasm32_copy/pkg
+tests/lenses/rust_wasm32_prepend/Cargo.lock
+tests/lenses/rust_wasm32_prepend/target
+tests/lenses/rust_wasm32_prepend/pkg
+tests/lenses/rust_wasm32_filter/Cargo.lock
+tests/lenses/rust_wasm32_filter/target
+tests/lenses/rust_wasm32_filter/pkg
 
 # Ignore OS X metadata files.
 .history

cli/view_add.go

@@ -10,34 +10,72 @@
 
 package cli
 
-import "github.com/spf13/cobra"
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/sourcenetwork/immutable"
+	"github.com/spf13/cobra"
+)
 
 func MakeViewAddCommand() *cobra.Command {
+	var lensFile string
 	var cmd = &cobra.Command{
-		Use:   "add [query] [sdl]",
+		Use:   "add [query] [sdl] [transform]",
 		Short: "Add new view",
 		Long: `Add new database view.
 
 Example: add from an argument string:
-  defradb client view add 'Foo { name, ...}' 'type Foo { ... }'
+  defradb client view add 'Foo { name, ...}' 'type Foo { ... }' '{"lenses": [...'
 
 Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`,
+		Args: cobra.RangeArgs(2, 4),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			store := mustGetStoreContext(cmd)
 
-			if len(args) != 2 {
-				return ErrViewAddMissingArgs
-			}
-
 			query := args[0]
 			sdl := args[1]
 
-			defs, err := store.AddView(cmd.Context(), query, sdl)
+			var lensCfgJson string
+			switch {
+			case lensFile != "":
+				data, err := os.ReadFile(lensFile)
+				if err != nil {
+					return err
+				}
+				lensCfgJson = string(data)
+			case len(args) == 3 && args[2] == "-":
+				data, err := io.ReadAll(cmd.InOrStdin())
+				if err != nil {
+					return err
+				}
+				lensCfgJson = string(data)
+			case len(args) == 3:
+				lensCfgJson = args[2]
+			}
+
+			var transform immutable.Option[model.Lens]
+			if lensCfgJson != "" {
+				decoder := json.NewDecoder(strings.NewReader(lensCfgJson))
+				decoder.DisallowUnknownFields()
+
+				var lensCfg model.Lens
+				if err := decoder.Decode(&lensCfg); err != nil {
+					return NewErrInvalidLensConfig(err)
+				}
+				transform = immutable.Some(lensCfg)
+			}
+
+			defs, err := store.AddView(cmd.Context(), query, sdl, transform)
 			if err != nil {
 				return err
 			}
 			return writeJSON(cmd, defs)
 		},
 	}
+	cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file")
 	return cmd
 }

client/db.go

@@ -155,7 +155,16 @@ type Store interface {
 	//
 	// It will return the collection definitions of the types defined in the SDL if successful, otherwise an error
 	// will be returned.  This function does not execute the given query.
-	AddView(ctx context.Context, gqlQuery string, sdl string) ([]CollectionDefinition, error)
+	//
+	// Optionally, a lens transform configuration may also be provided - it will execute after the query has run.
+	// The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out
+	// those passed in from the underlying query.
+	AddView(
+		ctx context.Context,
+		gqlQuery string,
+		sdl string,
+		transform immutable.Option[model.Lens],
+	) ([]CollectionDefinition, error)
 
 	// SetMigration sets the migration for all collections using the given source-destination schema version IDs.
 	//

client/descriptions.go

@@ -135,6 +135,13 @@ func sourcesOfType[ResultType any](col CollectionDescription) []ResultType {
 type QuerySource struct {
 	// Query contains the base query of this data source.
 	Query request.Select
+
+	// Transform is a optional Lens configuration.  If specified, data drawn from the [Query] will have the
+	// transform applied before being returned.
+	//
+	// The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out
+	// those passed in from the underlying query.
+	Transform immutable.Option[model.Lens]
 }
 
 // CollectionSource represents a collection data source from another collection instance.

db/txn_db.go

@@ -391,14 +391,19 @@ func (db *explicitTxnDB) SetMigration(ctx context.Context, cfg client.LensConfig
 	return db.setMigration(ctx, db.txn, cfg)
 }
 
-func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
+func (db *implicitTxnDB) AddView(
+	ctx context.Context,
+	query string,
+	sdl string,
+	transform immutable.Option[model.Lens],
+) ([]client.CollectionDefinition, error) {
 	txn, err := db.NewTxn(ctx, false)
 	if err != nil {
 		return nil, err
 	}
 	defer txn.Discard(ctx)
 
-	defs, err := db.addView(ctx, txn, query, sdl)
+	defs, err := db.addView(ctx, txn, query, sdl, transform)
 	if err != nil {
 		return nil, err
 	}
@@ -411,8 +416,13 @@ func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string)
 	return defs, nil
 }
 
-func (db *explicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
-	return db.addView(ctx, db.txn, query, sdl)
+func (db *explicitTxnDB) AddView(
+	ctx context.Context,
+	query string,
+	sdl string,
+	transform immutable.Option[model.Lens],
+) ([]client.CollectionDefinition, error) {
+	return db.addView(ctx, db.txn, query, sdl, transform)
 }
 
 // BasicImport imports a json dataset.

db/view.go

@@ -15,6 +15,9 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/sourcenetwork/immutable"
+
 	"github.com/sourcenetwork/defradb/client"
 	"github.com/sourcenetwork/defradb/client/request"
 	"github.com/sourcenetwork/defradb/datastore"
@@ -26,6 +29,7 @@ func (db *db) addView(
 	txn datastore.Txn,
 	inputQuery string,
 	sdl string,
+	transform immutable.Option[model.Lens],
 ) ([]client.CollectionDefinition, error) {
 	// Wrap the given query as part of the GQL query object - this simplifies the syntax for users
 	// and ensures that we can't be given mutations.  In the future this line should disappear along
@@ -57,7 +61,10 @@ func (db *db) addView(
 	}
 
 	for i := range newDefinitions {
-		source := client.QuerySource{Query: *baseQuery}
+		source := client.QuerySource{
+			Query:     *baseQuery,
+			Transform: transform,
+		}
 		newDefinitions[i].Description.Sources = append(newDefinitions[i].Description.Sources, &source)
 	}
 
@@ -78,6 +85,15 @@ func (db *db) addView(
 				return nil, err
 			}
 			returnDescriptions[i] = col.Definition()
+
+			for _, source := range col.Description().QuerySources() {
+				if source.Transform.HasValue() {
+					err = db.LensRegistry().SetMigration(ctx, col.ID(), source.Transform.Value())
+					if err != nil {
+						return nil, err
+					}
+				}
+			}
 		}
 	}
 

http/client.go

@@ -173,14 +173,20 @@ func (c *Client) SetActiveSchemaVersion(ctx context.Context, schemaVersionID str
 }
 
 type addViewRequest struct {
-	Query string
-	SDL   string
+	Query     string
+	SDL       string
+	Transform immutable.Option[model.Lens]
 }
 
-func (c *Client) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
+func (c *Client) AddView(
+	ctx context.Context,
+	query string,
+	sdl string,
+	transform immutable.Option[model.Lens],
+) ([]client.CollectionDefinition, error) {
 	methodURL := c.http.baseURL.JoinPath("view")
 
-	body, err := json.Marshal(addViewRequest{query, sdl})
+	body, err := json.Marshal(addViewRequest{query, sdl, transform})
 	if err != nil {
 		return nil, err
 	}

http/handler_store.go

@@ -117,7 +117,7 @@ func (s *storeHandler) AddView(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	defs, err := store.AddView(req.Context(), message.Query, message.SDL)
+	defs, err := store.AddView(req.Context(), message.Query, message.SDL, message.Transform)
 	if err != nil {
 		responseJSON(rw, http.StatusBadRequest, errorResponse{err})
 		return

planner/datasource.go

@@ -34,7 +34,7 @@ func (p *Planner) getCollectionScanPlan(mapperSelect *mapper.Select) (planSource
 	var plan planNode
 	if len(col.Description().QuerySources()) > 0 {
 		var err error
-		plan, err = p.View(mapperSelect, col.Description())
+		plan, err = p.View(mapperSelect, col)
 		if err != nil {
 			return planSource{}, err
 		}