feat: Allow new fields to be added locally to schema (#1139)

* Correct P2P error message

Spotted and quickly corrected.  Has nothing to do with this PR though, but is small enough to include.

* Remove field kind decimal

It is not supported and does not work.  Leaving it here will just confuse users, especially as they start to use the schema update system.

* Remove field kind bytes

It is not supported and does not work.  Leaving it here will just confuse users, especially as they start to use the schema update system.

* Remove field embedded object kinds

They are not supported and does not work.  Leaving it here will just confuse users, especially as they start to use the schema update system.

* Add documentation to FieldDesc.Kind

I'm not sure any more than this is really needed, especially given that we will largely be hiding this from users shortly after merge.

* Assert for collection name uniquenss within SDL

Previously this would partially succede - the gql types would be updated, but the saving of the collection descriptions would fail (as there is another uniqueness check there), leaving the database in an invalid state until restart.

* Correctly query all existing collections

Was incorrectly quering the version history, returning a collection per version, instead of just the current version.  Went unnoticed as previously each collection could only have a single version.

* Make collection persistance transactional

Collection stuff needs to be protected by transactions.

Partial success of either a create or a mutate cannot be permitted and the use of transactions protects against this.

The transactions are also needed to protect against the use of stale data, by including the collection in the transaction used for P2P and query/planner stuff we should ensure that stuff is done against a single, complete version of the collection, and that it is not possible for the collection to mutate whilst something is using the collection. At the moment such concurrent use should now result in a transaction conflict error - this is not ideal and the logic here should probably grow to permit the queuing of such clashes (e.g. through use of a mutex) instead of making the users retry until it succedes - such a change will very likely need to be done within the scope declared by the transaction anyway, so I see no wasted code/time by the changes in this commit here - it is a start, and prevents odd/damaging stuff from happening.

DB init was also covered almost as a side-affect, as the sequence needed protecting, and TBH it is probably a very good thing to protect against mutating the database state in the case of a failed init.

* Make SetSchema (GQL) transactional

Renames and changes AddSchema to SetSchema.  SetSchema is now transactional, GQL type changes will now only be 'commited' on transaction commit, whilst allowing SetSchema to be called safely at any point during the lifetime of the transaction - allowing for the schema to be validated against GQL constraints before any changes have been persisted else where, and allowing those other changes to be executed/validated before any changes have been made to the GQL types.

* Add support for adding fields to schema

Please note that the client interfaces will be reworked in the near future so that the transaction related items clutter the primary interface less, and are more consistent. For now I have just followed the existing `Txn` suffix naming.

client/db.go

@@ -22,10 +22,44 @@ import (
 type DB interface {
 	AddSchema(context.Context, string) error
 
+	// PatchSchema takes the given JSON patch string and applies it to the set of CollectionDescriptions
+	// present in the database.
+	//
+	// It will also update the GQL types used by the query system. It will error and not apply any of the
+	// requested, valid updates should the net result of the patch result in an invalid state.  The
+	// individual operations defined in the patch do not need to result in a valid state, only the net result
+	// of the full patch.
+	//
+	// The collections (including the schema version ID) will only be updated if any changes have actually
+	// been made, if the net result of the patch matches the current persisted description then no changes
+	// will be applied.
+	PatchSchema(context.Context, string) error
+
 	CreateCollection(context.Context, CollectionDescription) (Collection, error)
+	CreateCollectionTxn(context.Context, datastore.Txn, CollectionDescription) (Collection, error)
+
+	// UpdateCollectionTxn updates the persisted collection description matching the name of the given
+	// description, to the values in the given description.
+	//
+	// It will validate the given description using [ValidateUpdateCollectionTxn] before updating it.
+	//
+	// The collection (including the schema version ID) will only be updated if any changes have actually
+	// been made, if the given description matches the current persisted description then no changes will be
+	// applied.
+	UpdateCollectionTxn(context.Context, datastore.Txn, CollectionDescription) (Collection, error)
+
+	// ValidateUpdateCollectionTxn validates that the given collection description is a valid update.
+	//
+	// Will return true if the given desctiption differs from the current persisted state of the
+	// collection. Will return an error if it fails validation.
+	ValidateUpdateCollectionTxn(context.Context, datastore.Txn, CollectionDescription) (bool, error)
+
 	GetCollectionByName(context.Context, string) (Collection, error)
+	GetCollectionByNameTxn(context.Context, datastore.Txn, string) (Collection, error)
 	GetCollectionBySchemaID(context.Context, string) (Collection, error)
-	GetAllCollections(ctx context.Context) ([]Collection, error)
+	GetCollectionBySchemaIDTxn(context.Context, datastore.Txn, string) (Collection, error)
+	GetAllCollections(context.Context) ([]Collection, error)
+	GetAllCollectionsTxn(context.Context, datastore.Txn) ([]Collection, error)
 
 	Root() datastore.RootStore
 	Blockstore() blockstore.Blockstore

client/descriptions.go

@@ -136,18 +136,14 @@ const (
 	FieldKind_INT_ARRAY    FieldKind = 5
 	FieldKind_FLOAT        FieldKind = 6
 	FieldKind_FLOAT_ARRAY  FieldKind = 7
-	FieldKind_DECIMAL      FieldKind = 8
+	_                      FieldKind = 8 // safe to repurpose (was never used)
 	_                      FieldKind = 9 // safe to repurpose (previoulsy old field)
 	FieldKind_DATETIME     FieldKind = 10
 	FieldKind_STRING       FieldKind = 11
 	FieldKind_STRING_ARRAY FieldKind = 12
-	FieldKind_BYTES        FieldKind = 13
-
-	// Embedded object within the type
-	FieldKind_OBJECT FieldKind = 14
-
-	// Array of embedded objects
-	FieldKind_OBJECT_ARRAY FieldKind = 15
+	_                      FieldKind = 13 // safe to repurpose (was never used)
+	_                      FieldKind = 14 // safe to repurpose (was never used)
+	_                      FieldKind = 15 // safe to repurpose (was never used)
 
 	// Embedded object, but accessed via foreign keys
 	FieldKind_FOREIGN_OBJECT FieldKind = 16
@@ -185,8 +181,12 @@ func (f FieldID) String() string {
 
 // FieldDescription describes a field on a Schema and its associated metadata.
 type FieldDescription struct {
-	Name         string
-	ID           FieldID
+	Name string
+	ID   FieldID
+
+	// The data type that this field holds.
+	//
+	// Must contain a valid value.
 	Kind         FieldKind
 	Schema       string // If the field is an OBJECT type, then it has a target schema
 	RelationName string // The name of the relation index if the field is of type FOREIGN_OBJECT
@@ -201,7 +201,7 @@ type FieldDescription struct {
 
 // IsObject returns true if this field is an object type.
 func (f FieldDescription) IsObject() bool {
-	return (f.Kind == FieldKind_OBJECT) || (f.Kind == FieldKind_FOREIGN_OBJECT) ||
+	return (f.Kind == FieldKind_FOREIGN_OBJECT) ||
 		(f.Kind == FieldKind_FOREIGN_OBJECT_ARRAY)
 }
 

client/p2p.go

@@ -10,7 +10,11 @@
 
 package client
 
-import "context"
+import (
+	"context"
+
+	"github.com/sourcenetwork/defradb/datastore"
+)
 
 type P2P interface {
 	// SetReplicator adds a replicator to the persisted list or adds
@@ -27,10 +31,20 @@ type P2P interface {
 	// subscribes to to the the persisted list. It will error if the provided
 	// collection ID is invalid.
 	AddP2PCollection(ctx context.Context, collectionID string) error
+	// AddP2PCollectionTxn adds the given collection ID that the P2P system
+	// subscribes to to the the persisted list. It will error if the provided
+	// collection ID is invalid.
+	AddP2PCollectionTxn(ctx context.Context, txn datastore.Txn, collectionID string) error
+
 	// RemoveP2PCollection removes the given collection ID that the P2P system
 	// subscribes to from the the persisted list. It will error if the provided
 	// collection ID is invalid.
 	RemoveP2PCollection(ctx context.Context, collectionID string) error
+	// RemoveP2PCollectionTxn removes the given collection ID that the P2P system
+	// subscribes to from the the persisted list. It will error if the provided
+	// collection ID is invalid.
+	RemoveP2PCollectionTxn(ctx context.Context, txn datastore.Txn, collectionID string) error
+
 	// GetAllP2PCollections returns the list of persisted collection IDs that
 	// the P2P system subscribes to.
 	GetAllP2PCollections(ctx context.Context) ([]string, error)

core/parser.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/sourcenetwork/defradb/client"
 	"github.com/sourcenetwork/defradb/client/request"
+	"github.com/sourcenetwork/defradb/datastore"
 )
 
 // SchemaDefinition represents a schema definition.
@@ -49,5 +50,5 @@ type Parser interface {
 	ParseSDL(ctx context.Context, schemaString string) ([]client.CollectionDescription, error)
 
 	// Adds the given schema to this parser's model.
-	AddSchema(ctx context.Context, collections []client.CollectionDescription) error
+	SetSchema(ctx context.Context, txn datastore.Txn, collections []client.CollectionDescription) error
 }