fix: Move field id off of schema (#2336)

## Relevant issue(s)

Resolves #2334

## Description

Moves field id off of schema and onto collection.

It is now a local property and does not need to be the same across
multiple nodes. The new `Fields` property on `CollectionDescription`
will be where other local field stuff is move too from the schema
(relation_name, descriptions, secondary relation fields).

This change is a prerequisite of allowing fields to be deleted and
schema branching (see
#2333 for more info on
that if currious)

client/collection.go

@@ -18,14 +18,6 @@ import (
 	"github.com/sourcenetwork/defradb/datastore"
 )
 
-// CollectionDefinition contains the metadata defining what a Collection is.
-type CollectionDefinition struct {
-	// Description returns the CollectionDescription of this Collection.
-	Description CollectionDescription `json:"description"`
-	// Schema returns the SchemaDescription used to define this Collection.
-	Schema SchemaDescription `json:"schema"`
-}
-
 // Collection represents a defradb collection.
 //
 // A Collection is mostly analogous to a SQL table, however a collection is specific to its

client/definitions.go

@@ -0,0 +1,115 @@
+// Copyright 2024 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package client
+
+// CollectionDefinition contains the metadata defining what a Collection is.
+//
+// The definition types ([CollectionDefinition], [FieldDefinition]) are read-only types returned
+// from various functions as a convienient means to access the computated convergence of schema
+// and collection descriptions.
+type CollectionDefinition struct {
+	// Description returns the CollectionDescription of this Collection.
+	Description CollectionDescription `json:"description"`
+	// Schema returns the SchemaDescription used to define this Collection.
+	Schema SchemaDescription `json:"schema"`
+}
+
+// GetFieldByName returns the field for the given field name. If such a field is found it
+// will return it and true, if it is not found it will return false.
+func (def CollectionDefinition) GetFieldByName(fieldName string) (FieldDefinition, bool) {
+	collectionField, ok := def.Description.GetFieldByName(fieldName)
+	if ok {
+		schemaField, ok := def.Schema.GetFieldByName(fieldName)
+		if ok {
+			return NewFieldDefinition(
+				collectionField,
+				schemaField,
+			), true
+		}
+	}
+	return FieldDefinition{}, false
+}
+
+// GetFields returns the combined local and global field elements on this [CollectionDefinition]
+// as a single set.
+func (def CollectionDefinition) GetFields() []FieldDefinition {
+	fields := []FieldDefinition{}
+	for _, localField := range def.Description.Fields {
+		globalField, ok := def.Schema.GetFieldByName(localField.Name)
+		if ok {
+			fields = append(
+				fields,
+				NewFieldDefinition(localField, globalField),
+			)
+		}
+	}
+	return fields
+}
+
+// FieldDefinition describes the combined local and global set of properties that constitutes
+// a field on a collection.
+//
+// It draws it's information from the [CollectionFieldDescription] on the [CollectionDescription],
+// and the [SchemaFieldDescription] on the [SchemaDescription].
+//
+// It is to [CollectionFieldDescription] and [SchemaFieldDescription] what [CollectionDefinition]
+// is to [CollectionDescription] and [SchemaDescription].
+//
+// The definition types ([CollectionDefinition], [FieldDefinition]) are read-only types returned
+// from various functions as a convienient means to access the computated convergence of schema
+// and collection descriptions.
+type FieldDefinition struct {
+	// Name contains the name of this field.
+	Name string
+
+	// ID contains the local, internal ID of this field.
+	ID FieldID
+
+	// The data type that this field holds.
+	//
+	// Must contain a valid value. It is currently immutable.
+	Kind FieldKind
+
+	// Schema contains the schema name of the type this field contains if this field is
+	// a relation field.  Otherwise this will be empty.
+	Schema string
+
+	// RelationName the name of the relationship that this field represents if this field is
+	// a relation field.  Otherwise this will be empty.
+	RelationName string
+
+	// The CRDT Type of this field. If no type has been provided it will default to [LWW_REGISTER].
+	//
+	// It is currently immutable.
+	Typ CType
+
+	// If true, this is the primary half of a relation, otherwise is false.
+	IsPrimaryRelation bool
+}
+
+// NewFieldDefinition returns a new [FieldDefinition], combining the given local and global elements
+// into a single object.
+func NewFieldDefinition(local CollectionFieldDescription, global SchemaFieldDescription) FieldDefinition {
+	return FieldDefinition{
+		Name:              global.Name,
+		ID:                local.ID,
+		Kind:              global.Kind,
+		Schema:            global.Schema,
+		RelationName:      global.RelationName,
+		Typ:               global.Typ,
+		IsPrimaryRelation: global.IsPrimaryRelation,
+	}
+}
+
+// IsRelation returns true if this field is a relation.
+func (f FieldDefinition) IsRelation() bool {
+	return f.RelationName != ""
+}

client/descriptions.go

@@ -60,6 +60,9 @@ type CollectionDescription struct {
 	// - [CollectionSource]
 	Sources []any
 
+	// Fields contains the fields within this Collection.
+	Fields []CollectionFieldDescription
+
 	// Indexes contains the secondary indexes that this Collection has.
 	Indexes []IndexDescription
 }
@@ -69,26 +72,26 @@ func (col CollectionDescription) IDString() string {
 	return fmt.Sprint(col.ID)
 }
 
-// GetFieldByID searches for a field with the given ID. If such a field is found it
+// GetFieldByName returns the field for the given field name. If such a field is found it
 // will return it and true, if it is not found it will return false.
-func (col CollectionDescription) GetFieldByID(id FieldID, schema *SchemaDescription) (FieldDescription, bool) {
-	for _, field := range schema.Fields {
-		if field.ID == id {
+func (col CollectionDescription) GetFieldByName(fieldName string) (CollectionFieldDescription, bool) {
+	for _, field := range col.Fields {
+		if field.Name == fieldName {
 			return field, true
 		}
 	}
-	return FieldDescription{}, false
+	return CollectionFieldDescription{}, false
 }
 
 // GetFieldByName returns the field for the given field name. If such a field is found it
 // will return it and true, if it is not found it will return false.
-func (col CollectionDescription) GetFieldByName(fieldName string, schema *SchemaDescription) (FieldDescription, bool) {
-	for _, field := range schema.Fields {
+func (s SchemaDescription) GetFieldByName(fieldName string) (SchemaFieldDescription, bool) {
+	for _, field := range s.Fields {
 		if field.Name == fieldName {
 			return field, true
 		}
 	}
-	return FieldDescription{}, false
+	return SchemaFieldDescription{}, false
 }
 
 // GetFieldByRelation returns the field that supports the relation of the given name.
@@ -97,15 +100,15 @@ func (col CollectionDescription) GetFieldByRelation(
 	otherCollectionName string,
 	otherFieldName string,
 	schema *SchemaDescription,
-) (FieldDescription, bool) {
+) (SchemaFieldDescription, bool) {
 	for _, field := range schema.Fields {
 		if field.RelationName == relationName &&
 			!(col.Name.Value() == otherCollectionName && otherFieldName == field.Name) &&
 			field.Kind != FieldKind_DocID {
 			return field, true
 		}
 	}
-	return FieldDescription{}, false
+	return SchemaFieldDescription{}, false
 }
 
 // QuerySources returns all the Sources of type [QuerySource]
@@ -190,17 +193,7 @@ type SchemaDescription struct {
 	// Fields contains the fields within this Schema.
 	//
 	// Currently new fields may be added after initial declaration, but they cannot be removed.
-	Fields []FieldDescription
-}
-
-// GetField returns the field of the given name.
-func (sd SchemaDescription) GetField(name string) (FieldDescription, bool) {
-	for _, field := range sd.Fields {
-		if field.Name == name {
-			return field, true
-		}
-	}
-	return FieldDescription{}, false
+	Fields []SchemaFieldDescription
 }
 
 // FieldKind describes the type of a field.
@@ -245,6 +238,31 @@ func (f FieldKind) String() string {
 	}
 }
 
+// IsObject returns true if this FieldKind is an object type.
+func (f FieldKind) IsObject() bool {
+	return f == FieldKind_FOREIGN_OBJECT ||
+		f == FieldKind_FOREIGN_OBJECT_ARRAY
+}
+
+// IsObjectArray returns true if this FieldKind is an object array type.
+func (f FieldKind) IsObjectArray() bool {
+	return f == FieldKind_FOREIGN_OBJECT_ARRAY
+}
+
+// IsArray returns true if this FieldKind is an array type which includes inline arrays as well
+// as relation arrays.
+func (f FieldKind) IsArray() bool {
+	return f == FieldKind_BOOL_ARRAY ||
+		f == FieldKind_INT_ARRAY ||
+		f == FieldKind_FLOAT_ARRAY ||
+		f == FieldKind_STRING_ARRAY ||
+		f == FieldKind_FOREIGN_OBJECT_ARRAY ||
+		f == FieldKind_NILLABLE_BOOL_ARRAY ||
+		f == FieldKind_NILLABLE_INT_ARRAY ||
+		f == FieldKind_NILLABLE_FLOAT_ARRAY ||
+		f == FieldKind_NILLABLE_STRING_ARRAY
+}
+
 // Note: These values are serialized and persisted in the database, avoid modifying existing values.
 const (
 	FieldKind_None              FieldKind = 0
@@ -312,21 +330,13 @@ func (f FieldID) String() string {
 	return fmt.Sprint(uint32(f))
 }
 
-// FieldDescription describes a field on a Schema and its associated metadata.
-type FieldDescription struct {
+// SchemaFieldDescription describes a field on a Schema and its associated metadata.
+type SchemaFieldDescription struct {
 	// Name contains the name of this field.
 	//
 	// It is currently immutable.
 	Name string
 
-	// ID contains the internal ID of this field.
-	//
-	// Whilst this ID will typically match the field's index within the Schema's Fields
-	// slice, there is no guarantee that they will be the same.
-	//
-	// It is immutable.
-	ID FieldID
-
 	// The data type that this field holds.
 	//
 	// Must contain a valid value. It is currently immutable.
@@ -345,39 +355,24 @@ type FieldDescription struct {
 	// It is currently immutable.
 	Typ CType
 
+	// If true, this is the primary half of a relation, otherwise is false.
 	IsPrimaryRelation bool
 }
 
-// IsObject returns true if this field is an object type.
-func (f FieldDescription) IsObject() bool {
-	return (f.Kind == FieldKind_FOREIGN_OBJECT) ||
-		(f.Kind == FieldKind_FOREIGN_OBJECT_ARRAY)
-}
+// CollectionFieldDescription describes the local components of a field on a collection.
+type CollectionFieldDescription struct {
+	// Name contains the name of the [SchemaFieldDescription] that this field uses.
+	Name string
 
-// IsObjectArray returns true if this field is an object array type.
-func (f FieldDescription) IsObjectArray() bool {
-	return (f.Kind == FieldKind_FOREIGN_OBJECT_ARRAY)
+	// ID contains the local, internal ID of this field.
+	ID FieldID
 }
 
 // IsRelation returns true if this field is a relation.
-func (f FieldDescription) IsRelation() bool {
+func (f SchemaFieldDescription) IsRelation() bool {
 	return f.RelationName != ""
 }
 
-// IsArray returns true if this field is an array type which includes inline arrays as well
-// as relation arrays.
-func (f FieldDescription) IsArray() bool {
-	return f.Kind == FieldKind_BOOL_ARRAY ||
-		f.Kind == FieldKind_INT_ARRAY ||
-		f.Kind == FieldKind_FLOAT_ARRAY ||
-		f.Kind == FieldKind_STRING_ARRAY ||
-		f.Kind == FieldKind_FOREIGN_OBJECT_ARRAY ||
-		f.Kind == FieldKind_NILLABLE_BOOL_ARRAY ||
-		f.Kind == FieldKind_NILLABLE_INT_ARRAY ||
-		f.Kind == FieldKind_NILLABLE_FLOAT_ARRAY ||
-		f.Kind == FieldKind_NILLABLE_STRING_ARRAY
-}
-
 // IsSet returns true if the target relation type is set.
 func (m RelationType) IsSet(target RelationType) bool {
 	return m&target > 0
@@ -392,6 +387,7 @@ type collectionDescription struct {
 	RootID          uint32
 	SchemaVersionID string
 	Indexes         []IndexDescription
+	Fields          []CollectionFieldDescription
 
 	// Properties below this line are unmarshalled using custom logic in [UnmarshalJSON]
 	Sources []map[string]json.RawMessage
@@ -409,6 +405,7 @@ func (c *CollectionDescription) UnmarshalJSON(bytes []byte) error {
 	c.RootID = descMap.RootID
 	c.SchemaVersionID = descMap.SchemaVersionID
 	c.Indexes = descMap.Indexes
+	c.Fields = descMap.Fields
 	c.Sources = make([]any, len(descMap.Sources))
 
 	for i, source := range descMap.Sources {

client/document.go

@@ -187,7 +187,7 @@ func isNillableKind(kind FieldKind) bool {
 // and ensures it matches the supplied field description.
 // It will do any minor parsing, like dates, and return
 // the typed value again as an interface.
-func validateFieldSchema(val any, field FieldDescription) (any, error) {
+func validateFieldSchema(val any, field SchemaFieldDescription) (any, error) {
 	if isNillableKind(field.Kind) {
 		if val == nil {
 			return nil, nil
@@ -522,15 +522,15 @@ func (doc *Document) setWithFastJSONObject(obj *fastjson.Object) error {
 
 // Set the value of a field.
 func (doc *Document) Set(field string, value any) error {
-	fd, exists := doc.schemaDescription.GetField(field)
+	fd, exists := doc.schemaDescription.GetFieldByName(field)
 	if !exists {
 		return NewErrFieldNotExist(field)
 	}
-	if fd.IsRelation() && !fd.IsObjectArray() {
+	if fd.IsRelation() && !fd.Kind.IsObjectArray() {
 		if !strings.HasSuffix(field, request.RelatedObjectID) {
 			field = field + request.RelatedObjectID
 		}
-		fd, exists = doc.schemaDescription.GetField(field)
+		fd, exists = doc.schemaDescription.GetFieldByName(field)
 		if !exists {
 			return NewErrFieldNotExist(field)
 		}

client/document_test.go

@@ -30,7 +30,7 @@ var (
 	schemaDescriptions = []SchemaDescription{
 		{
 			Name: "User",
-			Fields: []FieldDescription{
+			Fields: []SchemaFieldDescription{
 				{
 					Name: "Name",
 					Typ:  LWW_REGISTER,

client/index.go

@@ -41,18 +41,19 @@ type IndexDescription struct {
 }
 
 // CollectIndexedFields returns all fields that are indexed by all collection indexes.
-func (d CollectionDescription) CollectIndexedFields(schema *SchemaDescription) []FieldDescription {
+func (d CollectionDefinition) CollectIndexedFields() []FieldDefinition {
 	fieldsMap := make(map[string]bool)
-	fields := make([]FieldDescription, 0, len(d.Indexes))
-	for _, index := range d.Indexes {
+	fields := make([]FieldDefinition, 0, len(d.Description.Indexes))
+	for _, index := range d.Description.Indexes {
 		for _, field := range index.Fields {
-			for i := range schema.Fields {
-				colField := schema.Fields[i]
-				if field.Name == colField.Name && !fieldsMap[field.Name] {
-					fieldsMap[field.Name] = true
-					fields = append(fields, colField)
-					break
-				}
+			if fieldsMap[field.Name] {
+				// If the FieldDescription has already been added to the result do not add it a second time
+				// this can happen if a field is referenced by multiple indexes
+				continue
+			}
+			colField, ok := d.GetFieldByName(field.Name)
+			if ok {
+				fields = append(fields, colField)
 			}
 		}
 	}