Merge branch 'master' into meshkit-minify

Signed-off-by: Mohd Uzair <[email protected]>

converter/k8s.go

@@ -17,6 +17,8 @@ func (k *K8sConverter) Convert(patternFile string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
+	patterns.ProcessAnnotations(pattern)
 	return NewK8sManifestsFromPatternfile(pattern)
 }
 

models/meshmodel/registry/v1beta1/model_filter.go

@@ -102,7 +102,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
 
 	if mf.Greedy {
 		if mf.Id != "" {
-			finder = finder.First("model_dbs.id = ?", mf.Id)
+			finder = finder.Where("model_dbs.id = ?", mf.Id)
 		}
 		if mf.Name != "" && mf.DisplayName != "" {
 			finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%")
@@ -182,7 +182,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
 		if includeComponents {
 			var components []component.ComponentDefinition
 			finder := db.Model(&component.ComponentDefinition{}).
-				Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles").
+				Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles,component_definition_dbs.capabilities").
 				Where("component_definition_dbs.model_id = ?", _modelDB.Id)
 			if err := finder.Scan(&components).Error; err != nil {
 				return nil, 0, 0, err

models/patterns/pattern.go

@@ -4,6 +4,7 @@ import (
 	"github.com/Masterminds/semver/v3"
 	"github.com/layer5io/meshkit/encoding"
 	"github.com/layer5io/meshkit/utils"
+	"github.com/meshery/schemas/models/v1beta1/component"
 	"github.com/meshery/schemas/models/v1beta1/pattern"
 )
 
@@ -36,3 +37,13 @@ func GetPatternFormat(patternFile string) (*pattern.PatternFile, error) {
 	}
 	return &pattern, nil
 }
+
+func ProcessAnnotations(pattern *pattern.PatternFile) {
+	components := []*component.ComponentDefinition{}
+	for _, component := range pattern.Components {
+		if !component.Metadata.IsAnnotation {
+			components = append(components, component)
+		}
+	}
+	pattern.Components = components
+}

models/registration/dir.go

@@ -7,7 +7,10 @@ import (
 	"reflect"
 
 	"github.com/layer5io/meshkit/models/meshmodel/entity"
+	"github.com/layer5io/meshkit/models/oci"
+
 	"github.com/layer5io/meshkit/utils"
+
 	"github.com/meshery/schemas/models/v1alpha3/relationship"
 	"github.com/meshery/schemas/models/v1beta1/component"
 	"github.com/meshery/schemas/models/v1beta1/model"
@@ -26,69 +29,185 @@ func NewDir(path string) Dir {
 }
 
 /*
-PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct.
+PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the PackagingUnit struct.
 Invalid definitions are stored in the regErrStore with error data.
 */
-func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) {
-	pkg := packagingUnit{}
-	// check if the given is a directory
-	_, err = os.ReadDir(d.dirpath)
+func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ PackagingUnit, err error) {
+	pkg := PackagingUnit{}
+
+	// Extract the filename to use as entityName in case of errors
+	filename := filepath.Base(d.dirpath)
+
+	// Check if the given path is accessible
+	_, err = os.Stat(d.dirpath)
 	if err != nil {
-		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err))
+		regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err)))
+		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err))
+	}
+
+	// Process the path (file or directory)
+	err = processDir(d.dirpath, &pkg, regErrStore)
+	if err != nil {
+		modelName := ""
+		if !reflect.ValueOf(pkg.Model).IsZero() {
+			modelName = pkg.Model.Name
+		}
+		regErrStore.InsertEntityRegError("", modelName, entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err)))
+		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err))
 	}
-	err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error {
+
+	if reflect.ValueOf(pkg.Model).IsZero() {
+		errMsg := fmt.Errorf("model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core must be present, otherwise the model package is considered malformed")
+		regErrStore.InsertEntityRegError("", "", entity.Model, filename, errMsg)
+		return pkg, errMsg
+	}
+
+	return pkg, nil
+}
+
+func processDir(dirPath string, pkg *PackagingUnit, regErrStore RegistrationErrorStore) error {
+	var tempDirs []string
+	defer func() {
+		for _, tempDir := range tempDirs {
+			os.RemoveAll(tempDir)
+		}
+	}()
+
+	return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
-			return err
+			err = utils.ErrFileWalkDir(fmt.Errorf("error accessing path: %w", err), path)
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+			regErrStore.AddInvalidDefinition(path, err)
+			return nil
 		}
-		if f.IsDir() {
+		if info.IsDir() {
 			return nil
 		}
-		byt, _ := os.ReadFile(path)
-		if byt == nil {
+
+		// Read the file content
+		data, err := os.ReadFile(path)
+		if err != nil {
+			err = oci.ErrReadingFile(err)
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+			regErrStore.AddInvalidDefinition(path, err)
 			return nil
 		}
 
-		var e entity.Entity
-		e, err = getEntity(byt)
+		// Check if the file is an OCI artifact
+		if oci.IsOCIArtifact(data) {
+			// Extract the OCI artifact
+			tempDir, err := oci.CreateTempOCIContentDir()
+			if err != nil {
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			tempDirs = append(tempDirs, tempDir)
+			err = oci.UnCompressOCIArtifact(path, tempDir)
+			if err != nil {
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			// Recursively process the extracted directory
+			if err := processDir(tempDir, pkg, regErrStore); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		// Check if the file is a zip or tar file
+		if utils.IsZip(path) || utils.IsTarGz(path) {
+			tempDir, err := os.MkdirTemp("", "nested-extract-")
+			if err != nil {
+				err = utils.ErrCreateDir(fmt.Errorf("error creating temp directory for nested archive extraction: %w", err), tempDir)
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			tempDirs = append(tempDirs, tempDir)
+			if err := utils.ExtractFile(path, tempDir); err != nil {
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			// Recursively process the extracted directory
+			if err := processDir(tempDir, pkg, regErrStore); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		content := data
+		content, err = utils.YAMLToJSON(content)
 		if err != nil {
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+			return nil
+		}
+		// Determine the entity type
+		entityType, err := utils.FindEntityType(content)
+		if err != nil {
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
 			regErrStore.AddInvalidDefinition(path, err)
 			return nil
 		}
 
-		// set it to pkgunit
+		if entityType == "" {
+			// Not an entity we care about
+			return nil
+		}
+
+		// Get the entity
+		var e entity.Entity
+		e, err = getEntity(content)
+		if err != nil {
+			regErrStore.InsertEntityRegError("", "", entityType, filepath.Base(path), fmt.Errorf("could not get entity: %w", err))
+			regErrStore.AddInvalidDefinition(path, fmt.Errorf("could not get entity: %w", err))
+			return nil
+		}
+
+		// Add the entity to the packaging unit
 		switch e.Type() {
 		case entity.Model:
-			if !reflect.ValueOf(pkg.model).IsZero() {
-				// currently models inside models are not handled
-				return nil
-			}
 			model, err := utils.Cast[*model.ModelDefinition](e)
 			if err != nil {
+				modelName := ""
+				if model != nil {
+					modelName = model.Name
+				}
+				regErrStore.InsertEntityRegError("", modelName, entityType, modelName, ErrGetEntity(err))
 				regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				return nil
 			}
-			pkg.model = *model
+			pkg.Model = *model
 		case entity.ComponentDefinition:
 			comp, err := utils.Cast[*component.ComponentDefinition](e)
 			if err != nil {
+				componentName := ""
+				if comp != nil {
+					componentName = comp.Component.Kind
+				}
+				regErrStore.InsertEntityRegError("", "", entityType, componentName, ErrGetEntity(err))
 				regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				return nil
 			}
-			pkg.components = append(pkg.components, *comp)
+			pkg.Components = append(pkg.Components, *comp)
 		case entity.RelationshipDefinition:
 			rel, err := utils.Cast[*relationship.RelationshipDefinition](e)
 			if err != nil {
+				relationshipName := ""
+				if rel != nil {
+					relationshipName = rel.Model.Name
+				}
+				regErrStore.InsertEntityRegError("", "", entityType, relationshipName, ErrGetEntity(err))
 				regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				return nil
 			}
-			pkg.relationships = append(pkg.relationships, *rel)
+			pkg.Relationships = append(pkg.Relationships, *rel)
+		default:
+			// Unhandled entity type
+			return nil
 		}
 		return nil
 	})
-	if err != nil {
-		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err))
-	}
-	if reflect.ValueOf(pkg.model).IsZero() {
-		err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..")
-		regErrStore.AddInvalidDefinition(d.dirpath, err)
-		return pkg, err
-	}
-	return pkg, err
 }

models/registration/interface.go

@@ -12,11 +12,11 @@ type RegistrationErrorStore interface {
 	InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error)
 }
 
-// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server
+// Anything that can be parsed into a PackagingUnit is a RegisterableEntity in Meshery server
 type RegisterableEntity interface {
 	/*
 		1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered
 		2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore
 	*/
-	PkgUnit(RegistrationErrorStore) (packagingUnit, error)
+	PkgUnit(RegistrationErrorStore) (PackagingUnit, error)
 }

models/registration/oci.go

@@ -8,7 +8,7 @@ type OCIImage struct {
 	_ gcrv1.Image
 }
 
-func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) {
-	pkg := packagingUnit{}
+func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) {
+	pkg := PackagingUnit{}
 	return pkg, nil
 }