diff --git a/encoding/convert.go b/encoding/convert.go new file mode 100644 index 00000000..5dc810eb --- /dev/null +++ b/encoding/convert.go @@ -0,0 +1,15 @@ +package encoding + +import ( + "gopkg.in/yaml.v3" +) + +func ToYaml(data []byte) ([]byte, error) { + var out map[string]interface{} + err := Unmarshal(data, &out) + if err != nil { + return nil, err + } + + return yaml.Marshal(out) +} diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 61b3d8be..e40509d8 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -38,6 +38,14 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } +func (pkg AhPackage) GetSourceURL() string { + return pkg.ChartUrl +} + +func (pkg AhPackage) GetName() string { + return pkg.Name +} + func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration diff --git a/generators/github/package.go b/generators/github/package.go index 0c632ef9..e7efea6c 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -6,6 +6,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" + "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" @@ -25,6 +26,14 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } +func (gp GitHubPackage) GetSourceURL() string { + return gp.SourceURL +} + +func (gp GitHubPackage) GetName() string { + return gp.Name +} + func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) @@ -34,28 +43,40 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, } manifestBytes := bytes.Split(data, []byte("\n---\n")) - crds, errs := component.FilterCRDs(manifestBytes) + errs := []error{} - for _, crd := range crds { - comp, err := component.Generate(crd) - if err != nil { - continue - } - if comp.Model.Metadata == nil { - comp.Model.Metadata = &model.ModelDefinition_Metadata{} - } - if comp.Model.Metadata.AdditionalProperties == nil { - comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) - } + for _, crd := range manifestBytes { + isCrd := kubernetes.IsCRD(string(crd)) + if !isCrd { - comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - comp.Model.Version = gp.version - comp.Model.Name = gp.Name - comp.Model.Category = category.CategoryDefinition{ - Name: "", + comps, err := component.GenerateFromOpenAPI(string(crd), gp) + if err != nil { + errs = append(errs, component.ErrGetSchema(err)) + continue + } + components = append(components, comps...) + } else { + comp, err := component.Generate(string(crd)) + if err != nil { + continue + } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) + } + + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + comp.Model.Version = gp.version + comp.Model.Name = gp.Name + comp.Model.Category = category.CategoryDefinition{ + Name: "", + } + comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + components = append(components, comp) } - comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - components = append(components, comp) + } return components, utils.CombineErrors(errs, "\n") diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 19f61177..c6dcd3eb 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -13,6 +13,8 @@ type Validator interface { type Package interface { GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string + GetSourceURL() string + GetName() string } // Supports pulling packages from Artifact Hub and other sources like Docker Hub. diff --git a/utils/component/error.go b/utils/component/error.go index b70e9a00..a37a07bb 100644 --- a/utils/component/error.go +++ b/utils/component/error.go @@ -9,6 +9,8 @@ const ( ErrUpdateSchemaCode = "meshkit-11158" ) +var ErrNoSchemasFound = errors.New(ErrGetSchemaCode, errors.Alert, []string{"Could not get schema for the given openapi spec"}, []string{"The OpenAPI spec doesn't include \"components.schemas\" path"}, []string{"The spec doesn't have include any schema"}, []string{"Verify the spec has valid schema."}) + // No reference usage found. Also check in adapters before deleting func ErrCrdGenerate(err error) error { return errors.New(ErrCrdGenerateCode, errors.Alert, []string{"Could not generate component with the given CRD"}, []string{err.Error()}, []string{""}, []string{"Verify CRD has valid schema."}) diff --git a/utils/component/generator.go b/utils/component/generator.go index 288f2780..b228b363 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -45,17 +45,28 @@ var DefaultPathConfig2 = CuePathConfig{ SpecPath: "spec.validation.openAPIV3Schema", } +var OpenAPISpecPathConfig = CuePathConfig{ + NamePath: `x-kubernetes-group-version-kind"[0].kind`, + IdentifierPath: "spec.names.kind", + VersionPath: `"x-kubernetes-group-version-kind"[0].version`, + GroupPath: `"x-kubernetes-group-version-kind"[0].group`, + ScopePath: "spec.scope", + SpecPath: "spec.versions[0].schema.openAPIV3Schema", + PropertiesPath: "properties", +} + var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(crd string) (component.ComponentDefinition, error) { +func Generate(resource string) (component.ComponentDefinition, error) { cmp := component.ComponentDefinition{} cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - crdCue, err := utils.YamlToCue(crd) + crdCue, err := utils.YamlToCue(resource) if err != nil { return cmp, err } + var schema string for _, cfg := range Configs { schema, err = getSchema(crdCue, cfg) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go new file mode 100644 index 00000000..045fcd6f --- /dev/null +++ b/utils/component/openapi_generator.go @@ -0,0 +1,165 @@ +package component + +import ( + "encoding/json" + "errors" + "fmt" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + cueJson "cuelang.org/go/encoding/json" + "github.com/layer5io/meshkit/generators/models" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/manifests" + + "gopkg.in/yaml.v3" + + "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" +) + +func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) { + if resource == "" { + return nil, nil + } + resource, err := getResolvedManifest(resource) + if err != nil && errors.Is(err, ErrNoSchemasFound) { + return nil, nil + } + if err != nil { + return nil, err + } + cuectx := cuecontext.New() + cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) + if err != nil { + return nil, err + } + + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) + definitions, err := utils.Lookup(parsedManifest, "components.schemas") + + if err != nil { + return nil, err + } + + fields, err := definitions.Fields() + if err != nil { + fmt.Printf("%v\n", err) + return nil, err + } + components := make([]component.ComponentDefinition, 0) + + for fields.Next() { + fieldVal := fields.Value() + kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`) + if err != nil { + continue + } + kind, err := kindCue.String() + if err != nil { + fmt.Printf("%v", err) + continue + } + + crd, err := fieldVal.MarshalJSON() + if err != nil { + fmt.Printf("%v", err) + continue + } + versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`) + if err != nil { + continue + } + + groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`) + if err != nil { + continue + } + + apiVersion, _ := versionCue.String() + if g, _ := groupCue.String(); g != "" { + apiVersion = g + "/" + apiVersion + } + modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata) + err = json.Unmarshal(crd, &modified) + if err != nil { + fmt.Printf("%v", err) + continue + } + + modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion) + if err == nil { + modified = modifiedProps + } + + DeleteFields(modified) + crd, err = json.Marshal(modified) + if err != nil { + fmt.Printf("%v", err) + continue + } + + c := component.ComponentDefinition{ + SchemaVersion: v1beta1.ComponentSchemaVersion, + Format: component.JSON, + Component: component.Component{ + Kind: kind, + Version: apiVersion, + Schema: string(crd), + }, + DisplayName: manifests.FormatToReadableString(kind), + Model: model.ModelDefinition{ + SchemaVersion: v1beta1.ModelSchemaVersion, + Model: model.Model{ + Version: pkg.GetVersion(), + }, + Name: pkg.GetName(), + DisplayName: manifests.FormatToReadableString(pkg.GetName()), + Metadata: &model.ModelDefinition_Metadata{ + AdditionalProperties: map[string]interface{}{ + "source_uri": pkg.GetSourceURL(), + }, + }, + }, + } + + components = append(components, c) + } + return components, nil + +} + +func getResolvedManifest(manifest string) (string, error) { + var m map[string]interface{} + + err := yaml.Unmarshal([]byte(manifest), &m) + if err != nil { + return "", utils.ErrDecodeYaml(err) + } + + byt, err := json.Marshal(m) + if err != nil { + return "", utils.ErrMarshal(err) + } + + cuectx := cuecontext.New() + cueParsedManExpr, err := cueJson.Extract("", byt) + if err != nil { + return "", ErrGetSchema(err) + } + + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) + definitions, err := utils.Lookup(parsedManifest, "components.schemas") + if err != nil { + return "", ErrNoSchemasFound + } + resol := manifests.ResolveOpenApiRefs{} + cache := make(map[string][]byte) + resolved, err := resol.ResolveReferences(byt, definitions, cache) + if err != nil { + return "", err + } + manifest = string(resolved) + return manifest, nil +} diff --git a/utils/component/utils.go b/utils/component/utils.go index f63d6068..f04ded45 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -7,7 +7,6 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" - "gopkg.in/yaml.v2" ) // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata) @@ -81,19 +80,10 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - - var crd map[string]interface{} - err := yaml.Unmarshal(m, &crd) - if err != nil { - errs = append(errs, err) - continue - } - - isCrd := kubernetes.IsCRD(crd) - if !isCrd { - continue + isCrd := kubernetes.IsCRD(string(m)) + if isCrd { + filteredManifests = append(filteredManifests, string(m)) } - filteredManifests = append(filteredManifests, string(m)) } return filteredManifests, errs } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 4af65eed..091e5902 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -108,7 +109,13 @@ func writeToFile(w io.Writer, path string) error { if err != nil { return utils.ErrReadFile(err, path) } - _, err = w.Write(data) + + byt, err := encoding.ToYaml(data) + if err != nil { + return utils.ErrWriteFile(err, path) + } + + _, err = w.Write(byt) if err != nil { return utils.ErrWriteFile(err, path) } diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 0525a624..9bf8d991 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -4,6 +4,7 @@ import ( "context" "github.com/layer5io/meshkit/encoding" + "github.com/layer5io/meshkit/utils" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -53,7 +54,19 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } } -func IsCRD(manifest map[string]interface{}) bool { - kind, ok := manifest["kind"].(string) - return ok && kind == "CustomResourceDefinition" +func IsCRD(manifest string) bool { + cueValue, err := utils.YamlToCue(manifest) + if err != nil { + return false + } + kind, err := utils.Lookup(cueValue, "kind") + if err != nil { + return false + } + kindStr, err := kind.String() + if err != nil { + return false + } + + return kindStr == "CustomResourceDefinition" } diff --git a/utils/manifests/utils.go b/utils/manifests/utils.go index fb3990f7..2935f1c0 100644 --- a/utils/manifests/utils.go +++ b/utils/manifests/utils.go @@ -9,6 +9,7 @@ import ( "strings" "cuelang.org/go/cue" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/oam/core/v1alpha1" ) @@ -254,7 +255,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue cache = make(map[string][]byte) } var val map[string]interface{} - err := json.Unmarshal(manifest, &val) + err := encoding.Unmarshal(manifest, &val) if err != nil { return nil, err } @@ -266,7 +267,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if ro.isInsideJsonSchemaProps && (ref == JsonSchemaPropsRef) { // hack so that the UI doesn't crash val["$ref"] = "string" - marVal, errJson := json.Marshal(val) + marVal, errJson := encoding.Marshal(val) if errJson != nil { return manifest, nil } @@ -282,13 +283,13 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue newval = append(newval, v0) continue } - byt, _ := json.Marshal(v0) + byt, _ := encoding.Marshal(v0) byt, err = ro.ResolveReferences(byt, definitions, cache) if err != nil { return nil, err } var newvalmap map[string]interface{} - _ = json.Unmarshal(byt, &newvalmap) + _ = encoding.Unmarshal(byt, &newvalmap) newval = append(newval, newvalmap) } val[k] = newval @@ -333,7 +334,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if reflect.ValueOf(v).Kind() == reflect.Map { var marVal []byte var def []byte - marVal, err = json.Marshal(v) + marVal, err = encoding.Marshal(v) if err != nil { return nil, err } @@ -349,7 +350,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue } } } - res, err := json.Marshal(val) + res, err := encoding.Marshal(val) if err != nil { return nil, err } @@ -358,7 +359,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue func replaceRefWithVal(def []byte, val map[string]interface{}, k string) error { var defVal map[string]interface{} - err := json.Unmarshal([]byte(def), &defVal) + err := encoding.Unmarshal([]byte(def), &defVal) if err != nil { return err }