Add reference data for secrets store connectors

Signed-off-by: Mandy Chessell <[email protected]>
mandy-chessell · Sep 25, 2024 · 84f6efb · 84f6efb
1 parent 49acb3b
commit 84f6efb
Show file tree

Hide file tree

Showing 80 changed files with 1,938 additions and 296 deletions.
diff --git a/EgeriaContentPacksGUIDMap.json b/EgeriaContentPacksGUIDMap.json
diff --git a/content-packs/ApacheAtlasContentPack.omarchive b/content-packs/ApacheAtlasContentPack.omarchive
diff --git a/content-packs/ApacheKafkaContentPack.omarchive b/content-packs/ApacheKafkaContentPack.omarchive
diff --git a/content-packs/CocoBusinessSystemsArchive.omarchive b/content-packs/CocoBusinessSystemsArchive.omarchive
diff --git a/content-packs/CocoClinicalTrialsTemplatesArchive.omarchive b/content-packs/CocoClinicalTrialsTemplatesArchive.omarchive
diff --git a/content-packs/CocoComboArchive.omarchive b/content-packs/CocoComboArchive.omarchive
diff --git a/content-packs/CocoGovernanceEngineDefinitionsArchive.omarchive b/content-packs/CocoGovernanceEngineDefinitionsArchive.omarchive
diff --git a/content-packs/CocoGovernanceProgramArchive.omarchive b/content-packs/CocoGovernanceProgramArchive.omarchive
diff --git a/content-packs/CocoOrganizationArchive.omarchive b/content-packs/CocoOrganizationArchive.omarchive
diff --git a/content-packs/CocoSustainabilityArchive.omarchive b/content-packs/CocoSustainabilityArchive.omarchive
diff --git a/content-packs/CocoTypesArchive.omarchive b/content-packs/CocoTypesArchive.omarchive
diff --git a/content-packs/CoreContentPack.omarchive b/content-packs/CoreContentPack.omarchive
diff --git a/content-packs/ObservabilityContentPack.omarchive b/content-packs/ObservabilityContentPack.omarchive
diff --git a/content-packs/PostgresContentPack.omarchive b/content-packs/PostgresContentPack.omarchive
diff --git a/content-packs/UnityCatalogContentPack.omarchive b/content-packs/UnityCatalogContentPack.omarchive
diff --git a/...mance-suite-client/src/main/java/org/odpi/openmetadata/conformance/client/RESTClient.java b/...mance-suite-client/src/main/java/org/odpi/openmetadata/conformance/client/RESTClient.java
@@ -79,7 +79,8 @@ class RESTClient
         RESTClientFactory  factory = new RESTClientFactory(serverName,
                                                            serverPlatformURLRoot,
                                                            userId,
-                                                           password);
+                                                           password,
+                                                           null);
 
         try
         {

diff --git a/open-metadata-distribution/omag-server-platform/build.gradle b/open-metadata-distribution/omag-server-platform/build.gradle
@@ -56,6 +56,7 @@ dependencies {
     implementation project(':open-metadata-implementation:adapters:open-connectors:data-manager-connectors:postgres-server-connectors')
     implementation project(':open-metadata-implementation:adapters:open-connectors:data-manager-connectors:unity-catalog-connectors')
     implementation project(':open-metadata-implementation:adapters:open-connectors:secrets-store-connectors:env-variable-secrets-store-connector')
+    implementation project(':open-metadata-implementation:adapters:open-connectors:secrets-store-connectors:yaml-secrets-store-connector')
     implementation project(':open-metadata-implementation:access-services:asset-consumer:asset-consumer-client')
     implementation project(':open-metadata-implementation:access-services:asset-manager:asset-manager-client')
     implementation project(':open-metadata-implementation:access-services:asset-owner:asset-owner-client')
@@ -163,6 +164,11 @@ distributions {
                 from { "$rootProject.projectDir/open-metadata-distribution/omag-server-platform/docs/platform" }
                 fileMode = 0755
             }
+            into('assembly/platform/loading-bay/secrets') {
+                // Identify how the platform is started
+                from { "$rootProject.projectDir/open-metadata-resources/open-metadata-deployment/secrets" }
+                fileMode = 0755
+            }
             into('assembly/platform/data') {
                 // Describe the data files
                 from { "$rootProject.projectDir/open-metadata-distribution/omag-server-platform/docs/data" }
@@ -220,6 +226,7 @@ distributions {
                 from { project(':open-metadata-resources:open-metadata-samples:governance-action-samples').jar }
                 from { project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:spring-rest-client-connector').jar }
                 from { project(':open-metadata-implementation:adapters:open-connectors:secrets-store-connectors:env-variable-secrets-store-connector').jar }
+                from { project(':open-metadata-implementation:adapters:open-connectors:secrets-store-connectors:yaml-secrets-store-connector').jar }
                 from { project(':open-metadata-implementation:adapters:open-connectors:system-connectors:apache-atlas-connectors').jar }
                 from { project(':open-metadata-implementation:adapters:open-connectors:system-connectors:apache-kafka-connectors').jar }
                 from { project(':open-metadata-implementation:adapters:open-connectors:system-connectors:egeria-system-connectors').jar }

diff --git a/...ata/adapters/connectors/unitycatalog/controls/UnityCatalogDeployedImplementationType.java b/...ata/adapters/connectors/unitycatalog/controls/UnityCatalogDeployedImplementationType.java
@@ -78,6 +78,67 @@ public enum UnityCatalogDeployedImplementationType implements DeployedImplementa
                              "The OSS Unity Catalog (UC) Server is an operational data platform 'catalog of catalogs' that supports controlled access to data managed through a related data platforms.",
                              "https://github.com/unitycatalog/unitycatalog"),
 
+    /**
+     * A collection of related data files within the Unity Catalog (UC) 'catalog of catalogs'.
+     */
+    DB_UC_VOLUME("Unity Catalog Volume",
+                  DeployedImplementationType.DATA_FOLDER,
+                  OpenMetadataType.DATA_FOLDER.typeName,
+                  null,
+                  "A collection of related data files within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
+                  "https://github.com/unitycatalog/unitycatalog"),
+
+
+    /**
+     * A function found in Unity Catalog (UC) that is working with data..
+     */
+    DB_UC_FUNCTION("Databricks Unity Catalog Function",
+                    OSS_UC_FUNCTION,
+                    OpenMetadataType.DEPLOYED_API.typeName,
+                    null,
+                    "A function found in Databricks Unity Catalog (UC) that is working with data.",
+                    "https://github.com/unitycatalog/unitycatalog"),
+
+    /**
+     * A relational table within the Unity Catalog (UC) 'catalog of catalogs'.
+     */
+    DB_UC_TABLE("Databricks Unity Catalog Table",
+                 OSS_UC_TABLE,
+                 OpenMetadataType.VIRTUAL_RELATIONAL_TABLE.typeName,
+                 null,
+                 "A relational table within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
+                 "https://github.com/unitycatalog/unitycatalog"),
+
+
+    /**
+     * A schema that organizes data assets for an operational data platform catalog within the Unity Catalog (UC) 'catalog of catalogs'.
+     */
+    DB_UC_SCHEMA("Databricks Unity Catalog Schema",
+                  OSS_UC_SCHEMA,
+                  OpenMetadataType.DEPLOYED_DATABASE_SCHEMA.typeName,
+                  null,
+                  "A schema that organizes data assets for an operational data platform catalog within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
+                  "https://github.com/unitycatalog/unitycatalog"),
+
+    /**
+     * An operational data platform catalog within the Unity Catalog (UC) 'catalog of catalogs'.
+     */
+    DB_UC_CATALOG("Databricks Unity Catalog Catalog",
+                   OSS_UC_CATALOG,
+                   OpenMetadataType.CATALOG.typeName,
+                   OpenMetadataType.ASSET_MANAGER.typeName,
+                   "An operational data platform catalog within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
+                   "https://github.com/unitycatalog/unitycatalog"),
+
+    /**
+     * An operational metadata catalog that supports controlled access to data managed through a data platform.
+     */
+    DB_UNITY_CATALOG_SERVER("Databricks Unity Catalog Server",
+                             OSS_UNITY_CATALOG_SERVER,
+                             OpenMetadataType.SOFTWARE_SERVER.typeName,
+                             OpenMetadataType.METADATA_SERVER_CLASSIFICATION.typeName,
+                             "The Databricks Unity Catalog (UC) Server is an operational data platform 'catalog of catalogs' that supports controlled access to data managed through a related data platforms.",
+                             "https://github.com/unitycatalog/unitycatalog"),
 
     ;
 

diff --git a/...odpi/openmetadata/adapters/connectors/unitycatalog/controls/UnityCatalogTemplateType.java b/...odpi/openmetadata/adapters/connectors/unitycatalog/controls/UnityCatalogTemplateType.java
@@ -14,47 +14,89 @@
  */
 public enum UnityCatalogTemplateType
 {
-    UC_SERVER_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UNITY_CATALOG_SERVER.getDeployedImplementationType(),
-                       "A SoftwareServer asset with connection and REST API Manager software capability.",
-                       UnityCatalogDeployedImplementationType.OSS_UNITY_CATALOG_SERVER.getAssociatedTypeName(),
-                       false,
-                       "dcca9788-b30f-4007-b1ac-ec634aff6879",
-                       null),
-
-    UC_CATALOG_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_CATALOG.getDeployedImplementationType(),
-                        "Create a " + UnityCatalogDeployedImplementationType.OSS_UC_CATALOG.getAssociatedTypeName() + " SoftwareCapability.",
-                        UnityCatalogDeployedImplementationType.OSS_UC_CATALOG.getAssociatedTypeName(),
-                        false,
-                        "5ee006aa-a6d6-411b-9b8d-5f720c079cae",
-                        null),
-
-    UC_SCHEMA_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_SCHEMA.getDeployedImplementationType(),
-                       "Create a DeployedDatabaseSchema to represent a schema from the Unity Catalog (UC).",
-                       UnityCatalogDeployedImplementationType.OSS_UC_SCHEMA.getAssociatedTypeName(),
-                       false,
-                       "5bf92b0f-3970-41ea-b0a3-aacfbf6fd92e",
-                       null),
-
-    UC_VOLUME_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_VOLUME.getDeployedImplementationType(),
-                       "Create a DataFolder asset to represent a volume from the Unity Catalog (UC).",
-                       UnityCatalogDeployedImplementationType.OSS_UC_VOLUME.getAssociatedTypeName(),
-                       false,
-                       "92d2d2dc-0798-41f0-9512-b10548d312b7",
-                       null),
-
-    UC_TABLE_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_TABLE.getDeployedImplementationType(),
-                      "Create a VirtualTableAsset asset to represent a table from the Unity Catalog (UC).",
-                      UnityCatalogDeployedImplementationType.OSS_UC_TABLE.getAssociatedTypeName(),
-                      false,
-                      "6cc1e5f5-4c1e-4290-a80e-e06643ffb13d",
-                      null),
-
-    UC_FUNCTION_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_FUNCTION.getDeployedImplementationType(),
-                         "Create a DeployedSoftwareComponent asset to represent a function from the Unity Catalog (UC).",
-                         UnityCatalogDeployedImplementationType.OSS_UC_FUNCTION.getAssociatedTypeName(),
-                         false,
-                         "a490ba65-6104-4213-9be9-524e16fed8aa",
-                         null),
+    OSS_UC_SERVER_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UNITY_CATALOG_SERVER.getDeployedImplementationType(),
+                           "A SoftwareServer asset with connection and REST API Manager software capability representing a Unity Catalog (UC) Server.",
+                           UnityCatalogDeployedImplementationType.OSS_UNITY_CATALOG_SERVER.getAssociatedTypeName(),
+                           false,
+                           "dcca9788-b30f-4007-b1ac-ec634aff6879",
+                           null),
+
+    DB_UC_SERVER_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UNITY_CATALOG_SERVER.getDeployedImplementationType(),
+                           "A SoftwareServer asset with connection and REST API Manager software capability representing a Unity Catalog (UC) Server running in a Databricks managed platform.",
+                           UnityCatalogDeployedImplementationType.DB_UNITY_CATALOG_SERVER.getAssociatedTypeName(),
+                           false,
+                           "3f7f62f6-4abc-424e-9f92-523306e7d5d5",
+                           null),
+
+    OSS_UC_CATALOG_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_CATALOG.getDeployedImplementationType(),
+                            "Create a " + UnityCatalogDeployedImplementationType.OSS_UC_CATALOG.getAssociatedTypeName() + " SoftwareCapability.",
+                            UnityCatalogDeployedImplementationType.OSS_UC_CATALOG.getAssociatedTypeName(),
+                            false,
+                            "5ee006aa-a6d6-411b-9b8d-5f720c079cae",
+                            null),
+
+    DB_UC_CATALOG_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_CATALOG.getDeployedImplementationType(),
+                            "Create a " + UnityCatalogDeployedImplementationType.DB_UC_CATALOG.getAssociatedTypeName() + " SoftwareCapability.",
+                            UnityCatalogDeployedImplementationType.DB_UC_CATALOG.getAssociatedTypeName(),
+                            false,
+                            "32a82fe1-b1f5-4d86-b54a-b19d0be87e6b",
+                            null),
+
+    OSS_UC_SCHEMA_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_SCHEMA.getDeployedImplementationType(),
+                           "Create a DeployedDatabaseSchema to represent a schema from the Unity Catalog (UC).",
+                           UnityCatalogDeployedImplementationType.OSS_UC_SCHEMA.getAssociatedTypeName(),
+                           false,
+                           "5bf92b0f-3970-41ea-b0a3-aacfbf6fd92e",
+                           null),
+
+    DB_UC_SCHEMA_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_SCHEMA.getDeployedImplementationType(),
+                           "Create a DeployedDatabaseSchema to represent a schema from the Unity Catalog (UC) running on a Databricks managed platform.",
+                           UnityCatalogDeployedImplementationType.DB_UC_SCHEMA.getAssociatedTypeName(),
+                           false,
+                           "844631dc-dc8a-4e8e-b285-56165917de09",
+                           null),
+
+    OSS_UC_VOLUME_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_VOLUME.getDeployedImplementationType(),
+                           "Create a DataFolder asset to represent a volume from the Unity Catalog (UC).",
+                           UnityCatalogDeployedImplementationType.OSS_UC_VOLUME.getAssociatedTypeName(),
+                           false,
+                           "92d2d2dc-0798-41f0-9512-b10548d312b7",
+                           null),
+
+    DB_UC_VOLUME_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_VOLUME.getDeployedImplementationType(),
+                           "Create a DataFolder asset to represent a volume from the Unity Catalog (UC) running on a Databricks managed platform.",
+                           UnityCatalogDeployedImplementationType.DB_UC_VOLUME.getAssociatedTypeName(),
+                           false,
+                           "a639212a-5c2d-464c-933f-aa612b9a4706",
+                           null),
+
+    OSS_UC_TABLE_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_TABLE.getDeployedImplementationType(),
+                          "Create a VirtualTableAsset asset to represent a table from the Unity Catalog (UC).",
+                          UnityCatalogDeployedImplementationType.OSS_UC_TABLE.getAssociatedTypeName(),
+                          false,
+                          "6cc1e5f5-4c1e-4290-a80e-e06643ffb13d",
+                          null),
+
+    DB_UC_TABLE_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_TABLE.getDeployedImplementationType(),
+                          "Create a VirtualTableAsset asset to represent a table from the Unity Catalog (UC) running on a Databricks managed platform.",
+                          UnityCatalogDeployedImplementationType.DB_UC_TABLE.getAssociatedTypeName(),
+                          false,
+                          "a1f8a374-3f78-4d06-8d8c-cc3322fc8b53",
+                          null),
+
+    OSS_UC_FUNCTION_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_FUNCTION.getDeployedImplementationType(),
+                             "Create a DeployedSoftwareComponent asset to represent a function from the Unity Catalog (UC).",
+                             UnityCatalogDeployedImplementationType.OSS_UC_FUNCTION.getAssociatedTypeName(),
+                             false,
+                             "a490ba65-6104-4213-9be9-524e16fed8aa",
+                             null),
+
+    DB_UC_FUNCTION_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_FUNCTION.getDeployedImplementationType(),
+                             "Create a DeployedSoftwareComponent asset to represent a function from the Unity Catalog (UC) running on a Databricks managed platform.",
+                             UnityCatalogDeployedImplementationType.DB_UC_FUNCTION.getAssociatedTypeName(),
+                             false,
+                             "d01bb98a-a6a1-490a-b935-31bae5974f6e",
+                             null),
 
     ;
 
@@ -212,11 +254,11 @@ public static List<TemplateType> getInsideCatalogTemplateTypes()
     {
         List<TemplateType> templateTypes = new ArrayList<>();
 
-        templateTypes.add(UC_CATALOG_TEMPLATE.getTemplateType());
-        templateTypes.add(UC_SCHEMA_TEMPLATE.getTemplateType());
-        templateTypes.add(UC_VOLUME_TEMPLATE.getTemplateType());
-        templateTypes.add(UC_TABLE_TEMPLATE.getTemplateType());
-        templateTypes.add(UC_FUNCTION_TEMPLATE.getTemplateType());
+        templateTypes.add(OSS_UC_CATALOG_TEMPLATE.getTemplateType());
+        templateTypes.add(OSS_UC_SCHEMA_TEMPLATE.getTemplateType());
+        templateTypes.add(OSS_UC_VOLUME_TEMPLATE.getTemplateType());
+        templateTypes.add(OSS_UC_TABLE_TEMPLATE.getTemplateType());
+        templateTypes.add(OSS_UC_FUNCTION_TEMPLATE.getTemplateType());
 
         return templateTypes;
     }

diff --git a/...nmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java b/...nmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java
@@ -117,7 +117,8 @@ public void start() throws ConnectorCheckedException
             RESTClientFactory factory = new RESTClientFactory(ucInstanceName,
                                                               targetRootURL,
                                                               connectionProperties.getUserId(),
-                                                              connectionProperties.getClearPassword());
+                                                              connectionProperties.getClearPassword(),
+                                                              secretsStoreConnectorMap);
 
             this.clientConnector = factory.getClientConnector();
 

diff --git a/...adata/adapters/connectors/integration/csvlineageimporter/CSVLineageImporterConnector.java b/...adata/adapters/connectors/integration/csvlineageimporter/CSVLineageImporterConnector.java
@@ -295,7 +295,7 @@ private CSVFileStoreConnector getConnectorWithOCF()
 
         try
         {
-            ConnectorBroker broker = new ConnectorBroker();
+            ConnectorBroker broker = new ConnectorBroker(auditLog);
 
             connector = (CSVFileStoreConnector) broker.getConnector(getHardCodedConnection(fileName));
         }

diff --git a/...enmetadata/adapters/connectors/integration/basicfiles/OMArchiveFilesMonitorForTarget.java b/...enmetadata/adapters/connectors/integration/basicfiles/OMArchiveFilesMonitorForTarget.java
@@ -31,7 +31,7 @@ public class OMArchiveFilesMonitorForTarget extends DataFilesMonitorForTarget
 {
     private static final Logger log = LoggerFactory.getLogger(OMArchiveFilesMonitorForTarget.class);
 
-    private static final ConnectorBroker connectorBroker = new ConnectorBroker();
+    private final ConnectorBroker connectorBroker = new ConnectorBroker(auditLog);
 
 
     /**
@@ -101,7 +101,7 @@ protected String addDataFileToCatalog(DataFileProperties properties) throws Conn
 
             if (propertyValue != null)
             {
-                properties.setResourceName(propertyValue);
+                properties.setName(propertyValue);
             }
 
             propertyValue = propertyHelper.getStringProperty(connectorName,

diff --git a/...adapters/connectors/integration/kafkaaudit/AuditLogDestinationCatalogTargetProcessor.java b/...adapters/connectors/integration/kafkaaudit/AuditLogDestinationCatalogTargetProcessor.java
@@ -63,7 +63,7 @@ else if (connectorToTarget instanceof BasicFolderConnector)
 
         try
         {
-            ConnectorBroker connectorBroker = new ConnectorBroker();
+            ConnectorBroker connectorBroker = new ConnectorBroker(auditLog);
 
             this.auditLogDestination = (OMRSAuditLogStoreConnectorBase) connectorBroker.getConnector(auditLogConnection);
         }

diff --git a/.../main/java/org/odpi/openmetadata/adapters/connectors/integration/openapis/RESTClient.java b/.../main/java/org/odpi/openmetadata/adapters/connectors/integration/openapis/RESTClient.java
@@ -90,7 +90,8 @@ protected RESTClient(String   serverName,
         RESTClientFactory  factory = new RESTClientFactory(serverName,
                                                            url,
                                                            userId,
-                                                           password);
+                                                           password,
+                                                           null);
 
         try
         {