Lb4/datahub rest emitter init

build.gradle.kts

@@ -70,3 +70,6 @@ allprojects {
     }
 
 }
+dependencies {
+    implementation("junit:junit:4.13.1")
+}

extensions/control-plane/api/management-api/build.gradle.kts

@@ -26,6 +26,7 @@ dependencies {
     api(project(":extensions:control-plane:api:management-api:contract-negotiation-api"))
     api(project(":extensions:control-plane:api:management-api:policy-definition-api"))
     api(project(":extensions:control-plane:api:management-api:transfer-process-api"))
+    api(project(":extensions:control-plane:api:management-api:dataspace-catalog-api"))
 }
 
 

extensions/control-plane/api/management-api/dataspace-catalog-api/build.gradle.kts

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2022 ZF Friedrichshafen AG
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ *    ZF Friedrichshafen AG - Initial API and Implementation
+ *    Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ */
+
+
+plugins {
+    `java-library`
+    id("io.swagger.core.v3.swagger-gradle-plugin")
+}
+
+dependencies {
+    api(project(":extensions:control-plane:api:management-api:asset-api"))
+    implementation("io.acryl:datahub-client:0.10.5-5")
+    implementation("org.apache.httpcomponents:httpclient:4.5")
+    implementation("org.apache.httpcomponents:httpasyncclient:4.1.5")
+    testImplementation("org.junit.jupiter:junit-jupiter-api:5.7.2")
+    testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
+    testImplementation(project(":core:common:transform-core"))
+    testImplementation(project(":core:control-plane:control-plane-core"))
+    testImplementation(project(":core:common:transform-core"))
+    testImplementation(project(":core:control-plane:control-plane-core"))
+    testImplementation(project(":core:data-plane-selector:data-plane-selector-core"))
+    testImplementation(project(":extensions:common:http"))
+    testImplementation(project(":core:common:junit"))
+    testImplementation(testFixtures(project(":extensions:common:http:jersey-core")))
+    testImplementation(libs.restAssured)
+    testImplementation(libs.awaitility)
+
+}
+
+edcBuild {
+    swagger {
+        apiGroup.set("management-api")
+    }
+}
+
+

extensions/control-plane/api/management-api/dataspace-catalog-api/src/main/java/be/imec/edit/ds/catalog/DatasetEntityIngestor.java

@@ -0,0 +1,143 @@
+package be.imec.edit.ds.catalog;
+
+import be.imec.edit.ds.catalog.common.DataSpaceCatalogIngestorBase;
+import com.linkedin.common.FabricType;
+import com.linkedin.common.urn.DatasetUrn;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.dataset.DatasetProperties;
+import com.linkedin.dataset.EditableDatasetProperties;
+import com.linkedin.schema.SchemaField;
+import com.linkedin.schema.SchemaFieldArray;
+import com.linkedin.schema.SchemaMetadata;
+import datahub.client.MetadataWriteResponse;
+import datahub.client.rest.RestEmitter;
+import datahub.shaded.org.apache.kafka.common.errors.ApiException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import org.eclipse.edc.spi.types.domain.asset.Asset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/***
+ * The class to ingest metadata about data asset (dataset) to data space catalog.
+ *
+* */
+
+public class DatasetEntityIngestor extends DataSpaceCatalogIngestorBase {
+  Logger log = LoggerFactory.getLogger(this.getClass().getName());
+
+
+  final String entityType = "dataset";
+  final String platformType = "dataPlatform";
+
+
+
+/***
+ * editableDatasetProperties aspect of dataset entity - see details on datahub documentation for dataset entity aspects
+ * */
+  private DatasetProperties _datasetProperties(Asset asset) {
+    var createdAt=new com.linkedin.common.TimeStamp();
+    createdAt.setTime(asset.getCreatedAt());
+    Map assetProps = (Map) ((Map) asset.getProperties().get("asset")).get("properties");
+    return new DatasetProperties()
+        .setDescription(assetProps.get("name").toString())
+        .setCreated(createdAt);
+  }
+
+  /***
+ * editableDatasetProperties aspect of dataset entity
+ * */
+  private EditableDatasetProperties _editableDatasetProperties(Asset asset) {
+    Map assetProps = (Map) ((Map) asset.getProperties().get("asset")).get("properties");
+    return new EditableDatasetProperties()
+            .setDescription(assetProps.get("name").toString());
+
+  }
+
+
+  /***
+   * schemaMetadata aspect of dataset entity
+   * */
+  public SchemaMetadata _schemaMetadata(Asset asset) { //todo: This should not be handcrafted, rather should come (if any) from an avro
+    SchemaFieldArray fields = new SchemaFieldArray();
+    fields.add(new SchemaField());
+    return new SchemaMetadata().setFields(fields);
+  }
+
+  /***
+   * Returns datahub style urn for an asset - includes `test` as platform, and includes EDC asset name and id within the urn.
+   * The FabricType is the environment type such as Dev, Prod, etc.
+   * */
+  public Urn _urn(Asset asset) throws URISyntaxException {
+    var assetName = ((Map) asset.getProperties().get("asset")).get("@id");
+    return new DatasetUrn(_platformUrn(platformType), "Data Asset: "+assetName, FabricType.DEV);
+  }
+
+  /***
+   * This method emits whole dataset, with all aspects (defined within) to dataspace catalog. To only ingest/emit a single aspect, see specs.
+   * In this method, we first create a dataset with a single aspect - datasetProperties Aspect. Then, we create other aspects such as
+   * schemaMetadata and editableProperties aspects, and ingest them in parallel.
+   * Usually it can be done sequentially, but this is to show that, if an entity already exists, then aspects can be pushed in parallel as well.
+   * Since the calls are asynchronous, the datahub api at the receiving end will respond asynchronously.
+   * */
+  public Urn emitMetadataChangeProposal(Asset asset)
+      throws URISyntaxException, IOException, ExecutionException, InterruptedException {
+
+    //Get the urn of the asset
+    Urn datasetUrn = _urn(asset);
+    log.info("Pushing dataset to data space catalog");
+
+    // Emit datasetProperties aspect
+    Future<MetadataWriteResponse> responseFuture = emitter.emit(_metadataChangeProposalWrapper(_datasetProperties(asset), entityType, datasetUrn));
+    var emitResponse = responseFuture.get();
+    // Emit other aspects in parallel
+    if(responseFuture.isDone() && emitResponse.isSuccess()){
+
+      //set and emit editable properties aspect to metadata model of datahub
+      Future<MetadataWriteResponse> editablePropsFut = emitter.emit(_metadataChangeProposalWrapper(_editableDatasetProperties(asset), entityType, datasetUrn));
+
+
+      //set and emit data schema aspect to metadata model of datahub
+      Future<MetadataWriteResponse> schemaPropsFut = emitter.emit(_metadataChangeProposalWrapper(_schemaMetadata(asset), entityType, datasetUrn));
+      var a = _editableDatasetProperties(asset);
+
+      int numThreads = 2; // Number of threads in the thread pool
+      ExecutorService executor = Executors.newFixedThreadPool(numThreads);
+      List<Future<MetadataWriteResponse>> futures = List.of(editablePropsFut, schemaPropsFut);
+      List<Future<Future<MetadataWriteResponse>>> responses = new ArrayList<>();
+      for (Future<MetadataWriteResponse> future: futures){
+        Callable<Future<MetadataWriteResponse>> callable = () -> future;
+        Future<Future<MetadataWriteResponse>> resp = executor.submit(callable);
+        responses.add(resp);
+      }
+      for (Future<Future<MetadataWriteResponse>> response: responses){
+        try{
+          Future<MetadataWriteResponse> fresp = response.get();
+          if(response.isDone() && fresp.isDone() && fresp.get().isSuccess()){
+            log.info("Success ingesting "+ fresp.get().getResponseContent());
+          }
+          else {
+            log.error(fresp.get().getResponseContent(), ApiException.class);
+          }
+        }
+        catch (Exception e) {
+          e.printStackTrace();
+          log.error(e.getMessage());
+          return null;
+        }
+      }
+    }
+    return datasetUrn;
+  }
+
+
+}

extensions/control-plane/api/management-api/dataspace-catalog-api/src/main/java/be/imec/edit/ds/catalog/DomainEntityIngestor.java

@@ -0,0 +1,20 @@
+package be.imec.edit.ds.catalog;
+
+import be.imec.edit.ds.catalog.common.DataSpaceCatalogIngestorBase;
+import com.linkedin.common.urn.Urn;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.concurrent.ExecutionException;
+import org.eclipse.edc.spi.types.domain.asset.Asset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class DomainEntityIngestor extends DataSpaceCatalogIngestorBase {
+  Logger log = LoggerFactory.getLogger(this.getClass().getName());
+  @Override
+  public Urn emitMetadataChangeProposal(Asset asset)
+      throws URISyntaxException, IOException, ExecutionException, InterruptedException {
+    return null;
+  }
+}

extensions/control-plane/api/management-api/dataspace-catalog-api/src/main/java/be/imec/edit/ds/catalog/PolicyEntityIngestor.java

@@ -0,0 +1,20 @@
+package be.imec.edit.ds.catalog;
+
+import be.imec.edit.ds.catalog.common.DataSpaceCatalogIngestorBase;
+import com.linkedin.common.urn.Urn;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.concurrent.ExecutionException;
+import org.eclipse.edc.spi.types.domain.asset.Asset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class PolicyEntityIngestor extends DataSpaceCatalogIngestorBase {
+  Logger log = LoggerFactory.getLogger(this.getClass().getName());
+  @Override
+  public Urn emitMetadataChangeProposal(Asset asset)
+      throws URISyntaxException, IOException, ExecutionException, InterruptedException {
+    return null;
+  }
+}

extensions/control-plane/api/management-api/dataspace-catalog-api/src/main/java/be/imec/edit/ds/catalog/RelationshipIngestor.java

@@ -0,0 +1,20 @@
+package be.imec.edit.ds.catalog;
+
+import be.imec.edit.ds.catalog.common.DataSpaceCatalogIngestorBase;
+import com.linkedin.common.urn.Urn;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.concurrent.ExecutionException;
+import org.eclipse.edc.spi.types.domain.asset.Asset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class RelationshipIngestor extends DataSpaceCatalogIngestorBase {
+  Logger log = LoggerFactory.getLogger(this.getClass().getName());
+  @Override
+  public Urn emitMetadataChangeProposal(Asset asset)
+      throws URISyntaxException, IOException, ExecutionException, InterruptedException {
+    return null;
+  }
+}

extensions/control-plane/api/management-api/dataspace-catalog-api/src/main/java/be/imec/edit/ds/catalog/UserEntityIngestor.java

@@ -0,0 +1,20 @@
+package be.imec.edit.ds.catalog;
+
+import be.imec.edit.ds.catalog.common.DataSpaceCatalogIngestorBase;
+import com.linkedin.common.urn.Urn;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.concurrent.ExecutionException;
+import org.eclipse.edc.spi.types.domain.asset.Asset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class UserEntityIngestor extends DataSpaceCatalogIngestorBase {
+  Logger log = LoggerFactory.getLogger(this.getClass().getName());
+  @Override
+  public Urn emitMetadataChangeProposal(Asset asset)
+      throws URISyntaxException, IOException, ExecutionException, InterruptedException {
+    return null;
+  }
+}

extensions/control-plane/api/management-api/dataspace-catalog-api/src/main/java/be/imec/edit/ds/catalog/common/DataSpaceCatalogIngestorBase.java

@@ -0,0 +1,51 @@
+package be.imec.edit.ds.catalog.common;
+
+import com.linkedin.common.urn.DataPlatformUrn;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.RecordTemplate;
+import datahub.client.rest.RestEmitter;
+import datahub.event.MetadataChangeProposalWrapper;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+import org.eclipse.edc.spi.types.domain.asset.Asset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+abstract public class DataSpaceCatalogIngestorBase {
+  Logger log = LoggerFactory.getLogger(this.getClass().getName());
+  /***
+  * To create an emitter that is pushing to datahub instance that is remote (though an IP or url), see RestEmitter class. It has examples for creating emitter with external urls. An example is shown below commented out
+   *
+  * */
+  protected RestEmitter emitter = RestEmitter.createWithDefaults();
+  //protected RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080")); // todo: replace the `localhost:8080` with ip address or address of the dathub gms.
+  /***
+   * Method to build change proposal for any entity. aspect represents an aspect, e.g. dataSetProperties or Ownership Aspect of dataset, entityType is e.g. dathahub entity `dataset` etc.
+   * And the urn is `datahub` style urn.
+   * */
+  public MetadataChangeProposalWrapper _metadataChangeProposalWrapper(RecordTemplate aspect, String entityType, Urn urn) {
+    return MetadataChangeProposalWrapper.builder()
+        .entityType(entityType)
+        .entityUrn(urn)
+        .upsert()
+        .aspect(aspect)
+        .build();
+  }
+  /**
+   * At the moment, edc connectors are experimental. The data platforms (bigquery, snowflake, hudi etc.) are unknown and not provided, so we use `imec-edc`. But we can
+   * Use `conf` files to configure this.
+   * */
+  public DataPlatformUrn _platformUrn(String entityType) throws URISyntaxException {
+    return DataPlatformUrn.createFromUrn(DataPlatformUrn.createFromTuple(entityType, "IMEC_EDC_PLATFORM"));
+  }
+
+  /***
+   * A method used by subclasses to implement entity specific changeproposals and emit to datahub (data space catalog)
+  * */
+  public abstract Urn emitMetadataChangeProposal(Asset asset) throws URISyntaxException, IOException, ExecutionException,
+                                                            InterruptedException;
+
+}