fix(ingest/nifi): remove duplicate upstream jobs

datahub-project · Jul 4, 2024 · 3892966 · 3892966
1 parent 8d5f0f3
commit 3892966
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -826,7 +826,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
             outgoing = list(
                 filter(lambda x: x[0] == component.id, self.nifi_flow.connections)
             )
-            inputJobs = []
+            inputJobs = set()
             jobProperties = None
 
             if component.nifi_type is NifiType.PROCESSOR:
@@ -877,7 +877,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                         dataset_urn,
                     )
                 else:
-                    inputJobs.append(
+                    inputJobs.add(
                         builder.make_data_job_urn_with_flow(flow_urn, incoming_from)
                     )
 
@@ -957,7 +957,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                 job_properties=jobProperties,
                 inlets=list(component.inlets.keys()),
                 outlets=list(component.outlets.keys()),
-                inputJobs=inputJobs,
+                inputJobs=list(inputJobs),
                 status=component.status,
             )