fix: pandas 'nan' values (#409)

Signed-off-by: bruno nirello <[email protected]> Co-authored-by: bruno nirello <[email protected]>
amundsen-io · Nov 10, 2020 · 3a28f46 · 3a28f46
1 parent a78a8d4
commit 3a28f46
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/databuilder/publisher/neo4j_csv_publisher.py b/databuilder/publisher/neo4j_csv_publisher.py
@@ -219,7 +219,7 @@ def _create_indices(self, node_file: str) -> None:
         LOGGER.info('Creating indices. (Existing indices will be ignored)')
 
         with open(node_file, 'r', encoding='utf8') as node_csv:
-            for node_record in pandas.read_csv(node_csv).to_dict(orient='records'):
+            for node_record in pandas.read_csv(node_csv, na_filter=False).to_dict(orient='records'):
                 label = node_record[NODE_LABEL_KEY]
                 if label not in self.labels:
                     self._try_create_index(label)
@@ -246,7 +246,7 @@ def _publish_node(self, node_file: str, tx: Transaction) -> Transaction:
         """
 
         with open(node_file, 'r', encoding='utf8') as node_csv:
-            for node_record in pandas.read_csv(node_csv).to_dict(orient="records"):
+            for node_record in pandas.read_csv(node_csv, na_filter=False).to_dict(orient="records"):
                 stmt = self.create_node_merge_statement(node_record=node_record)
                 params = self._create_props_param(node_record)
                 tx = self._execute_statement(stmt, tx, params)
@@ -301,7 +301,7 @@ def _publish_relation(self, relation_file: str, tx: Transaction) -> Transaction:
 
             count = 0
             with open(relation_file, 'r', encoding='utf8') as relation_csv:
-                for rel_record in pandas.read_csv(relation_csv).to_dict(orient="records"):
+                for rel_record in pandas.read_csv(relation_csv, na_filter=False).to_dict(orient="records"):
                     stmt, params = self._relation_preprocessor.preprocess_cypher(
                         start_label=rel_record[RELATION_START_LABEL],
                         end_label=rel_record[RELATION_END_LABEL],
@@ -317,7 +317,7 @@ def _publish_relation(self, relation_file: str, tx: Transaction) -> Transaction:
             LOGGER.info('Executed pre-processing Cypher statement {} times'.format(count))
 
         with open(relation_file, 'r', encoding='utf8') as relation_csv:
-            for rel_record in pandas.read_csv(relation_csv).to_dict(orient="records"):
+            for rel_record in pandas.read_csv(relation_csv, na_filter=False).to_dict(orient="records"):
                 stmt = self.create_relationship_merge_statement(rel_record=rel_record)
                 params = self._create_props_param(rel_record)
                 tx = self._execute_statement(stmt, tx, params,