#13 Spreadsheet download

ingest/downloader/downloader.py

@@ -0,0 +1,98 @@
+from typing import List
+
+from openpyxl import Workbook
+from openpyxl.worksheet.worksheet import Worksheet
+
+EXCLUDE_KEYS = ['describedBy', 'schema_type']
+
+
+class XlsDownloader:
+    def __init__(self):
+        self.workbook = {}
+        self.row = 1
+
+    def convert_json(self, entity_list: List[dict]):
+        self._flatten_object_list(entity_list)
+        return self.workbook
+
+    def _flatten_object_list(self, object_list: List[dict], object_key: str = ''):
+        for entity in object_list:
+            worksheet_name = object_key
+            row = {}
+            content = entity
+
+            if not object_key:
+                content = entity['content']
+                worksheet_name = self.get_concrete_entity(content)
+                row = {f'{worksheet_name}.uuid': entity['uuid']['uuid']}
+
+            if not worksheet_name:
+                raise Exception('There should be a worksheet name')
+
+            user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name)
+
+            rows = self.workbook.get(user_friendly_worksheet_name, [])
+            self.workbook[user_friendly_worksheet_name] = rows
+            self._flatten_object(content, row, parent_key=worksheet_name)
+            rows.append(row)
+
+    def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''):
+        if isinstance(object, dict):
+            for key in object:
+                full_key = f'{parent_key}.{key}' if parent_key else key
+                if key in EXCLUDE_KEYS:
+                    continue
+                value = object[key]
+                if isinstance(value, dict) or isinstance(value, list):
+                    self._flatten_object(value, flattened_object, parent_key=full_key)
+                else:
+                    flattened_object[full_key] = str(value)
+        elif isinstance(object, list):
+            if self.is_object_list(object):
+                self._flatten_object_list(object, parent_key)
+            else:
+                stringified = [str(e) for e in object]
+                flattened_object[parent_key] = '||'.join(stringified)
+
+    def _format_worksheet_name(self, worksheet_name):
+        names = worksheet_name.split('.')
+        names = [n.replace('_', ' ') for n in names]
+        new_worksheet_name = ' - '.join([n.capitalize() for n in names])
+        return new_worksheet_name
+
+    def is_object_list(self, content):
+        return content and isinstance(content[0], dict)
+
+    @staticmethod
+    def get_concrete_entity(content):
+        return content.get('describedBy').rsplit('/', 1)[-1]
+
+    def create_workbook(self, input_json: dict) -> Workbook:
+        workbook = Workbook()
+        workbook.remove(workbook.active)
+
+        for ws_title, ws_elements in input_json.items():
+            worksheet: Worksheet = workbook.create_sheet(title=ws_title)
+            self.add_worksheet_content(worksheet, ws_elements)
+
+        return workbook
+
+    def add_worksheet_content(self, worksheet, ws_elements: dict):
+        is_header = True
+        if isinstance(ws_elements, list):
+            for content in ws_elements:
+                self.add_row_content(worksheet, content, is_header)
+                is_header = False
+                self.row += 1
+        else:
+            self.add_row_content(worksheet, ws_elements)
+
+    def add_row_content(self, worksheet, content, is_header=True):
+        col = 1
+        for header, cell_value in content.items():
+            if is_header:
+                self.row = 1
+                worksheet.cell(row=self.row, column=col, value=header)
+                self.row += 1
+            worksheet.cell(row=self.row, column=col, value=cell_value)
+            col += 1

tests/unit/downloader/project-list-flattened.json

@@ -0,0 +1,87 @@
+{
+  "Project": [
+    {
+      "project.uuid": "3e329187-a9c4-48ec-90e3-cc45f7c2311c",
+      "project.project_core.project_short_name": "kriegsteinBrainOrganoids",
+      "project.project_core.project_title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution",
+      "project.project_core.project_description": "Direct comparisons of human and non-human primate brain tissue have the potential to reveal molecular pathways underlying remarkable specializations of the human brain. However, chimpanzee tissue is largely inaccessible during neocortical neurogenesis when differences in brain size first appear. To identify human-specific features of cortical development, we leveraged recent innovations that permit generating pluripotent stem cell-derived cerebral organoids from chimpanzee. First, we systematically evaluated the fidelity of organoid models to primary human and macaque cortex, finding organoid models preserve gene regulatory networks related to cell types and developmental processes but exhibit increased metabolic stress. Second, we identified 261 genes differentially expressed in human compared to chimpanzee organoids and macaque cortex. Many of these genes overlap with human-specific segmental duplications and a subset suggest increased PI3K/AKT/mTOR activation in human outer radial glia. Together, our findings establish a platform for systematic analysis of molecular changes contributing to human brain development and evolution. Overall design: Single cell mRNA sequencing of iPS-derived neural and glial progenitor cells using the Fluidigm C1 system This series includes re-analysis of publicly available data in accessions: phs000989.v3, GSE99951, GSE86207, GSE75140. Sample metadata and accession IDs for the re-analyzed samples are included in the file \"GSE124299_metadata_on_processed_samples.xlsx\" available on the foot of this record. The following samples have no raw data due to data loss: GSM3569728, GSM3569738, GSM3571601, GSM3571606, GSM3571615, GSM3571621, GSM3571625, and GSM3571631",
+      "project.insdc_project_accessions": "SRP180337",
+      "project.geo_series_accessions": "GSE124299",
+      "project.insdc_study_accessions": "PRJNA515930"
+    }
+  ],
+  "Project - Contributors": [
+    {
+      "project.contributors.name": "Alex A,,Pollen",
+      "project.contributors.email": "[email protected]",
+      "project.contributors.institution": "University of California, San Francisco (UCSF)",
+      "project.contributors.laboratory": "Department of Neurology",
+      "project.contributors.country": "USA",
+      "project.contributors.corresponding_contributor": "True",
+      "project.contributors.project_role.text": "experimental scientist",
+      "project.contributors.project_role.ontology": "EFO:0009741",
+      "project.contributors.project_role.ontology_label": "experimental scientist"
+    },
+    {
+      "project.contributors.name": "Parisa,,Nejad",
+      "project.contributors.email": "[email protected]",
+      "project.contributors.institution": "University of California, Santa Cruz",
+      "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform",
+      "project.contributors.country": "USA",
+      "project.contributors.corresponding_contributor": "False",
+      "project.contributors.project_role.text": "data wrangler",
+      "project.contributors.project_role.ontology": "EFO:0009737",
+      "project.contributors.project_role.ontology_label": "data curator"
+    },
+    {
+      "project.contributors.name": "Schwartz,,Rachel",
+      "project.contributors.email": "[email protected]",
+      "project.contributors.institution": "University of California, Santa Cruz",
+      "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform",
+      "project.contributors.country": "USA",
+      "project.contributors.corresponding_contributor": "False",
+      "project.contributors.project_role.text": "data wrangler",
+      "project.contributors.project_role.ontology": "EFO:0009737",
+      "project.contributors.project_role.ontology_label": "data curator"
+    }
+  ],
+  "Project - Publications": [
+    {
+      "project.publications.authors": "Pollen AA||Bhaduri A||Andrews MG||Nowakowski TJ||Meyerson OS||Mostajo-Radji MA||Di Lullo E||Alvarado B||Bedolli M||Dougherty ML||Fiddes IT||Kronenberg ZN||Shuga J||Leyrat AA||West JA||Bershteyn M||Lowe CB||Pavlovic BJ||Salama SR||Haussler D||Eichler EE||Kriegstein AR",
+      "project.publications.title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution.",
+      "project.publications.doi": "10.1016/j.cell.2019.01.017",
+      "project.publications.pmid": "30735633",
+      "project.publications.url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6544371/"
+    }
+  ],
+  "Project - Funders": [
+    {
+      "project.funders.grant_id": "U01 MH105989",
+      "project.funders.organization": "NIMH NIH HHS"
+    },
+    {
+      "project.funders.grant_id": "R35 NS097305",
+      "project.funders.organization": "NINDS NIH HHS"
+    },
+    {
+      "project.funders.grant_id": "T32 HD007470",
+      "project.funders.organization": "NICHD NIH HHS"
+    },
+    {
+      "project.funders.grant_id": "T32 GM007266",
+      "project.funders.organization": "NIGMS NIH HHS"
+    },
+    {
+      "project.funders.grant_id": "F32 NS103266",
+      "project.funders.organization": "NINDS NIH HHS"
+    },
+    {
+      "project.funders.grant_id": "NA",
+      "project.funders.organization": "Howard Hughes Medical Institute"
+    },
+    {
+      "project.funders.grant_id": "P51 OD011132",
+      "project.funders.organization": "NIH HHS"
+    }
+  ]
+}