From afc8e2772a76cb12b21356b336a4c35ae72ef5c7 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Thu, 8 Jul 2021 17:50:54 +0100 Subject: [PATCH 01/49] WIP - create workbook from given JSON --- ingest/downloader/downloader.py | 22 ++++++++++ tests/unit/downloader/test_xls_generation.py | 44 ++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 tests/unit/downloader/test_xls_generation.py diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index ae4738ac..7664bb00 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -1,3 +1,6 @@ +from openpyxl import Workbook +from openpyxl.worksheet.worksheet import Worksheet + EXCLUDE_KEYS = ['describedBy', 'schema_type'] @@ -37,3 +40,22 @@ def flatten(self, content, output, parent_key=''): def get_concrete_entity(self, content): return content.get('describedBy').rsplit('/', 1)[-1] + + @staticmethod + def create_workbook(input_json: dict) -> Workbook: + workbook = Workbook() + workbook.remove(workbook.active) + + for ws_title, ws_elements in input_json.items(): + worksheet: Worksheet = workbook.create_sheet(title=ws_title) + XlsDownloader.add_worksheet_content(worksheet, ws_elements) + + return workbook + + @staticmethod + def add_worksheet_content(worksheet, ws_elements: dict): + row = col = 1 + for header, cell_value in ws_elements.items(): + worksheet.cell(row=row, column=col, value=header) + worksheet.cell(row=row + 1, column=col, value=cell_value) + col += 1 diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py new file mode 100644 index 00000000..cdd6bc13 --- /dev/null +++ b/tests/unit/downloader/test_xls_generation.py @@ -0,0 +1,44 @@ +import unittest + +from openpyxl import Workbook +from openpyxl.worksheet.worksheet import Worksheet + +from ingest.downloader.downloader import XlsDownloader + + +class MyTestCase(unittest.TestCase): + def setUp(self) -> None: + self.downloader = XlsDownloader() + self.workbook = Workbook() + self.workbook.create_sheet(title='Project') + + def test_given_input_json_successfully_creates_a_workbook(self): + #given + project_sheet_title = 'Project' + input_json = { + project_sheet_title: { + 'project.uuid': '3e329187-a9c4-48ec-90e3-cc45f7c2311c', + 'project.project_core.project_short_name': 'kriegsteinBrainOrganoids', + 'project.project_core.project_title': 'Establishing Cerebral Organoids as Models' + } + } + + workbook: Workbook = self.downloader.create_workbook(input_json) + + #expect + self.assertEqual(len(workbook.worksheets), 1) + project_sheet: Worksheet = workbook[project_sheet_title] + self.assertEqual(project_sheet.title, project_sheet_title) + + rows = list(project_sheet.rows) + header_row = rows.pop(0) + for cell in header_row: + self.assertTrue(cell.value in input_json[project_sheet_title].keys()) + + for row in rows: + for cell in row: + self.assertTrue(cell.value in input_json[project_sheet_title].values()) + + +if __name__ == '__main__': + unittest.main() From 578b89ec1aa6c2a32892b53b872abf69b755c007 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 10:52:51 +0100 Subject: [PATCH 02/49] Added test cases --- tests/unit/downloader/test_downloader.py | 119 ++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index 7437525d..c3673817 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -6,8 +6,41 @@ class XlsDownloaderTest(TestCase): - def test_convert_json(self): - self.maxDiff = None + def test_convert_json__has_no_modules(self): + # given + entity_list = [{ + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "label", + "project_title": "title", + "project_description": "desc" + } + }, + 'uuid': { + 'uuid': 'uuid1' + } + }] + + # when + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + expected = { + 'project': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc' + } + ] + } + + self.assertEqual(actual, expected) + + def test_convert_json__has_string_arrays(self): # given entity_list = [{ 'content': { @@ -17,16 +50,96 @@ def test_convert_json(self): "project_short_name": "label", "project_title": "title", "project_description": "desc" + }, + "insdc_project_accessions": [ + "SRP180337" + ], + "geo_series_accessions": [ + "GSE124298", "GSE124299" + ] + }, + 'uuid': { + 'uuid': 'uuid1' + } + }] + + # when + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + expected = { + 'project': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc', + 'project.insdc_project_accessions': 'SRP180337', + 'project.geo_series_accessions': "GSE124298||GSE124299" } + ] + } + + self.assertEqual(actual, expected) + + def test_convert_json__has_modules(self): + # given + entity_list = [{ + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "label", + "project_title": "title", + "project_description": "desc" + }, + "contributors": [ + { + "name": "Alex A,,Pollen", + "email": "alex.pollen@ucsf.edu", + "institution": "University of California, San Francisco (UCSF)", + "laboratory": "Department of Neurology", + "country": "USA", + "corresponding_contributor": true, + "project_role": { + "text": "experimental scientist", + "ontology": "EFO:0009741", + "ontology_label": "experimental scientist" + } + } + ] }, 'uuid': { 'uuid': 'uuid1' } }] + # when + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + expected = { + 'project': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc' + } + ] + } + + # then + self.assertEqual(actual, expected) + + def test_convert_json__project_metadata(self): + self.maxDiff = None + + # given with open('project-list.json') as file: entity_list = json.load(file) + # when downloader = XlsDownloader() actual = downloader.convert_json(entity_list) @@ -44,3 +157,5 @@ def test_convert_json(self): # then self.assertEqual(actual, expected) + + From 357cd6c60d3f471f4b8cbcb57ea14e0b92ac3b50 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 12:20:50 +0100 Subject: [PATCH 03/49] Support object list with modules --- ingest/downloader/downloader.py | 56 +++++++----- .../downloader/project-list-flattened.json | 73 ++++++++++++++- tests/unit/downloader/test_downloader.py | 89 +++++++++++++++++-- 3 files changed, 190 insertions(+), 28 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 7664bb00..1ea468a5 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -1,3 +1,5 @@ +from typing import List + from openpyxl import Workbook from openpyxl.worksheet.worksheet import Worksheet @@ -5,20 +7,33 @@ class XlsDownloader: - def convert_json(self, entity_list): - workbook = {} - - for entity in entity_list: - content = entity['content'] - concrete_type = self.get_concrete_entity(content) - worksheet = workbook.get(concrete_type, []) - workbook[concrete_type] = worksheet - row = {f'{concrete_type}.uuid': entity['uuid']['uuid']} - self.flatten(content, row, parent_key='project') - worksheet.append(row) - return workbook + def __init__(self): + self.workbook = {} + + def convert_json(self, entity_list: List[dict]): + self.flatten_object_list(entity_list) + return self.workbook + + def flatten_object_list(self, object_list: List[dict], object_key: str = ''): + for entity in object_list: + worksheet_name = object_key + row = {} + content = entity + + if not object_key: + content = entity['content'] + worksheet_name = self.get_concrete_entity(content) + row = {f'{worksheet_name}.uuid': entity['uuid']['uuid']} - def flatten(self, content, output, parent_key=''): + if not worksheet_name: + raise Exception('There should be a worksheet name') + + rows = self.workbook.get(worksheet_name, []) + self.workbook[worksheet_name] = rows + self.flatten_object(content, row, parent_key=worksheet_name) + rows.append(row) + + def flatten_object(self, content: dict, output: dict, parent_key: str = ''): if isinstance(content, dict): for key in content: full_key = f'{parent_key}.{key}' if parent_key else key @@ -27,18 +42,19 @@ def flatten(self, content, output, parent_key=''): value = content[key] if isinstance(value, dict) or isinstance(value, list): - self.flatten(value, output, parent_key=full_key) + self.flatten_object(value, output, parent_key=full_key) else: - output[full_key] = value + output[full_key] = str(value) elif isinstance(content, list): - for elem in content: - if isinstance(elem, dict): - self.flatten(elem, output, parent_key=parent_key) - else: + if content and isinstance(content[0], dict): + self.flatten_object_list(content, parent_key) + else: + for elem in content: stringified = [str(e) for e in content] output[parent_key] = '||'.join(stringified) - def get_concrete_entity(self, content): + @staticmethod + def get_concrete_entity(content): return content.get('describedBy').rsplit('/', 1)[-1] @staticmethod diff --git a/tests/unit/downloader/project-list-flattened.json b/tests/unit/downloader/project-list-flattened.json index 0500ad8c..7c0dd7fa 100644 --- a/tests/unit/downloader/project-list-flattened.json +++ b/tests/unit/downloader/project-list-flattened.json @@ -11,6 +11,77 @@ } ], "project.contributors": [ - + { + "project.contributors.name": "Alex A,,Pollen", + "project.contributors.email": "alex.pollen@ucsf.edu", + "project.contributors.institution": "University of California, San Francisco (UCSF)", + "project.contributors.laboratory": "Department of Neurology", + "project.contributors.country": "USA", + "project.contributors.corresponding_contributor": "True", + "project.contributors.project_role.text": "experimental scientist", + "project.contributors.project_role.ontology": "EFO:0009741", + "project.contributors.project_role.ontology_label": "experimental scientist" + }, + { + "project.contributors.name": "Parisa,,Nejad", + "project.contributors.email": "pnejad@ucsc.edu", + "project.contributors.institution": "University of California, Santa Cruz", + "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform", + "project.contributors.country": "USA", + "project.contributors.corresponding_contributor": "False", + "project.contributors.project_role.text": "data wrangler", + "project.contributors.project_role.ontology": "EFO:0009737", + "project.contributors.project_role.ontology_label": "data curator" + }, + { + "project.contributors.name": "Schwartz,,Rachel", + "project.contributors.email": "raschwar@ucsc.edu", + "project.contributors.institution": "University of California, Santa Cruz", + "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform", + "project.contributors.country": "USA", + "project.contributors.corresponding_contributor": "False", + "project.contributors.project_role.text": "data wrangler", + "project.contributors.project_role.ontology": "EFO:0009737", + "project.contributors.project_role.ontology_label": "data curator" + } + ], + "project.publications": [ + { + "project.publications.authors": "Pollen AA||Bhaduri A||Andrews MG||Nowakowski TJ||Meyerson OS||Mostajo-Radji MA||Di Lullo E||Alvarado B||Bedolli M||Dougherty ML||Fiddes IT||Kronenberg ZN||Shuga J||Leyrat AA||West JA||Bershteyn M||Lowe CB||Pavlovic BJ||Salama SR||Haussler D||Eichler EE||Kriegstein AR", + "project.publications.title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution.", + "project.publications.doi": "10.1016/j.cell.2019.01.017", + "project.publications.pmid": "30735633", + "project.publications.url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6544371/" + } + ], + "project.funders": [ + { + "project.funders.grant_id": "U01 MH105989", + "project.funders.organization": "NIMH NIH HHS" + }, + { + "project.funders.grant_id": "R35 NS097305", + "project.funders.organization": "NINDS NIH HHS" + }, + { + "project.funders.grant_id": "T32 HD007470", + "project.funders.organization": "NICHD NIH HHS" + }, + { + "project.funders.grant_id": "T32 GM007266", + "project.funders.organization": "NIGMS NIH HHS" + }, + { + "project.funders.grant_id": "F32 NS103266", + "project.funders.organization": "NINDS NIH HHS" + }, + { + "project.funders.grant_id": "NA", + "project.funders.organization": "Howard Hughes Medical Institute" + }, + { + "project.funders.grant_id": "P51 OD011132", + "project.funders.organization": "NIH HHS" + } ] } \ No newline at end of file diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index c3673817..10adb459 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -100,7 +100,7 @@ def test_convert_json__has_modules(self): "institution": "University of California, San Francisco (UCSF)", "laboratory": "Department of Neurology", "country": "USA", - "corresponding_contributor": true, + "corresponding_contributor": True, "project_role": { "text": "experimental scientist", "ontology": "EFO:0009741", @@ -115,6 +115,7 @@ def test_convert_json__has_modules(self): }] # when + downloader = XlsDownloader() actual = downloader.convert_json(entity_list) @@ -126,19 +127,77 @@ def test_convert_json__has_modules(self): 'project.project_core.project_title': 'title', 'project.project_core.project_description': 'desc' } + ], + 'project.contributors': [ + {'project.contributors.corresponding_contributor': 'True', + 'project.contributors.country': 'USA', + 'project.contributors.email': 'alex.pollen@ucsf.edu', + 'project.contributors.institution': 'University of California, San Francisco (UCSF)', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.name': 'Alex A,,Pollen', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.project_role.text': 'experimental scientist'} ] } # then self.assertEqual(actual, expected) - def test_convert_json__project_metadata(self): - self.maxDiff = None - + def test_convert_json__has_boolean(self): # given - with open('project-list.json') as file: - entity_list = json.load(file) + entity_list = [{ + 'content': { + 'describedBy': 'https://schema.humancellatlas.org/type/project/14.2.0/project', + 'schema_type': 'project', + 'project_core': { + 'project_short_name': 'label', + 'project_title': 'title', + 'project_description': 'desc' + }, + 'boolean_field': True + }, + 'uuid': { + 'uuid': 'uuid1' + } + }] + + # when + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + expected = { + 'project': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc', + 'project.boolean_field': 'True' + } + ] + } + + # then + self.assertEqual(actual, expected) + def test_convert_json__has_integer(self): + # given + entity_list = [{ + 'content': { + 'describedBy': 'https://schema.humancellatlas.org/type/project/14.2.0/project', + 'schema_type': 'project', + 'project_core': { + 'project_short_name': 'label', + 'project_title': 'title', + 'project_description': 'desc' + }, + 'int_field': 1 + }, + 'uuid': { + 'uuid': 'uuid1' + } + }] # when downloader = XlsDownloader() @@ -150,7 +209,8 @@ def test_convert_json__project_metadata(self): 'project.uuid': 'uuid1', 'project.project_core.project_short_name': 'label', 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc' + 'project.project_core.project_description': 'desc', + 'project.int_field': '1' } ] } @@ -158,4 +218,19 @@ def test_convert_json__project_metadata(self): # then self.assertEqual(actual, expected) + def test_convert_json__project_metadata(self): + self.maxDiff = None + + # given + with open('project-list.json') as file: + entity_list = json.load(file) + + # when + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + with open('project-list-flattened.json') as file: + expected = json.load(file) + + # then + self.assertEqual(actual, expected) From f09205dd266204aca18ebfeef52955e9bdedcfe1 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 12:33:35 +0100 Subject: [PATCH 04/49] Optimise conversion of scalar list --- ingest/downloader/downloader.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 1ea468a5..005b843b 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -40,18 +40,19 @@ def flatten_object(self, content: dict, output: dict, parent_key: str = ''): if key in EXCLUDE_KEYS: continue value = content[key] - if isinstance(value, dict) or isinstance(value, list): self.flatten_object(value, output, parent_key=full_key) else: output[full_key] = str(value) elif isinstance(content, list): - if content and isinstance(content[0], dict): + if self.is_object_list(content): self.flatten_object_list(content, parent_key) else: - for elem in content: - stringified = [str(e) for e in content] - output[parent_key] = '||'.join(stringified) + stringified = [str(e) for e in content] + output[parent_key] = '||'.join(stringified) + + def is_object_list(self, content): + return content and isinstance(content[0], dict) @staticmethod def get_concrete_entity(content): From f606b50fd3264760c420aa4e76a6cede65656c9a Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 12:41:33 +0100 Subject: [PATCH 05/49] Made some functions private --- ingest/downloader/downloader.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 005b843b..e09a7e03 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -11,10 +11,10 @@ def __init__(self): self.workbook = {} def convert_json(self, entity_list: List[dict]): - self.flatten_object_list(entity_list) + self._flatten_object_list(entity_list) return self.workbook - def flatten_object_list(self, object_list: List[dict], object_key: str = ''): + def _flatten_object_list(self, object_list: List[dict], object_key: str = ''): for entity in object_list: worksheet_name = object_key row = {} @@ -30,26 +30,26 @@ def flatten_object_list(self, object_list: List[dict], object_key: str = ''): rows = self.workbook.get(worksheet_name, []) self.workbook[worksheet_name] = rows - self.flatten_object(content, row, parent_key=worksheet_name) + self._flatten_object(content, row, parent_key=worksheet_name) rows.append(row) - def flatten_object(self, content: dict, output: dict, parent_key: str = ''): - if isinstance(content, dict): - for key in content: + def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): + if isinstance(object, dict): + for key in object: full_key = f'{parent_key}.{key}' if parent_key else key if key in EXCLUDE_KEYS: continue - value = content[key] + value = object[key] if isinstance(value, dict) or isinstance(value, list): - self.flatten_object(value, output, parent_key=full_key) + self._flatten_object(value, flattened_object, parent_key=full_key) else: - output[full_key] = str(value) - elif isinstance(content, list): - if self.is_object_list(content): - self.flatten_object_list(content, parent_key) + flattened_object[full_key] = str(value) + elif isinstance(object, list): + if self.is_object_list(object): + self._flatten_object_list(object, parent_key) else: - stringified = [str(e) for e in content] - output[parent_key] = '||'.join(stringified) + stringified = [str(e) for e in object] + flattened_object[parent_key] = '||'.join(stringified) def is_object_list(self, content): return content and isinstance(content[0], dict) From eeee548bc36fdf25272f73f64360c41870ad9d69 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 13:13:08 +0100 Subject: [PATCH 06/49] Convert to user-friendly worksheet names --- ingest/downloader/downloader.py | 11 +++++++++-- tests/unit/downloader/project-list-flattened.json | 8 ++++---- tests/unit/downloader/test_downloader.py | 13 ++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index e09a7e03..db05faa0 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -28,8 +28,10 @@ def _flatten_object_list(self, object_list: List[dict], object_key: str = ''): if not worksheet_name: raise Exception('There should be a worksheet name') - rows = self.workbook.get(worksheet_name, []) - self.workbook[worksheet_name] = rows + user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) + + rows = self.workbook.get(user_friendly_worksheet_name, []) + self.workbook[user_friendly_worksheet_name] = rows self._flatten_object(content, row, parent_key=worksheet_name) rows.append(row) @@ -51,6 +53,11 @@ def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str stringified = [str(e) for e in object] flattened_object[parent_key] = '||'.join(stringified) + def _format_worksheet_name(self, worksheet_name): + names = worksheet_name.split('.') + new_worksheet_name = ' - '.join([n.capitalize() for n in names]) + return new_worksheet_name + def is_object_list(self, content): return content and isinstance(content[0], dict) diff --git a/tests/unit/downloader/project-list-flattened.json b/tests/unit/downloader/project-list-flattened.json index 7c0dd7fa..942b9163 100644 --- a/tests/unit/downloader/project-list-flattened.json +++ b/tests/unit/downloader/project-list-flattened.json @@ -1,5 +1,5 @@ { - "project": [ + "Project": [ { "project.uuid": "3e329187-a9c4-48ec-90e3-cc45f7c2311c", "project.project_core.project_short_name": "kriegsteinBrainOrganoids", @@ -10,7 +10,7 @@ "project.insdc_study_accessions": "PRJNA515930" } ], - "project.contributors": [ + "Project - Contributors": [ { "project.contributors.name": "Alex A,,Pollen", "project.contributors.email": "alex.pollen@ucsf.edu", @@ -45,7 +45,7 @@ "project.contributors.project_role.ontology_label": "data curator" } ], - "project.publications": [ + "Project - Publications": [ { "project.publications.authors": "Pollen AA||Bhaduri A||Andrews MG||Nowakowski TJ||Meyerson OS||Mostajo-Radji MA||Di Lullo E||Alvarado B||Bedolli M||Dougherty ML||Fiddes IT||Kronenberg ZN||Shuga J||Leyrat AA||West JA||Bershteyn M||Lowe CB||Pavlovic BJ||Salama SR||Haussler D||Eichler EE||Kriegstein AR", "project.publications.title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution.", @@ -54,7 +54,7 @@ "project.publications.url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6544371/" } ], - "project.funders": [ + "Project - Funders": [ { "project.funders.grant_id": "U01 MH105989", "project.funders.organization": "NIMH NIH HHS" diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index 10adb459..c9d15c39 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -5,7 +5,6 @@ class XlsDownloaderTest(TestCase): - def test_convert_json__has_no_modules(self): # given entity_list = [{ @@ -28,7 +27,7 @@ def test_convert_json__has_no_modules(self): actual = downloader.convert_json(entity_list) expected = { - 'project': [ + 'Project': [ { 'project.uuid': 'uuid1', 'project.project_core.project_short_name': 'label', @@ -68,7 +67,7 @@ def test_convert_json__has_string_arrays(self): actual = downloader.convert_json(entity_list) expected = { - 'project': [ + 'Project': [ { 'project.uuid': 'uuid1', 'project.project_core.project_short_name': 'label', @@ -120,7 +119,7 @@ def test_convert_json__has_modules(self): actual = downloader.convert_json(entity_list) expected = { - 'project': [ + 'Project': [ { 'project.uuid': 'uuid1', 'project.project_core.project_short_name': 'label', @@ -128,7 +127,7 @@ def test_convert_json__has_modules(self): 'project.project_core.project_description': 'desc' } ], - 'project.contributors': [ + 'Project - Contributors': [ {'project.contributors.corresponding_contributor': 'True', 'project.contributors.country': 'USA', 'project.contributors.email': 'alex.pollen@ucsf.edu', @@ -167,7 +166,7 @@ def test_convert_json__has_boolean(self): actual = downloader.convert_json(entity_list) expected = { - 'project': [ + 'Project': [ { 'project.uuid': 'uuid1', 'project.project_core.project_short_name': 'label', @@ -204,7 +203,7 @@ def test_convert_json__has_integer(self): actual = downloader.convert_json(entity_list) expected = { - 'project': [ + 'Project': [ { 'project.uuid': 'uuid1', 'project.project_core.project_short_name': 'label', From 5072a1c70c726388d51a20db2a7b1a103ffc71ef Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 13:29:33 +0100 Subject: [PATCH 07/49] Handle multiple entities of different types --- ingest/downloader/downloader.py | 1 + tests/unit/downloader/test_downloader.py | 83 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index db05faa0..d95f6676 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -55,6 +55,7 @@ def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str def _format_worksheet_name(self, worksheet_name): names = worksheet_name.split('.') + names = [n.replace('_', ' ') for n in names] new_worksheet_name = ' - '.join([n.capitalize() for n in names]) return new_worksheet_name diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index c9d15c39..3413901c 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -233,3 +233,86 @@ def test_convert_json__project_metadata(self): # then self.assertEqual(actual, expected) + + def test_convert_json__has_different_entities(self): + # given + entity_list = [{ + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "label", + "project_title": "title", + "project_description": "desc" + }, + "contributors": [ + { + "name": "Alex A,,Pollen", + "email": "alex.pollen@ucsf.edu", + "institution": "University of California, San Francisco (UCSF)", + "laboratory": "Department of Neurology", + "country": "USA", + "corresponding_contributor": True, + "project_role": { + "text": "experimental scientist", + "ontology": "EFO:0009741", + "ontology_label": "experimental scientist" + } + } + ] + }, + 'uuid': { + 'uuid': 'uuid1' + } + }, + { + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "label", + "biomaterial_description": "desc" + } + }, + 'uuid': { + 'uuid': 'uuid1' + } + } + ] + + # when + + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + expected = { + 'Project': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc' + } + ], + 'Project - Contributors': [ + {'project.contributors.corresponding_contributor': 'True', + 'project.contributors.country': 'USA', + 'project.contributors.email': 'alex.pollen@ucsf.edu', + 'project.contributors.institution': 'University of California, San Francisco (UCSF)', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.name': 'Alex A,,Pollen', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.project_role.text': 'experimental scientist'} + ], + 'Donor organism': [ + { + 'donor_organism.uuid': 'uuid1', + 'donor_organism.biomaterial_core.biomaterial_id': 'label', + 'donor_organism.biomaterial_core.biomaterial_description': 'desc' + } + ], + } + + # then + self.assertEqual(actual, expected) From db17cced64c307709d5212ce79b755ae5ca72a28 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 9 Jul 2021 13:42:34 +0100 Subject: [PATCH 08/49] Updated test data to have multiple entities of same type --- tests/unit/downloader/test_downloader.py | 40 +++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index 3413901c..8ce62862 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -265,19 +265,32 @@ def test_convert_json__has_different_entities(self): 'uuid': 'uuid1' } }, - { - 'content': { - "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "label", - "biomaterial_description": "desc" + { + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "label", + "biomaterial_description": "desc" + } + }, + 'uuid': { + 'uuid': 'uuid2' } }, - 'uuid': { - 'uuid': 'uuid1' + { + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "label", + "biomaterial_description": "desc" + } + }, + 'uuid': { + 'uuid': 'uuid3' + } } - } ] # when @@ -307,7 +320,12 @@ def test_convert_json__has_different_entities(self): ], 'Donor organism': [ { - 'donor_organism.uuid': 'uuid1', + 'donor_organism.uuid': 'uuid2', + 'donor_organism.biomaterial_core.biomaterial_id': 'label', + 'donor_organism.biomaterial_core.biomaterial_description': 'desc' + }, + { + 'donor_organism.uuid': 'uuid3', 'donor_organism.biomaterial_core.biomaterial_id': 'label', 'donor_organism.biomaterial_core.biomaterial_description': 'desc' } From 6fd08d763b711fbf37a9459c7afe6baf23739b75 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Fri, 9 Jul 2021 16:53:33 +0100 Subject: [PATCH 09/49] Test workbook creation with more than 1 data row --- ingest/downloader/downloader.py | 30 ++++++--- tests/unit/downloader/test_xls_generation.py | 70 +++++++++++++++++++- 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index d95f6676..5edff216 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -9,6 +9,7 @@ class XlsDownloader: def __init__(self): self.workbook = {} + self.row = 1 def convert_json(self, entity_list: List[dict]): self._flatten_object_list(entity_list) @@ -66,21 +67,32 @@ def is_object_list(self, content): def get_concrete_entity(content): return content.get('describedBy').rsplit('/', 1)[-1] - @staticmethod - def create_workbook(input_json: dict) -> Workbook: + def create_workbook(self, input_json: dict) -> Workbook: workbook = Workbook() workbook.remove(workbook.active) for ws_title, ws_elements in input_json.items(): worksheet: Worksheet = workbook.create_sheet(title=ws_title) - XlsDownloader.add_worksheet_content(worksheet, ws_elements) + self.add_worksheet_content(worksheet, ws_elements) return workbook - @staticmethod - def add_worksheet_content(worksheet, ws_elements: dict): - row = col = 1 - for header, cell_value in ws_elements.items(): - worksheet.cell(row=row, column=col, value=header) - worksheet.cell(row=row + 1, column=col, value=cell_value) + def add_worksheet_content(self, worksheet, ws_elements: dict): + is_header = True + if isinstance(ws_elements, list): + for content in ws_elements: + self.add_row_content(worksheet, content, is_header) + is_header = False + self.row += 1 + else: + self.add_row_content(worksheet, ws_elements) + + def add_row_content(self, worksheet, content, is_header=True): + col = 1 + for header, cell_value in content.items(): + if is_header: + self.row = 1 + worksheet.cell(row=self.row, column=col, value=header) + self.row += 1 + worksheet.cell(row=self.row, column=col, value=cell_value) col += 1 diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py index cdd6bc13..1f926338 100644 --- a/tests/unit/downloader/test_xls_generation.py +++ b/tests/unit/downloader/test_xls_generation.py @@ -12,7 +12,7 @@ def setUp(self) -> None: self.workbook = Workbook() self.workbook.create_sheet(title='Project') - def test_given_input_json_successfully_creates_a_workbook(self): + def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self): #given project_sheet_title = 'Project' input_json = { @@ -23,6 +23,7 @@ def test_given_input_json_successfully_creates_a_workbook(self): } } + # when workbook: Workbook = self.downloader.create_workbook(input_json) #expect @@ -39,6 +40,73 @@ def test_given_input_json_successfully_creates_a_workbook(self): for cell in row: self.assertTrue(cell.value in input_json[project_sheet_title].values()) + def test_given_input_has_many_rows_of_data_successfully_creates_a_workbook(self): + #given + contributors_sheet_title = 'Project - Contributors' + + input_json = { + contributors_sheet_title: [ + { + 'project.contributors.name': 'Karel Gott', + 'project.contributors.email': 'karel@gott.com', + 'project.contributors.institution': 'University of Prague', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.address': '123 Test Street', + 'project.contributors.country': 'UK', + 'project.contributors.corresponding_contributor': 'yes', + 'project.contributors.project_role.text': 'experimental scientist', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.orcid_id': 'https://orcid.org/1234-5678-9012-3456' + }, + { + 'project.contributors.name': 'Lady Carneval', + 'project.contributors.email': 'ladyc@gott.com', + 'project.contributors.institution': 'University of Prague', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.address': '123 Test Street', + 'project.contributors.country': 'UK', + 'project.contributors.corresponding_contributor': 'yes', + 'project.contributors.project_role.text': 'experimental scientist', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.orcid_id': 'https://orcid.org/9999-9999-9999-9999' + }, + { + 'project.contributors.name': 'Baby Shark', + 'project.contributors.email': 'sharkb@gott.com', + 'project.contributors.institution': 'University of Cambridge', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.address': '123 Ocean Drive', + 'project.contributors.country': 'USA', + 'project.contributors.corresponding_contributor': 'no', + 'project.contributors.project_role.text': 'data wrangler', + 'project.contributors.project_role.ontology': 'EFO:0009737', + 'project.contributors.project_role.ontology_label': 'data curator', + 'project.contributors.orcid_id': 'https://orcid.org/0000-0000-0000-0001' + }, + ] + } + + # when + workbook: Workbook = self.downloader.create_workbook(input_json) + + #expect + self.assertEqual(len(workbook.worksheets), 1) + contributors_sheet: Worksheet = workbook[contributors_sheet_title] + self.assertEqual(contributors_sheet.title, contributors_sheet_title) + + rows = list(contributors_sheet.rows) + header_row = rows.pop(0) + for cell in header_row: + self.assertTrue(cell.value in input_json[contributors_sheet_title][0].keys()) + + i = 0 + for row in rows: + for cell in row: + self.assertTrue(cell.value in input_json[contributors_sheet_title][i].values()) + i += 1 + if __name__ == '__main__': unittest.main() From fb4938b9cd963242cda72d261c185d885a5c35c0 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Mon, 12 Jul 2021 10:13:10 +0100 Subject: [PATCH 10/49] Add test for creating multiple worksheets --- tests/unit/downloader/test_xls_generation.py | 38 +++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py index 1f926338..3a8d68a7 100644 --- a/tests/unit/downloader/test_xls_generation.py +++ b/tests/unit/downloader/test_xls_generation.py @@ -1,3 +1,4 @@ +import json import unittest from openpyxl import Workbook @@ -28,6 +29,7 @@ def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self #expect self.assertEqual(len(workbook.worksheets), 1) + project_sheet: Worksheet = workbook[project_sheet_title] self.assertEqual(project_sheet.title, project_sheet_title) @@ -93,18 +95,44 @@ def test_given_input_has_many_rows_of_data_successfully_creates_a_workbook(self) #expect self.assertEqual(len(workbook.worksheets), 1) - contributors_sheet: Worksheet = workbook[contributors_sheet_title] - self.assertEqual(contributors_sheet.title, contributors_sheet_title) - rows = list(contributors_sheet.rows) + self.__assert_sheet(workbook, contributors_sheet_title, input_json) + + def test_given_input_has_many_rows_and_many_sheets_of_data_successfully_creates_a_workbook(self): + # given + project_sheet_title = 'Project' + contributors_sheet_title = 'Project - Contributors' + publications_sheet_title = 'Project - Publications' + funders_sheet_title = 'Project - Funders' + + # when + with open('project-list-flattened.json') as file: + input_json = json.load(file) + + #when + workbook: Workbook = self.downloader.create_workbook(input_json) + + # expect + self.assertEqual(len(workbook.worksheets), 4) + + self.__assert_sheet(workbook, project_sheet_title, input_json) + self.__assert_sheet(workbook, contributors_sheet_title, input_json) + self.__assert_sheet(workbook, publications_sheet_title, input_json) + self.__assert_sheet(workbook, funders_sheet_title, input_json) + + def __assert_sheet(self, workbook, sheet_title, input_json): + sheet: Worksheet = workbook[sheet_title] + self.assertEqual(sheet.title, sheet_title) + + rows = list(sheet.rows) header_row = rows.pop(0) for cell in header_row: - self.assertTrue(cell.value in input_json[contributors_sheet_title][0].keys()) + self.assertTrue(cell.value in input_json[sheet_title][0].keys()) i = 0 for row in rows: for cell in row: - self.assertTrue(cell.value in input_json[contributors_sheet_title][i].values()) + self.assertTrue(cell.value in input_json[sheet_title][i].values()) i += 1 From 37aa89548037e3cd58ebb720765823792fd43938 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Tue, 13 Jul 2021 10:55:26 +0100 Subject: [PATCH 11/49] Initial version of submission data collector --- ingest/downloader/data_collector.py | 17 ++ tests/unit/downloader/test_data_collector.py | 37 ++++ tests/unit/resources/mock_project.json | 175 +++++++++++++++++++ tests/unit/resources/mock_submission.json | 107 ++++++++++++ 4 files changed, 336 insertions(+) create mode 100644 ingest/downloader/data_collector.py create mode 100644 tests/unit/downloader/test_data_collector.py create mode 100644 tests/unit/resources/mock_project.json create mode 100644 tests/unit/resources/mock_submission.json diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py new file mode 100644 index 00000000..c36dc60a --- /dev/null +++ b/ingest/downloader/data_collector.py @@ -0,0 +1,17 @@ +from ingest.api.ingestapi import IngestApi + + +class DataCollector: + + def __init__(self, ingest_api: IngestApi): + self.api = ingest_api + + def collect_data_by_submission_uuid(self, submission_uuid): + submission = self.api.get_submission_by_uuid(submission_uuid).json + + project_json = self.api.get_related_entities('relatedProjects', submission, 'project').json + + data_by_submission = [ + project_json + ] + return data_by_submission diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py new file mode 100644 index 00000000..d6488a78 --- /dev/null +++ b/tests/unit/downloader/test_data_collector.py @@ -0,0 +1,37 @@ +import json +import unittest +from unittest.mock import MagicMock + +from ingest.api.ingestapi import IngestApi +from ingest.downloader.data_collector import DataCollector + + +class MyTestCase(unittest.TestCase): + + def setUp(self) -> None: + self.maxDiff = None + self.mock_ingest_api = MagicMock(spec=IngestApi) + self.data_collector = DataCollector(self.mock_ingest_api) + + def test_collected_data_by_submission_returns_correctly(self): + #given + project_uuid = '1234' + with open('../resources/mock_submission.json') as file: + mock_submission_json = json.load(file) + with open('../resources/mock_project.json') as file: + mock_project_json = json.load(file) + self.mock_ingest_api.get_submission_by_uuid.return_value.json = mock_submission_json + self.mock_ingest_api.get_related_entities.return_value.json = mock_project_json + expected_json = [ + mock_project_json + ] + + #when + result_json = self.data_collector.collect_data_by_submission_uuid(project_uuid) + + #then + self.assertEqual(result_json, expected_json) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/resources/mock_project.json b/tests/unit/resources/mock_project.json new file mode 100644 index 00000000..c36c1d7b --- /dev/null +++ b/tests/unit/resources/mock_project.json @@ -0,0 +1,175 @@ +{ + "content": { + "array_express_accessions": [ + "E-MTAB-1234" + ], + "contributors": [ + { + "name": "John Doe", + "institution": "Cambridge Genome Institute", + "orcid_id": "0000-0000-0000-0001" + }, + { + "name": "Jonas Smith", + "institution": "Eddington Research Centre", + "project_role": { + "text": "data curator", + "ontology": "EFO:0009737", + "ontology_label": "data curator" + }, + "orcid_id": "0000-0000-0000-0002" + }, + { + "name": "Lucy Orange", + "institution": "Molecular Digestive Oncology Centre", + "project_role": { + "text": "principal investigator", + "ontology": "EFO:0009736", + "ontology_label": "principal investigator" + }, + "orcid_id": "0000-0000-0000-0003" + } + ], + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "funders": [ + { + "grant_title": "Test Project 1", + "grant_id": "ABC-1234M3A9A78437847", + "organization": "Ministry of Science" + }, + { + "grant_title": "Test Project 2", + "grant_id": "HGHG847487JKJK", + "organization": "Hungarian Cancer Federation" + } + ], + "geo_series_accessions": [ + "GSE123456", + "GSE949548", + "GSE948958" + ], + "insdc_project_accessions": [ + "ERP123456" + ], + "insdc_study_accessions": [ + "PRJNA948595", + "PRJNA945859", + "PRJNA452145" + ], + "project_core": { + "project_short_name": "GSE123456_cancer", + "project_title": "Colorectal cancer.", + "project_description": "Immunotherapy for metastatic colorectal cancer is effective only for mismatch repair-deficient tumors with high microsatellite instability that demonstrate immune infiltration, suggesting that tumor cells can determine their immune microenvironment. To understand this cross-talk, we analyzed the transcriptome of 91,103 unsorted single cells from 23 Korean and 6 Belgian patients. Cancer cells displayed transcriptional features reminiscent of normal differentiation programs, and genetic alterations that apparently fostered immunosuppressive microenvironments directed by regulatory T cells, myofibroblasts and myeloid cells. Intercellular network reconstruction supported the association between cancer cell signatures and specific stromal or immune cell populations. Our collective view of the cellular landscape and intercellular interactions in colorectal cancer provide mechanistic information for the design of efficient immuno-oncology treatment strategies." + }, + "publications": [ + { + "authors": [ + "Karel G", + "Bela N", + "John S", + "Karen S" + ], + "title": "Lorem ipsum", + "doi": "11.2222/s5556-345-2445-g", + "pmid": 483782962, + "url": "https://doi.org/11.2222/s5556-345-2445-g" + } + ], + "schema_type": "project" + }, + "submissionDate": "2011-06-11T14:01:10.585Z", + "updateDate": "2011-07-09T17:47:29.091Z", + "user": "5ecce575524ebf7778be111b", + "lastModifiedUser": "5ecce575524ebf7778be111b", + "type": "Project", + "uuid": { + "uuid": "12345678-dc7c-44a6-7536-446363267351" + }, + "events": [], + "firstDcpVersion": "2011-06-11T14:01:10.585Z", + "dcpVersion": "2011-07-06T15:19:59.392Z", + "contentLastModified": "2011-07-06T15:19:59.392Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "releaseDate": null, + "accessionDate": null, + "technology": { + "ontologies": [ + { + "text": "10x 3' v2", + "ontology": "EFO:0009899", + "ontology_label": "10x 3' v2" + } + ] + }, + "organ": { + "ontologies": [ + { + "text": "colon", + "ontology": "UBERON:0001155", + "ontology_label": "colon" + } + ] + }, + "cellCount": 109512, + "dataAccess": { + "type": "A mixture of open and managed" + }, + "identifyingOrganisms": [ + "Human" + ], + "primaryWrangler": "5ecce575524ebf7778be111b", + "secondaryWrangler": null, + "wranglingState": "Eligible", + "wranglingPriority": 3, + "wranglingNotes": "tumor single cell atlas data; AutoImported Snapshot 2021-06-03", + "isInCatalogue": true, + "cataloguedDate": "2011-06-11T14:01:10.585Z", + "publicationsInfo": [ + { + "doi": "11.2222/s5556-345-2445-g", + "journalTitle": "Nature genetics", + "title": "Lorem ipsum", + "authors": [ + "Karel G", + "Bela N", + "John S", + "Karen S" + ], + "url": "https://doi.org/11.2222/s5556-345-2445-g" + } + ], + "hasOpenSubmission": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73", + "title": "A single project" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73/draftEvent" + }, + "bundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73/bundleManifests", + "title": "Access or create bundle manifests (describing which submitted contents went into which bundle in the datastore)" + }, + "supplementaryFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73/supplementaryFiles" + }, + "submissionEnvelopes": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73/submissionEnvelopes", + "title": "Access or create new submission envelopes" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/12c36ca6ac56ec2ecbb54a73/submissionEnvelope", + "title": "A single submission envelope" + } + } +} \ No newline at end of file diff --git a/tests/unit/resources/mock_submission.json b/tests/unit/resources/mock_submission.json new file mode 100644 index 00000000..42a6f8e0 --- /dev/null +++ b/tests/unit/resources/mock_submission.json @@ -0,0 +1,107 @@ +{ + "submissionDate": "2021-07-09T17:47:20.437Z", + "updateDate": "2021-07-09T17:47:39.422Z", + "user": "5ecce7cb524ebf7978be201d", + "lastModifiedUser": "anonymousUser", + "type": "Submission", + "uuid": { + "uuid": "2251fbc1-06a0-464d-a15d-158a3ef58b27" + }, + "events": [], + "stagingDetails": { + "stagingAreaUuid": { + "uuid": "2251fbc1-06a0-464d-a15d-158a3ef58b27" + }, + "stagingAreaLocation": { + "value": "s3://org-hca-data-archive-upload-prod/2251fbc1-06a0-464d-a15d-158a3ef58b27/" + } + }, + "submissionState": "Invalid", + "triggersAnalysis": true, + "isUpdate": false, + "submitActions": [], + "open": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c", + "title": "A single submission envelope" + }, + "biomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials" + }, + "processes": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/processes" + }, + "files": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/files", + "title": "Access or create, within a submission envelope, a new assay" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/protocols", + "title": "Access or create protocols" + }, + "bundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/bundleManifests", + "title": "Access or create bundle manifests (describing which submitted contents went into which bundle in the datastore)" + }, + "submissionManifest": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/submissionManifest" + }, + "exportJobs": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/exportJobs" + }, + "submissionEnvelopeErrors": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/submissionErrors", + "title": "All submission errors for a single envelope" + }, + "documentSmReport": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/documentSmReport" + }, + "summary": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/summary" + }, + "commitDraft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitDraftEvent" + }, + "commitValidating": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitValidatingEvent" + }, + "commitValid": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitValidEvent" + }, + "commitInvalid": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitInvalidEvent" + }, + "commitSubmit": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitSubmissionEvent" + }, + "commitProcessing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitProcessingEvent" + }, + "commitArchiving": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitArchivingEvent" + }, + "commitArchived": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitArchivedEvent" + }, + "commitExporting": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitExportingEvent" + }, + "commitExported": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitExportedEvent" + }, + "commitCleanup": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitCleanupEvent" + }, + "commitComplete": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/commitCompleteEvent" + } + } +} \ No newline at end of file From 29e1834a3333d1b022dd3239fb7f8bbfe3d17323 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Tue, 13 Jul 2021 11:00:50 +0100 Subject: [PATCH 12/49] Prefer if condition which controls flow to be done first in the for loop --- ingest/downloader/downloader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 5edff216..967b749e 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -39,10 +39,11 @@ def _flatten_object_list(self, object_list: List[dict], object_key: str = ''): def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): if isinstance(object, dict): for key in object: - full_key = f'{parent_key}.{key}' if parent_key else key if key in EXCLUDE_KEYS: continue + value = object[key] + full_key = f'{parent_key}.{key}' if parent_key else key if isinstance(value, dict) or isinstance(value, list): self._flatten_object(value, flattened_object, parent_key=full_key) else: From 74b103b39ee7cf77c08a1b20cb8263abedb183ca Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Tue, 13 Jul 2021 11:30:58 +0100 Subject: [PATCH 13/49] Refactor arrange part of unit tests --- tests/unit/downloader/project-list.json | 187 +++++++++++++++++++ tests/unit/downloader/test_downloader.py | 222 +++++++++-------------- 2 files changed, 274 insertions(+), 135 deletions(-) create mode 100644 tests/unit/downloader/project-list.json diff --git a/tests/unit/downloader/project-list.json b/tests/unit/downloader/project-list.json new file mode 100644 index 00000000..ad18bd43 --- /dev/null +++ b/tests/unit/downloader/project-list.json @@ -0,0 +1,187 @@ +[ + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "kriegsteinBrainOrganoids", + "project_title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution", + "project_description": "Direct comparisons of human and non-human primate brain tissue have the potential to reveal molecular pathways underlying remarkable specializations of the human brain. However, chimpanzee tissue is largely inaccessible during neocortical neurogenesis when differences in brain size first appear. To identify human-specific features of cortical development, we leveraged recent innovations that permit generating pluripotent stem cell-derived cerebral organoids from chimpanzee. First, we systematically evaluated the fidelity of organoid models to primary human and macaque cortex, finding organoid models preserve gene regulatory networks related to cell types and developmental processes but exhibit increased metabolic stress. Second, we identified 261 genes differentially expressed in human compared to chimpanzee organoids and macaque cortex. Many of these genes overlap with human-specific segmental duplications and a subset suggest increased PI3K/AKT/mTOR activation in human outer radial glia. Together, our findings establish a platform for systematic analysis of molecular changes contributing to human brain development and evolution. Overall design: Single cell mRNA sequencing of iPS-derived neural and glial progenitor cells using the Fluidigm C1 system This series includes re-analysis of publicly available data in accessions: phs000989.v3, GSE99951, GSE86207, GSE75140. Sample metadata and accession IDs for the re-analyzed samples are included in the file \"GSE124299_metadata_on_processed_samples.xlsx\" available on the foot of this record. The following samples have no raw data due to data loss: GSM3569728, GSM3569738, GSM3571601, GSM3571606, GSM3571615, GSM3571621, GSM3571625, and GSM3571631" + }, + "insdc_project_accessions": [ + "SRP180337" + ], + "geo_series_accessions": [ + "GSE124299" + ], + "insdc_study_accessions": [ + "PRJNA515930" + ], + "contributors": [ + { + "name": "Alex A,,Pollen", + "email": "alex.pollen@ucsf.edu", + "institution": "University of California, San Francisco (UCSF)", + "laboratory": "Department of Neurology", + "country": "USA", + "corresponding_contributor": true, + "project_role": { + "text": "experimental scientist", + "ontology": "EFO:0009741", + "ontology_label": "experimental scientist" + } + }, + { + "name": "Parisa,,Nejad", + "email": "pnejad@ucsc.edu", + "institution": "University of California, Santa Cruz", + "laboratory": "Human Cell Atlas Data Coordination Platform", + "country": "USA", + "corresponding_contributor": false, + "project_role": { + "text": "data wrangler", + "ontology": "EFO:0009737", + "ontology_label": "data curator" + } + }, + { + "name": "Schwartz,,Rachel", + "email": "raschwar@ucsc.edu", + "institution": "University of California, Santa Cruz", + "laboratory": "Human Cell Atlas Data Coordination Platform", + "country": "USA", + "corresponding_contributor": false, + "project_role": { + "text": "data wrangler", + "ontology": "EFO:0009737", + "ontology_label": "data curator" + } + } + ], + "publications": [ + { + "authors": [ + "Pollen AA", + "Bhaduri A", + "Andrews MG", + "Nowakowski TJ", + "Meyerson OS", + "Mostajo-Radji MA", + "Di Lullo E", + "Alvarado B", + "Bedolli M", + "Dougherty ML", + "Fiddes IT", + "Kronenberg ZN", + "Shuga J", + "Leyrat AA", + "West JA", + "Bershteyn M", + "Lowe CB", + "Pavlovic BJ", + "Salama SR", + "Haussler D", + "Eichler EE", + "Kriegstein AR" + ], + "title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution.", + "doi": "10.1016/j.cell.2019.01.017", + "pmid": 30735633, + "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6544371/" + } + ], + "funders": [ + { + "grant_id": "U01 MH105989", + "organization": "NIMH NIH HHS" + }, + { + "grant_id": "R35 NS097305", + "organization": "NINDS NIH HHS" + }, + { + "grant_id": "T32 HD007470", + "organization": "NICHD NIH HHS" + }, + { + "grant_id": "T32 GM007266", + "organization": "NIGMS NIH HHS" + }, + { + "grant_id": "F32 NS103266", + "organization": "NINDS NIH HHS" + }, + { + "grant_id": "NA", + "organization": "Howard Hughes Medical Institute" + }, + { + "grant_id": "P51 OD011132", + "organization": "NIH HHS" + } + ] + }, + "submissionDate": "2021-06-24T19:49:44.720Z", + "updateDate": "2021-06-24T19:53:07.441Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Project", + "uuid": { + "uuid": "3e329187-a9c4-48ec-90e3-cc45f7c2311c" + }, + "events": [], + "firstDcpVersion": "2021-06-24T19:49:44.720Z", + "dcpVersion": "2021-06-24T19:49:44.720Z", + "contentLastModified": "2021-06-24T19:49:44.720Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "releaseDate": null, + "accessionDate": null, + "technology": null, + "organ": null, + "cellCount": null, + "dataAccess": null, + "identifyingOrganisms": null, + "primaryWrangler": null, + "secondaryWrangler": null, + "wranglingState": null, + "wranglingPriority": null, + "wranglingNotes": null, + "isInCatalogue": null, + "cataloguedDate": null, + "publicationsInfo": null, + "hasOpenSubmission": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0", + "title": "A single project" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0/draftEvent" + }, + "bundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0/bundleManifests", + "title": "Access or create bundle manifests (describing which submitted contents went into which bundle in the datastore)" + }, + "supplementaryFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0/supplementaryFiles" + }, + "submissionEnvelopes": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0/submissionEnvelopes", + "title": "Access or create new submission envelopes" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/projects/60d4e1d8d20b7e03a5b35fa0/submissionEnvelope", + "title": "A single submission envelope" + } + } + } +] \ No newline at end of file diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index 8ce62862..661a02e5 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -5,28 +5,22 @@ class XlsDownloaderTest(TestCase): - def test_convert_json__has_no_modules(self): - # given - entity_list = [{ - 'content': { - "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", - "schema_type": "project", - "project_core": { - "project_short_name": "label", - "project_title": "title", - "project_description": "desc" - } - }, - 'uuid': { - 'uuid': 'uuid1' + def setUp(self) -> None: + self.content = { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "label", + "project_title": "title", + "project_description": "desc" } - }] + } - # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + self.uuid = { + 'uuid': 'uuid1' + } - expected = { + self.flattened_metadata_entity = { 'Project': [ { 'project.uuid': 'uuid1', @@ -37,96 +31,77 @@ def test_convert_json__has_no_modules(self): ] } - self.assertEqual(actual, expected) - - def test_convert_json__has_string_arrays(self): + def test_convert_json__has_no_modules(self): # given - entity_list = [{ - 'content': { - "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", - "schema_type": "project", - "project_core": { - "project_short_name": "label", - "project_title": "title", - "project_description": "desc" - }, - "insdc_project_accessions": [ - "SRP180337" - ], - "geo_series_accessions": [ - "GSE124298", "GSE124299" - ] - }, - 'uuid': { - 'uuid': 'uuid1' - } - }] + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid + } + entity_list = [self.metadata_entity] # when downloader = XlsDownloader() actual = downloader.convert_json(entity_list) - expected = { - 'Project': [ - { - 'project.uuid': 'uuid1', - 'project.project_core.project_short_name': 'label', - 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc', - 'project.insdc_project_accessions': 'SRP180337', - 'project.geo_series_accessions': "GSE124298||GSE124299" - } + self.assertEqual(actual, self.flattened_metadata_entity) + + def test_convert_json__has_string_arrays(self): + # given + self.content.update({ + "insdc_project_accessions": [ + "SRP180337" + ], + "geo_series_accessions": [ + "GSE124298", "GSE124299" ] + }) + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid } + entity_list = [self.metadata_entity] - self.assertEqual(actual, expected) + # when + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + self.flattened_metadata_entity['Project'][0].update({ + 'project.insdc_project_accessions': 'SRP180337', + 'project.geo_series_accessions': "GSE124298||GSE124299" + }) + + self.assertEqual(actual, self.flattened_metadata_entity) def test_convert_json__has_modules(self): # given - entity_list = [{ - 'content': { - "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", - "schema_type": "project", - "project_core": { - "project_short_name": "label", - "project_title": "title", - "project_description": "desc" - }, - "contributors": [ - { - "name": "Alex A,,Pollen", - "email": "alex.pollen@ucsf.edu", - "institution": "University of California, San Francisco (UCSF)", - "laboratory": "Department of Neurology", - "country": "USA", - "corresponding_contributor": True, - "project_role": { - "text": "experimental scientist", - "ontology": "EFO:0009741", - "ontology_label": "experimental scientist" - } - } - ] - }, - 'uuid': { - 'uuid': 'uuid1' + self.content.update({"contributors": [ + { + "name": "Alex A,,Pollen", + "email": "alex.pollen@ucsf.edu", + "institution": "University of California, San Francisco (UCSF)", + "laboratory": "Department of Neurology", + "country": "USA", + "corresponding_contributor": True, + "project_role": { + "text": "experimental scientist", + "ontology": "EFO:0009741", + "ontology_label": "experimental scientist" + } } - }] + ]}) + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid + } + + entity_list = [self.metadata_entity] # when downloader = XlsDownloader() actual = downloader.convert_json(entity_list) - expected = { - 'Project': [ - { - 'project.uuid': 'uuid1', - 'project.project_core.project_short_name': 'label', - 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc' - } - ], + self.flattened_metadata_entity.update({ 'Project - Contributors': [ {'project.contributors.corresponding_contributor': 'True', 'project.contributors.country': 'USA', @@ -138,65 +113,42 @@ def test_convert_json__has_modules(self): 'project.contributors.project_role.ontology_label': 'experimental scientist', 'project.contributors.project_role.text': 'experimental scientist'} ] - } + }) # then - self.assertEqual(actual, expected) + self.assertEqual(actual, self.flattened_metadata_entity) def test_convert_json__has_boolean(self): # given - entity_list = [{ - 'content': { - 'describedBy': 'https://schema.humancellatlas.org/type/project/14.2.0/project', - 'schema_type': 'project', - 'project_core': { - 'project_short_name': 'label', - 'project_title': 'title', - 'project_description': 'desc' - }, - 'boolean_field': True - }, - 'uuid': { - 'uuid': 'uuid1' - } - }] + self.content.update({ + 'boolean_field': True + }) + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid + } + entity_list = [self.metadata_entity] # when downloader = XlsDownloader() actual = downloader.convert_json(entity_list) - expected = { - 'Project': [ - { - 'project.uuid': 'uuid1', - 'project.project_core.project_short_name': 'label', - 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc', - 'project.boolean_field': 'True' - } - ] - } - + self.flattened_metadata_entity['Project'][0].update({ + 'project.boolean_field': 'True' + }) # then - self.assertEqual(actual, expected) + self.assertEqual(actual, self.flattened_metadata_entity) def test_convert_json__has_integer(self): # given - entity_list = [{ - 'content': { - 'describedBy': 'https://schema.humancellatlas.org/type/project/14.2.0/project', - 'schema_type': 'project', - 'project_core': { - 'project_short_name': 'label', - 'project_title': 'title', - 'project_description': 'desc' - }, - 'int_field': 1 - }, - 'uuid': { - 'uuid': 'uuid1' - } - }] + self.content.update({ + 'int_field': 1 + }) + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid + } + entity_list = [self.metadata_entity] # when downloader = XlsDownloader() From 9259ef08e101c31ac53ebca783ada7ad028d6ec9 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Tue, 13 Jul 2021 12:28:54 +0100 Subject: [PATCH 14/49] Add biomaterials to the collected data --- ingest/downloader/data_collector.py | 12 +- tests/unit/downloader/test_data_collector.py | 15 +- tests/unit/resources/mock_biomaterials.json | 1886 ++++++++++++++++++ 3 files changed, 1907 insertions(+), 6 deletions(-) create mode 100644 tests/unit/resources/mock_biomaterials.json diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index c36dc60a..347c7398 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -9,9 +9,17 @@ def __init__(self, ingest_api: IngestApi): def collect_data_by_submission_uuid(self, submission_uuid): submission = self.api.get_submission_by_uuid(submission_uuid).json - project_json = self.api.get_related_entities('relatedProjects', submission, 'project').json - + project_json = self.api.get_related_entities('relatedProjects', submission, 'projects').json data_by_submission = [ project_json ] + + self.__get_biomaterials(data_by_submission, submission) + return data_by_submission + + def __get_biomaterials(self, data_by_submission, submission): + biomaterials_json = self.api.get_related_entities('biomaterials', submission, 'biomaterials').json + if biomaterials_json: + biomaterials_json = biomaterials_json['_embedded']['biomaterials'] + data_by_submission.extend(biomaterials_json) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index d6488a78..cf62316a 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -1,6 +1,6 @@ import json import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock from ingest.api.ingestapi import IngestApi from ingest.downloader.data_collector import DataCollector @@ -13,18 +13,25 @@ def setUp(self) -> None: self.mock_ingest_api = MagicMock(spec=IngestApi) self.data_collector = DataCollector(self.mock_ingest_api) - def test_collected_data_by_submission_returns_correctly(self): + def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_correctly(self): #given project_uuid = '1234' with open('../resources/mock_submission.json') as file: mock_submission_json = json.load(file) with open('../resources/mock_project.json') as file: mock_project_json = json.load(file) + with open('../resources/mock_biomaterials.json') as file: + mock_biomaterials_json = json.load(file) self.mock_ingest_api.get_submission_by_uuid.return_value.json = mock_submission_json - self.mock_ingest_api.get_related_entities.return_value.json = mock_project_json + + mock_data = [Mock(), Mock()] + mock_data[0].json = mock_project_json + mock_data[1].json = mock_biomaterials_json + self.mock_ingest_api.get_related_entities.side_effect = mock_data expected_json = [ - mock_project_json + mock_project_json, ] + expected_json.extend(mock_biomaterials_json['_embedded']['biomaterials']) #when result_json = self.data_collector.collect_data_by_submission_uuid(project_uuid) diff --git a/tests/unit/resources/mock_biomaterials.json b/tests/unit/resources/mock_biomaterials.json new file mode 100644 index 00000000..588794a5 --- /dev/null +++ b/tests/unit/resources/mock_biomaterials.json @@ -0,0 +1,1886 @@ +{ + "_embedded": { + "biomaterials": [ + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC01", + "biomaterial_name": "donor SMC01", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.326Z", + "updateDate": "2021-07-09T17:47:29.160Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "8a357669-d82c-47a7-b3a9-6425ae1323a8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.326Z", + "dcpVersion": "2021-07-09T17:47:24.326Z", + "contentLastModified": "2021-07-09T17:47:24.326Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC02", + "biomaterial_name": "donor SMC02", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.336Z", + "updateDate": "2021-07-09T17:47:29.210Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "3238e232-da75-4728-adfa-c0e6a2c3bd7b" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.336Z", + "dcpVersion": "2021-07-09T17:47:24.336Z", + "contentLastModified": "2021-07-09T17:47:24.336Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC03", + "biomaterial_name": "donor SMC03", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.345Z", + "updateDate": "2021-07-09T17:47:29.263Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "822f5071-30ff-49cf-a8c6-1e2f12e3ee4c" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.345Z", + "dcpVersion": "2021-07-09T17:47:24.345Z", + "contentLastModified": "2021-07-09T17:47:24.344Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC04", + "biomaterial_name": "donor SMC04", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.354Z", + "updateDate": "2021-07-09T17:47:29.406Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "8a445012-bcbd-41dc-8fd4-d148a10933d7" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.354Z", + "dcpVersion": "2021-07-09T17:47:24.354Z", + "contentLastModified": "2021-07-09T17:47:24.354Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC05", + "biomaterial_name": "donor SMC05", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.366Z", + "updateDate": "2021-07-09T17:47:29.510Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "6996e821-5066-429e-94d6-dc5db6418e35" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.366Z", + "dcpVersion": "2021-07-09T17:47:24.366Z", + "contentLastModified": "2021-07-09T17:47:24.366Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC06", + "biomaterial_name": "donor SMC06", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.376Z", + "updateDate": "2021-07-09T17:47:29.626Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "a7e36003-1d79-46cd-b505-c1cd0fdecdcd" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.376Z", + "dcpVersion": "2021-07-09T17:47:24.376Z", + "contentLastModified": "2021-07-09T17:47:24.376Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC07", + "biomaterial_name": "donor SMC07", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.386Z", + "updateDate": "2021-07-09T17:47:29.762Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "91f9f0a9-9c5c-4b81-9639-fb8dcc95c38b" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.386Z", + "dcpVersion": "2021-07-09T17:47:24.386Z", + "contentLastModified": "2021-07-09T17:47:24.385Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC08", + "biomaterial_name": "donor SMC08", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.395Z", + "updateDate": "2021-07-09T17:47:29.826Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "429bf76a-a651-454a-b0a0-c8cc0102eab1" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.395Z", + "dcpVersion": "2021-07-09T17:47:24.395Z", + "contentLastModified": "2021-07-09T17:47:24.395Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC09", + "biomaterial_name": "donor SMC09", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.411Z", + "updateDate": "2021-07-09T17:47:29.954Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "1e6725b8-c939-4db1-8f39-98a48b1d8d34" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.411Z", + "dcpVersion": "2021-07-09T17:47:24.411Z", + "contentLastModified": "2021-07-09T17:47:24.410Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC10", + "biomaterial_name": "donor SMC10", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.420Z", + "updateDate": "2021-07-09T17:47:30.054Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "08aeb0ab-e002-4aa2-9453-80683827b006" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.420Z", + "dcpVersion": "2021-07-09T17:47:24.420Z", + "contentLastModified": "2021-07-09T17:47:24.419Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC11", + "biomaterial_name": "donor SMC11", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.428Z", + "updateDate": "2021-07-09T17:47:30.087Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "2a2cbf92-92c0-4626-bac7-cae204a22b5d" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.428Z", + "dcpVersion": "2021-07-09T17:47:24.428Z", + "contentLastModified": "2021-07-09T17:47:24.428Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC14", + "biomaterial_name": "donor SMC14", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.437Z", + "updateDate": "2021-07-09T17:47:30.121Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "f67f9936-f6ae-4c0c-98df-9445b27141a9" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.437Z", + "dcpVersion": "2021-07-09T17:47:24.437Z", + "contentLastModified": "2021-07-09T17:47:24.437Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC15", + "biomaterial_name": "donor SMC15", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.452Z", + "updateDate": "2021-07-09T17:47:30.153Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "3fb0a252-6571-46d6-ad84-24fafef15be4" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.452Z", + "dcpVersion": "2021-07-09T17:47:24.452Z", + "contentLastModified": "2021-07-09T17:47:24.451Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC16", + "biomaterial_name": "donor SMC16", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.463Z", + "updateDate": "2021-07-09T17:47:30.197Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "0b194892-ac38-47b5-a8f9-53948213fea8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.463Z", + "dcpVersion": "2021-07-09T17:47:24.463Z", + "contentLastModified": "2021-07-09T17:47:24.462Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC17", + "biomaterial_name": "donor SMC17", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.472Z", + "updateDate": "2021-07-09T17:47:30.239Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "0fee3104-0a2d-4b67-b1c5-1aa5966ab80c" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.472Z", + "dcpVersion": "2021-07-09T17:47:24.472Z", + "contentLastModified": "2021-07-09T17:47:24.471Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC18", + "biomaterial_name": "donor SMC18", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.480Z", + "updateDate": "2021-07-09T17:47:30.273Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "fe4dd742-4f82-4383-b7cc-ab3f8647c004" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.480Z", + "dcpVersion": "2021-07-09T17:47:24.480Z", + "contentLastModified": "2021-07-09T17:47:24.480Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC19", + "biomaterial_name": "donor SMC19", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.489Z", + "updateDate": "2021-07-09T17:47:30.304Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "e250b33e-594a-4987-a574-89d5c031286e" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.489Z", + "dcpVersion": "2021-07-09T17:47:24.489Z", + "contentLastModified": "2021-07-09T17:47:24.488Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC20", + "biomaterial_name": "donor SMC20", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.497Z", + "updateDate": "2021-07-09T17:47:30.362Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "e28ff73d-56a0-4ec3-9d88-964913a143c6" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.497Z", + "dcpVersion": "2021-07-09T17:47:24.497Z", + "contentLastModified": "2021-07-09T17:47:24.497Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC21", + "biomaterial_name": "donor SMC21", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.506Z", + "updateDate": "2021-07-09T17:47:30.404Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "9c4ccb9f-9655-4abd-84e1-376eae6eb8a4" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.506Z", + "dcpVersion": "2021-07-09T17:47:24.506Z", + "contentLastModified": "2021-07-09T17:47:24.506Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC22", + "biomaterial_name": "donor SMC22", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.515Z", + "updateDate": "2021-07-09T17:47:30.463Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "367e775d-c843-4958-9880-d78a5c299458" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.515Z", + "dcpVersion": "2021-07-09T17:47:24.515Z", + "contentLastModified": "2021-07-09T17:47:24.514Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/submissionEnvelope", + "title": "A single submission envelope" + } + } + } + ] + }, + "_links": { + "first": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=0&size=20" + }, + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=0&size=20" + }, + "next": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=1&size=20" + }, + "last": { + "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=7&size=20" + } + }, + "page": { + "size": 20, + "totalElements": 153, + "totalPages": 8, + "number": 0 + } +} \ No newline at end of file From c0efbac7f8803eb82af9d850d4039abcf59fa4f7 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Tue, 13 Jul 2021 12:35:50 +0100 Subject: [PATCH 15/49] Remove json call when gathering the submission data --- ingest/downloader/data_collector.py | 6 +++--- tests/unit/downloader/test_data_collector.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index 347c7398..58d0843f 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -7,9 +7,9 @@ def __init__(self, ingest_api: IngestApi): self.api = ingest_api def collect_data_by_submission_uuid(self, submission_uuid): - submission = self.api.get_submission_by_uuid(submission_uuid).json + submission = self.api.get_submission_by_uuid(submission_uuid) - project_json = self.api.get_related_entities('relatedProjects', submission, 'projects').json + project_json = self.api.get_related_entities('relatedProjects', submission, 'projects') data_by_submission = [ project_json ] @@ -19,7 +19,7 @@ def collect_data_by_submission_uuid(self, submission_uuid): return data_by_submission def __get_biomaterials(self, data_by_submission, submission): - biomaterials_json = self.api.get_related_entities('biomaterials', submission, 'biomaterials').json + biomaterials_json = self.api.get_related_entities('biomaterials', submission, 'biomaterials') if biomaterials_json: biomaterials_json = biomaterials_json['_embedded']['biomaterials'] data_by_submission.extend(biomaterials_json) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index cf62316a..9ff3d2d8 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -22,11 +22,11 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_project_json = json.load(file) with open('../resources/mock_biomaterials.json') as file: mock_biomaterials_json = json.load(file) - self.mock_ingest_api.get_submission_by_uuid.return_value.json = mock_submission_json + self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json mock_data = [Mock(), Mock()] - mock_data[0].json = mock_project_json - mock_data[1].json = mock_biomaterials_json + mock_data[0] = mock_project_json + mock_data[1] = mock_biomaterials_json self.mock_ingest_api.get_related_entities.side_effect = mock_data expected_json = [ mock_project_json, From 23994209335d37abe432610a6f448cddee8fb00d Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Tue, 13 Jul 2021 13:30:58 +0100 Subject: [PATCH 16/49] Fix errors in collecting data for a submission --- ingest/downloader/data_collector.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index 58d0843f..62302e45 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -8,11 +8,9 @@ def __init__(self, ingest_api: IngestApi): def collect_data_by_submission_uuid(self, submission_uuid): submission = self.api.get_submission_by_uuid(submission_uuid) - - project_json = self.api.get_related_entities('relatedProjects', submission, 'projects') - data_by_submission = [ - project_json - ] + related_projects_url = f"{submission['_links']['self']['href']}/relatedProjects" + project_json = self.api.get_all(related_projects_url, 'projects') + data_by_submission = list(project_json) self.__get_biomaterials(data_by_submission, submission) @@ -21,5 +19,4 @@ def collect_data_by_submission_uuid(self, submission_uuid): def __get_biomaterials(self, data_by_submission, submission): biomaterials_json = self.api.get_related_entities('biomaterials', submission, 'biomaterials') if biomaterials_json: - biomaterials_json = biomaterials_json['_embedded']['biomaterials'] - data_by_submission.extend(biomaterials_json) + data_by_submission.extend(list(biomaterials_json)) From 7d0b65359f24f2087c095e25e5a430c02e4cf649 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Tue, 13 Jul 2021 13:31:58 +0100 Subject: [PATCH 17/49] Remove validation of uuid columns until module entities are being handled --- ingest/importer/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ingest/importer/importer.py b/ingest/importer/importer.py index f7c830b2..9f191f0f 100644 --- a/ingest/importer/importer.py +++ b/ingest/importer/importer.py @@ -187,8 +187,8 @@ def __init__(self, template_mgr): def do_import(self, workbook: IngestWorkbook, is_update, project_uuid=None): registry = _ImportRegistry(self.template_mgr) importable_worksheets = workbook.importable_worksheets() - - workbook_errors = self.validate_worksheets(is_update, importable_worksheets) + workbook_errors = [] + # workbook_errors = self.validate_worksheets(is_update, importable_worksheets) if project_uuid: project_metadata = MetadataEntity(domain_type=_PROJECT_TYPE, From a10cf5b79a1cf53ba2bbdbd8efdee6b36941d0bb Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Wed, 14 Jul 2021 10:13:33 +0100 Subject: [PATCH 18/49] Fix failing tests as implementation changed --- tests/unit/downloader/test_data_collector.py | 9 +- tests/unit/resources/mock_biomaterials.json | 3620 +++++++++--------- 2 files changed, 1802 insertions(+), 1827 deletions(-) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index 9ff3d2d8..0a966481 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -22,16 +22,15 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_project_json = json.load(file) with open('../resources/mock_biomaterials.json') as file: mock_biomaterials_json = json.load(file) + self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json + self.mock_ingest_api.get_all.return_value = iter([mock_project_json]) + self.mock_ingest_api.get_related_entities.return_value = iter(mock_biomaterials_json) - mock_data = [Mock(), Mock()] - mock_data[0] = mock_project_json - mock_data[1] = mock_biomaterials_json - self.mock_ingest_api.get_related_entities.side_effect = mock_data expected_json = [ mock_project_json, ] - expected_json.extend(mock_biomaterials_json['_embedded']['biomaterials']) + expected_json.extend(mock_biomaterials_json) #when result_json = self.data_collector.collect_data_by_submission_uuid(project_uuid) diff --git a/tests/unit/resources/mock_biomaterials.json b/tests/unit/resources/mock_biomaterials.json index 588794a5..590abcb9 100644 --- a/tests/unit/resources/mock_biomaterials.json +++ b/tests/unit/resources/mock_biomaterials.json @@ -1,1886 +1,1862 @@ -{ - "_embedded": { - "biomaterials": [ - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC01", - "biomaterial_name": "donor SMC01", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.326Z", - "updateDate": "2021-07-09T17:47:29.160Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "8a357669-d82c-47a7-b3a9-6425ae1323a8" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.326Z", - "dcpVersion": "2021-07-09T17:47:24.326Z", - "contentLastModified": "2021-07-09T17:47:24.326Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/submissionEnvelope", - "title": "A single submission envelope" +[ + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC01", + "biomaterial_name": "donor SMC01", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC02", - "biomaterial_name": "donor SMC02", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.336Z", - "updateDate": "2021-07-09T17:47:29.210Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "3238e232-da75-4728-adfa-c0e6a2c3bd7b" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.336Z", - "dcpVersion": "2021-07-09T17:47:24.336Z", - "contentLastModified": "2021-07-09T17:47:24.336Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/submissionEnvelope", - "title": "A single submission envelope" - } + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.326Z", + "updateDate": "2021-07-09T17:47:29.160Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "8a357669-d82c-47a7-b3a9-6425ae1323a8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.326Z", + "dcpVersion": "2021-07-09T17:47:24.326Z", + "contentLastModified": "2021-07-09T17:47:24.326Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC03", - "biomaterial_name": "donor SMC03", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.345Z", - "updateDate": "2021-07-09T17:47:29.263Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "822f5071-30ff-49cf-a8c6-1e2f12e3ee4c" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.345Z", - "dcpVersion": "2021-07-09T17:47:24.345Z", - "contentLastModified": "2021-07-09T17:47:24.344Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/submissionEnvelope", - "title": "A single submission envelope" + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709e/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC02", + "biomaterial_name": "donor SMC02", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC04", - "biomaterial_name": "donor SMC04", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.354Z", - "updateDate": "2021-07-09T17:47:29.406Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "8a445012-bcbd-41dc-8fd4-d148a10933d7" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.354Z", - "dcpVersion": "2021-07-09T17:47:24.354Z", - "contentLastModified": "2021-07-09T17:47:24.354Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/submissionEnvelope", - "title": "A single submission envelope" - } + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.336Z", + "updateDate": "2021-07-09T17:47:29.210Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "3238e232-da75-4728-adfa-c0e6a2c3bd7b" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.336Z", + "dcpVersion": "2021-07-09T17:47:24.336Z", + "contentLastModified": "2021-07-09T17:47:24.336Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC05", - "biomaterial_name": "donor SMC05", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.366Z", - "updateDate": "2021-07-09T17:47:29.510Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "6996e821-5066-429e-94d6-dc5db6418e35" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.366Z", - "dcpVersion": "2021-07-09T17:47:24.366Z", - "contentLastModified": "2021-07-09T17:47:24.366Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/submissionEnvelope", - "title": "A single submission envelope" + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf709f/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC03", + "biomaterial_name": "donor SMC03", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC06", - "biomaterial_name": "donor SMC06", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.376Z", - "updateDate": "2021-07-09T17:47:29.626Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "a7e36003-1d79-46cd-b505-c1cd0fdecdcd" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.376Z", - "dcpVersion": "2021-07-09T17:47:24.376Z", - "contentLastModified": "2021-07-09T17:47:24.376Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/submissionEnvelope", - "title": "A single submission envelope" - } + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.345Z", + "updateDate": "2021-07-09T17:47:29.263Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "822f5071-30ff-49cf-a8c6-1e2f12e3ee4c" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.345Z", + "dcpVersion": "2021-07-09T17:47:24.345Z", + "contentLastModified": "2021-07-09T17:47:24.344Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC07", - "biomaterial_name": "donor SMC07", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.386Z", - "updateDate": "2021-07-09T17:47:29.762Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "91f9f0a9-9c5c-4b81-9639-fb8dcc95c38b" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.386Z", - "dcpVersion": "2021-07-09T17:47:24.386Z", - "contentLastModified": "2021-07-09T17:47:24.385Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/submissionEnvelope", - "title": "A single submission envelope" + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a0/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC04", + "biomaterial_name": "donor SMC04", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.354Z", + "updateDate": "2021-07-09T17:47:29.406Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "8a445012-bcbd-41dc-8fd4-d148a10933d7" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.354Z", + "dcpVersion": "2021-07-09T17:47:24.354Z", + "contentLastModified": "2021-07-09T17:47:24.354Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a1/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC05", + "biomaterial_name": "donor SMC05", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC08", - "biomaterial_name": "donor SMC08", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.395Z", - "updateDate": "2021-07-09T17:47:29.826Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "429bf76a-a651-454a-b0a0-c8cc0102eab1" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.395Z", - "dcpVersion": "2021-07-09T17:47:24.395Z", - "contentLastModified": "2021-07-09T17:47:24.395Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/submissionEnvelope", - "title": "A single submission envelope" + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.366Z", + "updateDate": "2021-07-09T17:47:29.510Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "6996e821-5066-429e-94d6-dc5db6418e35" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.366Z", + "dcpVersion": "2021-07-09T17:47:24.366Z", + "contentLastModified": "2021-07-09T17:47:24.366Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/draftEvent" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC09", - "biomaterial_name": "donor SMC09", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.411Z", - "updateDate": "2021-07-09T17:47:29.954Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "1e6725b8-c939-4db1-8f39-98a48b1d8d34" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.411Z", - "dcpVersion": "2021-07-09T17:47:24.411Z", - "contentLastModified": "2021-07-09T17:47:24.410Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/submissionEnvelope", - "title": "A single submission envelope" + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a2/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC06", + "biomaterial_name": "donor SMC06", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.376Z", + "updateDate": "2021-07-09T17:47:29.626Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "a7e36003-1d79-46cd-b505-c1cd0fdecdcd" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.376Z", + "dcpVersion": "2021-07-09T17:47:24.376Z", + "contentLastModified": "2021-07-09T17:47:24.376Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/derivedByProcesses" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC10", - "biomaterial_name": "donor SMC10", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.420Z", - "updateDate": "2021-07-09T17:47:30.054Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "08aeb0ab-e002-4aa2-9453-80683827b006" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.420Z", - "dcpVersion": "2021-07-09T17:47:24.420Z", - "contentLastModified": "2021-07-09T17:47:24.419Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/submissionEnvelope", - "title": "A single submission envelope" + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a3/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC07", + "biomaterial_name": "donor SMC07", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.386Z", + "updateDate": "2021-07-09T17:47:29.762Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "91f9f0a9-9c5c-4b81-9639-fb8dcc95c38b" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.386Z", + "dcpVersion": "2021-07-09T17:47:24.386Z", + "contentLastModified": "2021-07-09T17:47:24.385Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC11", - "biomaterial_name": "donor SMC11", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.428Z", - "updateDate": "2021-07-09T17:47:30.087Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "2a2cbf92-92c0-4626-bac7-cae204a22b5d" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.428Z", - "dcpVersion": "2021-07-09T17:47:24.428Z", - "contentLastModified": "2021-07-09T17:47:24.428Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/submissionEnvelope", - "title": "A single submission envelope" + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a4/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC08", + "biomaterial_name": "donor SMC08", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.395Z", + "updateDate": "2021-07-09T17:47:29.826Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "429bf76a-a651-454a-b0a0-c8cc0102eab1" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.395Z", + "dcpVersion": "2021-07-09T17:47:24.395Z", + "contentLastModified": "2021-07-09T17:47:24.395Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC14", - "biomaterial_name": "donor SMC14", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.437Z", - "updateDate": "2021-07-09T17:47:30.121Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "f67f9936-f6ae-4c0c-98df-9445b27141a9" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.437Z", - "dcpVersion": "2021-07-09T17:47:24.437Z", - "contentLastModified": "2021-07-09T17:47:24.437Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/submissionEnvelope", - "title": "A single submission envelope" + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a5/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC09", + "biomaterial_name": "donor SMC09", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.411Z", + "updateDate": "2021-07-09T17:47:29.954Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "1e6725b8-c939-4db1-8f39-98a48b1d8d34" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.411Z", + "dcpVersion": "2021-07-09T17:47:24.411Z", + "contentLastModified": "2021-07-09T17:47:24.410Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a6/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC10", + "biomaterial_name": "donor SMC10", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC15", - "biomaterial_name": "donor SMC15", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.452Z", - "updateDate": "2021-07-09T17:47:30.153Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "3fb0a252-6571-46d6-ad84-24fafef15be4" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.452Z", - "dcpVersion": "2021-07-09T17:47:24.452Z", - "contentLastModified": "2021-07-09T17:47:24.451Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/submissionEnvelope", - "title": "A single submission envelope" + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.420Z", + "updateDate": "2021-07-09T17:47:30.054Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "08aeb0ab-e002-4aa2-9453-80683827b006" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.420Z", + "dcpVersion": "2021-07-09T17:47:24.420Z", + "contentLastModified": "2021-07-09T17:47:24.419Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/draftEvent" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC16", - "biomaterial_name": "donor SMC16", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.463Z", - "updateDate": "2021-07-09T17:47:30.197Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "0b194892-ac38-47b5-a8f9-53948213fea8" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.463Z", - "dcpVersion": "2021-07-09T17:47:24.463Z", - "contentLastModified": "2021-07-09T17:47:24.462Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/submissionEnvelope", - "title": "A single submission envelope" + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a7/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC11", + "biomaterial_name": "donor SMC11", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC17", - "biomaterial_name": "donor SMC17", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.472Z", - "updateDate": "2021-07-09T17:47:30.239Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "0fee3104-0a2d-4b67-b1c5-1aa5966ab80c" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.472Z", - "dcpVersion": "2021-07-09T17:47:24.472Z", - "contentLastModified": "2021-07-09T17:47:24.471Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/submissionEnvelope", - "title": "A single submission envelope" + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.428Z", + "updateDate": "2021-07-09T17:47:30.087Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "2a2cbf92-92c0-4626-bac7-cae204a22b5d" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.428Z", + "dcpVersion": "2021-07-09T17:47:24.428Z", + "contentLastModified": "2021-07-09T17:47:24.428Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a8/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC14", + "biomaterial_name": "donor SMC14", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.437Z", + "updateDate": "2021-07-09T17:47:30.121Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "f67f9936-f6ae-4c0c-98df-9445b27141a9" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.437Z", + "dcpVersion": "2021-07-09T17:47:24.437Z", + "contentLastModified": "2021-07-09T17:47:24.437Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC18", - "biomaterial_name": "donor SMC18", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.480Z", - "updateDate": "2021-07-09T17:47:30.273Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "fe4dd742-4f82-4383-b7cc-ab3f8647c004" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.480Z", - "dcpVersion": "2021-07-09T17:47:24.480Z", - "contentLastModified": "2021-07-09T17:47:24.480Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/submissionEnvelope", - "title": "A single submission envelope" + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70a9/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC15", + "biomaterial_name": "donor SMC15", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.452Z", + "updateDate": "2021-07-09T17:47:30.153Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "3fb0a252-6571-46d6-ad84-24fafef15be4" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.452Z", + "dcpVersion": "2021-07-09T17:47:24.452Z", + "contentLastModified": "2021-07-09T17:47:24.451Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC19", - "biomaterial_name": "donor SMC19", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.489Z", - "updateDate": "2021-07-09T17:47:30.304Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "e250b33e-594a-4987-a574-89d5c031286e" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.489Z", - "dcpVersion": "2021-07-09T17:47:24.489Z", - "contentLastModified": "2021-07-09T17:47:24.488Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/submissionEnvelope", - "title": "A single submission envelope" + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70aa/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC16", + "biomaterial_name": "donor SMC16", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.463Z", + "updateDate": "2021-07-09T17:47:30.197Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "0b194892-ac38-47b5-a8f9-53948213fea8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.463Z", + "dcpVersion": "2021-07-09T17:47:24.463Z", + "contentLastModified": "2021-07-09T17:47:24.462Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ab/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC17", + "biomaterial_name": "donor SMC17", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC20", - "biomaterial_name": "donor SMC20", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.497Z", - "updateDate": "2021-07-09T17:47:30.362Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "e28ff73d-56a0-4ec3-9d88-964913a143c6" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.497Z", - "dcpVersion": "2021-07-09T17:47:24.497Z", - "contentLastModified": "2021-07-09T17:47:24.497Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/submissionEnvelope", - "title": "A single submission envelope" + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" } + ] + }, + "submissionDate": "2021-07-09T17:47:24.472Z", + "updateDate": "2021-07-09T17:47:30.239Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "0fee3104-0a2d-4b67-b1c5-1aa5966ab80c" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.472Z", + "dcpVersion": "2021-07-09T17:47:24.472Z", + "contentLastModified": "2021-07-09T17:47:24.471Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/draftEvent" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC21", - "biomaterial_name": "donor SMC21", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.506Z", - "updateDate": "2021-07-09T17:47:30.404Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "9c4ccb9f-9655-4abd-84e1-376eae6eb8a4" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.506Z", - "dcpVersion": "2021-07-09T17:47:24.506Z", - "contentLastModified": "2021-07-09T17:47:24.506Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/submissionEnvelope", - "title": "A single submission envelope" + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ac/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC18", + "biomaterial_name": "donor SMC18", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" }, - { - "content": { - "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", - "schema_type": "biomaterial", - "biomaterial_core": { - "biomaterial_id": "SMC22", - "biomaterial_name": "donor SMC22", - "biomaterial_description": "Donor has colorectal cancer.", - "ncbi_taxon_id": [ - 9606 - ] - }, - "human_specific": { - "ethnicity": [ - { - "text": "Korean", - "ontology": "HANCESTRO:0022", - "ontology_label": "Korean" - } - ] - }, - "genus_species": [ - { - "text": "Homo sapiens", - "ontology": "NCBITaxon:9606", - "ontology_label": "Homo sapiens" - } - ], - "sex": "unknown", - "is_living": "yes", - "development_stage": { - "text": "adult", - "ontology": "HsapDv:0000087", - "ontology_label": "human adult stage" - }, - "diseases": [ - { - "text": "colorectal cancer", - "ontology": "MONDO:0005575", - "ontology_label": "colorectal cancer" - } - ] - }, - "submissionDate": "2021-07-09T17:47:24.515Z", - "updateDate": "2021-07-09T17:47:30.463Z", - "user": null, - "lastModifiedUser": "anonymousUser", - "type": "Biomaterial", - "uuid": { - "uuid": "367e775d-c843-4958-9880-d78a5c299458" - }, - "events": [], - "firstDcpVersion": "2021-07-09T17:47:24.515Z", - "dcpVersion": "2021-07-09T17:47:24.515Z", - "contentLastModified": "2021-07-09T17:47:24.514Z", - "accession": null, - "validationState": "Valid", - "validationErrors": [], - "isUpdate": false, - "linked": true, - "_links": { - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1" - }, - "biomaterial": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1" - }, - "processing": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/processingEvent" - }, - "draft": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/draftEvent" - }, - "project": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/project", - "title": "A single project" - }, - "projects": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/projects", - "title": "Access or create projects. Creation can only be done inside a submission envelope" - }, - "inputToProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/inputToProcesses" - }, - "derivedByProcesses": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/derivedByProcesses" - }, - "submissionEnvelope": { - "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/submissionEnvelope", - "title": "A single submission envelope" + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.480Z", + "updateDate": "2021-07-09T17:47:30.273Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "fe4dd742-4f82-4383-b7cc-ab3f8647c004" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.480Z", + "dcpVersion": "2021-07-09T17:47:24.480Z", + "contentLastModified": "2021-07-09T17:47:24.480Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ad/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC19", + "biomaterial_name": "donor SMC19", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.489Z", + "updateDate": "2021-07-09T17:47:30.304Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "e250b33e-594a-4987-a574-89d5c031286e" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.489Z", + "dcpVersion": "2021-07-09T17:47:24.489Z", + "contentLastModified": "2021-07-09T17:47:24.488Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70ae/submissionEnvelope", + "title": "A single submission envelope" } - ] + } }, - "_links": { - "first": { - "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=0&size=20" + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC20", + "biomaterial_name": "donor SMC20", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] }, - "self": { - "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=0&size=20" + "submissionDate": "2021-07-09T17:47:24.497Z", + "updateDate": "2021-07-09T17:47:30.362Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "e28ff73d-56a0-4ec3-9d88-964913a143c6" }, - "next": { - "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=1&size=20" + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.497Z", + "dcpVersion": "2021-07-09T17:47:24.497Z", + "contentLastModified": "2021-07-09T17:47:24.497Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70af/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC21", + "biomaterial_name": "donor SMC21", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.506Z", + "updateDate": "2021-07-09T17:47:30.404Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "9c4ccb9f-9655-4abd-84e1-376eae6eb8a4" }, - "last": { - "href": "https://api.ingest.archive.data.humancellatlas.org/submissionEnvelopes/60e88ba8d5d575160aaf709c/biomaterials?page=7&size=20" + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.506Z", + "dcpVersion": "2021-07-09T17:47:24.506Z", + "contentLastModified": "2021-07-09T17:47:24.506Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b0/submissionEnvelope", + "title": "A single submission envelope" + } } }, - "page": { - "size": 20, - "totalElements": 153, - "totalPages": 8, - "number": 0 + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/biomaterial/15.5.0/donor_organism", + "schema_type": "biomaterial", + "biomaterial_core": { + "biomaterial_id": "SMC22", + "biomaterial_name": "donor SMC22", + "biomaterial_description": "Donor has colorectal cancer.", + "ncbi_taxon_id": [ + 9606 + ] + }, + "human_specific": { + "ethnicity": [ + { + "text": "Korean", + "ontology": "HANCESTRO:0022", + "ontology_label": "Korean" + } + ] + }, + "genus_species": [ + { + "text": "Homo sapiens", + "ontology": "NCBITaxon:9606", + "ontology_label": "Homo sapiens" + } + ], + "sex": "unknown", + "is_living": "yes", + "development_stage": { + "text": "adult", + "ontology": "HsapDv:0000087", + "ontology_label": "human adult stage" + }, + "diseases": [ + { + "text": "colorectal cancer", + "ontology": "MONDO:0005575", + "ontology_label": "colorectal cancer" + } + ] + }, + "submissionDate": "2021-07-09T17:47:24.515Z", + "updateDate": "2021-07-09T17:47:30.463Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Biomaterial", + "uuid": { + "uuid": "367e775d-c843-4958-9880-d78a5c299458" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:24.515Z", + "dcpVersion": "2021-07-09T17:47:24.515Z", + "contentLastModified": "2021-07-09T17:47:24.514Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1" + }, + "biomaterial": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/biomaterials/60e88bacd5d575160aaf70b1/submissionEnvelope", + "title": "A single submission envelope" + } + } } -} \ No newline at end of file +] From bb7b2ced85f1a7f39968a6ed6d731cd3e70b4e8e Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Wed, 14 Jul 2021 11:42:06 +0100 Subject: [PATCH 19/49] Add get related projects to IngestApi service class --- ingest/api/ingestapi.py | 12 +++++++++--- ingest/downloader/data_collector.py | 11 +++++++---- tests/unit/downloader/test_data_collector.py | 4 ++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ingest/api/ingestapi.py b/ingest/api/ingestapi.py index 18f342f2..9090696b 100644 --- a/ingest/api/ingestapi.py +++ b/ingest/api/ingestapi.py @@ -121,9 +121,15 @@ def getSubmissions(self): if r.status_code == requests.codes.ok: return json.loads(r.text)["_embedded"]["submissionEnvelopes"] - def getProjects(self, id): - submissionUrl = self.url + '/submissionEnvelopes/' + id + '/projects' - r = self.get(submissionUrl, headers=self.get_headers()) + def get_projects(self, submission_id): + return self.__get_projects_by_submission_id_and_type(submission_id, 'projects') + + def get_related_projects(self, submission_id): + return self.__get_projects_by_submission_id_and_type(submission_id, 'relatedProjects') + + def __get_projects_by_submission_id_and_type(self, submission_id, project_type): + submission_url = f'{self.url}/submissionEnvelopes/{submission_id}/{project_type}' + r = self.get(submission_url, headers=self.get_headers()) projects = [] if r.status_code == requests.codes.ok: projects = json.loads(r.text) diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index 62302e45..b4e2bbac 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -8,15 +8,18 @@ def __init__(self, ingest_api: IngestApi): def collect_data_by_submission_uuid(self, submission_uuid): submission = self.api.get_submission_by_uuid(submission_uuid) - related_projects_url = f"{submission['_links']['self']['href']}/relatedProjects" - project_json = self.api.get_all(related_projects_url, 'projects') - data_by_submission = list(project_json) + project_json = self.api.get_related_projects(submission_uuid) + + data_by_submission = [ + project_json + ] self.__get_biomaterials(data_by_submission, submission) return data_by_submission def __get_biomaterials(self, data_by_submission, submission): - biomaterials_json = self.api.get_related_entities('biomaterials', submission, 'biomaterials') + biomaterials_json = \ + self.api.get_related_entities('biomaterials', submission, 'biomaterials') if biomaterials_json: data_by_submission.extend(list(biomaterials_json)) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index 0a966481..fe413abd 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -1,6 +1,6 @@ import json import unittest -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock from ingest.api.ingestapi import IngestApi from ingest.downloader.data_collector import DataCollector @@ -24,7 +24,7 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_biomaterials_json = json.load(file) self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json - self.mock_ingest_api.get_all.return_value = iter([mock_project_json]) + self.mock_ingest_api.get_related_projects.return_value = mock_project_json self.mock_ingest_api.get_related_entities.return_value = iter(mock_biomaterials_json) expected_json = [ From 34c3271986b6928d8017d4eff33dae776e3fd1eb Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Thu, 15 Jul 2021 10:15:38 +0100 Subject: [PATCH 20/49] Write data header on row 4 --- ingest/downloader/downloader.py | 5 +++-- tests/unit/downloader/test_xls_generation.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 967b749e..1b110c02 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -4,12 +4,13 @@ from openpyxl.worksheet.worksheet import Worksheet EXCLUDE_KEYS = ['describedBy', 'schema_type'] +FIRST_DATA_ROW_NO = 4 class XlsDownloader: def __init__(self): self.workbook = {} - self.row = 1 + self.row = FIRST_DATA_ROW_NO def convert_json(self, entity_list: List[dict]): self._flatten_object_list(entity_list) @@ -92,7 +93,7 @@ def add_row_content(self, worksheet, content, is_header=True): col = 1 for header, cell_value in content.items(): if is_header: - self.row = 1 + self.row = FIRST_DATA_ROW_NO worksheet.cell(row=self.row, column=col, value=header) self.row += 1 worksheet.cell(row=self.row, column=col, value=cell_value) diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py index 3a8d68a7..7a6be44f 100644 --- a/tests/unit/downloader/test_xls_generation.py +++ b/tests/unit/downloader/test_xls_generation.py @@ -34,6 +34,7 @@ def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self self.assertEqual(project_sheet.title, project_sheet_title) rows = list(project_sheet.rows) + rows = rows[3:] header_row = rows.pop(0) for cell in header_row: self.assertTrue(cell.value in input_json[project_sheet_title].keys()) @@ -125,6 +126,7 @@ def __assert_sheet(self, workbook, sheet_title, input_json): self.assertEqual(sheet.title, sheet_title) rows = list(sheet.rows) + rows = rows[3:] header_row = rows.pop(0) for cell in header_row: self.assertTrue(cell.value in input_json[sheet_title][0].keys()) From 456a034f820b8da9fe9fa01eabdb773c5c457a2f Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Thu, 15 Jul 2021 15:06:49 +0100 Subject: [PATCH 21/49] Gathers data from protocols by submission id --- ingest/downloader/data_collector.py | 7 + tests/unit/downloader/test_data_collector.py | 7 +- tests/unit/resources/mock_protocols.json | 947 +++++++++++++++++++ 3 files changed, 959 insertions(+), 2 deletions(-) create mode 100644 tests/unit/resources/mock_protocols.json diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index b4e2bbac..74bc9765 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -15,6 +15,7 @@ def collect_data_by_submission_uuid(self, submission_uuid): ] self.__get_biomaterials(data_by_submission, submission) + self.__get_protocols(data_by_submission, submission) return data_by_submission @@ -23,3 +24,9 @@ def __get_biomaterials(self, data_by_submission, submission): self.api.get_related_entities('biomaterials', submission, 'biomaterials') if biomaterials_json: data_by_submission.extend(list(biomaterials_json)) + + def __get_protocols(self, data_by_submission, submission): + protocols_json = \ + self.api.get_related_entities('protocols', submission, 'protocols') + if protocols_json: + data_by_submission.extend(list(protocols_json)) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index fe413abd..6974febf 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -22,15 +22,18 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_project_json = json.load(file) with open('../resources/mock_biomaterials.json') as file: mock_biomaterials_json = json.load(file) + with open('../resources/mock_protocols.json') as file: + mock_protocols_json = json.load(file) self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json self.mock_ingest_api.get_related_projects.return_value = mock_project_json - self.mock_ingest_api.get_related_entities.return_value = iter(mock_biomaterials_json) + self.mock_ingest_api.get_related_entities.side_effect = \ + [iter(mock_biomaterials_json), iter(mock_protocols_json)] expected_json = [ mock_project_json, ] - expected_json.extend(mock_biomaterials_json) + expected_json.extend(mock_biomaterials_json + mock_protocols_json) #when result_json = self.data_collector.collect_data_by_submission_uuid(project_uuid) diff --git a/tests/unit/resources/mock_protocols.json b/tests/unit/resources/mock_protocols.json new file mode 100644 index 00000000..d1f57247 --- /dev/null +++ b/tests/unit/resources/mock_protocols.json @@ -0,0 +1,947 @@ +[ + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/biomaterial_collection/9.2.0/collection_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "collection_protocol", + "protocol_name": "Surgical resection", + "protocol_description": "After resection, samples from the tumor and non-malignant colon tissues were collected and immediately transferred for tissue preparation. Half of the tissues were subjected to single-cell isolation and the other half were cryopreserved for subsequent DNA and RNA extraction." + }, + "method": { + "text": "surgical resection", + "ontology": "EFO:0009744", + "ontology_label": "surgical resection" + } + }, + "submissionDate": "2021-07-09T17:47:25.900Z", + "updateDate": "2021-07-09T17:47:47.788Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "ecaa39ad-5cb2-4568-a19f-501ec59aedba" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.900Z", + "dcpVersion": "2021-07-09T17:47:25.900Z", + "contentLastModified": "2021-07-09T17:47:25.899Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7137" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7137", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7137/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7137/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7137/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7137/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/biomaterial_collection/6.2.0/dissociation_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "SMC_dissociation_protocol", + "protocol_name": "SMC protocol", + "protocol_description": "Samples using SMC tissue dissociation protocols used a tumor dissociation kit (Miltenyi Biotech, Germany), according to the manufacturer’s instructions. Briefly, tissues were cut into 2-4 mm pieces and transferred to C tubes containing an enzyme mix (Enzyme H, R, and A in RPMI1640 media)." + }, + "method": { + "text": "enzymatic dissociation", + "ontology": "EFO:0009128", + "ontology_label": "enzymatic dissociation" + } + }, + "submissionDate": "2021-07-09T17:47:25.909Z", + "updateDate": "2021-07-09T17:47:47.804Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "7606b04b-95bf-44ac-96a5-3205d2e0fa55" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.909Z", + "dcpVersion": "2021-07-09T17:47:25.909Z", + "contentLastModified": "2021-07-09T17:47:25.909Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7138" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7138", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7138/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7138/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7138/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7138/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/biomaterial_collection/6.2.0/dissociation_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "KUL_dissociation_protocol", + "protocol_name": "KUL protocol", + "protocol_description": "Samples using KUL3 tissue dissociation protocols were dissociated using an enzymatic manual tumor dissociation method. After rinsing with cold PBS, the tumor specimens were minced to obtain < 1mm3 pieces. The resulting pieces were incubated in the digestion buffer (2 mg/ml Collagenase P and 0.2 mg/ml DNase I in 10 ml DMEM) for 10 minutes at 37°C. The resulting suspension was filtered using a 40 um nylon mesh, mixed with 30 ml ice cold PBS/FBS and centrifuged at 300xG for 5 minutes at 4°C. The supernatant was removed and the pellet was resuspended with 1 ml RBC lysis buffer and incubated at RT for 5 minutes. The resulting suspension was centrifuged at 150xG for 5 minutes at 4°C and the supernatant is removed. The pellet is resuspended in 0.04% BSA and filtered using a 40um tip strainer. The number of cells in the resulting cell suspension was quantified by LUNA Counter." + }, + "method": { + "text": "enzymatic dissociation", + "ontology": "EFO:0009128", + "ontology_label": "enzymatic dissociation" + } + }, + "submissionDate": "2021-07-09T17:47:25.918Z", + "updateDate": "2021-07-09T17:47:47.814Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "d6cf99e7-5ef0-4065-9c3a-c6c7c136640f" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.918Z", + "dcpVersion": "2021-07-09T17:47:25.918Z", + "contentLastModified": "2021-07-09T17:47:25.918Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7139" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7139", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7139/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7139/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7139/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7139/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/biomaterial_collection/6.2.0/dissociation_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "Singapore_dissociation_protocol", + "protocol_name": "Singapore protocol", + "protocol_description": "The protocol consisted of tissue digestion by collagenase IV and DNases I in RPMI1640 media at 37°C for 1 h, followed by mechanical shearing with a 16 1/2 gauge needle, washing in PBS containing 1% BSA and 2 mM EDTA, and red blood cell lysis using ACK lysis buffer." + }, + "method": { + "text": "enzymatic dissociation", + "ontology": "EFO:0009128", + "ontology_label": "enzymatic dissociation" + } + }, + "submissionDate": "2021-07-09T17:47:25.929Z", + "updateDate": "2021-07-09T17:47:47.823Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "7e719ac2-b534-4522-8a4c-64d0de51f72d" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.929Z", + "dcpVersion": "2021-07-09T17:47:25.929Z", + "contentLastModified": "2021-07-09T17:47:25.928Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713a" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713a", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713a/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713a/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713a/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713a/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/biomaterial_collection/3.1.0/enrichment_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "SMC_enrichment_protocol1", + "protocol_name": "SMC tissue dissociation protocol – part 1", + "protocol_description": "Gentle MACS programs (h_tumor_01, 02, and 03) were run in a MACSmix tube rotator, with two 30-min incubation periods at 37°C between each run." + }, + "method": { + "text": "magnetic affinity cell sorting", + "ontology": "EFO:0009109", + "ontology_label": "magnetic affinity cell sorting" + } + }, + "submissionDate": "2021-07-09T17:47:25.938Z", + "updateDate": "2021-07-09T17:47:47.832Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "a9b6287d-af3f-4320-8004-c10692409e02" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.938Z", + "dcpVersion": "2021-07-09T17:47:25.938Z", + "contentLastModified": "2021-07-09T17:47:25.938Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713b" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713b", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713b/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713b/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713b/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713b/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/biomaterial_collection/3.1.0/enrichment_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "SMC_enrichment_protocol2", + "protocol_name": "SMC tissue dissociation protocol – part 2", + "protocol_description": "The digested samples were filtered through a 70-µm strainer, purified via a Ficoll-Paque PLUS (GE healthcare, USA) gradient, and cryopreserved in cell bank (Zenoaq, Japan) before sequencing." + }, + "method": { + "text": "density gradient centrifugation", + "ontology": "EFO:0009112", + "ontology_label": "density gradient centrifugation" + } + }, + "submissionDate": "2021-07-09T17:47:25.947Z", + "updateDate": "2021-07-09T17:47:47.842Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "4d19dc78-ed96-4652-a46f-73864bc1ec77" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.947Z", + "dcpVersion": "2021-07-09T17:47:25.947Z", + "contentLastModified": "2021-07-09T17:47:25.947Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713c" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713c", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713c/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713c/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713c/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713c/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/sequencing/6.2.0/library_preparation_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "library_preparation_protocol", + "protocol_name": "Chromium Single Cell 3’ v2 librarly preparation protocol.", + "protocol_description": "For freshly prepared samples, the single cell suspensions were loaded to Chromium system (10X genomics) immediately after cell dissociation targeting 5,000 cells per run. The cryopreserved single cell dissociates were loaded after a rapid thaw-wash cycle. Following the manufacturer’s instructions, barcoded sequencing libraries were generated using Chromium Single Cell 3’ v2 kits." + }, + "cell_barcode": { + "barcode_read": "Read 1", + "barcode_offset": 0, + "barcode_length": 16 + }, + "input_nucleic_acid_molecule": { + "text": "polyA RNA", + "ontology": "OBI:0000869", + "ontology_label": "polyA RNA" + }, + "nucleic_acid_source": "single cell", + "library_construction_method": { + "text": "10x 3' v2", + "ontology": "EFO:0009899", + "ontology_label": "10x 3' v2" + }, + "end_bias": "3 prime tag", + "primer": "poly-dT", + "strand": "first", + "umi_barcode": { + "barcode_read": "Read 1", + "barcode_offset": 16, + "barcode_length": 10 + } + }, + "submissionDate": "2021-07-09T17:47:25.957Z", + "updateDate": "2021-07-09T17:47:47.853Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "64db2348-3b6f-4ac1-8634-aaa2d03297d8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.957Z", + "dcpVersion": "2021-07-09T17:47:25.957Z", + "contentLastModified": "2021-07-09T17:47:25.957Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713d" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713d", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713d/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713d/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713d/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713d/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/sequencing/10.1.0/sequencing_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "GPL20301", + "protocol_name": "GPL20301 sequencing protocol.", + "protocol_description": "Sequenced across 8 lanes on a Hiseq 4000 platform (Illumina) targeting 100,000 reads per cell." + }, + "instrument_manufacturer_model": { + "text": "Illumina HiSeq 4000", + "ontology": "EFO:0008563", + "ontology_label": "Illumina HiSeq 4000" + }, + "paired_end": false, + "method": { + "text": "tag based single cell RNA sequencing", + "ontology": "EFO:0008440", + "ontology_label": "tag based single cell RNA sequencing" + }, + "10x": { + "fastq_method": "CellRanger", + "fastq_method_version": "2.1.0" + } + }, + "submissionDate": "2021-07-09T17:47:25.967Z", + "updateDate": "2021-07-09T17:47:47.864Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "f0101646-713d-4f4b-b561-d4e9fb4647f3" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.967Z", + "dcpVersion": "2021-07-09T17:47:25.967Z", + "contentLastModified": "2021-07-09T17:47:25.966Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713e" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713e", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713e/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713e/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713e/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713e/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/sequencing/10.1.0/sequencing_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "GPL16791", + "protocol_name": "GPL16791 sequencing protocol.", + "protocol_description": "Sequenced on a Hiseq 2500 platform (Illumina)." + }, + "instrument_manufacturer_model": { + "text": "Illumina HiSeq 2500", + "ontology": "EFO:0008565", + "ontology_label": "Illumina HiSeq 2500" + }, + "paired_end": false, + "method": { + "text": "tag based single cell RNA sequencing", + "ontology": "EFO:0008440", + "ontology_label": "tag based single cell RNA sequencing" + }, + "10x": { + "fastq_method": "CellRanger", + "fastq_method_version": "2.1.0" + } + }, + "submissionDate": "2021-07-09T17:47:25.976Z", + "updateDate": "2021-07-09T17:47:47.882Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "7875a3e9-5fdf-4f29-8ea7-78c416677568" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.976Z", + "dcpVersion": "2021-07-09T17:47:25.976Z", + "contentLastModified": "2021-07-09T17:47:25.975Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713f" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713f", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713f/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713f/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713f/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf713f/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/sequencing/10.1.0/sequencing_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "GPL24676", + "protocol_name": "GPL24676 sequencing protocol.", + "protocol_description": "Sequencing was performed on Illumina NextSeq 500 and Illumina NovaSeq 6000 for quality control and coverage respectively." + }, + "instrument_manufacturer_model": { + "text": "Illumina NovaSeq 6000", + "ontology": "EFO:0008637", + "ontology_label": "Illumina NovaSeq 6000" + }, + "paired_end": false, + "method": { + "text": "tag based single cell RNA sequencing", + "ontology": "EFO:0008440", + "ontology_label": "tag based single cell RNA sequencing" + }, + "10x": { + "fastq_method": "CellRanger", + "fastq_method_version": "2.1.0" + } + }, + "submissionDate": "2021-07-09T17:47:25.986Z", + "updateDate": "2021-07-09T17:47:47.895Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "8399a721-ef4f-4198-9db2-79cf8856bc72" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.986Z", + "dcpVersion": "2021-07-09T17:47:25.986Z", + "contentLastModified": "2021-07-09T17:47:25.985Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7140" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7140", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7140/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7140/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7140/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7140/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/analysis/9.2.0/analysis_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "expression_matrix_protocol1a", + "protocol_name": "Data processing protocol 1a.", + "protocol_description": "To eliminate doublets, cells with a number of genes exceeding the outliers were removed. The raw gene expression matrix from the CellRanger pipeline was filtered using the Seurat R package (version 2.3) and selected according to the following criteria: cells with > 1000 UMI counts, > 200 genes, and < 6,000 genes, and < 20% of mitochondrial gene expression in UMI counts. After filtering step, the raw gene expression matrix was normalized to the total UMI counts per cell." + }, + "type": { + "text": "data transformation", + "ontology": "OBI:0200000", + "ontology_label": "data transformation" + } + }, + "submissionDate": "2021-07-09T17:47:25.996Z", + "updateDate": "2021-07-09T17:47:47.916Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "4e32aaf5-6ae6-4e64-9155-74222fee8865" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:25.996Z", + "dcpVersion": "2021-07-09T17:47:25.996Z", + "contentLastModified": "2021-07-09T17:47:25.995Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7141" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7141", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7141/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7141/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7141/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88badd5d575160aaf7141/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/analysis/9.2.0/analysis_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "expression_matrix_protocol1b", + "protocol_name": "Data processing protocol 1b.", + "protocol_description": "To eliminate doublets, cells with a number of genes exceeding the outliers were removed. The raw gene expression matrix from the CellRanger pipeline was filtered using the Seurat R package (version 2.3) and selected according to the following criteria: cells with > 1000 UMI counts, > 200 genes, and < 6,000 genes, and < 20% of mitochondrial gene expression in UMI counts. After filtering step, the raw gene expression matrix was normalized to the total UMI counts per cell and transformed to the natural log scale." + }, + "type": { + "text": "data transformation", + "ontology": "OBI:0200000", + "ontology_label": "data transformation" + } + }, + "submissionDate": "2021-07-09T17:47:26.018Z", + "updateDate": "2021-07-09T17:47:47.925Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "6ccc8272-f922-4c8b-b211-810e8a6c430e" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.018Z", + "dcpVersion": "2021-07-09T17:47:26.018Z", + "contentLastModified": "2021-07-09T17:47:26.018Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7142" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7142", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7142/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7142/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7142/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7142/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/analysis/9.2.0/analysis_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "expression_matrix_protocol2a", + "protocol_name": "Data processing protocol 2a.", + "protocol_description": "The raw gene expression matrix from the CellRanger pipeline was filtered using the Seurat R package (version 2.3) and selected according to the following criteria: cells with > 1000 UMI counts, > 200 genes, and < 6,000 genes, and < 20% of mitochondrial gene expression in UMI counts." + }, + "type": { + "text": "data transformation", + "ontology": "OBI:0200000", + "ontology_label": "data transformation" + } + }, + "submissionDate": "2021-07-09T17:47:26.028Z", + "updateDate": "2021-07-09T17:47:47.935Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "05a895d9-2348-4a81-af29-c7c5a117b8fd" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.028Z", + "dcpVersion": "2021-07-09T17:47:26.028Z", + "contentLastModified": "2021-07-09T17:47:26.027Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7143" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7143", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7143/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7143/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7143/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7143/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/analysis/9.2.0/analysis_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "expression_matrix_protocol2b", + "protocol_name": "Data processing protocol 2b.", + "protocol_description": "The raw gene expression matrix from the CellRanger pipeline was filtered using the Seurat R package (version 2.3) and selected according to the following criteria: cells with > 1000 UMI counts, > 200 genes, and < 6,000 genes, and < 20% of mitochondrial gene expression in UMI counts." + }, + "type": { + "text": "data transformation", + "ontology": "OBI:0200000", + "ontology_label": "data transformation" + } + }, + "submissionDate": "2021-07-09T17:47:26.037Z", + "updateDate": "2021-07-09T17:47:47.945Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "c3c1ef30-e9df-4937-a07b-0e8f9303fce9" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.037Z", + "dcpVersion": "2021-07-09T17:47:26.037Z", + "contentLastModified": "2021-07-09T17:47:26.037Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7144" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7144", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7144/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7144/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7144/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7144/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/analysis/9.2.0/analysis_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "cell_type_annotation_protocol_1", + "protocol_name": "Cell type annotation.", + "protocol_description": "Seurat multiCCA and RCA pipelines were combined for initial clustering and cell type identification and removed discordant cells from 2 methods." + }, + "type": { + "text": "data transformation", + "ontology": "OBI:0200000", + "ontology_label": "data transformation" + } + }, + "submissionDate": "2021-07-09T17:47:26.047Z", + "updateDate": "2021-07-09T17:47:47.959Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "3c996506-da0e-4710-ae50-17c9a0cabdfe" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.047Z", + "dcpVersion": "2021-07-09T17:47:26.047Z", + "contentLastModified": "2021-07-09T17:47:26.047Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7145" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7145", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7145/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7145/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7145/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7145/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/protocol/analysis/9.2.0/analysis_protocol", + "schema_type": "protocol", + "protocol_core": { + "protocol_id": "cell_type_annotation_protocol_2", + "protocol_name": "Cell type annotation.", + "protocol_description": "Cell type annotation." + }, + "type": { + "text": "data transformation", + "ontology": "OBI:0200000", + "ontology_label": "data transformation" + } + }, + "submissionDate": "2021-07-09T17:47:26.056Z", + "updateDate": "2021-07-09T17:47:47.969Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Protocol", + "uuid": { + "uuid": "60c46f3c-d2fb-469c-ad25-b71ba233f44f" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.056Z", + "dcpVersion": "2021-07-09T17:47:26.056Z", + "contentLastModified": "2021-07-09T17:47:26.056Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7146" + }, + "protocol": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7146", + "title": "A single protocol" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7146/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7146/draftEvent" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7146/project", + "title": "A single project" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/protocols/60e88baed5d575160aaf7146/submissionEnvelope", + "title": "A single submission envelope" + } + } + } +] From 72fbb24fab11327a6175dfadd327d857c488709b Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Thu, 15 Jul 2021 15:25:00 +0100 Subject: [PATCH 22/49] Gathers data for files by submission id also DRY the data collector service --- ingest/downloader/data_collector.py | 21 +- tests/unit/downloader/test_data_collector.py | 6 +- tests/unit/resources/mock_files.json | 722 +++++++++++++++++++ 3 files changed, 734 insertions(+), 15 deletions(-) create mode 100644 tests/unit/resources/mock_files.json diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index 74bc9765..871061b4 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -14,19 +14,14 @@ def collect_data_by_submission_uuid(self, submission_uuid): project_json ] - self.__get_biomaterials(data_by_submission, submission) - self.__get_protocols(data_by_submission, submission) + self.__get_entities_by_submission_and_type(data_by_submission, submission, 'biomaterials') + self.__get_entities_by_submission_and_type(data_by_submission, submission, 'protocols') + self.__get_entities_by_submission_and_type(data_by_submission, submission, 'files') return data_by_submission - def __get_biomaterials(self, data_by_submission, submission): - biomaterials_json = \ - self.api.get_related_entities('biomaterials', submission, 'biomaterials') - if biomaterials_json: - data_by_submission.extend(list(biomaterials_json)) - - def __get_protocols(self, data_by_submission, submission): - protocols_json = \ - self.api.get_related_entities('protocols', submission, 'protocols') - if protocols_json: - data_by_submission.extend(list(protocols_json)) + def __get_entities_by_submission_and_type(self, data_by_submission, submission, entity_type): + entity_json = \ + self.api.get_related_entities(entity_type, submission, entity_type) + if entity_json: + data_by_submission.extend(list(entity_json)) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index 6974febf..a15bfc3a 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -24,16 +24,18 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_biomaterials_json = json.load(file) with open('../resources/mock_protocols.json') as file: mock_protocols_json = json.load(file) + with open('../resources/mock_files.json') as file: + mock_files_json = json.load(file) self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json self.mock_ingest_api.get_related_projects.return_value = mock_project_json self.mock_ingest_api.get_related_entities.side_effect = \ - [iter(mock_biomaterials_json), iter(mock_protocols_json)] + [iter(mock_biomaterials_json), iter(mock_protocols_json), iter(mock_files_json)] expected_json = [ mock_project_json, ] - expected_json.extend(mock_biomaterials_json + mock_protocols_json) + expected_json.extend(mock_biomaterials_json + mock_protocols_json + mock_files_json) #when result_json = self.data_collector.collect_data_by_submission_uuid(project_uuid) diff --git a/tests/unit/resources/mock_files.json b/tests/unit/resources/mock_files.json new file mode 100644 index 00000000..2402fdc9 --- /dev/null +++ b/tests/unit/resources/mock_files.json @@ -0,0 +1,722 @@ +[ + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3112", + "ontology_label": "Gene expression matrix", + "text": "Natural log TPM matrix." + } + ], + "file_name": "GSE132257_GEO_processed_protocol_and_fresh_frozen_natural_log_TPM_matrix.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE132257_GEO_processed_protocol_and_fresh_frozen_natural_log_TPM_matrix.txt.gz", + "submissionDate": "2021-07-09T17:47:26.069Z", + "updateDate": "2021-07-09T17:48:31.018Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "ea17a7bc-2853-453c-917b-3708ea96004c" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.069Z", + "dcpVersion": "2021-07-09T17:48:27.114Z", + "contentLastModified": "2021-07-09T17:48:27.114Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "4dc17209-1f7d-4e51-b13b-edfa1861ccd2", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7147/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3112", + "ontology_label": "Gene expression matrix", + "text": "Raw UMI count matrix." + } + ], + "file_name": "GSE132257_GEO_processed_protocol_and_fresh_frozen_raw_UMI_count_matrix.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE132257_GEO_processed_protocol_and_fresh_frozen_raw_UMI_count_matrix.txt.gz", + "submissionDate": "2021-07-09T17:47:26.082Z", + "updateDate": "2021-07-09T17:49:04.011Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "04fe5bad-8653-483d-a245-81a14a9c4438" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.082Z", + "dcpVersion": "2021-07-09T17:48:57.917Z", + "contentLastModified": "2021-07-09T17:48:57.917Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "260f51a8-340e-4d16-8dd5-fdfe9cc0b2fa", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7148/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3113", + "ontology_label": "Sample annotation", + "text": "Cell annotation." + } + ], + "file_name": "GSE132257_processed_protocol_and_fresh_frozen_cell_annotation.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE132257_processed_protocol_and_fresh_frozen_cell_annotation.txt.gz", + "submissionDate": "2021-07-09T17:47:26.094Z", + "updateDate": "2021-07-09T17:49:16.037Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "d4d1608d-c7d3-4f4a-9892-778c662b0dad" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.094Z", + "dcpVersion": "2021-07-09T17:49:12.049Z", + "contentLastModified": "2021-07-09T17:49:12.049Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "1416c582-52c7-44db-92f3-ff64020e52ee", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf7149/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3113", + "ontology_label": "Sample annotation", + "text": "Cell annotation." + } + ], + "file_name": "GSE132465_GEO_processed_CRC_10X_cell_annotation.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE132465_GEO_processed_CRC_10X_cell_annotation.txt.gz", + "submissionDate": "2021-07-09T17:47:26.106Z", + "updateDate": "2021-07-09T17:49:34.026Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "bb9c4eeb-6f69-48ac-a464-8f008891ef84" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.106Z", + "dcpVersion": "2021-07-09T17:49:28.790Z", + "contentLastModified": "2021-07-09T17:49:28.790Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "23b8fac5-c941-46eb-87b7-c08b585acadc", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714a/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3112", + "ontology_label": "Gene expression matrix", + "text": "Natural log TPM matrix." + } + ], + "file_name": "GSE132465_GEO_processed_CRC_10X_natural_log_TPM_matrix.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE132465_GEO_processed_CRC_10X_natural_log_TPM_matrix.txt.gz", + "submissionDate": "2021-07-09T17:47:26.118Z", + "updateDate": "2021-07-09T17:49:58.029Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "7739bad4-0f1d-42f4-8ca1-b325388c1907" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.118Z", + "dcpVersion": "2021-07-09T17:49:53.301Z", + "contentLastModified": "2021-07-09T17:49:53.301Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "f9ced301-0da3-4eeb-a823-7eb015c8bb62", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714b/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3112", + "ontology_label": "Gene expression matrix", + "text": "Raw UMI count matrix." + } + ], + "file_name": "GSE132465_GEO_processed_CRC_10X_raw_UMI_count_matrix.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE132465_GEO_processed_CRC_10X_raw_UMI_count_matrix.txt.gz", + "submissionDate": "2021-07-09T17:47:26.131Z", + "updateDate": "2021-07-09T17:50:16.033Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "52c8cbba-ee56-4848-a200-d739440f4a8a" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.131Z", + "dcpVersion": "2021-07-09T17:50:12.685Z", + "contentLastModified": "2021-07-09T17:50:12.685Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "de9ca0d1-6557-4556-b847-21854b475238", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714c/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3113", + "ontology_label": "Sample annotation", + "text": "Cell annotation." + } + ], + "file_name": "GSE144735_processed_KUL3_CRC_10X_annotation.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE144735_processed_KUL3_CRC_10X_annotation.txt.gz", + "submissionDate": "2021-07-09T17:47:26.146Z", + "updateDate": "2021-07-09T17:50:34.026Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "4e34a976-53ce-4aea-9f4c-3c98dd0d2604" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.146Z", + "dcpVersion": "2021-07-09T17:50:29.620Z", + "contentLastModified": "2021-07-09T17:50:29.620Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "00ca7055-e88e-49ac-8aa7-3c3b014be197", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714d/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3112", + "ontology_label": "Gene expression matrix", + "text": "Natural log TPM matrix." + } + ], + "file_name": "GSE144735_processed_KUL3_CRC_10X_natural_log_TPM_matrix.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE144735_processed_KUL3_CRC_10X_natural_log_TPM_matrix.txt.gz", + "submissionDate": "2021-07-09T17:47:26.171Z", + "updateDate": "2021-07-09T17:50:46.031Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "3ce04978-9781-45cd-a05f-753cff5d8601" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.171Z", + "dcpVersion": "2021-07-09T17:50:40.851Z", + "contentLastModified": "2021-07-09T17:50:40.851Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "6520ddbf-0439-40bb-a567-a9235b3aa702", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714e/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "describedBy": "https://schema.humancellatlas.org/type/file/6.3.0/analysis_file", + "file_core": { + "content_description": [ + { + "ontology": "data:3112", + "ontology_label": "Gene expression matrix", + "text": "Raw UMI count matrix." + } + ], + "file_name": "GSE144735_processed_KUL3_CRC_10X_raw_UMI_count_matrix.txt.gz", + "file_source": "GEO", + "format": "txt.gz" + }, + "schema_type": "file" + }, + "fileName": "GSE144735_processed_KUL3_CRC_10X_raw_UMI_count_matrix.txt.gz", + "submissionDate": "2021-07-09T17:47:26.184Z", + "updateDate": "2021-07-09T17:49:46.021Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "File", + "uuid": { + "uuid": "90ef9046-d112-42e2-aa0c-69744fb3d31d" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.184Z", + "dcpVersion": "2021-07-09T17:49:41.126Z", + "contentLastModified": "2021-07-09T17:49:41.126Z", + "accession": null, + "validationState": "Invalid", + "validationErrors": [ + { + "errorType": "FILE_NOT_UPLOADED", + "message": "File cloudUrl property not set.", + "userFriendlyMessage": "File not uploaded.", + "absoluteDataPath": "root of document" + } + ], + "isUpdate": false, + "cloudUrl": null, + "checksums": null, + "lastExportedChecksums": null, + "validationJob": null, + "validationId": null, + "dataFileUuid": "79c58532-be62-41fe-abae-042da9338cb7", + "size": null, + "fileContentType": null, + "linked": true, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f" + }, + "file": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f", + "title": "A single file" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f/draftEvent" + }, + "validationJob": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f/validationJob" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f/project", + "title": "A single project" + }, + "inputToProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f/inputToProcesses" + }, + "derivedByProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f/derivedByProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/files/60e88baed5d575160aaf714f/submissionEnvelope", + "title": "A single submission envelope" + } + } + } + ] From d000bc5c655622560b794546ec73e816e7d641a8 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Thu, 15 Jul 2021 15:41:58 +0100 Subject: [PATCH 23/49] Gathers data for processes by submission id --- ingest/downloader/data_collector.py | 1 + tests/unit/downloader/test_data_collector.py | 16 +- tests/unit/resources/mock_processes.json | 1702 ++++++++++++++++++ 3 files changed, 1717 insertions(+), 2 deletions(-) create mode 100644 tests/unit/resources/mock_processes.json diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index 871061b4..c7022673 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -15,6 +15,7 @@ def collect_data_by_submission_uuid(self, submission_uuid): ] self.__get_entities_by_submission_and_type(data_by_submission, submission, 'biomaterials') + self.__get_entities_by_submission_and_type(data_by_submission, submission, 'processes') self.__get_entities_by_submission_and_type(data_by_submission, submission, 'protocols') self.__get_entities_by_submission_and_type(data_by_submission, submission, 'files') diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index a15bfc3a..81d9cc6f 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -22,6 +22,8 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_project_json = json.load(file) with open('../resources/mock_biomaterials.json') as file: mock_biomaterials_json = json.load(file) + with open('../resources/mock_processes.json') as file: + mock_processes_json = json.load(file) with open('../resources/mock_protocols.json') as file: mock_protocols_json = json.load(file) with open('../resources/mock_files.json') as file: @@ -30,12 +32,22 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json self.mock_ingest_api.get_related_projects.return_value = mock_project_json self.mock_ingest_api.get_related_entities.side_effect = \ - [iter(mock_biomaterials_json), iter(mock_protocols_json), iter(mock_files_json)] + [ + iter(mock_biomaterials_json), + iter(mock_processes_json), + iter(mock_protocols_json), + iter(mock_files_json) + ] expected_json = [ mock_project_json, ] - expected_json.extend(mock_biomaterials_json + mock_protocols_json + mock_files_json) + expected_json.extend( + mock_biomaterials_json + + mock_processes_json + + mock_protocols_json + + mock_files_json + ) #when result_json = self.data_collector.collect_data_by_submission_uuid(project_uuid) diff --git a/tests/unit/resources/mock_processes.json b/tests/unit/resources/mock_processes.json new file mode 100644 index 00000000..fae602e1 --- /dev/null +++ b/tests/unit/resources/mock_processes.json @@ -0,0 +1,1702 @@ +[ + { + "content": { + "process_core": { + "process_id": "process_id_1" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.194Z", + "updateDate": "2021-07-09T17:47:48.210Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "88676915-d022-4706-be1d-04b8655b8595" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.194Z", + "dcpVersion": "2021-07-09T17:47:26.194Z", + "contentLastModified": "2021-07-09T17:47:26.194Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7150/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_2" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.204Z", + "updateDate": "2021-07-09T17:47:48.241Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "714b884b-950c-46ef-94a0-52b55af29476" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.204Z", + "dcpVersion": "2021-07-09T17:47:26.204Z", + "contentLastModified": "2021-07-09T17:47:26.204Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7151/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_3" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.216Z", + "updateDate": "2021-07-09T17:47:48.275Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "89de8fd1-c3e2-48fa-95e2-71c8f683b8e1" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.216Z", + "dcpVersion": "2021-07-09T17:47:26.216Z", + "contentLastModified": "2021-07-09T17:47:26.215Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7152/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_4" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.229Z", + "updateDate": "2021-07-09T17:47:48.306Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "73409f5e-e810-4502-8326-077f07895970" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.229Z", + "dcpVersion": "2021-07-09T17:47:26.229Z", + "contentLastModified": "2021-07-09T17:47:26.229Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7153/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_5" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.240Z", + "updateDate": "2021-07-09T17:47:48.335Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "52ee8311-5c28-4f9c-ad06-824ba8dc0cbc" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.240Z", + "dcpVersion": "2021-07-09T17:47:26.240Z", + "contentLastModified": "2021-07-09T17:47:26.239Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7154/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_6" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.252Z", + "updateDate": "2021-07-09T17:47:48.363Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "3efdee9a-2ad4-4e1a-b671-9871ef663432" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.252Z", + "dcpVersion": "2021-07-09T17:47:26.252Z", + "contentLastModified": "2021-07-09T17:47:26.251Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7155/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_7" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.262Z", + "updateDate": "2021-07-09T17:47:48.392Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "34f8b6f5-3f55-46fc-abdd-66aa97c6529b" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.262Z", + "dcpVersion": "2021-07-09T17:47:26.262Z", + "contentLastModified": "2021-07-09T17:47:26.262Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7156/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_8" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.272Z", + "updateDate": "2021-07-09T17:47:48.426Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "9ad6572f-e1e8-4e80-9311-0ea3451ae8a8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.272Z", + "dcpVersion": "2021-07-09T17:47:26.272Z", + "contentLastModified": "2021-07-09T17:47:26.272Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7157/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_9" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.283Z", + "updateDate": "2021-07-09T17:47:48.454Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "5607645e-8ec4-4e8a-bd0c-bd6b032b56a8" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.283Z", + "dcpVersion": "2021-07-09T17:47:26.283Z", + "contentLastModified": "2021-07-09T17:47:26.283Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7158/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_10" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.293Z", + "updateDate": "2021-07-09T17:47:48.482Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "1840cbb4-bcf6-4fa8-bc26-f5cae943f73d" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.293Z", + "dcpVersion": "2021-07-09T17:47:26.293Z", + "contentLastModified": "2021-07-09T17:47:26.292Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7159/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_11" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.303Z", + "updateDate": "2021-07-09T17:47:48.510Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "136f9dc3-ca89-4709-876a-e0be622a864b" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.303Z", + "dcpVersion": "2021-07-09T17:47:26.303Z", + "contentLastModified": "2021-07-09T17:47:26.302Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715a/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_12" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.343Z", + "updateDate": "2021-07-09T17:47:48.549Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "cd1c72e6-6ad5-4153-94c0-dd1a24c0d473" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.343Z", + "dcpVersion": "2021-07-09T17:47:26.343Z", + "contentLastModified": "2021-07-09T17:47:26.342Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715b/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_13" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.353Z", + "updateDate": "2021-07-09T17:47:48.585Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "b0f0ec00-a50e-4ea5-a817-4cb7f1584e8c" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.353Z", + "dcpVersion": "2021-07-09T17:47:26.353Z", + "contentLastModified": "2021-07-09T17:47:26.353Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715c/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_14" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.363Z", + "updateDate": "2021-07-09T17:47:48.615Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "ceb9120c-34ee-45ca-9137-4577c72df7dd" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.363Z", + "dcpVersion": "2021-07-09T17:47:26.363Z", + "contentLastModified": "2021-07-09T17:47:26.363Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715d/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_15" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.373Z", + "updateDate": "2021-07-09T17:47:48.647Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "dcad89e9-7c88-4191-8a4e-072fdfc29361" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.373Z", + "dcpVersion": "2021-07-09T17:47:26.373Z", + "contentLastModified": "2021-07-09T17:47:26.373Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715e/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_16" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.384Z", + "updateDate": "2021-07-09T17:47:48.679Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "41f990fb-23ab-446c-b1a5-16c7621f8d3e" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.384Z", + "dcpVersion": "2021-07-09T17:47:26.384Z", + "contentLastModified": "2021-07-09T17:47:26.384Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf715f/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_17" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.395Z", + "updateDate": "2021-07-09T17:47:48.723Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "38b16c73-d274-4755-aa06-9605249c2706" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.395Z", + "dcpVersion": "2021-07-09T17:47:26.395Z", + "contentLastModified": "2021-07-09T17:47:26.394Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7160/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_18" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.404Z", + "updateDate": "2021-07-09T17:47:48.753Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "f1749ba5-ef61-46cd-b3d3-3a6ecfea5f2a" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.404Z", + "dcpVersion": "2021-07-09T17:47:26.404Z", + "contentLastModified": "2021-07-09T17:47:26.404Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7161/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_19" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.414Z", + "updateDate": "2021-07-09T17:47:48.786Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "137e3e4b-2911-4b73-8db9-43aea72af412" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.414Z", + "dcpVersion": "2021-07-09T17:47:26.414Z", + "contentLastModified": "2021-07-09T17:47:26.414Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7162/submissionEnvelope", + "title": "A single submission envelope" + } + } + }, + { + "content": { + "process_core": { + "process_id": "process_id_20" + }, + "schema_type": "process", + "describedBy": "https://schema.humancellatlas.org/type/process/9.2.0/process" + }, + "submissionDate": "2021-07-09T17:47:26.424Z", + "updateDate": "2021-07-09T17:47:48.815Z", + "user": null, + "lastModifiedUser": "anonymousUser", + "type": "Process", + "uuid": { + "uuid": "25851209-764d-4e6e-a378-a6153f741c52" + }, + "events": [], + "firstDcpVersion": "2021-07-09T17:47:26.424Z", + "dcpVersion": "2021-07-09T17:47:26.424Z", + "contentLastModified": "2021-07-09T17:47:26.424Z", + "accession": null, + "validationState": "Valid", + "validationErrors": [], + "isUpdate": false, + "_links": { + "self": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163" + }, + "process": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163" + }, + "processing": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/processingEvent" + }, + "draft": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/draftEvent" + }, + "inputBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/inputBiomaterials" + }, + "derivedBiomaterials": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/derivedBiomaterials" + }, + "inputFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/inputFiles" + }, + "derivedFiles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/derivedFiles" + }, + "inputBundleReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/bundleReferences" + }, + "inputFileReferences": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/fileReference" + }, + "add-input-bundles": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/bundleReferences" + }, + "add-file-reference": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/fileReference" + }, + "project": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/project", + "title": "A single project" + }, + "projects": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/projects", + "title": "Access or create projects. Creation can only be done inside a submission envelope" + }, + "protocols": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/protocols", + "title": "Access or create protocols" + }, + "inputBundleManifests": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/inputBundleManifests" + }, + "chainedProcesses": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/chainedProcesses" + }, + "submissionEnvelope": { + "href": "https://api.ingest.archive.data.humancellatlas.org/processes/60e88baed5d575160aaf7163/submissionEnvelope", + "title": "A single submission envelope" + } + } + } +] From 600a9131c5d7555f8922295e9b6076516d1dd297 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 16:36:07 +0100 Subject: [PATCH 24/49] Fix getting of related project --- ingest/api/ingestapi.py | 5 +++-- ingest/downloader/data_collector.py | 14 +++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ingest/api/ingestapi.py b/ingest/api/ingestapi.py index 9090696b..707a43b1 100644 --- a/ingest/api/ingestapi.py +++ b/ingest/api/ingestapi.py @@ -124,8 +124,9 @@ def getSubmissions(self): def get_projects(self, submission_id): return self.__get_projects_by_submission_id_and_type(submission_id, 'projects') - def get_related_projects(self, submission_id): - return self.__get_projects_by_submission_id_and_type(submission_id, 'relatedProjects') + def get_related_project(self, submission_id): + projects = self.__get_projects_by_submission_id_and_type(submission_id, 'relatedProjects') + return projects[0] if projects else None def __get_projects_by_submission_id_and_type(self, submission_id, project_type): submission_url = f'{self.url}/submissionEnvelopes/{submission_id}/{project_type}' diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index c7022673..0fcd8e19 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -8,11 +8,15 @@ def __init__(self, ingest_api: IngestApi): def collect_data_by_submission_uuid(self, submission_uuid): submission = self.api.get_submission_by_uuid(submission_uuid) - project_json = self.api.get_related_projects(submission_uuid) - - data_by_submission = [ - project_json - ] + submission_id = submission['_links']['self']['href'].rsplit()[0] + project_json = self.api.get_related_project(submission_uuid) + + if project_json: + data_by_submission = [ + project_json + ] + else: + raise Exception('There should be a project') self.__get_entities_by_submission_and_type(data_by_submission, submission, 'biomaterials') self.__get_entities_by_submission_and_type(data_by_submission, submission, 'processes') From b567a7c772db1de5ea3633b10643a6f43eeebf78 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 16:41:37 +0100 Subject: [PATCH 25/49] Pass submission id not uuid --- ingest/downloader/data_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index 0fcd8e19..bce257e6 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -9,7 +9,7 @@ def __init__(self, ingest_api: IngestApi): def collect_data_by_submission_uuid(self, submission_uuid): submission = self.api.get_submission_by_uuid(submission_uuid) submission_id = submission['_links']['self']['href'].rsplit()[0] - project_json = self.api.get_related_project(submission_uuid) + project_json = self.api.get_related_project(submission_id) if project_json: data_by_submission = [ From 6e820484338bfb497dafe5f83901d3c29db13717 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 16:57:58 +0100 Subject: [PATCH 26/49] Further fixes on getting the related project --- ingest/api/ingestapi.py | 4 ++-- ingest/downloader/data_collector.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ingest/api/ingestapi.py b/ingest/api/ingestapi.py index 707a43b1..2256aad7 100644 --- a/ingest/api/ingestapi.py +++ b/ingest/api/ingestapi.py @@ -131,10 +131,10 @@ def get_related_project(self, submission_id): def __get_projects_by_submission_id_and_type(self, submission_id, project_type): submission_url = f'{self.url}/submissionEnvelopes/{submission_id}/{project_type}' r = self.get(submission_url, headers=self.get_headers()) - projects = [] + projects = {} if r.status_code == requests.codes.ok: projects = json.loads(r.text) - return projects + return projects.get('_embedded', {}).get('projects', []) def get_project_by_id(self, id): submission_url = self.url + '/projects/' + id diff --git a/ingest/downloader/data_collector.py b/ingest/downloader/data_collector.py index bce257e6..d07efd4b 100644 --- a/ingest/downloader/data_collector.py +++ b/ingest/downloader/data_collector.py @@ -8,7 +8,7 @@ def __init__(self, ingest_api: IngestApi): def collect_data_by_submission_uuid(self, submission_uuid): submission = self.api.get_submission_by_uuid(submission_uuid) - submission_id = submission['_links']['self']['href'].rsplit()[0] + submission_id = submission['_links']['self']['href'].split('/')[-1] project_json = self.api.get_related_project(submission_id) if project_json: From ba9bc9acbb92faeec3fcdb61646511f27ec65a15 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Thu, 15 Jul 2021 17:19:11 +0100 Subject: [PATCH 27/49] Fix data collector test --- tests/unit/downloader/test_data_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index 81d9cc6f..d3accf83 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -30,7 +30,7 @@ def test_collected_project_and_biomaterials_data_by_submission_uuid_returns_corr mock_files_json = json.load(file) self.mock_ingest_api.get_submission_by_uuid.return_value = mock_submission_json - self.mock_ingest_api.get_related_projects.return_value = mock_project_json + self.mock_ingest_api.get_related_project.return_value = mock_project_json self.mock_ingest_api.get_related_entities.side_effect = \ [ iter(mock_biomaterials_json), From 03b5336c8ee94cb5836063e3417f8302315a6742 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 20:21:12 +0100 Subject: [PATCH 28/49] Added tests for ontology properties which should be in the concrete entity worksheet --- tests/unit/downloader/test_downloader.py | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index 661a02e5..ebe34277 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -118,6 +118,79 @@ def test_convert_json__has_modules(self): # then self.assertEqual(actual, self.flattened_metadata_entity) + def test_convert_json__has_ontology_property_with_single_element(self): + # given + + self.content.update( + {"organ_parts": [ + { + "ontology": "UBERON:0000376", + "ontology_label": "hindlimb stylopod", + "text": "hindlimb stylopod" + } + ]}) + + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid + } + + entity_list = [self.metadata_entity] + + # when + + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + self.flattened_metadata_entity['Project'][0].update({ + 'project.organ_parts.ontology': 'UBERON:0000376', + 'project.organ_parts.ontology_label': 'hindlimb stylopod', + 'project.organ_parts.text': 'hindlimb stylopod', + + }) + + # then + self.assertEqual(actual, self.flattened_metadata_entity) + + def test_convert_json__has_ontology_property_with_multiple_elements(self): + # given + + self.content.update( + {'organ_parts': [ + { + 'ontology': 'UBERON:0000376', + 'ontology_label': 'dummylabel1', + 'text': 'dummytext1' + }, + { + 'ontology': 'UBERON:0002386', + 'ontology_label': 'dummylabel2', + 'text': 'dummytext2' + } + ]}) + + self.metadata_entity = { + 'content': self.content, + 'uuid': self.uuid + } + + entity_list = [self.metadata_entity] + + # when + + downloader = XlsDownloader() + actual = downloader.convert_json(entity_list) + + self.flattened_metadata_entity['Project'][0].update({ + 'project.organ_parts.ontology': 'UBERON:0000376||UBERON:0002386', + 'project.organ_parts.ontology_label': 'dummylabel1||dummylabel2', + 'project.organ_parts.text': 'dummytext1||dummytext2', + + }) + + # then + self.assertEqual(actual, self.flattened_metadata_entity) + def test_convert_json__has_boolean(self): # given self.content.update({ From 7f3ba1a268d8a50b6710df17f6531952221ac34a Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 20:44:50 +0100 Subject: [PATCH 29/49] Implement handling of list of ontology objects in metadata --- ingest/downloader/downloader.py | 21 ++++++++++++++++++--- tests/unit/downloader/test_downloader.py | 3 --- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 1b110c02..5f9724b4 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -3,6 +3,7 @@ from openpyxl import Workbook from openpyxl.worksheet.worksheet import Worksheet +ONTOLOGY_REQUIRED_PROPS = ['ontology', 'ontology_label'] EXCLUDE_KEYS = ['describedBy', 'schema_type'] FIRST_DATA_ROW_NO = 4 @@ -50,8 +51,13 @@ def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str else: flattened_object[full_key] = str(value) elif isinstance(object, list): - if self.is_object_list(object): - self._flatten_object_list(object, parent_key) + if self.is_list_of_objects(object): + if self.is_list_of_ontology_objects(object): + keys = self.get_keys_of_a_list_of_object(object) + for key in keys: + flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) + else: + self._flatten_object_list(object, parent_key) else: stringified = [str(e) for e in object] flattened_object[parent_key] = '||'.join(stringified) @@ -62,7 +68,7 @@ def _format_worksheet_name(self, worksheet_name): new_worksheet_name = ' - '.join([n.capitalize() for n in names]) return new_worksheet_name - def is_object_list(self, content): + def is_list_of_objects(self, content): return content and isinstance(content[0], dict) @staticmethod @@ -98,3 +104,12 @@ def add_row_content(self, worksheet, content, is_header=True): self.row += 1 worksheet.cell(row=self.row, column=col, value=cell_value) col += 1 + + def is_list_of_ontology_objects(self, object: dict): + first_elem = object[0] if object else {} + result = [prop in first_elem for prop in ONTOLOGY_REQUIRED_PROPS] + return all(result) + + def get_keys_of_a_list_of_object(self, object: dict): + first_elem = object[0] if object else {} + return list(first_elem.keys()) diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_downloader.py index ebe34277..34adcae2 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_downloader.py @@ -120,7 +120,6 @@ def test_convert_json__has_modules(self): def test_convert_json__has_ontology_property_with_single_element(self): # given - self.content.update( {"organ_parts": [ { @@ -154,7 +153,6 @@ def test_convert_json__has_ontology_property_with_single_element(self): def test_convert_json__has_ontology_property_with_multiple_elements(self): # given - self.content.update( {'organ_parts': [ { @@ -177,7 +175,6 @@ def test_convert_json__has_ontology_property_with_multiple_elements(self): entity_list = [self.metadata_entity] # when - downloader = XlsDownloader() actual = downloader.convert_json(entity_list) From 8fb05602eb4cd62061a192624f676132f5f06f31 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 20:59:15 +0100 Subject: [PATCH 30/49] Refactor flattening of object --- ingest/downloader/downloader.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 5f9724b4..d8f2c1f0 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -14,10 +14,10 @@ def __init__(self): self.row = FIRST_DATA_ROW_NO def convert_json(self, entity_list: List[dict]): - self._flatten_object_list(entity_list) + self._flatten(entity_list) return self.workbook - def _flatten_object_list(self, object_list: List[dict], object_key: str = ''): + def _flatten(self, object_list: List[dict], object_key: str = ''): for entity in object_list: worksheet_name = object_key row = {} @@ -52,15 +52,24 @@ def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str flattened_object[full_key] = str(value) elif isinstance(object, list): if self.is_list_of_objects(object): - if self.is_list_of_ontology_objects(object): - keys = self.get_keys_of_a_list_of_object(object) - for key in keys: - flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) - else: - self._flatten_object_list(object, parent_key) + self._flatten_object_list(flattened_object, object, parent_key) else: - stringified = [str(e) for e in object] - flattened_object[parent_key] = '||'.join(stringified) + self._flatten_scalar_list(flattened_object, object, parent_key) + + def _flatten_scalar_list(self, flattened_object, object, parent_key): + stringified = [str(e) for e in object] + flattened_object[parent_key] = '||'.join(stringified) + + def _flatten_object_list(self, flattened_object, object, parent_key): + if self.is_list_of_ontology_objects(object): + self._flatten_ontology_list(object, flattened_object, parent_key) + else: + self._flatten(object, parent_key) + + def _flatten_ontology_list(self, object, flattened_object, parent_key): + keys = self.get_keys_of_a_list_of_object(object) + for key in keys: + flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) def _format_worksheet_name(self, worksheet_name): names = worksheet_name.split('.') From 63ecd2bc96b43a51323ddf2430710531de8e919b Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 21:03:22 +0100 Subject: [PATCH 31/49] Grouped private functions for flattening logic --- ingest/downloader/downloader.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index d8f2c1f0..d9998987 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -51,7 +51,7 @@ def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str else: flattened_object[full_key] = str(value) elif isinstance(object, list): - if self.is_list_of_objects(object): + if self._is_list_of_objects(object): self._flatten_object_list(flattened_object, object, parent_key) else: self._flatten_scalar_list(flattened_object, object, parent_key) @@ -61,13 +61,13 @@ def _flatten_scalar_list(self, flattened_object, object, parent_key): flattened_object[parent_key] = '||'.join(stringified) def _flatten_object_list(self, flattened_object, object, parent_key): - if self.is_list_of_ontology_objects(object): + if self._is_list_of_ontology_objects(object): self._flatten_ontology_list(object, flattened_object, parent_key) else: self._flatten(object, parent_key) def _flatten_ontology_list(self, object, flattened_object, parent_key): - keys = self.get_keys_of_a_list_of_object(object) + keys = self._get_keys_of_a_list_of_object(object) for key in keys: flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) @@ -77,9 +77,18 @@ def _format_worksheet_name(self, worksheet_name): new_worksheet_name = ' - '.join([n.capitalize() for n in names]) return new_worksheet_name - def is_list_of_objects(self, content): + def _is_list_of_objects(self, content): return content and isinstance(content[0], dict) + def _is_list_of_ontology_objects(self, object: dict): + first_elem = object[0] if object else {} + result = [prop in first_elem for prop in ONTOLOGY_REQUIRED_PROPS] + return all(result) + + def _get_keys_of_a_list_of_object(self, object: dict): + first_elem = object[0] if object else {} + return list(first_elem.keys()) + @staticmethod def get_concrete_entity(content): return content.get('describedBy').rsplit('/', 1)[-1] @@ -113,12 +122,3 @@ def add_row_content(self, worksheet, content, is_header=True): self.row += 1 worksheet.cell(row=self.row, column=col, value=cell_value) col += 1 - - def is_list_of_ontology_objects(self, object: dict): - first_elem = object[0] if object else {} - result = [prop in first_elem for prop in ONTOLOGY_REQUIRED_PROPS] - return all(result) - - def get_keys_of_a_list_of_object(self, object: dict): - first_elem = object[0] if object else {} - return list(first_elem.keys()) From 321c5b8a625138549b1c78a5440bbf1f3806f3f0 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 21:07:23 +0100 Subject: [PATCH 32/49] Extracted flattening logic for lists --- ingest/downloader/downloader.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index d9998987..2ef585b4 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -51,10 +51,13 @@ def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str else: flattened_object[full_key] = str(value) elif isinstance(object, list): - if self._is_list_of_objects(object): - self._flatten_object_list(flattened_object, object, parent_key) - else: - self._flatten_scalar_list(flattened_object, object, parent_key) + self._flatten_list(flattened_object, object, parent_key) + + def _flatten_list(self, flattened_object, object, parent_key): + if self._is_list_of_objects(object): + self._flatten_object_list(flattened_object, object, parent_key) + else: + self._flatten_scalar_list(flattened_object, object, parent_key) def _flatten_scalar_list(self, flattened_object, object, parent_key): stringified = [str(e) for e in object] From 2b9bcd113e9f744fd2fc19c612b3a8136cc9c6d3 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Thu, 15 Jul 2021 21:53:53 +0100 Subject: [PATCH 33/49] Extracted flattening logic to a different class --- ingest/downloader/downloader.py | 90 ++----------------- ingest/downloader/flattener.py | 88 ++++++++++++++++++ .../{test_downloader.py => test_flattener.py} | 61 ++++++------- 3 files changed, 122 insertions(+), 117 deletions(-) create mode 100644 ingest/downloader/flattener.py rename tests/unit/downloader/{test_downloader.py => test_flattener.py} (88%) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 2ef585b4..7e4c0198 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -3,98 +3,18 @@ from openpyxl import Workbook from openpyxl.worksheet.worksheet import Worksheet -ONTOLOGY_REQUIRED_PROPS = ['ontology', 'ontology_label'] -EXCLUDE_KEYS = ['describedBy', 'schema_type'] +from ingest.downloader.flattener import Flattener + FIRST_DATA_ROW_NO = 4 class XlsDownloader: def __init__(self): - self.workbook = {} self.row = FIRST_DATA_ROW_NO + self.flattener = Flattener() - def convert_json(self, entity_list: List[dict]): - self._flatten(entity_list) - return self.workbook - - def _flatten(self, object_list: List[dict], object_key: str = ''): - for entity in object_list: - worksheet_name = object_key - row = {} - content = entity - - if not object_key: - content = entity['content'] - worksheet_name = self.get_concrete_entity(content) - row = {f'{worksheet_name}.uuid': entity['uuid']['uuid']} - - if not worksheet_name: - raise Exception('There should be a worksheet name') - - user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) - - rows = self.workbook.get(user_friendly_worksheet_name, []) - self.workbook[user_friendly_worksheet_name] = rows - self._flatten_object(content, row, parent_key=worksheet_name) - rows.append(row) - - def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): - if isinstance(object, dict): - for key in object: - if key in EXCLUDE_KEYS: - continue - - value = object[key] - full_key = f'{parent_key}.{key}' if parent_key else key - if isinstance(value, dict) or isinstance(value, list): - self._flatten_object(value, flattened_object, parent_key=full_key) - else: - flattened_object[full_key] = str(value) - elif isinstance(object, list): - self._flatten_list(flattened_object, object, parent_key) - - def _flatten_list(self, flattened_object, object, parent_key): - if self._is_list_of_objects(object): - self._flatten_object_list(flattened_object, object, parent_key) - else: - self._flatten_scalar_list(flattened_object, object, parent_key) - - def _flatten_scalar_list(self, flattened_object, object, parent_key): - stringified = [str(e) for e in object] - flattened_object[parent_key] = '||'.join(stringified) - - def _flatten_object_list(self, flattened_object, object, parent_key): - if self._is_list_of_ontology_objects(object): - self._flatten_ontology_list(object, flattened_object, parent_key) - else: - self._flatten(object, parent_key) - - def _flatten_ontology_list(self, object, flattened_object, parent_key): - keys = self._get_keys_of_a_list_of_object(object) - for key in keys: - flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) - - def _format_worksheet_name(self, worksheet_name): - names = worksheet_name.split('.') - names = [n.replace('_', ' ') for n in names] - new_worksheet_name = ' - '.join([n.capitalize() for n in names]) - return new_worksheet_name - - def _is_list_of_objects(self, content): - return content and isinstance(content[0], dict) - - def _is_list_of_ontology_objects(self, object: dict): - first_elem = object[0] if object else {} - result = [prop in first_elem for prop in ONTOLOGY_REQUIRED_PROPS] - return all(result) - - def _get_keys_of_a_list_of_object(self, object: dict): - first_elem = object[0] if object else {} - return list(first_elem.keys()) - - @staticmethod - def get_concrete_entity(content): - return content.get('describedBy').rsplit('/', 1)[-1] + def convert_json(self, metadata_list: List[dict]): + return self.flattener.flatten(metadata_list) def create_workbook(self, input_json: dict) -> Workbook: workbook = Workbook() diff --git a/ingest/downloader/flattener.py b/ingest/downloader/flattener.py new file mode 100644 index 00000000..644f6d31 --- /dev/null +++ b/ingest/downloader/flattener.py @@ -0,0 +1,88 @@ +from typing import List + +ONTOLOGY_REQUIRED_PROPS = ['ontology', 'ontology_label'] +EXCLUDE_KEYS = ['describedBy', 'schema_type'] + + +class Flattener: + def __init__(self): + self.workbook = {} + + def flatten(self, entity_list: List[dict], object_key: str = ''): + for entity in entity_list: + self.flatten_entity(entity, object_key) + return self.workbook + + def flatten_entity(self, entity, object_key): + worksheet_name = object_key + row = {} + content = entity + if not object_key: + content = entity['content'] + worksheet_name = self.get_concrete_entity(content) + row = {f'{worksheet_name}.uuid': entity['uuid']['uuid']} + if not worksheet_name: + raise Exception('There should be a worksheet name') + user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) + rows = self.workbook.get(user_friendly_worksheet_name, []) + self.workbook[user_friendly_worksheet_name] = rows + self.flatten_object(content, row, parent_key=worksheet_name) + rows.append(row) + + def flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): + if isinstance(object, dict): + for key in object: + if key in EXCLUDE_KEYS: + continue + + value = object[key] + full_key = f'{parent_key}.{key}' if parent_key else key + if isinstance(value, dict) or isinstance(value, list): + self.flatten_object(value, flattened_object, parent_key=full_key) + else: + flattened_object[full_key] = str(value) + elif isinstance(object, list): + self.flatten_list(flattened_object, object, parent_key) + + def flatten_list(self, flattened_object, object, parent_key): + if self._is_list_of_objects(object): + self.flatten_object_list(flattened_object, object, parent_key) + else: + self.flatten_scalar_list(flattened_object, object, parent_key) + + def flatten_scalar_list(self, flattened_object, object, parent_key): + stringified = [str(e) for e in object] + flattened_object[parent_key] = '||'.join(stringified) + + def flatten_object_list(self, flattened_object: dict, object: dict, parent_key: str): + if self._is_list_of_ontology_objects(object): + self.flatten_ontology_list(object, flattened_object, parent_key) + else: + self.flatten(object, parent_key) + + def flatten_ontology_list(self, object: dict, flattened_object:dict, parent_key: str): + keys = self._get_keys_of_a_list_of_object(object) + for key in keys: + flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) + + def _format_worksheet_name(self, worksheet_name): + names = worksheet_name.split('.') + names = [n.replace('_', ' ') for n in names] + new_worksheet_name = ' - '.join([n.capitalize() for n in names]) + return new_worksheet_name + + def _is_list_of_objects(self, content): + return content and isinstance(content[0], dict) + + def _is_list_of_ontology_objects(self, object: dict): + first_elem = object[0] if object else {} + result = [prop in first_elem for prop in ONTOLOGY_REQUIRED_PROPS] + return all(result) + + def _get_keys_of_a_list_of_object(self, object: dict): + first_elem = object[0] if object else {} + return list(first_elem.keys()) + + @staticmethod + def get_concrete_entity(content): + return content.get('describedBy').rsplit('/', 1)[-1] \ No newline at end of file diff --git a/tests/unit/downloader/test_downloader.py b/tests/unit/downloader/test_flattener.py similarity index 88% rename from tests/unit/downloader/test_downloader.py rename to tests/unit/downloader/test_flattener.py index 34adcae2..75602d3d 100644 --- a/tests/unit/downloader/test_downloader.py +++ b/tests/unit/downloader/test_flattener.py @@ -1,10 +1,10 @@ import json from unittest import TestCase -from ingest.downloader.downloader import XlsDownloader +from ingest.downloader.flattener import Flattener -class XlsDownloaderTest(TestCase): +class FlattenerTest(TestCase): def setUp(self) -> None: self.content = { "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", @@ -31,7 +31,7 @@ def setUp(self) -> None: ] } - def test_convert_json__has_no_modules(self): + def test_flatten__has_no_modules(self): # given self.metadata_entity = { 'content': self.content, @@ -40,12 +40,12 @@ def test_convert_json__has_no_modules(self): entity_list = [self.metadata_entity] # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) self.assertEqual(actual, self.flattened_metadata_entity) - def test_convert_json__has_string_arrays(self): + def test_flatten__has_string_arrays(self): # given self.content.update({ "insdc_project_accessions": [ @@ -62,8 +62,8 @@ def test_convert_json__has_string_arrays(self): entity_list = [self.metadata_entity] # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) self.flattened_metadata_entity['Project'][0].update({ 'project.insdc_project_accessions': 'SRP180337', @@ -72,7 +72,7 @@ def test_convert_json__has_string_arrays(self): self.assertEqual(actual, self.flattened_metadata_entity) - def test_convert_json__has_modules(self): + def test_flatten__has_modules(self): # given self.content.update({"contributors": [ { @@ -97,9 +97,8 @@ def test_convert_json__has_modules(self): entity_list = [self.metadata_entity] # when - - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) self.flattened_metadata_entity.update({ 'Project - Contributors': [ @@ -118,7 +117,7 @@ def test_convert_json__has_modules(self): # then self.assertEqual(actual, self.flattened_metadata_entity) - def test_convert_json__has_ontology_property_with_single_element(self): + def test_flatten__has_ontology_property_with_single_element(self): # given self.content.update( {"organ_parts": [ @@ -137,9 +136,8 @@ def test_convert_json__has_ontology_property_with_single_element(self): entity_list = [self.metadata_entity] # when - - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) self.flattened_metadata_entity['Project'][0].update({ 'project.organ_parts.ontology': 'UBERON:0000376', @@ -151,7 +149,7 @@ def test_convert_json__has_ontology_property_with_single_element(self): # then self.assertEqual(actual, self.flattened_metadata_entity) - def test_convert_json__has_ontology_property_with_multiple_elements(self): + def test_flatten__has_ontology_property_with_multiple_elements(self): # given self.content.update( {'organ_parts': [ @@ -175,8 +173,8 @@ def test_convert_json__has_ontology_property_with_multiple_elements(self): entity_list = [self.metadata_entity] # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) self.flattened_metadata_entity['Project'][0].update({ 'project.organ_parts.ontology': 'UBERON:0000376||UBERON:0002386', @@ -188,7 +186,7 @@ def test_convert_json__has_ontology_property_with_multiple_elements(self): # then self.assertEqual(actual, self.flattened_metadata_entity) - def test_convert_json__has_boolean(self): + def test_flatten__has_boolean(self): # given self.content.update({ 'boolean_field': True @@ -200,8 +198,8 @@ def test_convert_json__has_boolean(self): entity_list = [self.metadata_entity] # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) self.flattened_metadata_entity['Project'][0].update({ 'project.boolean_field': 'True' @@ -209,7 +207,7 @@ def test_convert_json__has_boolean(self): # then self.assertEqual(actual, self.flattened_metadata_entity) - def test_convert_json__has_integer(self): + def test_flatten__has_integer(self): # given self.content.update({ 'int_field': 1 @@ -221,8 +219,8 @@ def test_convert_json__has_integer(self): entity_list = [self.metadata_entity] # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) expected = { 'Project': [ @@ -239,7 +237,7 @@ def test_convert_json__has_integer(self): # then self.assertEqual(actual, expected) - def test_convert_json__project_metadata(self): + def test_flatten__project_metadata(self): self.maxDiff = None # given @@ -247,8 +245,8 @@ def test_convert_json__project_metadata(self): entity_list = json.load(file) # when - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) with open('project-list-flattened.json') as file: expected = json.load(file) @@ -256,7 +254,7 @@ def test_convert_json__project_metadata(self): # then self.assertEqual(actual, expected) - def test_convert_json__has_different_entities(self): + def test_flatten__has_different_entities(self): # given entity_list = [{ 'content': { @@ -316,9 +314,8 @@ def test_convert_json__has_different_entities(self): ] # when - - downloader = XlsDownloader() - actual = downloader.convert_json(entity_list) + flattener = Flattener() + actual = flattener.flatten(entity_list) expected = { 'Project': [ From 7abbdcf428e548cbc9e65cb36a549abbf98abe98 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 10:19:50 +0100 Subject: [PATCH 34/49] Set back helper functions as private --- ingest/downloader/flattener.py | 33 ++++++++++++++++----------------- ingest/importer/importer.py | 2 +- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/ingest/downloader/flattener.py b/ingest/downloader/flattener.py index 644f6d31..24427aff 100644 --- a/ingest/downloader/flattener.py +++ b/ingest/downloader/flattener.py @@ -10,26 +10,26 @@ def __init__(self): def flatten(self, entity_list: List[dict], object_key: str = ''): for entity in entity_list: - self.flatten_entity(entity, object_key) + self._flatten_entity(entity, object_key) return self.workbook - def flatten_entity(self, entity, object_key): + def _flatten_entity(self, entity, object_key): worksheet_name = object_key row = {} content = entity if not object_key: content = entity['content'] - worksheet_name = self.get_concrete_entity(content) + worksheet_name = self._get_concrete_entity(content) row = {f'{worksheet_name}.uuid': entity['uuid']['uuid']} if not worksheet_name: raise Exception('There should be a worksheet name') user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) rows = self.workbook.get(user_friendly_worksheet_name, []) self.workbook[user_friendly_worksheet_name] = rows - self.flatten_object(content, row, parent_key=worksheet_name) + self._flatten_object(content, row, parent_key=worksheet_name) rows.append(row) - def flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): + def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): if isinstance(object, dict): for key in object: if key in EXCLUDE_KEYS: @@ -38,29 +38,29 @@ def flatten_object(self, object: dict, flattened_object: dict, parent_key: str = value = object[key] full_key = f'{parent_key}.{key}' if parent_key else key if isinstance(value, dict) or isinstance(value, list): - self.flatten_object(value, flattened_object, parent_key=full_key) + self._flatten_object(value, flattened_object, parent_key=full_key) else: flattened_object[full_key] = str(value) elif isinstance(object, list): - self.flatten_list(flattened_object, object, parent_key) + self._flatten_list(flattened_object, object, parent_key) - def flatten_list(self, flattened_object, object, parent_key): + def _flatten_list(self, flattened_object, object, parent_key): if self._is_list_of_objects(object): - self.flatten_object_list(flattened_object, object, parent_key) + self._flatten_object_list(flattened_object, object, parent_key) else: - self.flatten_scalar_list(flattened_object, object, parent_key) + self._flatten_scalar_list(flattened_object, object, parent_key) - def flatten_scalar_list(self, flattened_object, object, parent_key): + def _flatten_scalar_list(self, flattened_object, object, parent_key): stringified = [str(e) for e in object] flattened_object[parent_key] = '||'.join(stringified) - def flatten_object_list(self, flattened_object: dict, object: dict, parent_key: str): + def _flatten_object_list(self, flattened_object: dict, object: dict, parent_key: str): if self._is_list_of_ontology_objects(object): - self.flatten_ontology_list(object, flattened_object, parent_key) + self._flatten_ontology_list(object, flattened_object, parent_key) else: self.flatten(object, parent_key) - def flatten_ontology_list(self, object: dict, flattened_object:dict, parent_key: str): + def _flatten_ontology_list(self, object: dict, flattened_object: dict, parent_key: str): keys = self._get_keys_of_a_list_of_object(object) for key in keys: flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) @@ -83,6 +83,5 @@ def _get_keys_of_a_list_of_object(self, object: dict): first_elem = object[0] if object else {} return list(first_elem.keys()) - @staticmethod - def get_concrete_entity(content): - return content.get('describedBy').rsplit('/', 1)[-1] \ No newline at end of file + def _get_concrete_entity(self, content: dict): + return content.get('describedBy').rsplit('/', 1)[-1] diff --git a/ingest/importer/importer.py b/ingest/importer/importer.py index 9f191f0f..adc87d9d 100644 --- a/ingest/importer/importer.py +++ b/ingest/importer/importer.py @@ -188,7 +188,7 @@ def do_import(self, workbook: IngestWorkbook, is_update, project_uuid=None): registry = _ImportRegistry(self.template_mgr) importable_worksheets = workbook.importable_worksheets() workbook_errors = [] - # workbook_errors = self.validate_worksheets(is_update, importable_worksheets) + workbook_errors = self.validate_worksheets(is_update, importable_worksheets) if project_uuid: project_metadata = MetadataEntity(domain_type=_PROJECT_TYPE, From 6269a3dd68ef349558957c04cef271d3458b6616 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 10:20:49 +0100 Subject: [PATCH 35/49] Undo unnecessary change --- ingest/importer/importer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ingest/importer/importer.py b/ingest/importer/importer.py index adc87d9d..80c234ca 100644 --- a/ingest/importer/importer.py +++ b/ingest/importer/importer.py @@ -187,7 +187,6 @@ def __init__(self, template_mgr): def do_import(self, workbook: IngestWorkbook, is_update, project_uuid=None): registry = _ImportRegistry(self.template_mgr) importable_worksheets = workbook.importable_worksheets() - workbook_errors = [] workbook_errors = self.validate_worksheets(is_update, importable_worksheets) if project_uuid: From 0807feccb68e091ebaf161da23dec18c865ff573 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 12:00:00 +0100 Subject: [PATCH 36/49] Changing the spreadsheet json --- .../downloader/project-list-flattened.json | 207 +++++++++++------- 1 file changed, 122 insertions(+), 85 deletions(-) diff --git a/tests/unit/downloader/project-list-flattened.json b/tests/unit/downloader/project-list-flattened.json index 942b9163..b015139b 100644 --- a/tests/unit/downloader/project-list-flattened.json +++ b/tests/unit/downloader/project-list-flattened.json @@ -1,87 +1,124 @@ { - "Project": [ - { - "project.uuid": "3e329187-a9c4-48ec-90e3-cc45f7c2311c", - "project.project_core.project_short_name": "kriegsteinBrainOrganoids", - "project.project_core.project_title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution", - "project.project_core.project_description": "Direct comparisons of human and non-human primate brain tissue have the potential to reveal molecular pathways underlying remarkable specializations of the human brain. However, chimpanzee tissue is largely inaccessible during neocortical neurogenesis when differences in brain size first appear. To identify human-specific features of cortical development, we leveraged recent innovations that permit generating pluripotent stem cell-derived cerebral organoids from chimpanzee. First, we systematically evaluated the fidelity of organoid models to primary human and macaque cortex, finding organoid models preserve gene regulatory networks related to cell types and developmental processes but exhibit increased metabolic stress. Second, we identified 261 genes differentially expressed in human compared to chimpanzee organoids and macaque cortex. Many of these genes overlap with human-specific segmental duplications and a subset suggest increased PI3K/AKT/mTOR activation in human outer radial glia. Together, our findings establish a platform for systematic analysis of molecular changes contributing to human brain development and evolution. Overall design: Single cell mRNA sequencing of iPS-derived neural and glial progenitor cells using the Fluidigm C1 system This series includes re-analysis of publicly available data in accessions: phs000989.v3, GSE99951, GSE86207, GSE75140. Sample metadata and accession IDs for the re-analyzed samples are included in the file \"GSE124299_metadata_on_processed_samples.xlsx\" available on the foot of this record. The following samples have no raw data due to data loss: GSM3569728, GSM3569738, GSM3571601, GSM3571606, GSM3571615, GSM3571621, GSM3571625, and GSM3571631", - "project.insdc_project_accessions": "SRP180337", - "project.geo_series_accessions": "GSE124299", - "project.insdc_study_accessions": "PRJNA515930" - } - ], - "Project - Contributors": [ - { - "project.contributors.name": "Alex A,,Pollen", - "project.contributors.email": "alex.pollen@ucsf.edu", - "project.contributors.institution": "University of California, San Francisco (UCSF)", - "project.contributors.laboratory": "Department of Neurology", - "project.contributors.country": "USA", - "project.contributors.corresponding_contributor": "True", - "project.contributors.project_role.text": "experimental scientist", - "project.contributors.project_role.ontology": "EFO:0009741", - "project.contributors.project_role.ontology_label": "experimental scientist" - }, - { - "project.contributors.name": "Parisa,,Nejad", - "project.contributors.email": "pnejad@ucsc.edu", - "project.contributors.institution": "University of California, Santa Cruz", - "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform", - "project.contributors.country": "USA", - "project.contributors.corresponding_contributor": "False", - "project.contributors.project_role.text": "data wrangler", - "project.contributors.project_role.ontology": "EFO:0009737", - "project.contributors.project_role.ontology_label": "data curator" - }, - { - "project.contributors.name": "Schwartz,,Rachel", - "project.contributors.email": "raschwar@ucsc.edu", - "project.contributors.institution": "University of California, Santa Cruz", - "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform", - "project.contributors.country": "USA", - "project.contributors.corresponding_contributor": "False", - "project.contributors.project_role.text": "data wrangler", - "project.contributors.project_role.ontology": "EFO:0009737", - "project.contributors.project_role.ontology_label": "data curator" - } - ], - "Project - Publications": [ - { - "project.publications.authors": "Pollen AA||Bhaduri A||Andrews MG||Nowakowski TJ||Meyerson OS||Mostajo-Radji MA||Di Lullo E||Alvarado B||Bedolli M||Dougherty ML||Fiddes IT||Kronenberg ZN||Shuga J||Leyrat AA||West JA||Bershteyn M||Lowe CB||Pavlovic BJ||Salama SR||Haussler D||Eichler EE||Kriegstein AR", - "project.publications.title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution.", - "project.publications.doi": "10.1016/j.cell.2019.01.017", - "project.publications.pmid": "30735633", - "project.publications.url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6544371/" - } - ], - "Project - Funders": [ - { - "project.funders.grant_id": "U01 MH105989", - "project.funders.organization": "NIMH NIH HHS" - }, - { - "project.funders.grant_id": "R35 NS097305", - "project.funders.organization": "NINDS NIH HHS" - }, - { - "project.funders.grant_id": "T32 HD007470", - "project.funders.organization": "NICHD NIH HHS" - }, - { - "project.funders.grant_id": "T32 GM007266", - "project.funders.organization": "NIGMS NIH HHS" - }, - { - "project.funders.grant_id": "F32 NS103266", - "project.funders.organization": "NINDS NIH HHS" - }, - { - "project.funders.grant_id": "NA", - "project.funders.organization": "Howard Hughes Medical Institute" - }, - { - "project.funders.grant_id": "P51 OD011132", - "project.funders.organization": "NIH HHS" - } - ] + "Project": { + "headers": [ + "project.uuid", + "project.project_core.project_short_name", + "project.project_core.project_title", + "project.project_core.project_description", + "project.insdc_project_accessions", + "project.geo_series_accessions", + "project.insdc_study_accessions" + ], + "values": [ + { + "project.uuid": "3e329187-a9c4-48ec-90e3-cc45f7c2311c", + "project.project_core.project_short_name": "kriegsteinBrainOrganoids", + "project.project_core.project_title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution", + "project.project_core.project_description": "Direct comparisons of human and non-human primate brain tissue have the potential to reveal molecular pathways underlying remarkable specializations of the human brain. However, chimpanzee tissue is largely inaccessible during neocortical neurogenesis when differences in brain size first appear. To identify human-specific features of cortical development, we leveraged recent innovations that permit generating pluripotent stem cell-derived cerebral organoids from chimpanzee. First, we systematically evaluated the fidelity of organoid models to primary human and macaque cortex, finding organoid models preserve gene regulatory networks related to cell types and developmental processes but exhibit increased metabolic stress. Second, we identified 261 genes differentially expressed in human compared to chimpanzee organoids and macaque cortex. Many of these genes overlap with human-specific segmental duplications and a subset suggest increased PI3K/AKT/mTOR activation in human outer radial glia. Together, our findings establish a platform for systematic analysis of molecular changes contributing to human brain development and evolution. Overall design: Single cell mRNA sequencing of iPS-derived neural and glial progenitor cells using the Fluidigm C1 system This series includes re-analysis of publicly available data in accessions: phs000989.v3, GSE99951, GSE86207, GSE75140. Sample metadata and accession IDs for the re-analyzed samples are included in the file \"GSE124299_metadata_on_processed_samples.xlsx\" available on the foot of this record. The following samples have no raw data due to data loss: GSM3569728, GSM3569738, GSM3571601, GSM3571606, GSM3571615, GSM3571621, GSM3571625, and GSM3571631", + "project.insdc_project_accessions": "SRP180337", + "project.geo_series_accessions": "GSE124299", + "project.insdc_study_accessions": "PRJNA515930" + } + ] + }, + "Project - Contributors": { + "headers": [ + "project.uuid", + "project.project_core.project_short_name", + "project.project_core.project_title", + "project.project_core.project_description", + "project.insdc_project_accessions", + "project.geo_series_accessions", + "project.insdc_study_accessions" + ], + "values": [ + { + "project.contributors.name": "Alex A,,Pollen", + "project.contributors.email": "alex.pollen@ucsf.edu", + "project.contributors.institution": "University of California, San Francisco (UCSF)", + "project.contributors.laboratory": "Department of Neurology", + "project.contributors.country": "USA", + "project.contributors.corresponding_contributor": "True", + "project.contributors.project_role.text": "experimental scientist", + "project.contributors.project_role.ontology": "EFO:0009741", + "project.contributors.project_role.ontology_label": "experimental scientist" + }, + { + "project.contributors.name": "Parisa,,Nejad", + "project.contributors.email": "pnejad@ucsc.edu", + "project.contributors.institution": "University of California, Santa Cruz", + "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform", + "project.contributors.country": "USA", + "project.contributors.corresponding_contributor": "False", + "project.contributors.project_role.text": "data wrangler", + "project.contributors.project_role.ontology": "EFO:0009737", + "project.contributors.project_role.ontology_label": "data curator" + }, + { + "project.contributors.name": "Schwartz,,Rachel", + "project.contributors.email": "raschwar@ucsc.edu", + "project.contributors.institution": "University of California, Santa Cruz", + "project.contributors.laboratory": "Human Cell Atlas Data Coordination Platform", + "project.contributors.country": "USA", + "project.contributors.corresponding_contributor": "False", + "project.contributors.project_role.text": "data wrangler", + "project.contributors.project_role.ontology": "EFO:0009737", + "project.contributors.project_role.ontology_label": "data curator" + } + ] + }, + "Project - Publications": { + "headers": [ + "project.publications.authors", + "project.publications.title", + "project.publications.doi", + "project.publications.pmid", + "project.publications.url" + ], + "values": [ + { + "project.publications.authors": "Pollen AA||Bhaduri A||Andrews MG||Nowakowski TJ||Meyerson OS||Mostajo-Radji MA||Di Lullo E||Alvarado B||Bedolli M||Dougherty ML||Fiddes IT||Kronenberg ZN||Shuga J||Leyrat AA||West JA||Bershteyn M||Lowe CB||Pavlovic BJ||Salama SR||Haussler D||Eichler EE||Kriegstein AR", + "project.publications.title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution.", + "project.publications.doi": "10.1016/j.cell.2019.01.017", + "project.publications.pmid": "30735633", + "project.publications.url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6544371/" + } + ] + }, + "Project - Funders": { + "headers": [ + "project.funders.grant_id", + "project.funders.organization" + ], + "values": [ + { + "project.funders.grant_id": "U01 MH105989", + "project.funders.organization": "NIMH NIH HHS" + }, + { + "project.funders.grant_id": "R35 NS097305", + "project.funders.organization": "NINDS NIH HHS" + }, + { + "project.funders.grant_id": "T32 HD007470", + "project.funders.organization": "NICHD NIH HHS" + }, + { + "project.funders.grant_id": "T32 GM007266", + "project.funders.organization": "NIGMS NIH HHS" + }, + { + "project.funders.grant_id": "F32 NS103266", + "project.funders.organization": "NINDS NIH HHS" + }, + { + "project.funders.grant_id": "NA", + "project.funders.organization": "Howard Hughes Medical Institute" + }, + { + "project.funders.grant_id": "P51 OD011132", + "project.funders.organization": "NIH HHS" + } + ] + } } \ No newline at end of file From 78e732dd7c374ad96f1c036f05f4ed14899f8f80 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 12:09:00 +0100 Subject: [PATCH 37/49] Updating tests with the new output format --- tests/unit/downloader/test_flattener.py | 136 +++++++++++++++--------- 1 file changed, 84 insertions(+), 52 deletions(-) diff --git a/tests/unit/downloader/test_flattener.py b/tests/unit/downloader/test_flattener.py index 75602d3d..a10818df 100644 --- a/tests/unit/downloader/test_flattener.py +++ b/tests/unit/downloader/test_flattener.py @@ -21,14 +21,18 @@ def setUp(self) -> None: } self.flattened_metadata_entity = { - 'Project': [ - { - 'project.uuid': 'uuid1', - 'project.project_core.project_short_name': 'label', - 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc' - } - ] + 'Project': { + 'headers': ['project.uuid', 'project.project_core.project_short_name', + 'project.project_core.project_title', 'project.project_core.project_description'], + 'values': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc' + } + ] + } } def test_flatten__has_no_modules(self): @@ -65,7 +69,7 @@ def test_flatten__has_string_arrays(self): flattener = Flattener() actual = flattener.flatten(entity_list) - self.flattened_metadata_entity['Project'][0].update({ + self.flattened_metadata_entity['Project']['values'][0].update({ 'project.insdc_project_accessions': 'SRP180337', 'project.geo_series_accessions': "GSE124298||GSE124299" }) @@ -139,7 +143,7 @@ def test_flatten__has_ontology_property_with_single_element(self): flattener = Flattener() actual = flattener.flatten(entity_list) - self.flattened_metadata_entity['Project'][0].update({ + self.flattened_metadata_entity['Project']['values'][0].update({ 'project.organ_parts.ontology': 'UBERON:0000376', 'project.organ_parts.ontology_label': 'hindlimb stylopod', 'project.organ_parts.text': 'hindlimb stylopod', @@ -176,7 +180,7 @@ def test_flatten__has_ontology_property_with_multiple_elements(self): flattener = Flattener() actual = flattener.flatten(entity_list) - self.flattened_metadata_entity['Project'][0].update({ + self.flattened_metadata_entity['Project']['values'][0].update({ 'project.organ_parts.ontology': 'UBERON:0000376||UBERON:0002386', 'project.organ_parts.ontology_label': 'dummylabel1||dummylabel2', 'project.organ_parts.text': 'dummytext1||dummytext2', @@ -201,7 +205,7 @@ def test_flatten__has_boolean(self): flattener = Flattener() actual = flattener.flatten(entity_list) - self.flattened_metadata_entity['Project'][0].update({ + self.flattened_metadata_entity['Project']['values'][0].update({ 'project.boolean_field': 'True' }) # then @@ -223,15 +227,24 @@ def test_flatten__has_integer(self): actual = flattener.flatten(entity_list) expected = { - 'Project': [ - { - 'project.uuid': 'uuid1', - 'project.project_core.project_short_name': 'label', - 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc', - 'project.int_field': '1' - } - ] + 'Project': { + 'headers': [ + 'project.uuid', + 'project.project_core.project_short_name', + 'project.project_core.project_title', + 'project.project_core.project_description', + 'project.int_field' + ], + 'values': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc', + 'project.int_field': '1' + } + ] + } } # then @@ -318,37 +331,56 @@ def test_flatten__has_different_entities(self): actual = flattener.flatten(entity_list) expected = { - 'Project': [ - { - 'project.uuid': 'uuid1', - 'project.project_core.project_short_name': 'label', - 'project.project_core.project_title': 'title', - 'project.project_core.project_description': 'desc' - } - ], - 'Project - Contributors': [ - {'project.contributors.corresponding_contributor': 'True', - 'project.contributors.country': 'USA', - 'project.contributors.email': 'alex.pollen@ucsf.edu', - 'project.contributors.institution': 'University of California, San Francisco (UCSF)', - 'project.contributors.laboratory': 'Department of Neurology', - 'project.contributors.name': 'Alex A,,Pollen', - 'project.contributors.project_role.ontology': 'EFO:0009741', - 'project.contributors.project_role.ontology_label': 'experimental scientist', - 'project.contributors.project_role.text': 'experimental scientist'} - ], - 'Donor organism': [ - { - 'donor_organism.uuid': 'uuid2', - 'donor_organism.biomaterial_core.biomaterial_id': 'label', - 'donor_organism.biomaterial_core.biomaterial_description': 'desc' - }, - { - 'donor_organism.uuid': 'uuid3', - 'donor_organism.biomaterial_core.biomaterial_id': 'label', - 'donor_organism.biomaterial_core.biomaterial_description': 'desc' - } - ], + 'Project': { + 'headers': ['project.uuid', 'project.project_core.project_short_name', + 'project.project_core.project_title', 'project.project_core.project_description'], + 'values': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc' + } + ]}, + 'Project - Contributors': { + 'headers': ['project.contributors.corresponding_contributor', + 'project.contributors.country', + 'project.contributors.email', + 'project.contributors.institution', + 'project.contributors.laboratory', + 'project.contributors.name', + 'project.contributors.project_role.ontology', + 'project.contributors.project_role.ontology_label', + 'project.contributors.project_role.text' + ], + 'values': [ + {'project.contributors.corresponding_contributor': 'True', + 'project.contributors.country': 'USA', + 'project.contributors.email': 'alex.pollen@ucsf.edu', + 'project.contributors.institution': 'University of California, San Francisco (UCSF)', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.name': 'Alex A,,Pollen', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.project_role.text': 'experimental scientist'} + ] + }, + 'Donor organism': { + 'headers': ['donor_organism.uuid', 'donor_organism.biomaterial_core.biomaterial_id', + 'donor_organism.biomaterial_core.biomaterial_description'], + 'values': [ + { + 'donor_organism.uuid': 'uuid2', + 'donor_organism.biomaterial_core.biomaterial_id': 'label', + 'donor_organism.biomaterial_core.biomaterial_description': 'desc' + }, + { + 'donor_organism.uuid': 'uuid3', + 'donor_organism.biomaterial_core.biomaterial_id': 'label', + 'donor_organism.biomaterial_core.biomaterial_description': 'desc' + } + ] + }, } # then From e75724469bc5a60024ba3135ea5701ad53f8d2b2 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 12:41:56 +0100 Subject: [PATCH 38/49] Updated flattener tests expected output --- ingest/downloader/flattener.py | 12 ++- .../downloader/project-list-flattened.json | 16 ++-- tests/unit/downloader/test_flattener.py | 94 ++++++++++++++----- 3 files changed, 88 insertions(+), 34 deletions(-) diff --git a/ingest/downloader/flattener.py b/ingest/downloader/flattener.py index 24427aff..ccd4baa3 100644 --- a/ingest/downloader/flattener.py +++ b/ingest/downloader/flattener.py @@ -24,10 +24,18 @@ def _flatten_entity(self, entity, object_key): if not worksheet_name: raise Exception('There should be a worksheet name') user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) - rows = self.workbook.get(user_friendly_worksheet_name, []) - self.workbook[user_friendly_worksheet_name] = rows self._flatten_object(content, row, parent_key=worksheet_name) + worksheet = self.workbook.get(user_friendly_worksheet_name, {'headers': [], 'values': []}) + rows = worksheet.get('values') rows.append(row) + headers = worksheet.get('headers') + for key in row.keys(): + if key not in headers: + headers.append(key) + self.workbook[user_friendly_worksheet_name] = { + 'headers': headers, + 'values': rows + } def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): if isinstance(object, dict): diff --git a/tests/unit/downloader/project-list-flattened.json b/tests/unit/downloader/project-list-flattened.json index b015139b..8aed9478 100644 --- a/tests/unit/downloader/project-list-flattened.json +++ b/tests/unit/downloader/project-list-flattened.json @@ -23,13 +23,15 @@ }, "Project - Contributors": { "headers": [ - "project.uuid", - "project.project_core.project_short_name", - "project.project_core.project_title", - "project.project_core.project_description", - "project.insdc_project_accessions", - "project.geo_series_accessions", - "project.insdc_study_accessions" + "project.contributors.name", + "project.contributors.email", + "project.contributors.institution", + "project.contributors.laboratory", + "project.contributors.country", + "project.contributors.corresponding_contributor", + "project.contributors.project_role.text", + "project.contributors.project_role.ontology", + "project.contributors.project_role.ontology_label" ], "values": [ { diff --git a/tests/unit/downloader/test_flattener.py b/tests/unit/downloader/test_flattener.py index a10818df..0d6340e9 100644 --- a/tests/unit/downloader/test_flattener.py +++ b/tests/unit/downloader/test_flattener.py @@ -74,6 +74,13 @@ def test_flatten__has_string_arrays(self): 'project.geo_series_accessions': "GSE124298||GSE124299" }) + self.flattened_metadata_entity['Project']['headers'].extend( + [ + 'project.insdc_project_accessions', + 'project.geo_series_accessions' + ] + ) + self.assertEqual(actual, self.flattened_metadata_entity) def test_flatten__has_modules(self): @@ -105,17 +112,30 @@ def test_flatten__has_modules(self): actual = flattener.flatten(entity_list) self.flattened_metadata_entity.update({ - 'Project - Contributors': [ - {'project.contributors.corresponding_contributor': 'True', - 'project.contributors.country': 'USA', - 'project.contributors.email': 'alex.pollen@ucsf.edu', - 'project.contributors.institution': 'University of California, San Francisco (UCSF)', - 'project.contributors.laboratory': 'Department of Neurology', - 'project.contributors.name': 'Alex A,,Pollen', - 'project.contributors.project_role.ontology': 'EFO:0009741', - 'project.contributors.project_role.ontology_label': 'experimental scientist', - 'project.contributors.project_role.text': 'experimental scientist'} - ] + 'Project - Contributors': { + 'headers': [ + 'project.contributors.name', + 'project.contributors.email', + 'project.contributors.institution', + 'project.contributors.laboratory', + 'project.contributors.country', + 'project.contributors.corresponding_contributor', + 'project.contributors.project_role.text', + 'project.contributors.project_role.ontology', + 'project.contributors.project_role.ontology_label' + ], + 'values': [ + {'project.contributors.corresponding_contributor': 'True', + 'project.contributors.country': 'USA', + 'project.contributors.email': 'alex.pollen@ucsf.edu', + 'project.contributors.institution': 'University of California, San Francisco (UCSF)', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.name': 'Alex A,,Pollen', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.project_role.text': 'experimental scientist'} + ] + } }) # then @@ -149,6 +169,13 @@ def test_flatten__has_ontology_property_with_single_element(self): 'project.organ_parts.text': 'hindlimb stylopod', }) + self.flattened_metadata_entity['Project']['headers'].extend( + [ + 'project.organ_parts.ontology', + 'project.organ_parts.ontology_label', + 'project.organ_parts.text' + ] + ) # then self.assertEqual(actual, self.flattened_metadata_entity) @@ -186,6 +213,13 @@ def test_flatten__has_ontology_property_with_multiple_elements(self): 'project.organ_parts.text': 'dummytext1||dummytext2', }) + self.flattened_metadata_entity['Project']['headers'].extend( + [ + 'project.organ_parts.ontology', + 'project.organ_parts.ontology_label', + 'project.organ_parts.text' + ] + ) # then self.assertEqual(actual, self.flattened_metadata_entity) @@ -208,6 +242,8 @@ def test_flatten__has_boolean(self): self.flattened_metadata_entity['Project']['values'][0].update({ 'project.boolean_field': 'True' }) + self.flattened_metadata_entity['Project']['headers'].append('project.boolean_field') + # then self.assertEqual(actual, self.flattened_metadata_entity) @@ -332,8 +368,12 @@ def test_flatten__has_different_entities(self): expected = { 'Project': { - 'headers': ['project.uuid', 'project.project_core.project_short_name', - 'project.project_core.project_title', 'project.project_core.project_description'], + 'headers': [ + 'project.uuid', + 'project.project_core.project_short_name', + 'project.project_core.project_title', + 'project.project_core.project_description' + ], 'values': [ { 'project.uuid': 'uuid1', @@ -343,16 +383,17 @@ def test_flatten__has_different_entities(self): } ]}, 'Project - Contributors': { - 'headers': ['project.contributors.corresponding_contributor', - 'project.contributors.country', - 'project.contributors.email', - 'project.contributors.institution', - 'project.contributors.laboratory', - 'project.contributors.name', - 'project.contributors.project_role.ontology', - 'project.contributors.project_role.ontology_label', - 'project.contributors.project_role.text' - ], + 'headers': [ + 'project.contributors.name', + 'project.contributors.email', + 'project.contributors.institution', + 'project.contributors.laboratory', + 'project.contributors.country', + 'project.contributors.corresponding_contributor', + 'project.contributors.project_role.text', + 'project.contributors.project_role.ontology', + 'project.contributors.project_role.ontology_label' + ], 'values': [ {'project.contributors.corresponding_contributor': 'True', 'project.contributors.country': 'USA', @@ -366,8 +407,11 @@ def test_flatten__has_different_entities(self): ] }, 'Donor organism': { - 'headers': ['donor_organism.uuid', 'donor_organism.biomaterial_core.biomaterial_id', - 'donor_organism.biomaterial_core.biomaterial_description'], + 'headers': [ + 'donor_organism.uuid', + 'donor_organism.biomaterial_core.biomaterial_id', + 'donor_organism.biomaterial_core.biomaterial_description' + ], 'values': [ { 'donor_organism.uuid': 'uuid2', From dd56ad40bcfe03827d465f4e0f0e06256b5a94de Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 12:47:55 +0100 Subject: [PATCH 39/49] Add test for entity rows with different columns --- tests/unit/downloader/test_flattener.py | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/unit/downloader/test_flattener.py b/tests/unit/downloader/test_flattener.py index 0d6340e9..72944d2b 100644 --- a/tests/unit/downloader/test_flattener.py +++ b/tests/unit/downloader/test_flattener.py @@ -429,3 +429,65 @@ def test_flatten__has_different_entities(self): # then self.assertEqual(actual, expected) + + def test_flatten__rows_have_different_columns(self): + # given + entity_list = [ + { + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "label1", + } + }, + 'uuid': { + 'uuid': 'uuid1' + } + }, + { + 'content': { + "describedBy": "https://schema.humancellatlas.org/type/project/14.2.0/project", + "schema_type": "project", + "project_core": { + "project_short_name": "label2", + "project_title": "title", + "project_description": "desc" + } + }, + 'uuid': { + 'uuid': 'uuid2' + } + }, + + ] + + # when + flattener = Flattener() + actual = flattener.flatten(entity_list) + + expected = { + 'Project': { + 'headers': [ + 'project.uuid', + 'project.project_core.project_short_name', + 'project.project_core.project_title', + 'project.project_core.project_description' + ], + 'values': [ + { + 'project.uuid': 'uuid1', + 'project.project_core.project_short_name': 'label1' + }, + { + 'project.uuid': 'uuid2', + 'project.project_core.project_short_name': 'label2', + 'project.project_core.project_title': 'title', + 'project.project_core.project_description': 'desc' + } + ] + } + } + + # then + self.assertEqual(actual, expected) From e782e6d0ca139eca0589062345c8e481dd824123 Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Fri, 16 Jul 2021 12:54:07 +0100 Subject: [PATCH 40/49] Refactor flatten method --- ingest/downloader/flattener.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/ingest/downloader/flattener.py b/ingest/downloader/flattener.py index ccd4baa3..ffa89926 100644 --- a/ingest/downloader/flattener.py +++ b/ingest/downloader/flattener.py @@ -17,25 +17,39 @@ def _flatten_entity(self, entity, object_key): worksheet_name = object_key row = {} content = entity + if not object_key: content = entity['content'] worksheet_name = self._get_concrete_entity(content) row = {f'{worksheet_name}.uuid': entity['uuid']['uuid']} + if not worksheet_name: raise Exception('There should be a worksheet name') - user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) + self._flatten_object(content, row, parent_key=worksheet_name) + + user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) worksheet = self.workbook.get(user_friendly_worksheet_name, {'headers': [], 'values': []}) + + rows = self._update_rows(row, worksheet) + headers = self._update_headers(row, worksheet) + + self.workbook[user_friendly_worksheet_name] = { + 'headers': headers, + 'values': rows + } + + def _update_rows(self, row, worksheet): rows = worksheet.get('values') rows.append(row) + return rows + + def _update_headers(self, row, worksheet): headers = worksheet.get('headers') for key in row.keys(): if key not in headers: headers.append(key) - self.workbook[user_friendly_worksheet_name] = { - 'headers': headers, - 'values': rows - } + return headers def _flatten_object(self, object: dict, flattened_object: dict, parent_key: str = ''): if isinstance(object, dict): From 099e5fc90761090c637c46665a82f6e81da32bbb Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Fri, 16 Jul 2021 15:59:15 +0100 Subject: [PATCH 41/49] Adjust the spreadsheet generation to the new JSON contract --- ingest/downloader/downloader.py | 37 ++--- tests/unit/downloader/test_data_collector.py | 2 +- tests/unit/downloader/test_xls_generation.py | 140 +++++++++++-------- 3 files changed, 101 insertions(+), 78 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 7e4c0198..0534abb5 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -5,12 +5,12 @@ from ingest.downloader.flattener import Flattener -FIRST_DATA_ROW_NO = 4 +HEADER_ROW_NO = 4 class XlsDownloader: def __init__(self): - self.row = FIRST_DATA_ROW_NO + self.row = HEADER_ROW_NO self.flattener = Flattener() def convert_json(self, metadata_list: List[dict]): @@ -27,21 +27,22 @@ def create_workbook(self, input_json: dict) -> Workbook: return workbook def add_worksheet_content(self, worksheet, ws_elements: dict): - is_header = True - if isinstance(ws_elements, list): - for content in ws_elements: - self.add_row_content(worksheet, content, is_header) - is_header = False - self.row += 1 - else: - self.add_row_content(worksheet, ws_elements) - - def add_row_content(self, worksheet, content, is_header=True): + headers = ws_elements.get('headers') + self.__add_header_row(worksheet, headers) + all_values = ws_elements.get('values') + + for row_values in all_values: + self.row += 1 + self.__add_row_content(worksheet, headers, row_values) + + def __add_header_row(self, worksheet, headers: list): + self.row = HEADER_ROW_NO col = 1 - for header, cell_value in content.items(): - if is_header: - self.row = FIRST_DATA_ROW_NO - worksheet.cell(row=self.row, column=col, value=header) - self.row += 1 - worksheet.cell(row=self.row, column=col, value=cell_value) + for header in headers: + worksheet.cell(row=self.row, column=col, value=header) col += 1 + + def __add_row_content(self, worksheet, headers: list, values: dict): + for header, value in values.items(): + index = headers.index(header) + worksheet.cell(row=self.row, column=index+1, value=value) diff --git a/tests/unit/downloader/test_data_collector.py b/tests/unit/downloader/test_data_collector.py index d3accf83..639494e1 100644 --- a/tests/unit/downloader/test_data_collector.py +++ b/tests/unit/downloader/test_data_collector.py @@ -6,7 +6,7 @@ from ingest.downloader.data_collector import DataCollector -class MyTestCase(unittest.TestCase): +class DataCollectorTest(unittest.TestCase): def setUp(self) -> None: self.maxDiff = None diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py index 7a6be44f..da435921 100644 --- a/tests/unit/downloader/test_xls_generation.py +++ b/tests/unit/downloader/test_xls_generation.py @@ -7,7 +7,7 @@ from ingest.downloader.downloader import XlsDownloader -class MyTestCase(unittest.TestCase): +class XLSGenerationTest(unittest.TestCase): def setUp(self) -> None: self.downloader = XlsDownloader() self.workbook = Workbook() @@ -18,9 +18,26 @@ def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self project_sheet_title = 'Project' input_json = { project_sheet_title: { - 'project.uuid': '3e329187-a9c4-48ec-90e3-cc45f7c2311c', - 'project.project_core.project_short_name': 'kriegsteinBrainOrganoids', - 'project.project_core.project_title': 'Establishing Cerebral Organoids as Models' + "headers": [ + "project.uuid", + "project.project_core.project_short_name", + "project.project_core.project_title", + "project.project_core.project_description", + "project.insdc_project_accessions", + "project.geo_series_accessions", + "project.insdc_study_accessions" + ], + "values": [ + { + "project.uuid": "3e329187-a9c4-48ec-90e3-cc45f7c2311c", + "project.project_core.project_short_name": "kriegsteinBrainOrganoids", + "project.project_core.project_title": "Establishing Cerebral Organoids as Models of Human-Specific Brain Evolution", + "project.project_core.project_description": "Direct comparisons of human and non-human primate brain tissue have the potential to reveal molecular pathways underlying remarkable specializations of the human brain. However, chimpanzee tissue is largely inaccessible during neocortical neurogenesis when differences in brain size first appear. To identify human-specific features of cortical development, we leveraged recent innovations that permit generating pluripotent stem cell-derived cerebral organoids from chimpanzee. First, we systematically evaluated the fidelity of organoid models to primary human and macaque cortex, finding organoid models preserve gene regulatory networks related to cell types and developmental processes but exhibit increased metabolic stress. Second, we identified 261 genes differentially expressed in human compared to chimpanzee organoids and macaque cortex. Many of these genes overlap with human-specific segmental duplications and a subset suggest increased PI3K/AKT/mTOR activation in human outer radial glia. Together, our findings establish a platform for systematic analysis of molecular changes contributing to human brain development and evolution. Overall design: Single cell mRNA sequencing of iPS-derived neural and glial progenitor cells using the Fluidigm C1 system This series includes re-analysis of publicly available data in accessions: phs000989.v3, GSE99951, GSE86207, GSE75140. Sample metadata and accession IDs for the re-analyzed samples are included in the file \"GSE124299_metadata_on_processed_samples.xlsx\" available on the foot of this record. The following samples have no raw data due to data loss: GSM3569728, GSM3569738, GSM3571601, GSM3571606, GSM3571615, GSM3571621, GSM3571625, and GSM3571631", + "project.insdc_project_accessions": "SRP180337", + "project.geo_series_accessions": "GSE124299", + "project.insdc_study_accessions": "PRJNA515930" + } + ] } } @@ -30,65 +47,69 @@ def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self #expect self.assertEqual(len(workbook.worksheets), 1) - project_sheet: Worksheet = workbook[project_sheet_title] - self.assertEqual(project_sheet.title, project_sheet_title) - - rows = list(project_sheet.rows) - rows = rows[3:] - header_row = rows.pop(0) - for cell in header_row: - self.assertTrue(cell.value in input_json[project_sheet_title].keys()) - - for row in rows: - for cell in row: - self.assertTrue(cell.value in input_json[project_sheet_title].values()) + self.__assert_sheet(workbook, project_sheet_title, input_json) def test_given_input_has_many_rows_of_data_successfully_creates_a_workbook(self): #given contributors_sheet_title = 'Project - Contributors' input_json = { - contributors_sheet_title: [ - { - 'project.contributors.name': 'Karel Gott', - 'project.contributors.email': 'karel@gott.com', - 'project.contributors.institution': 'University of Prague', - 'project.contributors.laboratory': 'Department of Neurology', - 'project.contributors.address': '123 Test Street', - 'project.contributors.country': 'UK', - 'project.contributors.corresponding_contributor': 'yes', - 'project.contributors.project_role.text': 'experimental scientist', - 'project.contributors.project_role.ontology': 'EFO:0009741', - 'project.contributors.project_role.ontology_label': 'experimental scientist', - 'project.contributors.orcid_id': 'https://orcid.org/1234-5678-9012-3456' - }, - { - 'project.contributors.name': 'Lady Carneval', - 'project.contributors.email': 'ladyc@gott.com', - 'project.contributors.institution': 'University of Prague', - 'project.contributors.laboratory': 'Department of Neurology', - 'project.contributors.address': '123 Test Street', - 'project.contributors.country': 'UK', - 'project.contributors.corresponding_contributor': 'yes', - 'project.contributors.project_role.text': 'experimental scientist', - 'project.contributors.project_role.ontology': 'EFO:0009741', - 'project.contributors.project_role.ontology_label': 'experimental scientist', - 'project.contributors.orcid_id': 'https://orcid.org/9999-9999-9999-9999' - }, - { - 'project.contributors.name': 'Baby Shark', - 'project.contributors.email': 'sharkb@gott.com', - 'project.contributors.institution': 'University of Cambridge', - 'project.contributors.laboratory': 'Department of Neurology', - 'project.contributors.address': '123 Ocean Drive', - 'project.contributors.country': 'USA', - 'project.contributors.corresponding_contributor': 'no', - 'project.contributors.project_role.text': 'data wrangler', - 'project.contributors.project_role.ontology': 'EFO:0009737', - 'project.contributors.project_role.ontology_label': 'data curator', - 'project.contributors.orcid_id': 'https://orcid.org/0000-0000-0000-0001' - }, - ] + contributors_sheet_title: { + 'headers': [ + "project.contributors.name", + "project.contributors.email", + "project.contributors.institution", + "project.contributors.laboratory", + "project.contributors.address", + "project.contributors.country", + "project.contributors.corresponding_contributor", + "project.contributors.project_role.text", + "project.contributors.project_role.ontology", + "project.contributors.project_role.ontology_label", + "project.contributors.orcid_id" + ], + 'values': [ + { + 'project.contributors.name': 'Karel Gott', + 'project.contributors.email': 'karel@gott.com', + 'project.contributors.institution': 'University of Prague', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.address': '123 Test Street', + 'project.contributors.country': 'UK', + 'project.contributors.corresponding_contributor': 'yes', + 'project.contributors.project_role.text': 'experimental scientist', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.orcid_id': 'https://orcid.org/1234-5678-9012-3456' + }, + { + 'project.contributors.name': 'Lady Carneval', + 'project.contributors.email': 'ladyc@gott.com', + 'project.contributors.institution': 'University of Prague', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.address': '123 Test Street', + 'project.contributors.country': 'UK', + 'project.contributors.corresponding_contributor': 'yes', + 'project.contributors.project_role.text': 'experimental scientist', + 'project.contributors.project_role.ontology': 'EFO:0009741', + 'project.contributors.project_role.ontology_label': 'experimental scientist', + 'project.contributors.orcid_id': 'https://orcid.org/9999-9999-9999-9999' + }, + { + 'project.contributors.name': 'Baby Shark', + 'project.contributors.email': 'sharkb@gott.com', + 'project.contributors.institution': 'University of Cambridge', + 'project.contributors.laboratory': 'Department of Neurology', + 'project.contributors.address': '123 Ocean Drive', + 'project.contributors.country': 'USA', + 'project.contributors.corresponding_contributor': 'no', + 'project.contributors.project_role.text': 'data wrangler', + 'project.contributors.project_role.ontology': 'EFO:0009737', + 'project.contributors.project_role.ontology_label': 'data curator', + 'project.contributors.orcid_id': 'https://orcid.org/0000-0000-0000-0001' + } + ] + } } # when @@ -129,12 +150,13 @@ def __assert_sheet(self, workbook, sheet_title, input_json): rows = rows[3:] header_row = rows.pop(0) for cell in header_row: - self.assertTrue(cell.value in input_json[sheet_title][0].keys()) + self.assertTrue(cell.value in input_json[sheet_title]['headers']) i = 0 + input_values = input_json[sheet_title]['values'] for row in rows: for cell in row: - self.assertTrue(cell.value in input_json[sheet_title][i].values()) + self.assertTrue(cell.value in input_values[i].values()) i += 1 From 781007f379a4dd876d1a235575ef8d44bd5e0083 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Mon, 19 Jul 2021 10:59:05 +0100 Subject: [PATCH 42/49] Fix 1st data row number in the spreadsheet --- ingest/downloader/downloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 0534abb5..3a110ff4 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -4,6 +4,7 @@ from openpyxl.worksheet.worksheet import Worksheet from ingest.downloader.flattener import Flattener +from ingest.importer.spreadsheet.ingest_worksheet import START_DATA_ROW HEADER_ROW_NO = 4 @@ -31,6 +32,7 @@ def add_worksheet_content(self, worksheet, ws_elements: dict): self.__add_header_row(worksheet, headers) all_values = ws_elements.get('values') + self.row = START_DATA_ROW - 1 for row_values in all_values: self.row += 1 self.__add_row_content(worksheet, headers, row_values) From da14c58f8eab13aecb2dbea447f0452303624581 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Mon, 19 Jul 2021 14:33:39 +0100 Subject: [PATCH 43/49] Project worksheet should go as first sheet --- ingest/downloader/downloader.py | 6 +++++- tests/unit/downloader/test_xls_generation.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 3a110ff4..c4ccf118 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -22,7 +22,11 @@ def create_workbook(self, input_json: dict) -> Workbook: workbook.remove(workbook.active) for ws_title, ws_elements in input_json.items(): - worksheet: Worksheet = workbook.create_sheet(title=ws_title) + if ws_title == 'Project': + worksheet: Worksheet = workbook.create_sheet(title=ws_title, index=0) + else: + worksheet: Worksheet = workbook.create_sheet(title=ws_title) + self.add_worksheet_content(worksheet, ws_elements) return workbook diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py index da435921..737690d4 100644 --- a/tests/unit/downloader/test_xls_generation.py +++ b/tests/unit/downloader/test_xls_generation.py @@ -152,6 +152,7 @@ def __assert_sheet(self, workbook, sheet_title, input_json): for cell in header_row: self.assertTrue(cell.value in input_json[sheet_title]['headers']) + rows.pop(0) i = 0 input_values = input_json[sheet_title]['values'] for row in rows: From 9342eb6927023a3b83512ef59bf72ff920c8215a Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Tue, 20 Jul 2021 10:38:35 +0100 Subject: [PATCH 44/49] Use a constant var for delimiter chars --- ingest/downloader/flattener.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ingest/downloader/flattener.py b/ingest/downloader/flattener.py index ffa89926..94fc994e 100644 --- a/ingest/downloader/flattener.py +++ b/ingest/downloader/flattener.py @@ -1,5 +1,7 @@ from typing import List +SCALAR_LIST_DELIMETER = '||' + ONTOLOGY_REQUIRED_PROPS = ['ontology', 'ontology_label'] EXCLUDE_KEYS = ['describedBy', 'schema_type'] @@ -74,7 +76,7 @@ def _flatten_list(self, flattened_object, object, parent_key): def _flatten_scalar_list(self, flattened_object, object, parent_key): stringified = [str(e) for e in object] - flattened_object[parent_key] = '||'.join(stringified) + flattened_object[parent_key] = SCALAR_LIST_DELIMETER.join(stringified) def _flatten_object_list(self, flattened_object: dict, object: dict, parent_key: str): if self._is_list_of_ontology_objects(object): @@ -85,7 +87,7 @@ def _flatten_object_list(self, flattened_object: dict, object: dict, parent_key: def _flatten_ontology_list(self, object: dict, flattened_object: dict, parent_key: str): keys = self._get_keys_of_a_list_of_object(object) for key in keys: - flattened_object[f'{parent_key}.{key}'] = '||'.join([elem[key] for elem in object]) + flattened_object[f'{parent_key}.{key}'] = SCALAR_LIST_DELIMETER.join([elem[key] for elem in object]) def _format_worksheet_name(self, worksheet_name): names = worksheet_name.split('.') From 2ccc5bd410a7017edea280d261c48378db3b3f7a Mon Sep 17 00:00:00 2001 From: Alegria Aclan Date: Tue, 20 Jul 2021 10:41:30 +0100 Subject: [PATCH 45/49] Applied more PR comments --- ingest/downloader/flattener.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ingest/downloader/flattener.py b/ingest/downloader/flattener.py index 94fc994e..6ac181c4 100644 --- a/ingest/downloader/flattener.py +++ b/ingest/downloader/flattener.py @@ -1,5 +1,6 @@ from typing import List +MODULE_WORKSHEET_NAME_CONNECTOR = ' - ' SCALAR_LIST_DELIMETER = '||' ONTOLOGY_REQUIRED_PROPS = ['ontology', 'ontology_label'] @@ -33,7 +34,7 @@ def _flatten_entity(self, entity, object_key): user_friendly_worksheet_name = self._format_worksheet_name(worksheet_name) worksheet = self.workbook.get(user_friendly_worksheet_name, {'headers': [], 'values': []}) - rows = self._update_rows(row, worksheet) + rows = self._append_row_to_worksheet(row, worksheet) headers = self._update_headers(row, worksheet) self.workbook[user_friendly_worksheet_name] = { @@ -41,7 +42,7 @@ def _flatten_entity(self, entity, object_key): 'values': rows } - def _update_rows(self, row, worksheet): + def _append_row_to_worksheet(self, row, worksheet): rows = worksheet.get('values') rows.append(row) return rows @@ -92,7 +93,7 @@ def _flatten_ontology_list(self, object: dict, flattened_object: dict, parent_ke def _format_worksheet_name(self, worksheet_name): names = worksheet_name.split('.') names = [n.replace('_', ' ') for n in names] - new_worksheet_name = ' - '.join([n.capitalize() for n in names]) + new_worksheet_name = MODULE_WORKSHEET_NAME_CONNECTOR.join([n.capitalize() for n in names]) return new_worksheet_name def _is_list_of_objects(self, content): From 96cf2994a75a2e11627b819f7907ee9ea3caa215 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Tue, 20 Jul 2021 10:58:41 +0100 Subject: [PATCH 46/49] Code review fixes --- ingest/downloader/downloader.py | 6 +- tests/unit/downloader/test_xls_generation.py | 94 ++++++++++---------- 2 files changed, 49 insertions(+), 51 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index c4ccf118..c75bcade 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -43,10 +43,8 @@ def add_worksheet_content(self, worksheet, ws_elements: dict): def __add_header_row(self, worksheet, headers: list): self.row = HEADER_ROW_NO - col = 1 - for header in headers: - worksheet.cell(row=self.row, column=col, value=header) - col += 1 + for col, header in enumerate(headers): + worksheet.cell(row=self.row, column=col+1, value=header) def __add_row_content(self, worksheet, headers: list, values: dict): for header, value in values.items(): diff --git a/tests/unit/downloader/test_xls_generation.py b/tests/unit/downloader/test_xls_generation.py index 737690d4..ab41b2fa 100644 --- a/tests/unit/downloader/test_xls_generation.py +++ b/tests/unit/downloader/test_xls_generation.py @@ -16,7 +16,49 @@ def setUp(self) -> None: def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self): #given project_sheet_title = 'Project' - input_json = { + input_json = self.__input_json1(project_sheet_title) + + # when + workbook: Workbook = self.downloader.create_workbook(input_json) + + #expect + self.assertEqual(len(workbook.worksheets), 1) + + self.__assert_sheet(workbook, project_sheet_title, input_json) + + def test_given_input_has_many_rows_of_data_successfully_creates_a_workbook(self): + #given + contributors_sheet_title = 'Project - Contributors' + + input_json = self.__input_json2(contributors_sheet_title) + + # when + workbook: Workbook = self.downloader.create_workbook(input_json) + + #expect + self.assertEqual(len(workbook.worksheets), 1) + + self.__assert_sheet(workbook, contributors_sheet_title, input_json) + + def test_given_input_has_many_rows_and_many_sheets_of_data_successfully_creates_a_workbook(self): + # given + sheet_titles = ['Project', 'Project - Contributors', 'Project - Publications', 'Project - Funders'] + + # when + with open('project-list-flattened.json') as file: + input_json = json.load(file) + + #when + workbook: Workbook = self.downloader.create_workbook(input_json) + + # expect + self.assertEqual(len(workbook.worksheets), 4) + + map(lambda title: self.__assert_sheet(workbook, title, input_json), sheet_titles) + + @staticmethod + def __input_json1(project_sheet_title): + return { project_sheet_title: { "headers": [ "project.uuid", @@ -41,19 +83,9 @@ def test_given_input_has_only_1_set_of_data_successfully_creates_a_workbook(self } } - # when - workbook: Workbook = self.downloader.create_workbook(input_json) - - #expect - self.assertEqual(len(workbook.worksheets), 1) - - self.__assert_sheet(workbook, project_sheet_title, input_json) - - def test_given_input_has_many_rows_of_data_successfully_creates_a_workbook(self): - #given - contributors_sheet_title = 'Project - Contributors' - - input_json = { + @staticmethod + def __input_json2(contributors_sheet_title): + return { contributors_sheet_title: { 'headers': [ "project.contributors.name", @@ -112,36 +144,6 @@ def test_given_input_has_many_rows_of_data_successfully_creates_a_workbook(self) } } - # when - workbook: Workbook = self.downloader.create_workbook(input_json) - - #expect - self.assertEqual(len(workbook.worksheets), 1) - - self.__assert_sheet(workbook, contributors_sheet_title, input_json) - - def test_given_input_has_many_rows_and_many_sheets_of_data_successfully_creates_a_workbook(self): - # given - project_sheet_title = 'Project' - contributors_sheet_title = 'Project - Contributors' - publications_sheet_title = 'Project - Publications' - funders_sheet_title = 'Project - Funders' - - # when - with open('project-list-flattened.json') as file: - input_json = json.load(file) - - #when - workbook: Workbook = self.downloader.create_workbook(input_json) - - # expect - self.assertEqual(len(workbook.worksheets), 4) - - self.__assert_sheet(workbook, project_sheet_title, input_json) - self.__assert_sheet(workbook, contributors_sheet_title, input_json) - self.__assert_sheet(workbook, publications_sheet_title, input_json) - self.__assert_sheet(workbook, funders_sheet_title, input_json) - def __assert_sheet(self, workbook, sheet_title, input_json): sheet: Worksheet = workbook[sheet_title] self.assertEqual(sheet.title, sheet_title) @@ -153,12 +155,10 @@ def __assert_sheet(self, workbook, sheet_title, input_json): self.assertTrue(cell.value in input_json[sheet_title]['headers']) rows.pop(0) - i = 0 input_values = input_json[sheet_title]['values'] - for row in rows: + for i, row in enumerate(rows): for cell in row: self.assertTrue(cell.value in input_values[i].values()) - i += 1 if __name__ == '__main__': From 0c89235197c02e1bc71aee429a65959021c6161e Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Tue, 20 Jul 2021 11:56:08 +0100 Subject: [PATCH 47/49] enumeration fix --- ingest/downloader/downloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index c75bcade..6bf9b97f 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -43,8 +43,8 @@ def add_worksheet_content(self, worksheet, ws_elements: dict): def __add_header_row(self, worksheet, headers: list): self.row = HEADER_ROW_NO - for col, header in enumerate(headers): - worksheet.cell(row=self.row, column=col+1, value=header) + for col, header in enumerate(headers, start=1): + worksheet.cell(row=self.row, column=col, value=header) def __add_row_content(self, worksheet, headers: list, values: dict): for header, value in values.items(): From b0fba31a19d33e82c11b15351863dc6b2191519e Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Wed, 21 Jul 2021 10:17:11 +0100 Subject: [PATCH 48/49] Remove row instance variable and use enumeration instead --- ingest/downloader/downloader.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 6bf9b97f..75bccd75 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -11,7 +11,6 @@ class XlsDownloader: def __init__(self): - self.row = HEADER_ROW_NO self.flattener = Flattener() def convert_json(self, metadata_list: List[dict]): @@ -36,17 +35,16 @@ def add_worksheet_content(self, worksheet, ws_elements: dict): self.__add_header_row(worksheet, headers) all_values = ws_elements.get('values') - self.row = START_DATA_ROW - 1 - for row_values in all_values: - self.row += 1 - self.__add_row_content(worksheet, headers, row_values) + for row_no, row_values in enumerate(all_values, start=START_DATA_ROW): + self.__add_row_content(worksheet, headers, row_no, row_values) - def __add_header_row(self, worksheet, headers: list): - self.row = HEADER_ROW_NO + @staticmethod + def __add_header_row(worksheet, headers: list): for col, header in enumerate(headers, start=1): - worksheet.cell(row=self.row, column=col, value=header) + worksheet.cell(row=HEADER_ROW_NO, column=col, value=header) - def __add_row_content(self, worksheet, headers: list, values: dict): + @staticmethod + def __add_row_content(worksheet, headers: list, row_no: int, values: dict): for header, value in values.items(): index = headers.index(header) - worksheet.cell(row=self.row, column=index+1, value=value) + worksheet.cell(row=row_no, column=index+1, value=value) From 09d7a8f29f23ba0a8deca5a7581952e46b825c76 Mon Sep 17 00:00:00 2001 From: Karoly Erdos Date: Wed, 21 Jul 2021 10:35:57 +0100 Subject: [PATCH 49/49] Change variable naming --- ingest/downloader/downloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ingest/downloader/downloader.py b/ingest/downloader/downloader.py index 75bccd75..1e51ece7 100644 --- a/ingest/downloader/downloader.py +++ b/ingest/downloader/downloader.py @@ -35,8 +35,8 @@ def add_worksheet_content(self, worksheet, ws_elements: dict): self.__add_header_row(worksheet, headers) all_values = ws_elements.get('values') - for row_no, row_values in enumerate(all_values, start=START_DATA_ROW): - self.__add_row_content(worksheet, headers, row_no, row_values) + for row_number, row_values in enumerate(all_values, start=START_DATA_ROW): + self.__add_row_content(worksheet, headers, row_number, row_values) @staticmethod def __add_header_row(worksheet, headers: list): @@ -44,7 +44,7 @@ def __add_header_row(worksheet, headers: list): worksheet.cell(row=HEADER_ROW_NO, column=col, value=header) @staticmethod - def __add_row_content(worksheet, headers: list, row_no: int, values: dict): + def __add_row_content(worksheet, headers: list, row_number: int, values: dict): for header, value in values.items(): index = headers.index(header) - worksheet.cell(row=row_no, column=index+1, value=value) + worksheet.cell(row=row_number, column=index + 1, value=value)