From b01934d0053d97e5891bc28ea6cb178842d5d971 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 22 Feb 2024 11:45:37 +0100 Subject: [PATCH 1/2] catalog: increment sample version when new file is added, #TASK-5668 --- .../db/mongodb/FileMongoDBAdaptor.java | 43 ++++++----- .../db/mongodb/SampleMongoDBAdaptor.java | 73 +++++++++---------- .../mongodb/converters/SampleConverter.java | 5 ++ .../catalog/managers/FileManagerTest.java | 39 ++++++++++ .../managers/IndividualManagerTest.java | 41 ++++++++++- 5 files changed, 139 insertions(+), 62 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 3e6e48743c6..4d0e93e23c5 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.FileConverter; +import org.opencb.opencga.catalog.db.mongodb.converters.SampleConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.FileCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; @@ -209,19 +210,15 @@ long insert(ClientSession clientSession, long studyId, File file, List e ObjectMap params = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getId()); ObjectMap actionMap = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), BasicUpdateAction.ADD.name()); QueryOptions sampleUpdateOptions = new QueryOptions(Constants.ACTIONS, actionMap); - UpdateDocument sampleUpdateDocument = dbAdaptorFactory.getCatalogSampleDBAdaptor() - .updateFileReferences(params, sampleUpdateOptions); for (List sampleList : sampleListList) { logger.debug("Updating list of fileIds in batch of {} samples...", sampleList.size()); - // Update list of fileIds from sample - Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) - .append(SampleDBAdaptor.QueryParams.UID.key(), - sampleList.stream().map(Sample::getUid).collect(Collectors.toList())); - dbAdaptorFactory.getCatalogSampleDBAdaptor().getCollection().update(clientSession, - dbAdaptorFactory.getCatalogSampleDBAdaptor().parseQuery(query, null), - sampleUpdateDocument.toFinalUpdateDocument(), new QueryOptions("multi", true)); + for (Sample sample : sampleList) { + SampleConverter sampleConverter = dbAdaptorFactory.getCatalogSampleDBAdaptor().getSampleConverter(); + Document sampleDocument = sampleConverter.convertToStorageType(sample); + dbAdaptorFactory.getCatalogSampleDBAdaptor().privateUpdate(clientSession, sampleDocument, params, null, + sampleUpdateOptions); + } // Add sample to sampleList samples.addAll(sampleList); @@ -438,8 +435,8 @@ private void updateSampleReferences(ClientSession clientSession, File file, Upda throw new CatalogDBException("Internal error: Expected a list of added, removed or set samples"); } - Bson sampleBsonQuery = null; - UpdateDocument sampleUpdate = null; + Bson sampleBsonQuery; + UpdateDocument sampleUpdate; ObjectMap params = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getId()); if (!setSamples.isEmpty()) { @@ -466,29 +463,31 @@ private void updateSampleReferences(ClientSession clientSession, File file, Upda Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), file.getStudyUid()) .append(SampleDBAdaptor.QueryParams.UID.key(), addedSamples.getAsLongList(file.getId())); - sampleBsonQuery = dbAdaptorFactory.getCatalogSampleDBAdaptor().parseQuery(query, null); + List sampleList = dbAdaptorFactory.getCatalogSampleDBAdaptor().nativeGet(clientSession, query, + dbAdaptorFactory.getCatalogSampleDBAdaptor().SAMPLE_FETCH_FOR_UPDATE_OPTIONS).getResults(); ObjectMap actionMap = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), BasicUpdateAction.ADD.name()); QueryOptions sampleUpdateOptions = new QueryOptions(Constants.ACTIONS, actionMap); - sampleUpdate = dbAdaptorFactory.getCatalogSampleDBAdaptor().updateFileReferences(params, sampleUpdateOptions); - - dbAdaptorFactory.getCatalogSampleDBAdaptor().getCollection().update(clientSession, sampleBsonQuery, - sampleUpdate.toFinalUpdateDocument(), new QueryOptions(MongoDBCollection.MULTI, true)); + for (Document sampleDocument : sampleList) { + dbAdaptorFactory.getCatalogSampleDBAdaptor().privateUpdate(clientSession, sampleDocument, params, null, + sampleUpdateOptions); + } } if (removedSamples != null && !removedSamples.isEmpty()) { Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), file.getStudyUid()) .append(SampleDBAdaptor.QueryParams.UID.key(), removedSamples.getAsLongList(file.getId())); - sampleBsonQuery = dbAdaptorFactory.getCatalogSampleDBAdaptor().parseQuery(query, null); + List sampleList = dbAdaptorFactory.getCatalogSampleDBAdaptor().nativeGet(clientSession, query, + dbAdaptorFactory.getCatalogSampleDBAdaptor().SAMPLE_FETCH_FOR_UPDATE_OPTIONS).getResults(); ObjectMap actionMap = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), BasicUpdateAction.REMOVE.name()); QueryOptions sampleUpdateOptions = new QueryOptions(Constants.ACTIONS, actionMap); - sampleUpdate = dbAdaptorFactory.getCatalogSampleDBAdaptor().updateFileReferences(params, sampleUpdateOptions); - - dbAdaptorFactory.getCatalogSampleDBAdaptor().getCollection().update(clientSession, sampleBsonQuery, - sampleUpdate.toFinalUpdateDocument(), new QueryOptions(MongoDBCollection.MULTI, true)); + for (Document sampleDocument : sampleList) { + dbAdaptorFactory.getCatalogSampleDBAdaptor().privateUpdate(clientSession, sampleDocument, params, null, + sampleUpdateOptions); + } } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index 6d7bd764c57..22ffb20c77e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -83,6 +83,10 @@ public class SampleMongoDBAdaptor extends AnnotationMongoDBAdaptor imple private final IndividualMongoDBAdaptor individualDBAdaptor; private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; + final QueryOptions SAMPLE_FETCH_FOR_UPDATE_OPTIONS = new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.STUDY_UID.key(), + PRIVATE_INDIVIDUAL_UID)); + public SampleMongoDBAdaptor(MongoDBCollection sampleCollection, MongoDBCollection archiveSampleCollection, MongoDBCollection deletedSampleCollection, Configuration configuration, MongoDBAdaptorFactory dbAdaptorFactory) { @@ -91,7 +95,7 @@ public SampleMongoDBAdaptor(MongoDBCollection sampleCollection, MongoDBCollectio this.sampleCollection = sampleCollection; this.archiveSampleCollection = archiveSampleCollection; this.deletedSampleCollection = deletedSampleCollection; - sampleConverter = new SampleConverter(); + this.sampleConverter = new SampleConverter(); individualDBAdaptor = dbAdaptorFactory.getCatalogIndividualDBAdaptor(); this.versionedMongoDBAdaptor = new VersionedMongoDBAdaptor(sampleCollection, archiveSampleCollection, deletedSampleCollection); } @@ -105,6 +109,10 @@ public MongoDBCollection getArchiveSampleCollection() { return archiveSampleCollection; } + public SampleConverter getSampleConverter() { + return sampleConverter; + } + /* * Samples methods * *************************** @@ -245,10 +253,7 @@ public OpenCGAResult update(long id, ObjectMap parameters, QueryOptions queryOpt public OpenCGAResult update(long uid, ObjectMap parameters, List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query query = new Query(QueryParams.UID.key(), uid); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.STUDY_UID.key(), - PRIVATE_INDIVIDUAL_UID)); - OpenCGAResult documentResult = nativeGet(query, options); + OpenCGAResult documentResult = nativeGet(query, SAMPLE_FETCH_FOR_UPDATE_OPTIONS); if (documentResult.getNumResults() == 0) { throw new CatalogDBException("Could not update sample. Sample uid '" + uid + "' not found."); } @@ -310,7 +315,6 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample long sampleUid = sampleDocument.getLong(QueryParams.UID.key()); int version = sampleDocument.getInteger(QueryParams.VERSION.key()); long studyUid = sampleDocument.getLong(QueryParams.STUDY_UID.key()); - long individualUid = sampleDocument.getLong(PRIVATE_INDIVIDUAL_UID); Query tmpQuery = new Query() .append(QueryParams.STUDY_UID.key(), studyUid) @@ -335,9 +339,10 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample Bson finalQuery = parseQuery(tmpQuery); logger.debug("Sample update: query : {}, update: {}", finalQuery.toBsonDocument(), sampleUpdate.toBsonDocument()); - result = sampleCollection.update(clientSession, finalQuery, sampleUpdate, new QueryOptions("multi", true)); + result = sampleCollection.update(clientSession, finalQuery, sampleUpdate, new QueryOptions(MongoDBCollection.MULTI, true)); if (updateParams.getSet().containsKey(PRIVATE_INDIVIDUAL_UID)) { + long individualUid = sampleDocument.getLong(PRIVATE_INDIVIDUAL_UID); long newIndividualUid = updateParams.getSet().getLong(PRIVATE_INDIVIDUAL_UID); // If the sample has been associated a different individual @@ -612,6 +617,30 @@ UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, long st document.getSet().put(QueryParams.ID.key(), parameters.get(QueryParams.ID.key())); } + // Check if the tags exist. + if (parameters.containsKey(QueryParams.FILE_IDS.key())) { + List fileIdList = parameters.getAsStringList(QueryParams.FILE_IDS.key()); + + if (!fileIdList.isEmpty()) { + Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); + ParamUtils.BasicUpdateAction operation = + ParamUtils.BasicUpdateAction.from(actionMap, QueryParams.FILE_IDS.key(), ParamUtils.BasicUpdateAction.ADD); + switch (operation) { + case SET: + document.getSet().put(QueryParams.FILE_IDS.key(), fileIdList); + break; + case REMOVE: + document.getPullAll().put(QueryParams.FILE_IDS.key(), fileIdList); + break; + case ADD: + document.getAddToSet().put(QueryParams.FILE_IDS.key(), fileIdList); + break; + default: + throw new IllegalArgumentException("Unknown update action " + operation); + } + } + } + if (parameters.containsKey(QueryParams.INTERNAL_RGA.key())) { RgaIndex rgaIndex = parameters.get(QueryParams.INTERNAL_RGA.key(), RgaIndex.class); rgaIndex.setDate(TimeUtils.getTime()); @@ -698,36 +727,6 @@ void fixPhenotypesForRemoval(ObjectMap parameters) { parameters.put(QueryParams.PHENOTYPES.key(), phenotypeParamList); } - UpdateDocument updateFileReferences(ObjectMap parameters, QueryOptions queryOptions) { - UpdateDocument document = new UpdateDocument(); - - // Check if the tags exist. - if (parameters.containsKey(QueryParams.FILE_IDS.key())) { - List fileIdList = parameters.getAsStringList(QueryParams.FILE_IDS.key()); - - if (!fileIdList.isEmpty()) { - Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); - ParamUtils.BasicUpdateAction operation = - ParamUtils.BasicUpdateAction.from(actionMap, QueryParams.FILE_IDS.key(), ParamUtils.BasicUpdateAction.ADD); - switch (operation) { - case SET: - document.getSet().put(QueryParams.FILE_IDS.key(), fileIdList); - break; - case REMOVE: - document.getPullAll().put(QueryParams.FILE_IDS.key(), fileIdList); - break; - case ADD: - document.getAddToSet().put(QueryParams.FILE_IDS.key(), fileIdList); - break; - default: - throw new IllegalArgumentException("Unknown update action " + operation); - } - } - } - - return document; - } - @Override public long getStudyId(long sampleId) throws CatalogDBException { Bson query = new Document(PRIVATE_UID, sampleId); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java index dad025bae39..582509ad581 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java @@ -37,6 +37,11 @@ public SampleConverter() { individualConverter = new IndividualConverter(); } + @Override + public Document convertToStorageType(Sample object) { + return convertToStorageType(object, null); + } + @Override public Document convertToStorageType(Sample object, List variableSetList) { Document document = super.convertToStorageType(object, variableSetList); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index 77703e0c2ad..c6c8dc91c61 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -648,6 +648,45 @@ public void testLinkFilePassingNoDirectoryPath() throws CatalogException, URISyn assertEquals(File.Type.FILE, link.first().getType()); } + @Test + public void changeNameTest() throws CatalogException { + // Link VCF file. This VCF file will automatically create sample NA19600 + String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); + catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "data/", "", "", null, null, null, null, null), true, token); + + Query query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), "variant-test-file.vcf.gz"); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()); + OpenCGAResult sampleResult = catalogManager.getSampleManager().search(studyFqn, query, options, token); + assertEquals(4, sampleResult.getNumResults()); + for (Sample sample : sampleResult.getResults()) { + assertEquals(1, sample.getFileIds().size()); + assertEquals("data:variant-test-file.vcf.gz", sample.getFileIds().get(0)); + assertEquals(1, sample.getVersion()); + } + + // Rename file + FileUpdateParams updateParams = new FileUpdateParams().setName("variant_test.vcf.gz"); + catalogManager.getFileManager().update(studyFqn, "variant-test-file.vcf.gz", updateParams, null, token); + + assertThrows("not found", CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, "variant-test-file.vcf.gz", null, token)); + File file = catalogManager.getFileManager().get(studyFqn, updateParams.getName(), FileManager.INCLUDE_FILE_URI_PATH, token).first(); + assertEquals("data:" + updateParams.getName(), file.getId()); + assertEquals("data/" + updateParams.getName(), file.getPath()); + assertEquals(updateParams.getName(), file.getName()); + + sampleResult = catalogManager.getSampleManager().search(studyFqn, query, options, token); + assertEquals(0, sampleResult.getNumResults()); + + query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), updateParams.getName()); + sampleResult = catalogManager.getSampleManager().search(studyFqn, query, options, token); + assertEquals(4, sampleResult.getNumResults()); + for (Sample sample : sampleResult.getResults()) { + assertEquals(1, sample.getFileIds().size()); + assertEquals("data:" + updateParams.getName(), sample.getFileIds().get(0)); + assertEquals(2, sample.getVersion()); + } + } + @Test public void testAssociateSamples() throws CatalogException, URISyntaxException { URI uri = getClass().getResource("/biofiles/variant-test-file-dot-names.vcf.gz").toURI(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java index 0ac883fe91a..1f01b9d1acd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java @@ -13,6 +13,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; +import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -21,6 +22,7 @@ import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.family.Family; +import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualQualityControl; import org.opencb.opencga.core.models.individual.IndividualReferenceParam; @@ -972,7 +974,7 @@ public void memberReferenceTest() throws CatalogException { // Update individual 2 individual2 = catalogManager.getIndividualManager().update(studyFqn, individual2.getId(), new IndividualUpdateParams() - .setName("blabla"), options, token).first(); + .setName("blabla"), options, token).first(); assertEquals(2, individual2.getSamples().size()); assertEquals(3, individual2.getVersion()); assertEquals(2, individual2.getSamples().stream().map(Sample::getVersion).filter(v -> v == 1).count()); @@ -996,7 +998,7 @@ public void memberReferenceTest() throws CatalogException { // Update id from individual1 individual1 = catalogManager.getIndividualManager().update(studyFqn, individual1.getId(), new IndividualUpdateParams() - .setId("blabla"), options, token).first(); + .setId("blabla"), options, token).first(); assertEquals(2, individual1.getSamples().size()); assertEquals(3, individual1.getVersion()); assertEquals(2, individual1.getSamples().stream().map(Sample::getVersion).filter(v -> v == 2).count()); @@ -1038,7 +1040,7 @@ public void updateInUseInCATest() throws CatalogException { // locked true ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null) - .setLocked(true); + .setLocked(true); case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, token).first(); assertFalse(case1.isLocked()); @@ -1129,4 +1131,37 @@ public void updateDeleteInUseInCATest() throws CatalogException { } } + @Test + // TASK-5668 + public void viewSampleFilesFromIndividualTest() throws CatalogException { + // Link VCF file. This VCF file will automatically create sample NA19600 + String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); + catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, token); + + Sample sample = catalogManager.getSampleManager().get(studyFqn, "NA19600", + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), token).first(); + assertEquals(1, sample.getFileIds().size()); + assertEquals("variant-test-file.vcf.gz", sample.getFileIds().get(0)); + + // Create individual + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("individual"), Collections.singletonList(sample.getId()), + QueryOptions.empty(), token); + Individual individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), token).first(); + assertEquals(1, individual.getSamples().get(0).getFileIds().size()); + assertEquals("variant-test-file.vcf.gz", individual.getSamples().get(0).getFileIds().get(0)); + + // Link BAM file (related to NA19600 sample) + String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); + catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, token); + + sample = catalogManager.getSampleManager().get(studyFqn, "NA19600", + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), token).first(); + assertEquals(2, sample.getFileIds().size()); + assertTrue(Arrays.asList("variant-test-file.vcf.gz", "NA19600.chrom20.small.bam").containsAll(sample.getFileIds())); + + individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), token).first(); + assertEquals(2, individual.getSamples().get(0).getFileIds().size()); + assertTrue(Arrays.asList("variant-test-file.vcf.gz", "NA19600.chrom20.small.bam").containsAll(individual.getSamples().get(0).getFileIds())); + } + } From 41fb753bbee38f65126061cfbc36dd82d9de76c1 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 28 Feb 2024 11:27:46 +0100 Subject: [PATCH 2/2] catalog: calls to updates are always sent to the DBAdaptors, #TASK-5668 --- .../db/api/AnnotationSetDBAdaptor.java | 20 +- .../opencga/catalog/db/api/DBAdaptor.java | 15 - .../catalog/db/api/ProjectDBAdaptor.java | 11 - .../catalog/db/api/StudyDBAdaptor.java | 19 +- .../db/mongodb/AnnotationMongoDBAdaptor.java | 310 ++++++++--------- .../db/mongodb/CatalogMongoDBAdaptor.java | 35 ++ .../ClinicalAnalysisMongoDBAdaptor.java | 51 ++- .../db/mongodb/CohortMongoDBAdaptor.java | 28 +- .../db/mongodb/FamilyMongoDBAdaptor.java | 94 +++-- .../db/mongodb/FileMongoDBAdaptor.java | 85 +++-- .../db/mongodb/IndividualMongoDBAdaptor.java | 324 ++++++++++++------ .../mongodb/InterpretationMongoDBAdaptor.java | 12 +- .../catalog/db/mongodb/JobMongoDBAdaptor.java | 2 +- .../catalog/db/mongodb/MongoDBAdaptor.java | 2 +- .../db/mongodb/PanelMongoDBAdaptor.java | 6 +- .../db/mongodb/ProjectMongoDBAdaptor.java | 4 +- .../db/mongodb/SampleMongoDBAdaptor.java | 190 +++++++--- .../db/mongodb/StudyMongoDBAdaptor.java | 34 +- .../db/mongodb/UserMongoDBAdaptor.java | 2 +- .../db/mongodb/VersionedMongoDBAdaptor.java | 49 ++- .../mongodb/converters/SampleConverter.java | 1 - .../opencga/catalog/managers/FileManager.java | 2 +- .../catalog/managers/StudyManager.java | 4 +- .../db/mongodb/StudyMongoDBAdaptorTest.java | 14 +- 24 files changed, 822 insertions(+), 492 deletions(-) create mode 100644 opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CatalogMongoDBAdaptor.java diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java index e52ca8a9f84..a06f3762b4d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java @@ -58,12 +58,16 @@ OpenCGAResult update(Query query, ObjectMap parameters, List variab /** * Add the variable to all the possible annotations from the variableSetId using the default value. * + * @param studyUid Study uid. * @param variableSetId variable set id to identify the annotations that will add a new annotation. - * @param variable new variable that will be added. + * @param variable new variable that will be added. * @return a OpenCGAResult object. * @throws CatalogDBException if the variable could not be added to an existing annotationSet. + * @throws CatalogParameterException if there is any unexpected parameter. + * @throws CatalogAuthorizationException if the operation is not authorized. */ - OpenCGAResult addVariableToAnnotations(long variableSetId, Variable variable) throws CatalogDBException; + OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; // /** // * This method will rename the id of all the annotations corresponding to the variableSetId changing oldName per newName. @@ -80,21 +84,25 @@ OpenCGAResult update(Query query, ObjectMap parameters, List variab /** * Remove the annotation with annotationName from the annotation set. * - * @param variableSetId variable set id for which the annotationSets have to delete the annotation. + * @param studyUid Study uid. + * @param variableSetId variable set id for which the annotationSets have to delete the annotation. * @param annotationName Annotation name. * @return a OpenCGAResult object. * @throws CatalogDBException when there is an error in the database. + * @throws CatalogParameterException if there is any unexpected parameter. + * @throws CatalogAuthorizationException if the operation is not authorized. */ - OpenCGAResult removeAnnotationField(long variableSetId, String annotationName) throws CatalogDBException; + OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; /** * Makes a groupBy to obtain the different values that every annotation has and the total number of each. * * - * @param studyId study id. + * @param studyUid study uid. * @param variableSetId variable set id for which the group by will be done. * @return a list of Feature count with every different value. * @throws CatalogDBException when there is an error in the database. */ - OpenCGAResult getAnnotationSummary(long studyId, long variableSetId) throws CatalogDBException; + OpenCGAResult getAnnotationSummary(long studyUid, long variableSetId) throws CatalogDBException; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/DBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/DBAdaptor.java index 2b0347446ea..cd410a5dcbd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/DBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/DBAdaptor.java @@ -25,9 +25,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.response.OpenCGAResult; -import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.function.Consumer; /** @@ -61,19 +59,6 @@ default OpenCGAResult stats() { OpenCGAResult get(Query query, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; - OpenCGAResult nativeGet(Query query, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; - - default List nativeGet(List queries, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - Objects.requireNonNull(queries); - List queryResults = new ArrayList<>(queries.size()); - for (Query query : queries) { - queryResults.add(nativeGet(query, options)); - } - return queryResults; - } - OpenCGAResult update(long id, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java index ac605873666..e41e9029160 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java @@ -186,20 +186,9 @@ default List> get(List queries, QueryOptions optio return queryResults; } - OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException; - OpenCGAResult nativeGet(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException; - default List nativeGet(List queries, QueryOptions options) throws CatalogDBException { - Objects.requireNonNull(queries); - List queryResults = new ArrayList<>(queries.size()); - for (Query query : queries) { - queryResults.add(nativeGet(query, options)); - } - return queryResults; - } - OpenCGAResult update(long id, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index f74d1a843b7..7ac9ad282ff 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -82,20 +82,9 @@ default List> get(List queries, QueryOptions options return queryResults; } - OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException; - OpenCGAResult nativeGet(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException; - default List nativeGet(List queries, QueryOptions options) throws CatalogDBException { - Objects.requireNonNull(queries); - List queryResults = new ArrayList<>(queries.size()); - for (Query query : queries) { - queryResults.add(nativeGet(query, options)); - } - return queryResults; - } - OpenCGAResult update(long id, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -395,14 +384,14 @@ default void checkVariableSetExists(String variableSetId, long studyId) throws C OpenCGAResult createVariableSet(long studyId, VariableSet variableSet) throws CatalogDBException; - OpenCGAResult addFieldToVariableSet(long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException; + OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) + throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; OpenCGAResult renameFieldVariableSet(long variableSetId, String oldName, String newName, String user) throws CatalogDBException, CatalogAuthorizationException; - OpenCGAResult removeFieldFromVariableSet(long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException; + OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) + throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; OpenCGAResult getVariableSet(long variableSetUid, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java index e8cfa1d0ad1..d1e48be9331 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java @@ -20,7 +20,6 @@ import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.StringUtils; @@ -59,7 +58,7 @@ /** * Created by pfurio on 07/07/16. */ -public abstract class AnnotationMongoDBAdaptor extends MongoDBAdaptor implements AnnotationSetDBAdaptor { +public abstract class AnnotationMongoDBAdaptor extends CatalogMongoDBAdaptor implements AnnotationSetDBAdaptor { private final AnnotationConverter annotationConverter; @@ -70,6 +69,13 @@ public abstract class AnnotationMongoDBAdaptor extends MongoDBAdaptor impleme protected abstract MongoDBCollection getCollection(); + abstract OpenCGAResult transactionalUpdate(ClientSession clientSession, T entry, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException; + + abstract OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException; + public enum AnnotationSetParams implements QueryParam { INTERNAL_ANNOTATION_SETS("_ias", TEXT_ARRAY, ""), ANNOTATION_SETS("_as", TEXT_ARRAY, ""), @@ -136,47 +142,6 @@ public static AnnotationSetParams getParam(String key) { } } - public void createAnnotationSetForMigration(Object id, VariableSet variableSet, AnnotationSet annotationSet) - throws CatalogDBException { - // Check if there already exists an annotation set with the same name - DataResult count = getCollection().count( - new Document() - .append(AnnotationSetParams.ANNOTATION_SET_NAME.key(), annotationSet.getId()) - .append("_id", id)); - if (count.getNumMatches() > 0) { - throw CatalogDBException.alreadyExists("AnnotationSet", "name", annotationSet.getId()); - } - - if (variableSet.isUnique()) { - // Check if the variableset has been already annotated with a different annotation set - count = getCollection().count( - new Document() - .append(AnnotationSetParams.ANNOTATION_SETS_VARIABLE_SET_ID.key(), annotationSet.getVariableSetId()) - .append("_id", id)); - if (count.getNumMatches() > 0) { - throw new CatalogDBException("Repeated annotation for a unique VariableSet"); - } - } - - List documentList = annotationConverter.annotationToDB(variableSet, annotationSet.getId(), - annotationSet.getAnnotations()); - - // Insert the annotation set in the database - Bson query = Filters.and( - Filters.eq("_id", id), - Filters.eq(AnnotationSetParams.ANNOTATION_SET_NAME.key(), new Document("$ne", annotationSet.getId())) - ); - Bson update = new Document() - .append("$addToSet", new Document(AnnotationSetParams.ANNOTATION_SETS.key(), new Document("$each", documentList))) - .append("$set", new Document(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key() + "." + variableSet.getUid(), - variableSet.getId())); - DataResult result = getCollection().update(query, update, null); - - if (result.getNumUpdated() != 1) { - throw CatalogDBException.alreadyExists("AnnotationSet", "name", annotationSet.getId()); - } - } - /** * Remove all possible include/exclude annotation options from the query options to do the query properly. * @@ -262,7 +227,23 @@ public boolean containsAnnotationQuery(Query query) { return query.containsKey(Constants.ANNOTATION); } - OpenCGAResult updateAnnotationSets(ClientSession clientSession, long entryUid, ObjectMap parameters, + OpenCGAResult updateAnnotationSets(ClientSession clientSession, long studyUid, List entryList, + ObjectMap parameters, List variableSetList, QueryOptions options, + boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + OpenCGAResult result = OpenCGAResult.empty(Annotable.class); + if (parameters.containsKey(ANNOTATION_SETS)) { + for (Document entry : entryList) { + long entryUid = entry.get(PRIVATE_UID, Number.class).longValue(); + OpenCGAResult tmpResult = updateAnnotationSets(clientSession, studyUid, entryUid, parameters, + variableSetList, options, isVersioned); + result.append(tmpResult); + } + } + return result; + } + + OpenCGAResult updateAnnotationSets(ClientSession clientSession, long studyUid, long entryUid, ObjectMap parameters, List variableSetList, QueryOptions options, boolean isVersioned) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Map actionMap = options.getMap(Constants.ACTIONS, new HashMap<>()); @@ -290,27 +271,27 @@ OpenCGAResult updateAnnotationSets(ClientSession clientSess if (action == ParamUtils.BasicUpdateAction.SET) { if (CollectionUtils.isEmpty(internalAnnotationDocumentList)) { // 2.1 Remove all user existing annotations - removeAllAnnotationSets(clientSession, entryUid, isVersioned); + removeAllAnnotationSets(clientSession, studyUid, entryUid, isVersioned); } else { // 2.1 Remove all internal existing annotations - removeAllAnnotationSets(clientSession, entryUid, isVersioned, true); + removeAllAnnotationSets(clientSession, studyUid, entryUid, isVersioned, true); } } if (CollectionUtils.isEmpty(internalAnnotationDocumentList)) { // 3. Insert the list of documents - addNewAnnotations(clientSession, entryUid, annotationDocumentList, isVersioned); + addNewAnnotations(clientSession, studyUid, entryUid, annotationDocumentList, isVersioned); // 4. Set variable set map uid - id - addPrivateVariableMap(clientSession, entryUid, getPrivateVariableMapToSet(annotationSetList, variableSetList), + addPrivateVariableMap(clientSession, studyUid, entryUid, getPrivateVariableMapToSet(annotationSetList, variableSetList), isVersioned); } else { // 3. Insert the list of documents - addNewAnnotations(clientSession, entryUid, internalAnnotationDocumentList, isVersioned, true); + addNewAnnotations(clientSession, studyUid, entryUid, internalAnnotationDocumentList, isVersioned, true); // 4. Set variable set map uid - id - addPrivateVariableMap(clientSession, entryUid, getPrivateVariableMapToSet(annotationSetList, variableSetList), - isVersioned, true); + addPrivateVariableMap(clientSession, studyUid, entryUid, + getPrivateVariableMapToSet(annotationSetList, variableSetList), isVersioned, true); } } else if (action == ParamUtils.BasicUpdateAction.REMOVE) { @@ -373,7 +354,7 @@ OpenCGAResult updateAnnotationSets(ClientSession clientSess } // 1. Remove annotationSet - removeAnnotationSetByAnnotationSetId(clientSession, entryUid, annotationSet.getId(), isVersioned); + removeAnnotationSetByAnnotationSetId(clientSession, studyUid, entryUid, annotationSet.getId(), isVersioned); String variableSetId = annotationSetIdVariableSetUidMap.get(annotationSet.getId()); // Remove the annotation set from the variableSetAnnotationsets @@ -384,7 +365,7 @@ OpenCGAResult updateAnnotationSets(ClientSession clientSess // 2. Unset variable set map uid - id Map variableSetMapToRemove = new HashMap<>(); variableSetMapToRemove.put(variableSetId, null); - removePrivateVariableMap(clientSession, entryUid, variableSetMapToRemove, isVersioned); + removePrivateVariableMap(clientSession, studyUid, entryUid, variableSetMapToRemove, isVersioned); } } else if (StringUtils.isNotEmpty(annotationSet.getVariableSetId())) { VariableSet variableSet = variableSetMap.get(annotationSet.getVariableSetId()); @@ -402,12 +383,13 @@ OpenCGAResult updateAnnotationSets(ClientSession clientSess if (!variableSet.isInternal()) { // Remove all annotationSets - removeAnnotationSetByVariableSetId(clientSession, entryUid, variableSet.getUid(), isVersioned); - removePrivateVariableMap(clientSession, entryUid, variableSetMapToRemove, isVersioned); + removeAnnotationSetByVariableSetId(clientSession, studyUid, entryUid, variableSet.getUid(), isVersioned); + removePrivateVariableMap(clientSession, studyUid, entryUid, variableSetMapToRemove, isVersioned); } else { // Remove all annotationSets - removeAnnotationSetByVariableSetId(clientSession, entryUid, variableSet.getUid(), isVersioned, true); - removePrivateVariableMap(clientSession, entryUid, variableSetMapToRemove, isVersioned, true); + removeAnnotationSetByVariableSetId(clientSession, studyUid, entryUid, variableSet.getUid(), isVersioned, + true); + removePrivateVariableMap(clientSession, studyUid, entryUid, variableSetMapToRemove, isVersioned, true); } } } else { @@ -423,22 +405,26 @@ OpenCGAResult updateAnnotationSets(ClientSession clientSess List annotationDocumentList = getNewAnnotationList(Collections.singletonList(annotationSet), variableSetList); // 2. Remove all the existing annotations of the annotation set - removeAnnotationSetByAnnotationSetId(clientSession, entryUid, annotationSet.getId(), isVersioned); + removeAnnotationSetByAnnotationSetId(clientSession, studyUid, entryUid, annotationSet.getId(), isVersioned); // 3. Add new list of annotations - addNewAnnotations(clientSession, entryUid, annotationDocumentList, isVersioned); + addNewAnnotations(clientSession, studyUid, entryUid, annotationDocumentList, isVersioned); + } else { + return endWrite(startTime, 1, 0, new ArrayList<>()); } return endWrite(startTime, 1, 1, new ArrayList<>()); } - private void removePrivateVariableMap(ClientSession clientSession, long entryId, Map privateVariableMapToSet, - boolean isVersioned) throws CatalogDBException { - removePrivateVariableMap(clientSession, entryId, privateVariableMapToSet, isVersioned, false); + private void removePrivateVariableMap(ClientSession clientSession, long studyUid, long entryId, + Map privateVariableMapToSet, boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + removePrivateVariableMap(clientSession, studyUid, entryId, privateVariableMapToSet, isVersioned, false); } - private void removePrivateVariableMap(ClientSession clientSession, long entryId, Map privateVariableMapToSet, - boolean isVersioned, boolean isInternal) throws CatalogDBException { + private void removePrivateVariableMap(ClientSession clientSession, long studyUid, long entryId, + Map privateVariableMapToSet, boolean isVersioned, boolean isInternal) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Document queryDocument = new Document(PRIVATE_UID, entryId); if (isVersioned) { queryDocument.append(LAST_OF_VERSION, true); @@ -448,27 +434,29 @@ private void removePrivateVariableMap(ClientSession clientSession, long entryId, // We only want to remove the private variable map if it is not currently in use by any annotation set queryDocument.append(AnnotationSetParams.VARIABLE_SET_ID.key(), new Document("$ne", Long.parseLong(entry.getKey()))); - Bson unset; + UpdateDocument updateDocument = new UpdateDocument(); if (!isInternal) { - unset = Updates.unset(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key() + "." + entry.getKey()); + updateDocument.getUnset().add(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key() + "." + entry.getKey()); } else { - unset = Updates.unset(AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key() + "." + entry.getKey()); + updateDocument.getUnset().add(AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key() + "." + entry.getKey()); } - DataResult result = getCollection().update(clientSession, queryDocument, unset, new QueryOptions()); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumUpdated() < 1 && result.getNumMatches() == 1) { throw new CatalogDBException("Could not remove private map information"); } } } - private void addPrivateVariableMap(ClientSession clientSession, long entryId, Map variableMap, boolean isVersioned) - throws CatalogDBException { - addPrivateVariableMap(clientSession, entryId, variableMap, isVersioned, false); + private void addPrivateVariableMap(ClientSession clientSession, long studyUid, long entryId, Map variableMap, + boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + addPrivateVariableMap(clientSession, studyUid, entryId, variableMap, isVersioned, false); } - private void addPrivateVariableMap(ClientSession clientSession, long entryId, Map variableMap, boolean isVersioned, - boolean isInternal) throws CatalogDBException { + private void addPrivateVariableMap(ClientSession clientSession, long studyUid, long entryId, Map variableMap, + boolean isVersioned, boolean isInternal) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Document queryDocument = new Document(PRIVATE_UID, entryId); if (isVersioned) { queryDocument.append(LAST_OF_VERSION, true); @@ -476,12 +464,12 @@ private void addPrivateVariableMap(ClientSession clientSession, long entryId, Ma String key = isInternal ? AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key() : AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key(); - List setMap = new ArrayList<>(variableMap.size()); + UpdateDocument updateDocument = new UpdateDocument(); for (Map.Entry entry : variableMap.entrySet()) { - setMap.add(Updates.set(key + "." + entry.getKey(), entry.getValue())); + updateDocument.getSet().put(key + "." + entry.getKey(), entry.getValue()); } - DataResult result = getCollection().update(clientSession, queryDocument, Updates.combine(setMap), new QueryOptions("multi", true)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumUpdated() < 1 && result.getNumMatches() == 0) { throw new CatalogDBException("Could not add new private map information"); } @@ -502,103 +490,108 @@ private Map getPrivateVariableMapToSet(List annot return privateVariableMap; } - private void removeAllAnnotationSets(ClientSession clientSession, long entryId, boolean isVersioned) throws CatalogDBException { - removeAllAnnotationSets(clientSession, entryId, isVersioned, false); + private void removeAllAnnotationSets(ClientSession clientSession, long studyUid, long entryId, boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + removeAllAnnotationSets(clientSession, studyUid, entryId, isVersioned, false); } - private void removeAllAnnotationSets(ClientSession clientSession, long entryId, boolean isVersioned, boolean internal) - throws CatalogDBException { + private void removeAllAnnotationSets(ClientSession clientSession, long studyUid, long entryId, boolean isVersioned, boolean internal) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Document queryDocument = new Document(PRIVATE_UID, entryId); if (isVersioned) { queryDocument.append(LAST_OF_VERSION, true); } // We empty the annotation sets list and the private map - Bson bsonUpdate; + UpdateDocument updateDocument = new UpdateDocument(); if (!internal) { - bsonUpdate = Updates.combine( - Updates.set(AnnotationSetParams.ANNOTATION_SETS.key(), Collections.emptyList()), - Updates.set(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key(), Collections.emptyMap()) - ); + updateDocument.getSet().put(AnnotationSetParams.ANNOTATION_SETS.key(), Collections.emptyList()); + updateDocument.getSet().put(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key(), Collections.emptyMap()); } else { - bsonUpdate = Updates.combine( - Updates.set(AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key(), Collections.emptyList()), - Updates.set(AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key(), Collections.emptyMap()) - ); + updateDocument.getSet().put(AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key(), Collections.emptyList()); + updateDocument.getSet().put(AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key(), Collections.emptyMap()); } - DataResult result = getCollection().update(clientSession, queryDocument, bsonUpdate, new QueryOptions()); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumUpdated() < 1 && result.getNumMatches() == 0) { throw new CatalogDBException("Could not remove all annotationSets"); } } - private void addNewAnnotations(ClientSession clientSession, long entryId, List annotationDocumentList, boolean isVersioned) - throws CatalogDBException { - addNewAnnotations(clientSession, entryId, annotationDocumentList, isVersioned, false); + private void addNewAnnotations(ClientSession clientSession, long studyUid, long entryId, List annotationDocumentList, + boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + addNewAnnotations(clientSession, studyUid, entryId, annotationDocumentList, isVersioned, false); } - private void addNewAnnotations(ClientSession clientSession, long entryId, List annotationDocumentList, boolean isVersioned, - boolean isInternal) throws CatalogDBException { + private void addNewAnnotations(ClientSession clientSession, long studyUid, long entryId, List annotationDocumentList, + boolean isVersioned, boolean isInternal) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Document queryDocument = new Document(PRIVATE_UID, entryId); if (isVersioned) { queryDocument.append(LAST_OF_VERSION, true); } String key = isInternal ? AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key() : AnnotationSetParams.ANNOTATION_SETS.key(); - Bson push = Updates.addEachToSet(key, annotationDocumentList); + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getAddToSet().put(key, annotationDocumentList); - DataResult result = getCollection().update(clientSession, queryDocument, push, new QueryOptions("multi", true)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumUpdated() < 1) { throw new CatalogDBException("Could not add new annotations"); } } - private void removeAnnotationSetByAnnotationSetId(ClientSession clientSession, long entryId, String annotationSetId, - boolean isVersioned) throws CatalogDBException { + private void removeAnnotationSetByAnnotationSetId(ClientSession clientSession, long studyUid, long entryId, String annotationSetId, + boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Document queryDocument = new Document(PRIVATE_UID, entryId); if (isVersioned) { queryDocument.append(LAST_OF_VERSION, true); } - Bson pull = Updates.pull(AnnotationSetParams.ANNOTATION_SETS.key(), + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getPull().put(AnnotationSetParams.ANNOTATION_SETS.key(), new Document(AnnotationSetParams.ANNOTATION_SET_NAME.key(), annotationSetId)); - DataResult result = getCollection().update(clientSession, queryDocument, pull, new QueryOptions("multi", true)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumUpdated() < 1) { throw new CatalogDBException("Could not delete the annotation set"); } } - private void removeAnnotationSetByVariableSetId(ClientSession clientSession, long entryId, long variableSetUid, boolean isVersioned) - throws CatalogDBException { - removeAnnotationSetByVariableSetId(clientSession, entryId, variableSetUid, isVersioned, false); + private void removeAnnotationSetByVariableSetId(ClientSession clientSession, long studyUid, long entryId, long variableSetUid, + boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + removeAnnotationSetByVariableSetId(clientSession, studyUid, entryId, variableSetUid, isVersioned, false); } - private void removeAnnotationSetByVariableSetId(ClientSession clientSession, long entryUid, long variableSetUid, boolean isVersioned, - boolean isInternal) throws CatalogDBException { + private void removeAnnotationSetByVariableSetId(ClientSession clientSession, long studyUid, long entryUid, long variableSetUid, + boolean isVersioned, boolean isInternal) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Document queryDocument = new Document(PRIVATE_UID, entryUid); if (isVersioned) { queryDocument.append(LAST_OF_VERSION, true); } - Bson pull; + UpdateDocument updateDocument = new UpdateDocument(); if (!isInternal) { - pull = Updates.pull(AnnotationSetParams.ANNOTATION_SETS.key(), + updateDocument.getPull().put(AnnotationSetParams.ANNOTATION_SETS.key(), new Document(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetUid)); } else { - pull = Updates.pull(AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key(), + updateDocument.getPull().put(AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key(), new Document(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetUid)); } - DataResult result = getCollection().update(clientSession, queryDocument, pull, new QueryOptions("multi", true)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumMatches() > 0 && result.getNumUpdated() < 1) { throw new CatalogDBException("Could not delete the annotation set"); } } protected void removeAllAnnotationSetsByVariableSetId(ClientSession clientSession, long studyUid, VariableSet variableSet, - boolean isVersioned) throws CatalogDBException { + boolean isVersioned) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { String annotationSetKey = variableSet.isInternal() ? AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key() : AnnotationSetParams.ANNOTATION_SETS.key(); @@ -610,12 +603,12 @@ protected void removeAllAnnotationSetsByVariableSetId(ClientSession clientSessio queryDocument.append(LAST_OF_VERSION, true); } - Bson pull = new Document() - .append("$pull", new Document(annotationSetKey, - new Document(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSet.getUid()))) - .append("$unset", new Document(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key() + "." + variableSet.getUid(), "")); + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getPull().put(annotationSetKey, + new Document(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSet.getUid())); + updateDocument.getUnset().add(AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key() + "." + variableSet.getUid()); - DataResult result = getCollection().update(clientSession, queryDocument, pull, new QueryOptions("multi", true)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, queryDocument, updateDocument); if (result.getNumMatches() > 0 && result.getNumUpdated() < 1) { throw new CatalogDBException("Could not delete the annotation set"); } @@ -694,7 +687,8 @@ private List getNewAnnotationList(List annotationSetLis return annotationList; } - public OpenCGAResult addVariableToAnnotations(long variableSetId, Variable variable) throws CatalogDBException { + public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long startTime = startQuery(); // We generate the generic document that should be inserted @@ -731,30 +725,32 @@ public OpenCGAResult addVariableToAnnotations(long variableSetId, Variable varia } // Construct the query dynamically for each different annotation set and make the update - long matchCount = 0; - long modifiedCount = 0; - Bson bsonQuery; - for (String annotationId : annotationNames) { - bsonQuery = Filters.elemMatch(AnnotationSetParams.ANNOTATION_SETS.key(), Filters.and( - Filters.eq(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetId), - Filters.eq(AnnotationSetParams.ANNOTATION_SET_NAME.key(), annotationId) - )); - - // Add the annotation set key-value to the documents that will be pushed - for (Document document : documentList) { - document.put(AnnotationSetParams.ANNOTATION_SET_NAME.key(), annotationId); - } + return runTransaction(session -> { + long matchCount = 0; + long modifiedCount = 0; + Bson bsonQuery; + for (String annotationId : annotationNames) { + bsonQuery = Filters.elemMatch(AnnotationSetParams.ANNOTATION_SETS.key(), Filters.and( + Filters.eq(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetId), + Filters.eq(AnnotationSetParams.ANNOTATION_SET_NAME.key(), annotationId) + )); - // Prepare the update event - Bson update = new Document("$addToSet", new Document(AnnotationSetParams.ANNOTATION_SETS.key(), - new Document("$each", documentList))); + // Add the annotation set key-value to the documents that will be pushed + for (Document document : documentList) { + document.put(AnnotationSetParams.ANNOTATION_SET_NAME.key(), annotationId); + } - DataResult result = getCollection().update(bsonQuery, update, new QueryOptions(MongoDBCollection.MULTI, true)); - modifiedCount += result.getNumUpdated(); - matchCount += result.getNumMatches(); - } + // Prepare the update event + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getAddToSet().put(AnnotationSetParams.ANNOTATION_SETS.key(), documentList); + + OpenCGAResult result = transactionalUpdate(session, studyUid, bsonQuery, updateDocument); + modifiedCount += result.getNumUpdated(); + matchCount += result.getNumMatches(); + } - return endWrite(startTime, matchCount, modifiedCount, new ArrayList<>()); + return endWrite(startTime, matchCount, modifiedCount, new ArrayList<>()); + }); } /** @@ -764,41 +760,45 @@ public OpenCGAResult addVariableToAnnotations(long variableSetId, Variable varia * - If fieldId is personal, it will remove all the existing entries for personal.name, personal.telephone, personal.address, etc. * - If fieldId is personal.name, it will only remove the existing entries for personal.name * + * @param studyUid Study uid. * @param variableSetId Variable set id. * @param fieldId Field id corresponds with the variable name whose annotations have to be removed. * @return A OpenCGAResult object. * @throws CatalogDBException if there is any unexpected error. + * @throws CatalogParameterException if there is any unexpected parameter. + * @throws CatalogAuthorizationException if the operation is not authorized. */ - public OpenCGAResult removeAnnotationField(long variableSetId, String fieldId) throws CatalogDBException { + public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long startTime = startQuery(); -// List aggregateResult = getAnnotationDocuments(variableSetId, fieldId); - Document pull = new Document("$pull", - new Document(AnnotationSetParams.ANNOTATION_SETS.key(), - new Document() - .append(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetId) - .append(AnnotationSetParams.ID.key(), Pattern.compile("^" + fieldId)) - )); + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getPull().put(AnnotationSetParams.ANNOTATION_SETS.key(), + new Document() + .append(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetId) + .append(AnnotationSetParams.ID.key(), Pattern.compile("^" + fieldId)) + ); Document query = new Document(AnnotationSetParams.ANNOTATION_SETS.key(), new Document("$elemMatch", new Document() .append(AnnotationSetParams.VARIABLE_SET_ID.key(), variableSetId) .append(AnnotationSetParams.ID.key(), Pattern.compile("^" + fieldId)))); - DataResult result = getCollection().update(query, pull, new QueryOptions("multi", true)); - if (result.getNumUpdated() == 0 && result.getNumMatches() > 0) { - throw new CatalogDBException("VariableSet {id: " + variableSetId + "}: An unexpected error happened when extracting the " - + "annotations for the variable " + fieldId + ". Please, report this error to the OpenCGA developers."); - } - - return new OpenCGAResult(result); + return runTransaction(clientSession -> { + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, query, updateDocument); + if (result.getNumUpdated() == 0 && result.getNumMatches() > 0) { + throw new CatalogDBException("VariableSet {id: " + variableSetId + "}: An unexpected error happened when extracting the " + + "annotations for the variable " + fieldId + ". Please, report this error to the OpenCGA developers."); + } + return endWrite(startTime, result); + }); } - public OpenCGAResult getAnnotationSummary(long studyId, long variableSetId) throws CatalogDBException { + public OpenCGAResult getAnnotationSummary(long studyUid, long variableSetId) throws CatalogDBException { long startTime = startQuery(); List aggregation = new ArrayList<>(6); - aggregation.add(new Document("$match", new Document(PRIVATE_STUDY_UID, studyId))); + aggregation.add(new Document("$match", new Document(PRIVATE_STUDY_UID, studyUid))); aggregation.add(new Document("$project", new Document(AnnotationSetParams.ANNOTATION_SETS.key(), 1))); aggregation.add(new Document("$unwind", "$" + AnnotationSetParams.ANNOTATION_SETS.key())); // aggregation.add(new Document("$unwind", "$" + AnnotationSetParams.ANNOTATION_SETS_ANNOTATIONS.key())); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CatalogMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CatalogMongoDBAdaptor.java new file mode 100644 index 00000000000..fa7e803d985 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CatalogMongoDBAdaptor.java @@ -0,0 +1,35 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import org.bson.Document; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public abstract class CatalogMongoDBAdaptor extends MongoDBAdaptor { + + public CatalogMongoDBAdaptor(Configuration configuration, Logger logger) { + super(configuration, logger); + } + + abstract OpenCGAResult nativeGet(Query query, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + + List> nativeGet(List queries, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Objects.requireNonNull(queries); + List> queryResults = new ArrayList<>(queries.size()); + for (Query query : queries) { + queryResults.add(nativeGet(query, options)); + } + return queryResults; + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index c0ea18866a4..dd5b90a22d8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -249,7 +249,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va String clinicalAnalysisId = result.first().getId(); try { - return runTransaction(clientSession -> privateUpdate(clientSession, result.first(), parameters, variableSetList, + return runTransaction(clientSession -> transactionalUpdate(clientSession, result.first(), parameters, variableSetList, clinicalAuditList, queryOptions)); } catch (CatalogDBException e) { logger.error("Could not update clinical analysis {}: {}", clinicalAnalysisId, e.getMessage(), e); @@ -275,14 +275,15 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List throw new NotImplementedException("Use other update method passing the ClinicalAuditList"); } - OpenCGAResult privateUpdate(ClientSession clientSession, ClinicalAnalysis clinical, ObjectMap parameters, - List variableSetList, List clinicalAuditList, QueryOptions queryOptions) + OpenCGAResult transactionalUpdate(ClientSession clientSession, ClinicalAnalysis clinical, ObjectMap parameters, + List variableSetList, List clinicalAuditList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); String clinicalAnalysisId = clinical.getId(); long clinicalAnalysisUid = clinical.getUid(); - DataResult result = updateAnnotationSets(clientSession, clinicalAnalysisUid, parameters, variableSetList, queryOptions, false); + DataResult result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, variableSetList, + queryOptions, false); // Perform the update Query query = new Query(QueryParams.UID.key(), clinicalAnalysisUid); @@ -338,6 +339,42 @@ OpenCGAResult privateUpdate(ClientSession clientSession, ClinicalAnalysis clinic return endWrite(tmpStartTime, 1, 1, events); } + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, ClinicalAnalysis entry, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + throw new NotImplementedException("Please call to the other transactionalUpdate method passing the ClinicalAudit list"); + } + + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, + UpdateDocument updateDocument) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + + Document updateOperation = updateDocument.toFinalUpdateDocument(); + if (!updateOperation.isEmpty()) { + logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); + DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); + + if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, + ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + while (iterator.hasNext()) { + ClinicalAnalysis clinical = iterator.next(); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + } + } + + logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); + return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + } else { + throw new CatalogDBException("Nothing to update"); + } + } + UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List clinicalAuditList, Query query, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { @@ -1166,8 +1203,8 @@ void updateClinicalAnalysisFamilyReferences(ClientSession clientSession, Family } ObjectMap params = new ObjectMap(QueryParams.FAMILY.key(), familyCopy); - OpenCGAResult result = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().privateUpdate(clientSession, clinicalAnalysis, - params, Collections.emptyList(), null, QueryOptions.empty()); + OpenCGAResult result = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().transactionalUpdate(clientSession, + clinicalAnalysis, params, Collections.emptyList(), null, QueryOptions.empty()); if (result.getNumUpdated() != 1) { throw new CatalogDBException("ClinicalAnalysis '" + clinicalAnalysis.getId() + "' could not be updated to the latest " + "family version of '" + family.getId() + "'"); @@ -1224,7 +1261,7 @@ void updateClinicalAnalysisPanelReferences(ClientSession clientSession, Panel pa actionMap.put(PANELS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions updateOptions = new QueryOptions(Constants.ACTIONS, actionMap); ObjectMap params = new ObjectMap(PANELS.key(), panelList); - privateUpdate(clientSession, clinicalAnalysis, params, Collections.emptyList(), null, updateOptions); + transactionalUpdate(clientSession, clinicalAnalysis, params, Collections.emptyList(), null, updateOptions); // Update references from Interpretations dbAdaptorFactory.getInterpretationDBAdaptor().updateInterpretationPanelReferences(clientSession, clinicalAnalysis, panel); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index 63ff15c96fb..4a1977e1433 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -246,7 +246,7 @@ public OpenCGAResult update(long cohortUid, ObjectMap parameters, List privateUpdate(clientSession, documentResult.first(), parameters, variableSetList, + return runTransaction(clientSession -> transactionalUpdate(clientSession, documentResult.first(), parameters, variableSetList, queryOptions)); } catch (CatalogDBException e) { logger.error("Could not update cohort {}: {}", cohortId, e.getMessage(), e); @@ -274,7 +274,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List while (iterator.hasNext()) { Cohort cohort = iterator.next(); try { - result.append(runTransaction(clientSession -> privateUpdate(clientSession, cohort, parameters, variableSetList, + result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, cohort, parameters, variableSetList, queryOptions))); } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { logger.error("Could not update cohort {}: {}", cohort.getId(), e.getMessage(), e); @@ -285,15 +285,17 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List return result; } - private OpenCGAResult privateUpdate(ClientSession clientSession, Cohort cohort, ObjectMap parameters, - List variableSetList, QueryOptions queryOptions) + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, Cohort cohort, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); Query tmpQuery = new Query() .append(QueryParams.STUDY_UID.key(), cohort.getStudyUid()) .append(QueryParams.UID.key(), cohort.getUid()); - DataResult result = updateAnnotationSets(clientSession, cohort.getUid(), parameters, variableSetList, queryOptions, false); + DataResult result = updateAnnotationSets(clientSession, cohort.getStudyUid(), cohort.getUid(), parameters, variableSetList, + queryOptions, false); UpdateDocument parseUpdateDocument = parseAndValidateUpdateParams(clientSession, parameters, tmpQuery, queryOptions); Document cohortUpdate = parseUpdateDocument.toFinalUpdateDocument(); @@ -339,6 +341,22 @@ private OpenCGAResult privateUpdate(ClientSession clientSession, Cohort return endWrite(tmpStartTime, 1, 1, events); } + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + + Document cohortUpdate = updateDocument.toFinalUpdateDocument(); + if (cohortUpdate.isEmpty()) { + throw new CatalogDBException("Nothing to be updated"); + } + + logger.debug("Cohort update: query : {}, update: {}", query.toBsonDocument(), cohortUpdate.toBsonDocument()); + DataResult result = cohortCollection.update(clientSession, query, cohortUpdate, null); + logger.debug("{} cohorts successfully updated", result.getNumUpdated()); + return endWrite(tmpStartTime, result.getNumMatches(), result.getNumUpdated(), Collections.emptyList()); + } + private void updateCohortReferenceInSamples(ClientSession clientSession, Cohort cohort, List samples, ParamUtils.BasicUpdateAction updateAction) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 3bb88c7042b..e2717dbddb9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -19,7 +19,6 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; @@ -332,7 +331,7 @@ public OpenCGAResult update(long familyUid, ObjectMap parameters, List privateUpdate(clientSession, familyDataResult.first(), parameters, variableSetList, queryOptions)); + -> transactionalUpdate(clientSession, familyDataResult.first(), parameters, variableSetList, queryOptions)); } catch (CatalogDBException e) { logger.error("Could not update family {}: {}", familyDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update family " + familyDataResult.first().getId() + ": " + e.getMessage(), @@ -367,7 +366,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List Family family = iterator.next(); try { result.append(runTransaction(clientSession -> - privateUpdate(clientSession, family, parameters, variableSetList, queryOptions))); + transactionalUpdate(clientSession, family, parameters, variableSetList, queryOptions))); } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { logger.error("Could not update family {}: {}", family.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, family.getId(), e.getMessage())); @@ -377,18 +376,31 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List return result; } - OpenCGAResult privateUpdate(ClientSession clientSession, Family family, ObjectMap parameters, List variableSetList, - QueryOptions queryOptions) + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, Family family, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); + long studyUid = family.getStudyUid(); + long familyUid = family.getUid(); + if (studyUid <= 0) { + throw new CatalogDBException("Unexpected studyUid value received"); + } + if (familyUid <= 0) { + throw new CatalogDBException("Unexpected familyUid value received"); + } + Query tmpQuery = new Query() - .append(QueryParams.STUDY_UID.key(), family.getStudyUid()) - .append(QueryParams.UID.key(), family.getUid()); + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.UID.key(), familyUid); + List fieldsToInclude = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key(), QueryParams.MEMBERS.key() + "." + IndividualDBAdaptor.QueryParams.ID.key()); Bson bsonQuery = parseQuery(tmpQuery); - return versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { - DataResult result = updateAnnotationSets(clientSession, family.getUid(), parameters, variableSetList, queryOptions, - true); + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, fieldsToInclude, (entrylist) -> { + String familyId = entrylist.get(0).getString(QueryParams.ID.key()); + DataResult result = updateAnnotationSets(clientSession, studyUid, familyUid, parameters, + variableSetList, queryOptions, true); List familyMemberIds = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); boolean updateRoles = queryOptions.getBoolean(ParamConstants.FAMILY_UPDATE_ROLES_PARAM); boolean updatePedigree = queryOptions.getBoolean(ParamConstants.FAMILY_UPDATE_PEDIGREEE_GRAPH_PARAM); @@ -425,15 +437,14 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Family family, if (!familyMemberIds.isEmpty()) { // Fetch individuals with relevant information to guess the relationship Query individualQuery = new Query() - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), family.getStudyUid()) + .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(IndividualDBAdaptor.QueryParams.ID.key(), familyMemberIds); QueryOptions relationshipOptions = dbAdaptorFactory.getCatalogIndividualDBAdaptor().fixOptionsForRelatives( null); OpenCGAResult memberResult = dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(clientSession, individualQuery, relationshipOptions); family.setMembers(memberResult.getResults()); - Map> roles = calculateRoles(clientSession, family - .getStudyUid(), family); + Map> roles = calculateRoles(clientSession, studyUid, family); parameters.put(QueryParams.ROLES.key(), roles); } else { parameters.put(QueryParams.ROLES.key(), Collections.emptyMap()); @@ -477,8 +488,8 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Family family, // Fetch members (we don't trust those from the Family object because they could have been updated previously) Query query = new Query() - .append(IndividualDBAdaptor.QueryParams.FAMILY_IDS.key(), family.getId()) - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), family.getStudyUid()); + .append(IndividualDBAdaptor.QueryParams.FAMILY_IDS.key(), familyId) + .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); OpenCGAResult individualResult = dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(clientSession, query, IndividualManager.INCLUDE_INDIVIDUAL_IDS); List memberIds = individualResult.getResults().stream().map(Individual::getId) @@ -491,12 +502,12 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Family family, } if (result.getNumMatches() == 0) { - throw new CatalogDBException("Family " + family.getId() + " not found"); + throw new CatalogDBException("Family " + familyId + " not found"); } if (result.getNumUpdated() == 0) { - events.add(new Event(Event.Type.WARNING, family.getId(), "Family was already updated")); + events.add(new Event(Event.Type.WARNING, familyId, "Family was already updated")); } - logger.debug("Family {} successfully updated", family.getId()); + logger.debug("Family {} successfully updated", familyId); } return endWrite(tmpStartTime, 1, 1, events); @@ -504,6 +515,29 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Family family, this::iterator, (DBIterator iterator) -> updateReferencesAfterFamilyVersionIncrement(clientSession, iterator)); } + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + List fieldsToInclude = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key(), QueryParams.MEMBERS.key() + "." + IndividualDBAdaptor.QueryParams.ID.key()); + + return versionedMongoDBAdaptor.update(clientSession, query, fieldsToInclude, (entrylist) -> { + Document familyUpdate = updateDocument.toFinalUpdateDocument(); + + if (familyUpdate.isEmpty()) { + throw new CatalogDBException("Nothing to be updated"); + } + + logger.debug("Family update: query : {}, update: {}", query.toBsonDocument(), familyUpdate.toBsonDocument()); + DataResult result = familyCollection.update(clientSession, query, familyUpdate, + new QueryOptions(MongoDBCollection.MULTI, true)); + logger.debug("{} families successfully updated", result.getNumUpdated()); + return endWrite(tmpStartTime, result.getNumMatches(), result.getNumUpdated(), Collections.emptyList()); + }, Arrays.asList(QueryParams.MEMBERS_ID.key(), QueryParams.MEMBERS_SAMPLES_ID.key()), + this::iterator, (DBIterator iterator) -> updateReferencesAfterFamilyVersionIncrement(clientSession, iterator)); + } + private PedigreeGraph computePedigreeGraph(ClientSession clientSession, Family family) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query query = new Query() @@ -590,18 +624,11 @@ void updateIndividualReferencesInFamily(ClientSession clientSession, long studyU ObjectMap params = new ObjectMap(FamilyDBAdaptor.QueryParams.MEMBERS.key(), members); UpdateDocument updateDocument = parseAndValidateUpdateParams(clientSession, params, tmpQuery); - Document bsonUpdate = updateDocument.toFinalUpdateDocument(); Bson bsonQuery = parseQuery(tmpQuery); - versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { - DataResult result = familyCollection.update(clientSession, bsonQuery, bsonUpdate, QueryOptions.empty()); - if (result.getNumUpdated() != 1) { - throw new CatalogDBException("Family '" + family.getId() + "' could not be updated to the latest member" - + " versions"); - } - return result; - }, Arrays.asList(QueryParams.MEMBERS_ID.key(), QueryParams.MEMBERS_SAMPLES_ID.key()), - this::iterator, - (DBIterator fIterator) -> updateReferencesAfterFamilyVersionIncrement(clientSession, fIterator)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); + if (result.getNumUpdated() != 1) { + throw new CatalogDBException("Family '" + family.getId() + "' could not be updated to the latest member versions"); + } } } } @@ -657,12 +684,9 @@ void updateIndividualIdFromFamilies(ClientSession clientSession, long studyUid, .append(QueryParams.STUDY_UID.key(), studyUid) .append(QueryParams.UID.key(), family.getUid()) ); - versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { - Bson update = Updates.set(QueryParams.ROLES.key(), getMongoDBDocument(roles, QueryParams.ROLES.key())); - return familyCollection.update(clientSession, bsonQuery, update, QueryOptions.empty()); - }, Arrays.asList(QueryParams.MEMBERS_ID.key(), QueryParams.MEMBERS_SAMPLES_ID.key()), - this::iterator, - (DBIterator fIterator) -> updateReferencesAfterFamilyVersionIncrement(clientSession, fIterator)); + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getSet().put(QueryParams.ROLES.key(), getMongoDBDocument(roles, QueryParams.ROLES.key())); + transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 4d0e93e23c5..d283a09fd2b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -31,7 +31,6 @@ import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.FileConverter; -import org.opencb.opencga.catalog.db.mongodb.converters.SampleConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.FileCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; @@ -153,14 +152,14 @@ public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtual ObjectMap params = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList( new FileRelatedFile(file, FileRelatedFile.Relation.MULTIPART) )); - privateUpdate(clientSession, virtualFile, params, null, qOptions); + transactionalUpdate(clientSession, virtualFile, params, null, qOptions); } // Add multipart file in physical file ObjectMap params = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList( new FileRelatedFile(virtualFile, FileRelatedFile.Relation.MULTIPART) )); - privateUpdate(clientSession, file, params, null, qOptions); + transactionalUpdate(clientSession, file, params, null, qOptions); return endWrite(tmpStartTime, 1, 1, 0, 0, null); }, @@ -214,9 +213,7 @@ long insert(ClientSession clientSession, long studyId, File file, List e logger.debug("Updating list of fileIds in batch of {} samples...", sampleList.size()); for (Sample sample : sampleList) { - SampleConverter sampleConverter = dbAdaptorFactory.getCatalogSampleDBAdaptor().getSampleConverter(); - Document sampleDocument = sampleConverter.convertToStorageType(sample); - dbAdaptorFactory.getCatalogSampleDBAdaptor().privateUpdate(clientSession, sampleDocument, params, null, + dbAdaptorFactory.getCatalogSampleDBAdaptor().transactionalUpdate(clientSession, sample, params, null, sampleUpdateOptions); } @@ -336,7 +333,7 @@ public OpenCGAResult update(long fileUid, ObjectMap parameters, List privateUpdate(clientSession, fileDataResult.first(), parameters, + return runTransaction(clientSession -> transactionalUpdate(clientSession, fileDataResult.first(), parameters, variableSetList, queryOptions)); } catch (CatalogDBException e) { logger.error("Could not update file {}: {}", fileDataResult.first().getPath(), e.getMessage(), e); @@ -362,7 +359,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List while (iterator.hasNext()) { File file = iterator.next(); try { - result.append(runTransaction(clientSession -> privateUpdate(clientSession, file, parameters, variableSetList, + result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, file, parameters, variableSetList, queryOptions))); } catch (CatalogDBException e) { logger.error("Could not update file {}: {}", file.getPath(), e.getMessage(), e); @@ -373,20 +370,24 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List return result; } - OpenCGAResult privateUpdate(ClientSession clientSession, File file, ObjectMap parameters, List variableSetList, - QueryOptions queryOptions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, File file, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { long tmpStartTime = startQuery(); + long studyUid = file.getStudyUid(); + long fileUid = file.getUid(); Query tmpQuery = new Query() - .append(QueryParams.STUDY_UID.key(), file.getStudyUid()) - .append(QueryParams.UID.key(), file.getUid()); + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.UID.key(), fileUid); // We perform the update. Bson queryBson = parseQuery(tmpQuery); - DataResult result = updateAnnotationSets(clientSession, file.getUid(), parameters, variableSetList, queryOptions, false); + DataResult result = updateAnnotationSets(clientSession, studyUid, fileUid, parameters, variableSetList, + queryOptions, false); - UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, file.getStudyUid(), parameters, tmpQuery, queryOptions); + UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, studyUid, parameters, tmpQuery, queryOptions); Document fileUpdate = updateDocument.toFinalUpdateDocument(); if (fileUpdate.isEmpty() && result.getNumUpdated() == 0) { @@ -406,7 +407,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, File file, Obje if (parameters.containsKey(QueryParams.SIZE.key())) { long newDiskUsage = parameters.getLong(QueryParams.SIZE.key()); long difDiskUsage = newDiskUsage - file.getSize(); - dbAdaptorFactory.getCatalogStudyDBAdaptor().updateDiskUsage(clientSession, file.getStudyUid(), difDiskUsage); + dbAdaptorFactory.getCatalogStudyDBAdaptor().updateDiskUsage(clientSession, studyUid, difDiskUsage); } updateSampleReferences(clientSession, file, updateDocument); @@ -423,6 +424,26 @@ OpenCGAResult privateUpdate(ClientSession clientSession, File file, Obje return endWrite(tmpStartTime, 1, 1, events); } + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument) + throws CatalogDBException { + long tmpStartTime = startQuery(); + + Document fileUpdate = updateDocument.toFinalUpdateDocument(); + + if (fileUpdate.isEmpty()) { + throw new CatalogDBException("Nothing to be updated"); + } + + List events = new ArrayList<>(); + logger.debug("Update file. Query: {}, Update: {}", query.toBsonDocument(), fileUpdate.toBsonDocument()); + + DataResult result = fileCollection.update(clientSession, query, fileUpdate, null); + logger.debug("{} file(s) successfully updated", result.getNumUpdated()); + + return endWrite(tmpStartTime, result.getNumMatches(), result.getNumUpdated(), events); + } + private void updateSampleReferences(ClientSession clientSession, File file, UpdateDocument updateDocument) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (!updateDocument.getAttributes().isEmpty()) { @@ -456,38 +477,36 @@ private void updateSampleReferences(ClientSession clientSession, File file, Upda sampleUpdate = new UpdateDocument(); sampleUpdate.getSet().append(SampleDBAdaptor.QueryParams.FILE_IDS.key() + ".$", newFileId); - dbAdaptorFactory.getCatalogSampleDBAdaptor().getCollection().update(clientSession, sampleBsonQuery, - sampleUpdate.toFinalUpdateDocument(), new QueryOptions(MongoDBCollection.MULTI, true)); + dbAdaptorFactory.getCatalogSampleDBAdaptor().transactionalUpdate(clientSession, file.getStudyUid(), sampleBsonQuery, + sampleUpdate); } if (addedSamples != null && !addedSamples.isEmpty()) { Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), file.getStudyUid()) .append(SampleDBAdaptor.QueryParams.UID.key(), addedSamples.getAsLongList(file.getId())); - List sampleList = dbAdaptorFactory.getCatalogSampleDBAdaptor().nativeGet(clientSession, query, - dbAdaptorFactory.getCatalogSampleDBAdaptor().SAMPLE_FETCH_FOR_UPDATE_OPTIONS).getResults(); + sampleBsonQuery = dbAdaptorFactory.getCatalogSampleDBAdaptor().parseQuery(query, null); ObjectMap actionMap = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), BasicUpdateAction.ADD.name()); QueryOptions sampleUpdateOptions = new QueryOptions(Constants.ACTIONS, actionMap); + updateDocument = dbAdaptorFactory.getCatalogSampleDBAdaptor().parseAndValidateUpdateParams(clientSession, + file.getStudyUid(), params, query, sampleUpdateOptions); - for (Document sampleDocument : sampleList) { - dbAdaptorFactory.getCatalogSampleDBAdaptor().privateUpdate(clientSession, sampleDocument, params, null, - sampleUpdateOptions); - } + dbAdaptorFactory.getCatalogSampleDBAdaptor().transactionalUpdate(clientSession, file.getStudyUid(), sampleBsonQuery, + updateDocument); } if (removedSamples != null && !removedSamples.isEmpty()) { Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), file.getStudyUid()) .append(SampleDBAdaptor.QueryParams.UID.key(), removedSamples.getAsLongList(file.getId())); - List sampleList = dbAdaptorFactory.getCatalogSampleDBAdaptor().nativeGet(clientSession, query, - dbAdaptorFactory.getCatalogSampleDBAdaptor().SAMPLE_FETCH_FOR_UPDATE_OPTIONS).getResults(); + sampleBsonQuery = dbAdaptorFactory.getCatalogSampleDBAdaptor().parseQuery(query, null); ObjectMap actionMap = new ObjectMap(SampleDBAdaptor.QueryParams.FILE_IDS.key(), BasicUpdateAction.REMOVE.name()); QueryOptions sampleUpdateOptions = new QueryOptions(Constants.ACTIONS, actionMap); + updateDocument = dbAdaptorFactory.getCatalogSampleDBAdaptor().parseAndValidateUpdateParams(clientSession, + file.getStudyUid(), params, query, sampleUpdateOptions); - for (Document sampleDocument : sampleList) { - dbAdaptorFactory.getCatalogSampleDBAdaptor().privateUpdate(clientSession, sampleDocument, params, null, - sampleUpdateOptions); - } + dbAdaptorFactory.getCatalogSampleDBAdaptor().transactionalUpdate(clientSession, file.getStudyUid(), sampleBsonQuery, + updateDocument); } } } @@ -719,6 +738,12 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon nestedPut(QueryParams.STATUS_DATE.key(), TimeUtils.getTime(), document.getSet()); } + if (parameters.containsKey(QueryParams.INTERNAL_STATUS_ID.key())) { + FileStatus fileStatus = new FileStatus(parameters.getString(QueryParams.INTERNAL_STATUS_ID.key())); + Document fileStatusDoc = getMongoDBDocument(fileStatus, QueryParams.INTERNAL_STATUS.key()); + document.getSet().put(QueryParams.INTERNAL_STATUS.key(), fileStatusDoc); + } + if (!document.toFinalUpdateDocument().isEmpty()) { String time = TimeUtils.getTime(); if (StringUtils.isEmpty(parameters.getString(MODIFICATION_DATE.key()))) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index d9c6857200a..f325a1cf6e8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -18,7 +18,6 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; @@ -214,13 +213,13 @@ Individual insert(ClientSession clientSession, long studyId, Individual individu void updateFamilyReferences(ClientSession clientSession, long studyUid, List individualIds, String familyId, ParamUtils.BasicUpdateAction action) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { - Bson bsonUpdate; + UpdateDocument updateDocument = new UpdateDocument(); switch (action) { case ADD: - bsonUpdate = Updates.addToSet(QueryParams.FAMILY_IDS.key(), familyId); + updateDocument.getAddToSet().put(QueryParams.FAMILY_IDS.key(), familyId); break; case REMOVE: - bsonUpdate = Updates.pull(QueryParams.FAMILY_IDS.key(), familyId); + updateDocument.getPull().put(QueryParams.FAMILY_IDS.key(), familyId); break; case SET: default: @@ -232,15 +231,7 @@ void updateFamilyReferences(ClientSession clientSession, long studyUid, List { - DataResult update = individualCollection.update(clientSession, bsonQuery, bsonUpdate, - new QueryOptions(MongoDBCollection.MULTI, true)); - if (update.getNumMatches() == 0) { - throw new CatalogDBException("Could not update family references in individuals"); - } - return null; - }, Collections.singletonList(QueryParams.SAMPLES_IDS.key()), this::iterator, - (DBIterator iterator) -> updateReferencesAfterIndividualVersionIncrement(clientSession, studyUid, iterator)); + transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); } @Override @@ -335,20 +326,11 @@ public OpenCGAResult update(long id, ObjectMap parameters, QueryOptions queryOpt @Override public OpenCGAResult update(long individualUid, ObjectMap parameters, List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.STUDY_UID.key())); - OpenCGAResult dataResult = get(individualUid, options); - - if (dataResult.getNumResults() == 0) { - throw new CatalogDBException("Could not update individual. Individual uid '" + individualUid + "' not found."); - } - try { - return runTransaction(clientSession -> privateUpdate(clientSession, dataResult.first(), parameters, variableSetList, + return runTransaction(clientSession -> transactionalUpdate(clientSession, individualUid, parameters, variableSetList, queryOptions)); } catch (CatalogDBException e) { - logger.error("Could not update individual {}: {}", dataResult.first().getId(), e.getMessage(), e); - throw new CatalogDBException("Could not update individual " + dataResult.first().getId() + ": " + e.getMessage(), e.getCause()); + throw new CatalogDBException("Could not update individual: " + e.getMessage(), e.getCause()); } } @@ -379,7 +361,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List Individual individual = iterator.next(); try { - result.append(runTransaction(clientSession -> privateUpdate(clientSession, individual, parameters, variableSetList, + result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, individual, parameters, variableSetList, queryOptions))); } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { logger.error("Could not update individual {}: {}", individual.getId(), e.getMessage(), e); @@ -390,82 +372,215 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List return result; } - OpenCGAResult privateUpdate(ClientSession clientSession, Individual individual, ObjectMap parameters, - List variableSetList, QueryOptions queryOptions) + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, Individual individual, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); + long studyUid = individual.getStudyUid(); + long individualUid = individual.getUid(); + if (studyUid <= 0) { + throw new CatalogDBException("Unexpected studyUid value received"); + } + if (individualUid <= 0) { + throw new CatalogDBException("Unexpected individualUid value received"); + } Query tmpQuery = new Query() - .append(QueryParams.STUDY_UID.key(), individual.getStudyUid()) - .append(QueryParams.UID.key(), individual.getUid()); + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.UID.key(), individualUid); + List fieldsToInclude = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key()); + Bson bson = parseQuery(tmpQuery); - return versionedMongoDBAdaptor.update(clientSession, bson, () -> { - DataResult result = updateAnnotationSets(clientSession, individual.getUid(), parameters, variableSetList, queryOptions, true); - UpdateDocument updateDocument = parseAndValidateUpdateParams(clientSession, parameters, tmpQuery, queryOptions); - Document individualUpdate = updateDocument.toFinalUpdateDocument(); - - if (individualUpdate.isEmpty() && result.getNumUpdated() == 0) { - if (!parameters.isEmpty()) { - logger.error("Non-processed update parameters: {}", parameters.keySet()); - } - throw new CatalogDBException("Nothing to be updated"); - } + return versionedMongoDBAdaptor.update(clientSession, bson, fieldsToInclude, (entryList) -> { + String individualId = entryList.get(0).getString(QueryParams.ID.key()); + DataResult result = updateAnnotationSets(clientSession, studyUid, individualUid, parameters, + variableSetList, queryOptions, true); + UpdateDocument updateDocument = parseAndValidateUpdateParams(clientSession, parameters, tmpQuery, queryOptions); + Document individualUpdate = updateDocument.toFinalUpdateDocument(); + + if (individualUpdate.isEmpty() && result.getNumUpdated() == 0) { + if (!parameters.isEmpty()) { + logger.error("Non-processed update parameters: {}", parameters.keySet()); + } + throw new CatalogDBException("Nothing to be updated"); + } - List events = new ArrayList<>(); - if (!individualUpdate.isEmpty()) { - Bson finalQuery = parseQuery(tmpQuery); + List events = new ArrayList<>(); + if (!individualUpdate.isEmpty()) { + Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Individual update: query : {}, update: {}", finalQuery.toBsonDocument(), individualUpdate.toBsonDocument()); + logger.debug("Individual update: query : {}, update: {}", finalQuery.toBsonDocument(), + individualUpdate.toBsonDocument()); - result = individualCollection.update(clientSession, finalQuery, individualUpdate, new QueryOptions("multi", true)); + result = individualCollection.update(clientSession, finalQuery, individualUpdate, new QueryOptions("multi", true)); - if (result.getNumMatches() == 0) { - throw new CatalogDBException("Individual " + individual.getId() + " not found"); - } - if (result.getNumUpdated() == 0) { - events.add(new Event(Event.Type.WARNING, individual.getId(), "Individual was already updated")); - } + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Individual " + individualId + " not found"); + } + if (result.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, individualId, "Individual was already updated")); + } - if (!updateDocument.getAttributes().isEmpty()) { - List addedSamples = updateDocument.getAttributes().getAsLongList("ADDED_SAMPLES"); - List removedSamples = updateDocument.getAttributes().getAsLongList("REMOVED_SAMPLES"); + if (!updateDocument.getAttributes().isEmpty()) { + List addedSamples = updateDocument.getAttributes().getAsLongList("ADDED_SAMPLES"); + List removedSamples = updateDocument.getAttributes().getAsLongList("REMOVED_SAMPLES"); - // Set new individual reference - dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualFromSampleCollection(clientSession, - individual.getStudyUid(), addedSamples, individual.getId()); + // Set new individual reference + dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualFromSampleCollection(clientSession, + studyUid, addedSamples, individualId); - // Set individual reference to "" - dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualFromSampleCollection(clientSession, - individual.getStudyUid(), removedSamples, ""); - } + // Set individual reference to "" + dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualFromSampleCollection(clientSession, + studyUid, removedSamples, ""); + } - // If the list of disorders or phenotypes is altered, we will need to update the corresponding effective lists - // of the families associated (if any) - if (parameters.containsKey(QueryParams.DISORDERS.key()) || parameters.containsKey(QueryParams.PHENOTYPES.key())) { - recalculateFamilyDisordersPhenotypes(clientSession, individual); - } + // If the list of disorders or phenotypes is altered, we will need to update the corresponding effective lists + // of the families associated (if any) + if (parameters.containsKey(QueryParams.DISORDERS.key()) || parameters.containsKey(QueryParams.PHENOTYPES.key())) { + recalculateFamilyDisordersPhenotypes(clientSession, studyUid, individualUid); + } - if (StringUtils.isNotEmpty(parameters.getString(QueryParams.ID.key()))) { - // We need to update the individual id reference in all its samples - dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualIdFromSamples(clientSession, individual.getStudyUid(), - individual.getId(), parameters.getString(QueryParams.ID.key())); + if (StringUtils.isNotEmpty(parameters.getString(QueryParams.ID.key()))) { + // We need to update the individual id reference in all its samples + dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualIdFromSamples(clientSession, studyUid, + individualId, parameters.getString(QueryParams.ID.key())); - // Update the family roles - familyDBAdaptor.updateIndividualIdFromFamilies(clientSession, individual.getStudyUid(), - individual.getUid(), individual.getId(), parameters.getString(QueryParams.ID.key())); - } + // Update the family roles + familyDBAdaptor.updateIndividualIdFromFamilies(clientSession, studyUid, individualUid, individualId, + parameters.getString(QueryParams.ID.key())); + } - if (parameters.containsKey(QueryParams.FATHER_UID.key()) || parameters.containsKey(QueryParams.MOTHER_UID.key())) { - // If the parents have changed, we need to check family roles - recalculateFamilyRolesForMember(clientSession, individual.getStudyUid(), individual.getUid()); - } + if (parameters.containsKey(QueryParams.FATHER_UID.key()) || parameters.containsKey(QueryParams.MOTHER_UID.key())) { + // If the parents have changed, we need to check family roles + recalculateFamilyRolesForMember(clientSession, studyUid, individualUid); + } - logger.debug("Individual {} successfully updated", individual.getId()); - } + logger.debug("Individual {} successfully updated", individualId); + } - return endWrite(tmpStartTime, 1, 1, events); - }, Collections.singletonList(QueryParams.SAMPLES_IDS.key()), this::iterator, - (DBIterator iterator) -> updateReferencesAfterIndividualVersionIncrement(clientSession, - individual.getStudyUid(), iterator)); + return endWrite(tmpStartTime, 1, 1, events); + }, Collections.singletonList(QueryParams.SAMPLES_IDS.key()), this::iterator, + (DBIterator iterator) -> updateReferencesAfterIndividualVersionIncrement(clientSession, studyUid, iterator)); + } + + OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + List fieldsToInclude = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key()); + + return versionedMongoDBAdaptor.update(clientSession, query, fieldsToInclude, (entryList) -> { + List events = new ArrayList<>(); + Document update = updateDocument.toFinalUpdateDocument(); + if (!update.isEmpty()) { + logger.debug("Individual update: query : {}, update: {}", query.toBsonDocument(), update.toBsonDocument()); + DataResult result = individualCollection.update(clientSession, query, update, + new QueryOptions(MongoDBCollection.MULTI, true)); + List individualIds = entryList.stream().map(x -> x.getString(QueryParams.ID.key())) + .collect(Collectors.toList()); + if (result.getNumUpdated() == 0) { + for (String individualId : individualIds) { + events.add(new Event(Event.Type.WARNING, individualId, "Individual was already updated")); + } + } + + logger.debug("Individual(s) {} successfully updated", StringUtils.join(individualIds, ", ")); + return endWrite(tmpStartTime, result.getNumMatches(), result.getNumUpdated(), events); + } else { + return endWrite(tmpStartTime, entryList.size(), 0, events); + } + }, Collections.singletonList(QueryParams.SAMPLES_IDS.key()), this::iterator, + (DBIterator iterator) -> updateReferencesAfterIndividualVersionIncrement(clientSession, studyUid, iterator)); + } + + // If we know the study uid, we should be calling to the other transactionalUpdate method that receives the entire object instead of + // the uid + @Deprecated + OpenCGAResult transactionalUpdate(ClientSession clientSession, long individualUid, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + Query tmpQuery = new Query() + .append(QueryParams.UID.key(), individualUid); + Bson bson = parseQuery(tmpQuery); + List fieldsToInclude = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key()); + // TODO: Quick query simply to obtain the study uid. This should be avoided. The method should already receive the study uid + Document individualDocument = nativeGet(tmpQuery, new QueryOptions(QueryOptions.INCLUDE, fieldsToInclude)).first(); + long studyUid = individualDocument.get(QueryParams.STUDY_UID.key(), Number.class).longValue(); + return versionedMongoDBAdaptor.update(clientSession, bson, fieldsToInclude, entryList -> { + Document individual = entryList.get(0); + String individualId = individual.getString(QueryParams.ID.key()); + + DataResult result = updateAnnotationSets(clientSession, studyUid, individualUid, parameters, variableSetList, + queryOptions, true); + UpdateDocument updateDocument = parseAndValidateUpdateParams(clientSession, parameters, tmpQuery, queryOptions); + Document individualUpdate = updateDocument.toFinalUpdateDocument(); + + if (individualUpdate.isEmpty() && result.getNumUpdated() == 0) { + if (!parameters.isEmpty()) { + logger.error("Non-processed update parameters: {}", parameters.keySet()); + } + throw new CatalogDBException("Nothing to be updated"); + } + + List events = new ArrayList<>(); + if (!individualUpdate.isEmpty()) { + Bson finalQuery = parseQuery(tmpQuery); + + logger.debug("Individual update: query : {}, update: {}", finalQuery.toBsonDocument(), + individualUpdate.toBsonDocument()); + + result = individualCollection.update(clientSession, finalQuery, individualUpdate, new QueryOptions("multi", true)); + + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Individual " + individualId + " not found"); + } + if (result.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, individualId, "Individual was already updated")); + } + + if (!updateDocument.getAttributes().isEmpty()) { + List addedSamples = updateDocument.getAttributes().getAsLongList("ADDED_SAMPLES"); + List removedSamples = updateDocument.getAttributes().getAsLongList("REMOVED_SAMPLES"); + + // Set new individual reference + dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualFromSampleCollection(clientSession, studyUid, + addedSamples, individualId); + + // Set individual reference to "" + dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualFromSampleCollection(clientSession, studyUid, + removedSamples, ""); + } + + // If the list of disorders or phenotypes is altered, we will need to update the corresponding effective lists + // of the families associated (if any) + if (parameters.containsKey(QueryParams.DISORDERS.key()) || parameters.containsKey(QueryParams.PHENOTYPES.key())) { + recalculateFamilyDisordersPhenotypes(clientSession, studyUid, individualUid); + } + + if (StringUtils.isNotEmpty(parameters.getString(QueryParams.ID.key()))) { + // We need to update the individual id reference in all its samples + dbAdaptorFactory.getCatalogSampleDBAdaptor().updateIndividualIdFromSamples(clientSession, studyUid, + individualId, parameters.getString(QueryParams.ID.key())); + + // Update the family roles + familyDBAdaptor.updateIndividualIdFromFamilies(clientSession, studyUid, individualUid, individualId, + parameters.getString(QueryParams.ID.key())); + } + + if (parameters.containsKey(QueryParams.FATHER_UID.key()) || parameters.containsKey(QueryParams.MOTHER_UID.key())) { + // If the parents have changed, we need to check family roles + recalculateFamilyRolesForMember(clientSession, studyUid, individualUid); + } + + logger.debug("Individual {} successfully updated", individualId); + } + + return endWrite(tmpStartTime, 1, 1, events); + }, Collections.singletonList(QueryParams.SAMPLES_IDS.key()), this::iterator, + (DBIterator iterator) -> updateReferencesAfterIndividualVersionIncrement(clientSession, studyUid, iterator)); } private void recalculateFamilyRolesForMember(ClientSession clientSession, long studyUid, long memberUid) @@ -480,7 +595,7 @@ private void recalculateFamilyRolesForMember(ClientSession clientSession, long s try (DBIterator iterator = familyDBAdaptor.iterator(clientSession, query, options)) { while (iterator.hasNext()) { Family family = iterator.next(); - familyDBAdaptor.privateUpdate(clientSession, family, new ObjectMap(), null, + familyDBAdaptor.transactionalUpdate(clientSession, family, new ObjectMap(), null, new QueryOptions(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true)); } } @@ -501,18 +616,18 @@ private void updateReferencesAfterIndividualVersionIncrement(ClientSession clien } } - private void recalculateFamilyDisordersPhenotypes(ClientSession clientSession, Individual individual) + private void recalculateFamilyDisordersPhenotypes(ClientSession clientSession, long studyUid, long individualUid) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { // We fetch the current updated individual to know how the current list of disorders and phenotypes QueryOptions individualOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.PHENOTYPES.key(), QueryParams.DISORDERS.key())); Individual currentIndividual = get(clientSession, new Query() - .append(QueryParams.STUDY_UID.key(), individual.getStudyUid()) - .append(QueryParams.UID.key(), individual.getUid()), individualOptions).first(); + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.UID.key(), individualUid), individualOptions).first(); Query familyQuery = new Query() - .append(FamilyDBAdaptor.QueryParams.MEMBER_UID.key(), individual.getUid()) - .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), individual.getStudyUid()); + .append(FamilyDBAdaptor.QueryParams.MEMBER_UID.key(), individualUid) + .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); QueryOptions familyOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( FamilyDBAdaptor.QueryParams.UID.key(), FamilyDBAdaptor.QueryParams.STUDY_UID.key(), FamilyDBAdaptor.QueryParams.MEMBERS.key())); @@ -530,7 +645,7 @@ private void recalculateFamilyDisordersPhenotypes(ClientSession clientSession, I ObjectMap params = getNewFamilyDisordersAndPhenotypesToUpdate(family, currentIndividual.getDisorders(), currentIndividual.getPhenotypes(), currentIndividual.getUid()); - familyDBAdaptor.privateUpdate(clientSession, family, params, null, familyUpdateOptions); + familyDBAdaptor.transactionalUpdate(clientSession, family, params, null, familyUpdateOptions); } } @@ -625,7 +740,7 @@ private void updateClinicalAnalysisIndividualReferences(ClientSession clientSess ObjectMap params = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.PROBAND.key(), individualCopy); - OpenCGAResult result = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().privateUpdate(clientSession, clinicalAnalysis, + OpenCGAResult result = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().transactionalUpdate(clientSession, clinicalAnalysis, params, Collections.emptyList(), null, QueryOptions.empty()); if (result.getNumUpdated() != 1) { throw new CatalogDBException("ClinicalAnalysis '" + clinicalAnalysis.getId() + "' could not be updated to the latest " @@ -1078,12 +1193,12 @@ public OpenCGAResult get(ClientSession clientSession, Query query, Q } @Override - public OpenCGAResult nativeGet(Query query, QueryOptions options) + public OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { return nativeGet(null, query, options); } - public OpenCGAResult nativeGet(ClientSession clientSession, Query query, QueryOptions options) + public OpenCGAResult nativeGet(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long startTime = startQuery(); try (DBIterator dbIterator = nativeIterator(clientSession, query, options)) { @@ -1092,12 +1207,12 @@ public OpenCGAResult nativeGet(ClientSession clientSession, Query query, QueryOp } @Override - public OpenCGAResult nativeGet(long studyUid, Query query, QueryOptions options, String user) + public OpenCGAResult nativeGet(long studyUid, Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { return nativeGet(null, studyUid, query, options, user); } - public OpenCGAResult nativeGet(ClientSession clientSession, long studyUid, Query query, QueryOptions options, String user) + public OpenCGAResult nativeGet(ClientSession clientSession, long studyUid, Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { long startTime = startQuery(); try (DBIterator dbIterator = nativeIterator(clientSession, studyUid, query, options, user)) { @@ -1669,9 +1784,9 @@ void removeSampleReferences(ClientSession clientSession, long studyUid, long sam QueryOptions queryOptions = new QueryOptions(Constants.ACTIONS, new ObjectMap(QueryParams.SAMPLES.key(), ParamUtils.BasicUpdateAction.REMOVE.name())); - Bson update; + UpdateDocument updateDocument; try { - update = parseAndValidateUpdateParams(clientSession, params, query, queryOptions).toFinalUpdateDocument(); + updateDocument = parseAndValidateUpdateParams(clientSession, params, query, queryOptions); } catch (CatalogDBException e) { if (e.getMessage().contains("No individual found to be updated")) { return; @@ -1681,16 +1796,9 @@ void removeSampleReferences(ClientSession clientSession, long studyUid, long sam } Bson bsonQuery = parseQuery(query); - versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { - QueryOptions multi = new QueryOptions(MongoDBCollection.MULTI, true); - logger.debug("Sample references extraction. Query: {}, update: {}", bsonQuery.toBsonDocument(), - update.toBsonDocument()); - DataResult updateResult = individualCollection.update(clientSession, bsonQuery, update, multi); - logger.debug("Sample uid '" + sampleUid + "' references removed from " + updateResult.getNumUpdated() + " out of " - + updateResult.getNumMatches() + " individuals"); - return null; - }, Collections.singletonList(QueryParams.SAMPLES_IDS.key()), this::iterator, - (DBIterator iterator) -> updateReferencesAfterIndividualVersionIncrement(clientSession, studyUid, iterator)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); + logger.debug("Sample uid '" + sampleUid + "' references removed from " + result.getNumUpdated() + " out of " + + result.getNumMatches() + " individuals"); } public MongoDBCollection getIndividualCollection() { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java index 97b1d50546b..af0cc61e95a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java @@ -67,7 +67,7 @@ import static org.opencb.opencga.catalog.db.mongodb.ClinicalAnalysisMongoDBAdaptor.fixCommentsForRemoval; import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.*; -public class InterpretationMongoDBAdaptor extends MongoDBAdaptor implements InterpretationDBAdaptor { +public class InterpretationMongoDBAdaptor extends CatalogMongoDBAdaptor implements InterpretationDBAdaptor { private final MongoDBCollection interpretationCollection; private final MongoDBCollection archiveInterpretationCollection; @@ -200,7 +200,7 @@ private void updateClinicalAnalysisReferences(ClientSession clientSession, Inter } // Update interpretation(s) in ClinicalAnalysis - clinicalDBAdaptor.privateUpdate(clientSession, ca, params, Collections.emptyList(), clinicalAuditList, options); + clinicalDBAdaptor.transactionalUpdate(clientSession, ca, params, Collections.emptyList(), clinicalAuditList, options); break; case SECONDARY: // Add to secondaryInterpretations array in ClinicalAnalysis @@ -215,7 +215,7 @@ private void updateClinicalAnalysisReferences(ClientSession clientSession, Inter params.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERPRETATION.key(), null); } - clinicalDBAdaptor.privateUpdate(clientSession, ca, params, Collections.emptyList(), clinicalAuditList, options); + clinicalDBAdaptor.transactionalUpdate(clientSession, ca, params, Collections.emptyList(), clinicalAuditList, options); break; default: throw new IllegalStateException("Unknown action " + action); @@ -686,7 +686,7 @@ OpenCGAResult update(ClientSession clientSession, Interpretation if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { // Updates to Interpretation data model -> increment version - return versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, (entrylist) -> { DataResult update = DataResult.empty(); // Because it will generate a new interpretation version, we set version to +1 so the reference in clinical is also @@ -785,7 +785,7 @@ private void updateClinicalAnalysisInterpretationReference(ClientSession clientS params = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.SECONDARY_INTERPRETATIONS.key(), interpretationList); } - OpenCGAResult update = clinicalDBAdaptor.privateUpdate(clientSession, ca, params, Collections.emptyList(), clinicalAuditList, + OpenCGAResult update = clinicalDBAdaptor.transactionalUpdate(clientSession, ca, params, Collections.emptyList(), clinicalAuditList, options); if (update.getNumUpdated() != 1) { throw new CatalogDBException("Could not update interpretation reference in Clinical Analysis to new version"); @@ -878,7 +878,7 @@ OpenCGAResult delete(ClientSession clientSession, Interpretation interpretation, actions.put(ClinicalAnalysisDBAdaptor.QueryParams.SECONDARY_INTERPRETATIONS.key(), ParamUtils.BasicUpdateAction.REMOVE); clinicalOptions.put(Constants.ACTIONS, actions); } - clinicalDBAdaptor.privateUpdate(clientSession, clinicalAnalysis, clinicalParams, Collections.emptyList(), clinicalAuditList, + clinicalDBAdaptor.transactionalUpdate(clientSession, clinicalAnalysis, clinicalParams, Collections.emptyList(), clinicalAuditList, clinicalOptions); Query query = new Query() diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 48bedd2acaf..9f201fa9cad 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -64,7 +64,7 @@ /** * Created by pfurio on 08/01/16. */ -public class JobMongoDBAdaptor extends MongoDBAdaptor implements JobDBAdaptor { +public class JobMongoDBAdaptor extends CatalogMongoDBAdaptor implements JobDBAdaptor { private static final String PRIVATE_PRIORITY = "_priority"; private static final String PRIVATE_STUDY_UIDS = "_studyUids"; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index a3b47c037dc..0c577c0a9da 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -45,7 +45,7 @@ /** * Created by jacobo on 12/09/14. */ -public class MongoDBAdaptor extends AbstractDBAdaptor { +public abstract class MongoDBAdaptor extends AbstractDBAdaptor { public static final String PRIVATE_UID = "uid"; public static final String PRIVATE_UUID = "uuid"; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index 35b290c064d..9ad5b57fccf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -55,7 +55,7 @@ import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.*; -public class PanelMongoDBAdaptor extends MongoDBAdaptor implements PanelDBAdaptor { +public class PanelMongoDBAdaptor extends CatalogMongoDBAdaptor implements PanelDBAdaptor { private final MongoDBCollection panelCollection; private final MongoDBCollection panelArchiveCollection; @@ -326,7 +326,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer return result; } - private OpenCGAResult privateUpdate(ClientSession clientSession, Panel panel, ObjectMap parameters, QueryOptions queryOptions) + private OpenCGAResult privateUpdate(ClientSession clientSession, Panel panel, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); Query tmpQuery = new Query() @@ -343,7 +343,7 @@ private OpenCGAResult privateUpdate(ClientSession clientSession, Panel p } Bson finalQuery = parseQuery(tmpQuery); - return versionedMongoDBAdaptor.update(clientSession, finalQuery, () -> { + return versionedMongoDBAdaptor.update(clientSession, finalQuery, (entrylist) -> { logger.debug("Panel update: query : {}, update: {}", finalQuery.toBsonDocument(), panelUpdate.toBsonDocument()); DataResult result = panelCollection.update(clientSession, finalQuery, new Document("$set", panelUpdate), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index 1474f1ed4e1..85304c18e71 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -60,7 +60,7 @@ /** * Created by imedina on 08/01/16. */ -public class ProjectMongoDBAdaptor extends MongoDBAdaptor implements ProjectDBAdaptor { +public class ProjectMongoDBAdaptor extends CatalogMongoDBAdaptor implements ProjectDBAdaptor { private final MongoDBCollection userCollection; private final MongoDBCollection deletedUserCollection; @@ -604,7 +604,7 @@ public OpenCGAResult get(Query query, QueryOptions options, String user } @Override - public OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException { + public OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); try (DBIterator dbIterator = nativeIterator(query, options)) { return endQuery(startTime, dbIterator); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index 22ffb20c77e..da6dca117ab 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -21,7 +21,6 @@ import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; @@ -63,6 +62,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor.QueryParams.MODIFICATION_DATE; import static org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBUtils.filterAnnotationSets; @@ -83,10 +83,6 @@ public class SampleMongoDBAdaptor extends AnnotationMongoDBAdaptor imple private final IndividualMongoDBAdaptor individualDBAdaptor; private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; - final QueryOptions SAMPLE_FETCH_FOR_UPDATE_OPTIONS = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.STUDY_UID.key(), - PRIVATE_INDIVIDUAL_UID)); - public SampleMongoDBAdaptor(MongoDBCollection sampleCollection, MongoDBCollection archiveSampleCollection, MongoDBCollection deletedSampleCollection, Configuration configuration, MongoDBAdaptorFactory dbAdaptorFactory) { @@ -253,7 +249,10 @@ public OpenCGAResult update(long id, ObjectMap parameters, QueryOptions queryOpt public OpenCGAResult update(long uid, ObjectMap parameters, List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query query = new Query(QueryParams.UID.key(), uid); - OpenCGAResult documentResult = nativeGet(query, SAMPLE_FETCH_FOR_UPDATE_OPTIONS); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.STUDY_UID.key(), + PRIVATE_INDIVIDUAL_UID)); + OpenCGAResult documentResult = nativeGet(query, options); if (documentResult.getNumResults() == 0) { throw new CatalogDBException("Could not update sample. Sample uid '" + uid + "' not found."); } @@ -307,7 +306,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List return result; } - OpenCGAResult privateUpdate(ClientSession clientSession, Document sampleDocument, ObjectMap parameters, + OpenCGAResult privateUpdate(ClientSession clientSession, Document sampleDocument, ObjectMap parameters, List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); @@ -320,9 +319,9 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample .append(QueryParams.STUDY_UID.key(), studyUid) .append(QueryParams.UID.key(), sampleUid); Bson bsonQuery = parseQuery(tmpQuery); - return versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, (entrylist) -> { // Perform the update - DataResult result = updateAnnotationSets(clientSession, sampleUid, parameters, variableSetList, queryOptions, true); + DataResult result = updateAnnotationSets(clientSession, studyUid, sampleUid, parameters, variableSetList, queryOptions, true); UpdateDocument updateParams = parseAndValidateUpdateParams(clientSession, studyUid, parameters, tmpQuery, queryOptions); Document sampleUpdate = updateParams.toFinalUpdateDocument(); @@ -342,21 +341,104 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample result = sampleCollection.update(clientSession, finalQuery, sampleUpdate, new QueryOptions(MongoDBCollection.MULTI, true)); if (updateParams.getSet().containsKey(PRIVATE_INDIVIDUAL_UID)) { - long individualUid = sampleDocument.getLong(PRIVATE_INDIVIDUAL_UID); + long oldIndividualUid = sampleDocument.getLong(PRIVATE_INDIVIDUAL_UID); long newIndividualUid = updateParams.getSet().getLong(PRIVATE_INDIVIDUAL_UID); // If the sample has been associated a different individual - if (newIndividualUid != individualUid) { + if (newIndividualUid != oldIndividualUid) { Sample sample = new Sample().setUid(sampleUid).setVersion(version).setStudyUid(studyUid); + if (oldIndividualUid > 0) { + // Remove the sample from the individual where it was associated + updateSampleFromIndividualCollection(clientSession, sample, oldIndividualUid, + ParamUtils.BasicUpdateAction.REMOVE); + } if (newIndividualUid > 0) { // Add the sample to the list of samples of new individual updateSampleFromIndividualCollection(clientSession, sample, newIndividualUid, ParamUtils.BasicUpdateAction.ADD); } + } + } + + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Sample " + sampleId + " not found"); + } + if (result.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, sampleId, "Sample was already updated")); + } + logger.debug("Sample {} successfully updated", sampleId); + } + + return endWrite(tmpStartTime, 1, 1, events); + }, this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); + } + + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, Sample sample, ObjectMap parameters, + List variableSetList, QueryOptions queryOptions) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + long studyUid = sample.getStudyUid(); + long sampleUid = sample.getUid(); + if (studyUid <= 0) { + throw new CatalogDBException("Unexpected studyUid value received"); + } + if (sampleUid <= 0) { + throw new CatalogDBException("Unexpected sampleUid value received"); + } + + Query tmpQuery = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.UID.key(), sampleUid); + List fieldsToInclude = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key(), PRIVATE_INDIVIDUAL_UID); + + Bson bsonQuery = parseQuery(tmpQuery); + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, fieldsToInclude, (entrylist) -> { + String sampleId = entrylist.get(0).getString(QueryParams.ID.key()); + // Perform the update + DataResult result = updateAnnotationSets(clientSession, studyUid, sampleUid, parameters, variableSetList, queryOptions, + true); + + UpdateDocument updateParams = parseAndValidateUpdateParams(clientSession, studyUid, parameters, tmpQuery, queryOptions); + Document sampleUpdate = updateParams.toFinalUpdateDocument(); + + if (sampleUpdate.isEmpty() && result.getNumUpdated() == 0) { + if (!parameters.isEmpty()) { + logger.error("Non-processed update parameters: {}", parameters.keySet()); + } + throw new CatalogDBException("Nothing to be updated"); + } + + List events = new ArrayList<>(); + if (!sampleUpdate.isEmpty()) { + Bson finalQuery = parseQuery(tmpQuery); + + logger.debug("Sample update: query : {}, update: {}", finalQuery.toBsonDocument(), sampleUpdate.toBsonDocument()); + result = sampleCollection.update(clientSession, finalQuery, sampleUpdate, new QueryOptions(MongoDBCollection.MULTI, true)); + + if (updateParams.getSet().containsKey(PRIVATE_INDIVIDUAL_UID)) { + long individualUid = entrylist.get(0).getLong(PRIVATE_INDIVIDUAL_UID); + long newIndividualUid = updateParams.getSet().getLong(PRIVATE_INDIVIDUAL_UID); + + // If the sample has been associated a different individual + if (newIndividualUid != individualUid) { + int version = entrylist.get(0).getInteger(QueryParams.VERSION.key()); + Sample tmpSample = new Sample() + .setUid(sampleUid) + .setVersion(version) + .setStudyUid(studyUid); + + if (newIndividualUid > 0) { + // Add the sample to the list of samples of new individual + updateSampleFromIndividualCollection(clientSession, tmpSample, newIndividualUid, + ParamUtils.BasicUpdateAction.ADD); + } if (individualUid > 0) { // Remove the sample from the individual where it was associated - updateSampleFromIndividualCollection(clientSession, sample, individualUid, ParamUtils.BasicUpdateAction.REMOVE); + updateSampleFromIndividualCollection(clientSession, tmpSample, individualUid, + ParamUtils.BasicUpdateAction.REMOVE); } } } @@ -374,6 +456,32 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample }, this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); } + @Override + OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + + List includeIds = Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), + QueryParams.STUDY_UID.key(), PRIVATE_INDIVIDUAL_UID); + return versionedMongoDBAdaptor.update(clientSession, query, includeIds, (sampleList) -> { + List events = new ArrayList<>(); + Document update = updateDocument.toFinalUpdateDocument(); + if (!update.isEmpty()) { + logger.debug("Sample update: query : {}, update: {}", query.toBsonDocument(), update.toBsonDocument()); + DataResult result = sampleCollection.update(clientSession, query, update, + new QueryOptions(MongoDBCollection.MULTI, true)); + List sampleIds = sampleList.stream().map(x -> x.getString(QueryParams.ID.key())).collect(Collectors.toList()); + if (result.getNumUpdated() == 0) { + for (String sampleId : sampleIds) { + events.add(new Event(Event.Type.WARNING, sampleId, "Sample was already updated")); + } + } + logger.debug("Samples {} successfully updated", StringUtils.join(sampleIds, ", ")); + } + return endWrite(tmpStartTime, sampleList.size(), sampleList.size(), events); + }, this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); + } + private void updateReferencesAfterSampleVersionIncrement(ClientSession clientSession, DBIterator iterator) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { while (iterator.hasNext()) { @@ -388,28 +496,25 @@ void updateIndividualFromSampleCollection(ClientSession clientSession, long stud } ObjectMap params = new ObjectMap(QueryParams.INDIVIDUAL_ID.key(), individualId); - Document update = parseAndValidateUpdateParams(clientSession, studyId, params, null, QueryOptions.empty()).toFinalUpdateDocument(); + UpdateDocument updateDocument = parseAndValidateUpdateParams(clientSession, studyId, params, null, QueryOptions.empty()); Bson query = parseQuery(new Query() .append(QueryParams.STUDY_UID.key(), studyId) .append(QueryParams.UID.key(), sampleUids)); - versionedMongoDBAdaptor.update(clientSession, query, () -> { - QueryOptions options = new QueryOptions(MongoDBCollection.MULTI, true); - return sampleCollection.update(clientSession, query, update, options); - }, this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); + transactionalUpdate(clientSession, studyId, query, updateDocument); } void updateCohortReferences(ClientSession clientSession, long studyUid, List sampleUids, String cohortId, ParamUtils.BasicUpdateAction action) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { - Bson bsonUpdate; + UpdateDocument updateDocument = new UpdateDocument(); switch (action) { case ADD: - bsonUpdate = Updates.addToSet(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId); + updateDocument.getAddToSet().put(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId); break; case REMOVE: - bsonUpdate = Updates.pull(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId); + updateDocument.getPull().put(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId); break; case SET: default: @@ -421,14 +526,7 @@ void updateCohortReferences(ClientSession clientSession, long studyUid, List { - DataResult update = sampleCollection.update(clientSession, bsonQuery, bsonUpdate, - new QueryOptions(MongoDBCollection.MULTI, true)); - if (update.getNumMatches() == 0) { - throw new CatalogDBException("Could not update cohort references in samples"); - } - return update; - }, this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); + transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); } /** @@ -466,8 +564,8 @@ private void updateIndividualSampleReferences(ClientSession clientSession, Sampl options = new QueryOptions() .append(Constants.ACTIONS, action); - OpenCGAResult result = dbAdaptorFactory.getCatalogIndividualDBAdaptor().privateUpdate(clientSession, individual, params, null, - options); + OpenCGAResult result = dbAdaptorFactory.getCatalogIndividualDBAdaptor().transactionalUpdate(clientSession, individual, + params, null, options); if (result.getNumUpdated() != 1) { throw new CatalogDBException("Individual '" + individual.getId() + "' could not be updated to the latest sample version" + " of '" + sample.getId() + "'"); @@ -489,11 +587,10 @@ void updateIndividualIdFromSamples(ClientSession clientSession, long studyUid, S .append(QueryParams.INDIVIDUAL_ID.key(), oldIndividualId); Bson bsonQuery = parseQuery(query); - Bson update = Updates.set(QueryParams.INDIVIDUAL_ID.key(), newIndividualId); + UpdateDocument updateDocument = new UpdateDocument(); + updateDocument.getSet().put(QueryParams.INDIVIDUAL_ID.key(), newIndividualId); - versionedMongoDBAdaptor.update(clientSession, bsonQuery, - () -> sampleCollection.update(clientSession, bsonQuery, update, new QueryOptions(MongoDBCollection.MULTI, true)), - this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); + transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); } /** @@ -528,13 +625,9 @@ private void updateSampleFromIndividualCollection(ClientSession clientSession, S actionMap.put(IndividualDBAdaptor.QueryParams.SAMPLES.key(), updateAction.name()); options.put(Constants.ACTIONS, actionMap); - Query query = new Query(IndividualDBAdaptor.QueryParams.UID.key(), individualUid); - Document update = individualDBAdaptor.parseAndValidateUpdateParams(clientSession, params, query, options).toFinalUpdateDocument(); - Bson bsonQuery = individualDBAdaptor.parseQuery(new Query() - .append(IndividualDBAdaptor.QueryParams.UID.key(), individualUid) - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), sample.getStudyUid()), null); - - individualDBAdaptor.getCollection().update(clientSession, bsonQuery, update, null); + Individual individual = new Individual().setUid(individualUid) + .setStudyUid(sample.getStudyUid()); + individualDBAdaptor.transactionalUpdate(clientSession, individual, params, null, options); } UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, long studyUid, ObjectMap parameters, Query query, @@ -782,14 +875,8 @@ public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, Bson query = Filters.and(filters); UpdateDocument updateDocument = new UpdateDocument().setSet(rootDocument); - Document bsonUpdate = updateDocument.toFinalUpdateDocument(); - - return runTransaction( - (ClientSession clientSession) -> versionedMongoDBAdaptor.update(clientSession, query, - () -> new OpenCGAResult<>(sampleCollection.update(query, bsonUpdate, new QueryOptions("multi", true))), - this::iterator, - (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator))); + return runTransaction(session -> transactionalUpdate(session, studyUid, query, updateDocument)); } @Override @@ -932,11 +1019,8 @@ void removeFileReferences(ClientSession clientSession, long studyUid, String fil logger.debug("Removing file from sample '{}' field. Query: {}, Update: {}", QueryParams.FILE_IDS.key(), bsonQuery.toBsonDocument(), updateDocument.toBsonDocument()); - versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { - DataResult result = sampleCollection.update(clientSession, bsonQuery, updateDocument, new QueryOptions("multi", true)); - logger.debug("File '{}' removed from {} samples", fileId, result.getNumUpdated()); - return result; - }, this::iterator, (DBIterator iterator) -> updateReferencesAfterSampleVersionIncrement(clientSession, iterator)); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, document); + logger.debug("File '{}' removed from {} samples", fileId, result.getNumUpdated()); } // TODO: Check clean diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index d1648813ee8..b0d67dc923a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -71,7 +71,7 @@ * * @author Jacobo Coll <jacobo167@gmail.com> */ -public class StudyMongoDBAdaptor extends MongoDBAdaptor implements StudyDBAdaptor { +public class StudyMongoDBAdaptor extends CatalogMongoDBAdaptor implements StudyDBAdaptor { private final MongoDBCollection studyCollection; private final MongoDBCollection deletedStudyCollection; @@ -863,8 +863,8 @@ public OpenCGAResult createVariableSet(long studyId, VariableSet va } @Override - public OpenCGAResult addFieldToVariableSet(long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException { + public OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) + throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableNotInVariableSet(variableSet.first(), variable.getId()); @@ -876,11 +876,12 @@ public OpenCGAResult addFieldToVariableSet(long variableSetId, Vari throw CatalogDBException.updateError("VariableSet", variableSetId); } if (variable.isRequired()) { - dbAdaptorFactory.getCatalogSampleDBAdaptor().addVariableToAnnotations(variableSetId, variable); - dbAdaptorFactory.getCatalogCohortDBAdaptor().addVariableToAnnotations(variableSetId, variable); - dbAdaptorFactory.getCatalogIndividualDBAdaptor().addVariableToAnnotations(variableSetId, variable); - dbAdaptorFactory.getCatalogFamilyDBAdaptor().addVariableToAnnotations(variableSetId, variable); - dbAdaptorFactory.getCatalogFileDBAdaptor().addVariableToAnnotations(variableSetId, variable); + dbAdaptorFactory.getCatalogSampleDBAdaptor().addVariableToAnnotations(studyUid, variableSetId, variable); + dbAdaptorFactory.getCatalogCohortDBAdaptor().addVariableToAnnotations(studyUid, variableSetId, variable); + dbAdaptorFactory.getCatalogIndividualDBAdaptor().addVariableToAnnotations(studyUid, variableSetId, variable); + dbAdaptorFactory.getCatalogFamilyDBAdaptor().addVariableToAnnotations(studyUid, variableSetId, variable); + dbAdaptorFactory.getCatalogFileDBAdaptor().addVariableToAnnotations(studyUid, variableSetId, variable); + dbAdaptorFactory.getClinicalAnalysisDBAdaptor().addVariableToAnnotations(studyUid, variableSetId, variable); } return new OpenCGAResult<>(result); @@ -939,8 +940,8 @@ public OpenCGAResult renameFieldVariableSet(long variableSetId, Str } @Override - public OpenCGAResult removeFieldFromVariableSet(long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException { + public OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) + throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { long startTime = startQuery(); OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); @@ -956,11 +957,12 @@ public OpenCGAResult removeFieldFromVariableSet(long variableSetId, } // Remove all the annotations from that field - dbAdaptorFactory.getCatalogSampleDBAdaptor().removeAnnotationField(variableSetId, name); - dbAdaptorFactory.getCatalogCohortDBAdaptor().removeAnnotationField(variableSetId, name); - dbAdaptorFactory.getCatalogIndividualDBAdaptor().removeAnnotationField(variableSetId, name); - dbAdaptorFactory.getCatalogFamilyDBAdaptor().removeAnnotationField(variableSetId, name); - dbAdaptorFactory.getCatalogFileDBAdaptor().removeAnnotationField(variableSetId, name); + dbAdaptorFactory.getCatalogSampleDBAdaptor().removeAnnotationField(studyUid, variableSetId, name); + dbAdaptorFactory.getCatalogCohortDBAdaptor().removeAnnotationField(studyUid, variableSetId, name); + dbAdaptorFactory.getCatalogIndividualDBAdaptor().removeAnnotationField(studyUid, variableSetId, name); + dbAdaptorFactory.getCatalogFamilyDBAdaptor().removeAnnotationField(studyUid, variableSetId, name); + dbAdaptorFactory.getCatalogFileDBAdaptor().removeAnnotationField(studyUid, variableSetId, name); + dbAdaptorFactory.getClinicalAnalysisDBAdaptor().removeAnnotationField(studyUid, variableSetId, name); return new OpenCGAResult<>(result); } @@ -1234,7 +1236,7 @@ public OpenCGAResult deleteVariableSet(long studyUid, VariableSet v } private void deleteAllAnnotationSetsByVariableSet(ClientSession session, long studyUid, VariableSet variableSet) - throws CatalogDBException { + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { List entities = variableSet.getEntities(); if (CollectionUtils.isEmpty(entities)) { entities = new ArrayList<>(EnumSet.allOf(VariableSet.AnnotableDataModels.class)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index a25d2eabd10..0ba4f454192 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -65,7 +65,7 @@ /** * @author Jacobo Coll <jacobo167@gmail.com> */ -public class UserMongoDBAdaptor extends MongoDBAdaptor implements UserDBAdaptor { +public class UserMongoDBAdaptor extends CatalogMongoDBAdaptor implements UserDBAdaptor { private final MongoDBCollection userCollection; private final MongoDBCollection deletedUserCollection; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java index abafe45bf38..11318fe3fd9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java @@ -16,6 +16,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,6 +118,10 @@ private String getClientSessionUuid(ClientSession session) { } public interface VersionedModelExecution { + T execute(List entries) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + } + + public interface NonVersionedModelExecution { T execute() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; } @@ -137,29 +142,46 @@ protected void insert(ClientSession session, Document document) { archiveCollection.insert(session, document, QueryOptions.empty()); } - protected T update(ClientSession session, Bson sourceQuery, VersionedModelExecution update, - PostVersionIncrementIterator postVersionIncrementIterator, - ReferenceModelExecution postVersionIncrementExecution) + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, VersionedModelExecution> update, + PostVersionIncrementIterator postVersionIncrementIterator, + ReferenceModelExecution postVersionIncrementExecution) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return update(session, sourceQuery, Collections.emptyList(), update, Collections.emptyList(), postVersionIncrementIterator, + postVersionIncrementExecution); + } + + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, List fieldsToInclude, + VersionedModelExecution> update, PostVersionIncrementIterator postVersionIncrementIterator, + ReferenceModelExecution postVersionIncrementExecution) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - return update(session, sourceQuery, update, Collections.emptyList(), postVersionIncrementIterator, postVersionIncrementExecution); + return update(session, sourceQuery, fieldsToInclude, update, Collections.emptyList(), postVersionIncrementIterator, + postVersionIncrementExecution); } - protected T update(ClientSession session, Bson sourceQuery, VersionedModelExecution update, - List postVersionIncrementAdditionalIncludeFields, PostVersionIncrementIterator dbIterator, - ReferenceModelExecution postVersionIncrementExecution) + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, List fieldsToInclude, + VersionedModelExecution> update, + List postVersionIncrementAdditionalIncludeFields, + PostVersionIncrementIterator dbIterator, + ReferenceModelExecution postVersionIncrementExecution) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { String uuid = getClientSessionUuid(session); // 1. Increment version // 1.1 Only increase version of those documents not already increased by same transaction id - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(PRIVATE_UID, VERSION, RELEASE_FROM_VERSION, PRIVATE_TRANSACTION_ID)); + Set includeFields = new HashSet<>(Arrays.asList(PRIVATE_UID, VERSION, RELEASE_FROM_VERSION, PRIVATE_TRANSACTION_ID)); + if (fieldsToInclude != null) { + includeFields.addAll(fieldsToInclude); + } + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, includeFields); + List entryList = new LinkedList<>(); List allUids = new LinkedList<>(); List uidsChanged = new LinkedList<>(); try (MongoDBIterator iterator = collection.iterator(session, sourceQuery, null, null, options)) { while (iterator.hasNext()) { Document result = iterator.next(); + entryList.add(result); + long uid = result.get(PRIVATE_UID, Number.class).longValue(); int version = result.get(VERSION, Number.class).intValue(); String transactionId = result.getString(PRIVATE_TRANSACTION_ID); @@ -192,8 +214,13 @@ protected T update(ClientSession session, Bson sourceQuery, VersionedMode } } + if (entryList.isEmpty()) { + logger.warn("Update not executed. No entries could be found for query '{}'", sourceQuery.toBsonDocument()); + return OpenCGAResult.empty(); + } + // 2. Execute main update - T executionResult = update.execute(); + OpenCGAResult executionResult = update.execute(entryList); // 3. Fetch document containing update and copy into the archive collection Bson bsonQuery = Filters.in(PRIVATE_UID, allUids); @@ -245,7 +272,7 @@ protected T update(ClientSession session, Bson sourceQuery, VersionedMode return executionResult; } - protected T updateWithoutVersionIncrement(Bson sourceQuery, VersionedModelExecution update) + protected T updateWithoutVersionIncrement(Bson sourceQuery, NonVersionedModelExecution update) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { // Execute main update T executionResult = update.execute(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java index 582509ad581..20ca29506c6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/SampleConverter.java @@ -49,7 +49,6 @@ public Document convertToStorageType(Sample object, List variableSe document.put("uid", object.getUid()); document.put("studyUid", object.getStudyUid()); - document.put("individual", new Document()); return document; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index d7cbd3cc475..3d643c17bb2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -1558,7 +1558,7 @@ public OpenCGAResult delete(String studyStr, List fileIds, ObjectMap par boolean physicalDelete = params.getBoolean(Constants.SKIP_TRASH, false); auditManager.initAuditBatch(operationUuid); - OpenCGAResult result = OpenCGAResult.empty(); + OpenCGAResult result = OpenCGAResult.empty(File.class); for (String id : fileIds) { String fileId = id; String fileUuid = ""; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 184bbe18b67..9c22e95f0bd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -1429,7 +1429,7 @@ public OpenCGAResult addFieldToVariableSet(String studyId, String v } authorizationManager.checkCanCreateUpdateDeleteVariableSets(study.getUid(), userId); - OpenCGAResult result = studyDBAdaptor.addFieldToVariableSet(variableSet.getUid(), variable, userId); + OpenCGAResult result = studyDBAdaptor.addFieldToVariableSet(study.getUid(), variableSet.getUid(), variable, userId); auditManager.audit(userId, Enums.Action.ADD_VARIABLE_TO_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); @@ -1457,7 +1457,7 @@ public OpenCGAResult removeFieldFromVariableSet(String studyId, Str try { authorizationManager.checkCanCreateUpdateDeleteVariableSets(study.getUid(), userId); - OpenCGAResult result = studyDBAdaptor.removeFieldFromVariableSet(variableSet.getUid(), variableId, userId); + OpenCGAResult result = studyDBAdaptor.removeFieldFromVariableSet(study.getUid(), variableSet.getUid(), variableId, userId); auditManager.audit(userId, Enums.Action.REMOVE_VARIABLE_FROM_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index 5791005394c..480866110ae 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -100,10 +100,10 @@ public void createVariableSetTest() throws CatalogDBException { } @Test - public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogAuthorizationException { + public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); DataResult result = - catalogStudyDBAdaptor.removeFieldFromVariableSet(variableSetDataResult.first().getUid(), "NAME", user3.getId()); + catalogStudyDBAdaptor.removeFieldFromVariableSet(5L, variableSetDataResult.first().getUid(), "NAME", user3.getId()); assertEquals(1, result.getNumUpdated()); VariableSet variableSet = catalogStudyDBAdaptor.getVariableSet(variableSetDataResult.first().getUid(), QueryOptions.empty()).first(); @@ -153,12 +153,12 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAuthorizationException { + public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { createExampleVariableSet("VARSET_1", false); createExampleVariableSet("VARSET_2", true); Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()); - DataResult result = catalogStudyDBAdaptor.addFieldToVariableSet(18, variable, user3.getId()); + DataResult result = catalogStudyDBAdaptor.addFieldToVariableSet(5L, 18, variable, user3.getId()); assertEquals(1, result.getNumUpdated()); DataResult queryResult = catalogStudyDBAdaptor.getVariableSet(18L, QueryOptions.empty()); @@ -170,7 +170,7 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho // We try to insert the same one again. thrown.expect(CatalogDBException.class); thrown.expectMessage("already exist"); - catalogStudyDBAdaptor.addFieldToVariableSet(18, variable, user3.getId()); + catalogStudyDBAdaptor.addFieldToVariableSet(5L, 18, variable, user3.getId()); } /** @@ -179,12 +179,12 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest2() throws CatalogDBException, CatalogAuthorizationException { + public void addFieldToVariableSetTest2() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()); thrown.expect(CatalogDBException.class); thrown.expectMessage("not found"); - catalogStudyDBAdaptor.addFieldToVariableSet(18, variable, user3.getId()); + catalogStudyDBAdaptor.addFieldToVariableSet(5L, 18, variable, user3.getId()); } @Test