Skip to content

Commit

Permalink
Merge pull request #2448 from opencb/TASK-6231
Browse files Browse the repository at this point in the history 
TASK-6231 - Port Patch 1.10.4.1 -> 1.10.5
  • Loading branch information
@juanfeSanahuja
juanfeSanahuja authored May 14, 2024
2 parents f295315 + 28e916f commit 422cfb8
Show file tree
Hide file tree
Showing 14 changed files with 233 additions and 39 deletions.
4 changes: 2 additions & 2 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,9 @@ public static void checkValidUserId(String userId) throws CatalogParameterExcept
if (userId.equals(ParamConstants.ANONYMOUS_USER_ID) || userId.equals(ParamConstants.REGISTERED_USERS)) {
throw new CatalogParameterException("User id cannot be one of the reserved OpenCGA users.");
}
if (!userId.matches("^[A-Za-z0-9]([-_.]?[A-Za-z0-9])*$")) {
if (!userId.matches("^[A-Za-z0-9]([-_.@]?[A-Za-z0-9])*$")) {
throw new CatalogParameterException("Invalid user id. Id needs to start by any character and might contain single '-', '_', "
+ "'.', symbols followed by any character or number.");
+ "'.' or '@' symbols followed by any character or number.");
}
}

Expand Down
11 changes: 11 additions & 0 deletions opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.opencb.opencga.core.models.sample.*;
import org.opencb.opencga.core.models.study.*;
import org.opencb.opencga.core.models.user.Account;
import org.opencb.opencga.core.models.user.AuthenticationResponse;
import org.opencb.opencga.core.models.user.User;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.testclassification.duration.MediumTests;
Expand Down Expand Up @@ -276,6 +277,16 @@ private String getAdminToken() throws CatalogException, IOException {
return catalogManager.getUserManager().loginAsAdmin("admin").getToken();
}

@Test
public void createUserUsingMailAsId() throws CatalogException {
catalogManager.getUserManager().create(new User().setId("[email protected]").setName("Hello")
.setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, opencgaToken);
AuthenticationResponse login = catalogManager.getUserManager().login("[email protected]", TestParamConstants.PASSWORD);
assertNotNull(login);
User user = catalogManager.getUserManager().get("[email protected]", new QueryOptions(), login.getToken()).first();
assertEquals("[email protected]", user.getId());
}

@Test
public void getGroupsTest() throws CatalogException {
Group group = new Group("groupId", Arrays.asList("user2", "user3")).setSyncedFrom(new Group.Sync("ldap", "bio"));
Expand Down
19 changes: 17 additions & 2 deletions opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.opencb.cellbase.core.config.SpeciesProperties;
import org.opencb.cellbase.core.models.DataRelease;
import org.opencb.cellbase.core.result.CellBaseDataResponse;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.utils.VersionUtils;

Expand Down Expand Up @@ -307,16 +308,30 @@ private static String majorMinor(String version) {
public String getVersionFromServer() throws IOException {
if (serverVersion == null) {
synchronized (this) {
String serverVersion = cellBaseClient.getMetaClient().about().firstResult().getString("Version");
ObjectMap result = retryMetaAbout(3);
if (result == null) {
throw new IOException("Unable to get version from server for cellbase " + toString());
}
String serverVersion = result.getString("Version");
if (StringUtils.isEmpty(serverVersion)) {
serverVersion = cellBaseClient.getMetaClient().about().firstResult().getString("Version: ");
serverVersion = result.getString("Version: ");
}
this.serverVersion = serverVersion;
}
}
return serverVersion;
}

private ObjectMap retryMetaAbout(int retries) throws IOException {
ObjectMap result = cellBaseClient.getMetaClient().about().firstResult();
if (result == null && retries > 0) {
// Retry
logger.warn("Unable to get version from server for cellbase " + toString() + ". Retrying...");
result = retryMetaAbout(retries - 1);
}
return result;
}

public boolean isMinVersion(String minVersion) throws IOException {
String serverVersion = getVersionFromServer();
return VersionUtils.isMinVersion(minVersion, serverVersion);
Expand Down
9 changes: 7 additions & 2 deletions ...src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1135,9 +1135,14 @@ public Iterable<CohortMetadata> getInvalidCohorts(int studyId) {
return () -> Iterators.filter(cohortIterator(studyId), CohortMetadata::isInvalid);
}

public Iterable<CohortMetadata> getCalculatedOrInvalidCohorts(int studyId) {
public Iterable<CohortMetadata> getCalculatedOrPartialCohorts(int studyId) {
return () -> Iterators.filter(cohortIterator(studyId),
cohortMetadata -> cohortMetadata.isStatsReady() || cohortMetadata.isInvalid());
cohortMetadata -> {
TaskMetadata.Status status = cohortMetadata.getStatsStatus();
return status == TaskMetadata.Status.READY
|| status == TaskMetadata.Status.RUNNING
|| status == TaskMetadata.Status.ERROR;
});
}

public CohortMetadata setSamplesToCohort(int studyId, String cohortName, Collection<Integer> samples) throws StorageEngineException {
Expand Down
74 changes: 59 additions & 15 deletions ...pencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package org.opencb.opencga.storage.core.variant.annotation.converters;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.models.variant.avro.*;
import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils;

import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class VariantAnnotationModelUtils {

Expand All @@ -21,54 +25,88 @@ public class VariantAnnotationModelUtils {
* - annotation.consequenceTypes.transcriptId
* - annotation.consequenceTypes.ensemblTranscriptId
* - annotation.consequenceTypes.hgvs
* - annotation.consequenceTypes.proteinVariantAnnotation.proteinId
* - annotation.consequenceTypes.proteinVariantAnnotation.uniprotAccession
* - annotation.consequenceTypes.proteinVariantAnnotation.uniprotName
* - annotation.consequenceTypes.proteinVariantAnnotation.uniprotVariantId
* - annotation.consequenceTypes.proteinVariantAnnotation.features.id
* - annotation.traitAssociation.id
* - annotation.geneTraitAssociation.hpo
* - annotation.geneTraitAssociation.id
* - annotation.geneTraitAssociation.hpo
* - annotation.pharmacogenomics.id
* - annotation.pharmacogenomics.name
*
* @param variantAnnotation VariantAnnotation object
* @return Set of XRefs
*/
private static final Pattern HGVS_PATTERN = Pattern.compile("\\([^()]*\\)");

public Set<String> extractXRefs(VariantAnnotation variantAnnotation) {
Set<String> xrefs = new HashSet<>();
Set<String> xrefs = new HashSet<>(100);

if (variantAnnotation == null) {
return xrefs;
}

xrefs.add(variantAnnotation.getId());

if (variantAnnotation.getXrefs() != null) {
if (CollectionUtils.isNotEmpty(variantAnnotation.getXrefs())) {
for (Xref xref : variantAnnotation.getXrefs()) {
if (xref != null) {
xrefs.add(xref.getId());
}
}
}

if (variantAnnotation.getHgvs() != null) {
List<ConsequenceType> consequenceTypes = variantAnnotation.getConsequenceTypes();

if (CollectionUtils.isNotEmpty(variantAnnotation.getHgvs())) {
xrefs.addAll(variantAnnotation.getHgvs());

// TODO Remove this code when CellBase 6.4.0 returns the expected HGVS
for (String hgvs: variantAnnotation.getHgvs()) {
if (VariantQueryUtils.isTranscript(hgvs)) {
// 1. Remove the content between parentheses, e.g. ENST00000680783.1(ENSG00000135744):c.776T>C
if (hgvs.contains("(")) {
Matcher matcher = HGVS_PATTERN.matcher(hgvs);
StringBuffer result = new StringBuffer();
while (matcher.find()) {
matcher.appendReplacement(result, "");
}
matcher.appendTail(result);
xrefs.add(result.toString());
}

// 2. Add the HGVS with the Ensembl and gene name, e.g. ENSG00000135744:c.776T>C, AGT:c.776T>C
if (CollectionUtils.isNotEmpty(consequenceTypes)) {
for (ConsequenceType conseqType : consequenceTypes) {
if (conseqType != null && conseqType.getHgvs() != null && conseqType.getHgvs().contains(hgvs)) {
String[] fields = hgvs.split(":", 2);
if (StringUtils.isNotEmpty(conseqType.getGeneId())) {
xrefs.add(conseqType.getGeneId() + ":" + fields[1]);
}
if (StringUtils.isNotEmpty(conseqType.getGeneName())) {
xrefs.add(conseqType.getGeneName() + ":" + fields[1]);
}
break;
}
}
}
}
}
}

List<ConsequenceType> consequenceTypes = variantAnnotation.getConsequenceTypes();
if (consequenceTypes != null) {
if (CollectionUtils.isNotEmpty(consequenceTypes)) {
for (ConsequenceType conseqType : consequenceTypes) {
xrefs.add(conseqType.getGeneName());
xrefs.add(conseqType.getGeneId());
xrefs.add(conseqType.getEnsemblGeneId());
xrefs.add(conseqType.getTranscriptId());
xrefs.add(conseqType.getEnsemblGeneId());
xrefs.add(conseqType.getEnsemblTranscriptId());

if (conseqType.getHgvs() != null) {
xrefs.addAll(conseqType.getHgvs());
}

ProteinVariantAnnotation protVarAnnotation = conseqType.getProteinVariantAnnotation();
if (protVarAnnotation != null) {

xrefs.add(protVarAnnotation.getProteinId());
xrefs.add(protVarAnnotation.getUniprotAccession());
xrefs.add(protVarAnnotation.getUniprotName());
xrefs.add(protVarAnnotation.getUniprotVariantId());
Expand All @@ -80,7 +118,6 @@ public Set<String> extractXRefs(VariantAnnotation variantAnnotation) {
}
}
}

}

if (CollectionUtils.isNotEmpty(variantAnnotation.getTraitAssociation())) {
Expand All @@ -89,10 +126,17 @@ public Set<String> extractXRefs(VariantAnnotation variantAnnotation) {
}
}

if (variantAnnotation.getGeneTraitAssociation() != null) {
if (CollectionUtils.isNotEmpty(variantAnnotation.getGeneTraitAssociation())) {
for (GeneTraitAssociation geneTrait : variantAnnotation.getGeneTraitAssociation()) {
xrefs.add(geneTrait.getHpo());
xrefs.add(geneTrait.getId());
xrefs.add(geneTrait.getHpo());
}
}

if (CollectionUtils.isNotEmpty(variantAnnotation.getPharmacogenomics())) {
for (Pharmacogenomics pharmacogenomics : variantAnnotation.getPharmacogenomics()) {
xrefs.add(pharmacogenomics.getId());
xrefs.add(pharmacogenomics.getName());
}
}

Expand Down
13 changes: 13 additions & 0 deletions ...e-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,19 @@ public static boolean isVariantAccession(String value) {
return value.startsWith("rs") || value.startsWith("VAR_");
}

/**
* Determines if the given value might be a known transcript or not.
* Ensembl transcripts start with `ENST`
* RefSeq transcripts start with `NM_` and `XM_`
* See <a href="https://www.ncbi.nlm.nih.gov/books/NBK21091/table/ch18.T.refseq_accession_numbers_and_mole/?report=objectonly">...</a>
*
* @param value Value to check
* @return If is a known transcript
*/
public static boolean isTranscript(String value) {
return value.startsWith("ENST") || value.startsWith("NM_") || value.startsWith("XM_");
}

/**
* Determines if the given value is a HGVS.
*
Expand Down
10 changes: 8 additions & 2 deletions ...rg/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.opencb.opencga.storage.core.metadata.models.CohortMetadata;
import org.opencb.opencga.storage.core.metadata.models.FileMetadata;
import org.opencb.opencga.storage.core.metadata.models.StudyMetadata;
import org.opencb.opencga.storage.core.metadata.models.TaskMetadata;
import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
import org.opencb.opencga.storage.core.variant.adaptors.VariantField;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException;
Expand Down Expand Up @@ -134,9 +135,10 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti
for (VariantQueryProjection.StudyVariantQueryProjection study : studies.values()) {
int studyId = study.getId();
List<Integer> cohorts = new LinkedList<>();
for (CohortMetadata cohort : metadataManager.getCalculatedOrInvalidCohorts(studyId)) {
for (CohortMetadata cohort : metadataManager.getCalculatedOrPartialCohorts(studyId)) {
cohorts.add(cohort.getId());
if (cohort.isInvalid()) {
TaskMetadata.Status status = cohort.getStatsStatus();
if (status == TaskMetadata.Status.ERROR) {
String message = "Please note that the Cohort Stats for "
+ "'" + study.getName() + ":" + cohort.getName() + "' are currently outdated.";
int numSampmles = cohort.getSamples().size();
Expand All @@ -147,6 +149,10 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti
}
message += " To display updated statistics, please execute variant-stats-index.";
events.add(new Event(Event.Type.WARNING, message));
} else if (status == TaskMetadata.Status.RUNNING) {
String message = "Please note that the Cohort Stats for "
+ "'" + study.getName() + ":" + cohort.getName() + "' are currently being calculated.";
events.add(new Event(Event.Type.WARNING, message));
}
}
study.setCohorts(cohorts);
Expand Down
1 change: 1 addition & 0 deletions ...n/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,7 @@ protected VariantStatsDBWriter newVariantStatisticsDBWriter(VariantDBAdaptor dbA
//
// }

@Deprecated
void checkAndUpdateCalculatedCohorts(StudyMetadata studyMetadata, URI uri, boolean updateStats)
throws IOException, StorageEngineException {
Set<String> cohortNames = readCohortsFromStatsFile(uri);
Expand Down
1 change: 1 addition & 0 deletions ...src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ public void postCalculateStats(
}
}

@Deprecated
public static void checkAndUpdateCalculatedCohorts(
VariantStorageMetadataManager metadataManager, StudyMetadata studyMetadata, Collection<String> cohorts, boolean updateStats)
throws StorageEngineException {
Expand Down
46 changes: 46 additions & 0 deletions ...b/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtilsTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package org.opencb.opencga.storage.core.variant.annotation.converters;

import junit.framework.TestCase;
import org.junit.experimental.categories.Category;
import org.opencb.biodata.models.variant.avro.ConsequenceType;
import org.opencb.biodata.models.variant.avro.VariantAnnotation;
import org.opencb.biodata.models.variant.avro.Xref;
import org.opencb.opencga.core.testclassification.duration.ShortTests;

import java.util.Arrays;
import java.util.Collections;
import java.util.Set;

@Category(ShortTests.class)
public class VariantAnnotationModelUtilsTest extends TestCase {

public void testXrefsHgvs() throws Exception {
VariantAnnotation variantAnnotation = new VariantAnnotation();
variantAnnotation.setId("id");
variantAnnotation.setXrefs(Collections.singletonList(new Xref("xref1", "source")));
variantAnnotation.setHgvs(Arrays.asList(
"ENST00000680783.1(ENSG00000135744):c.776T>C",
"ENSP00000451720.1:p.Asn134Lys"
));
ConsequenceType ct = new ConsequenceType();
ct.setGeneName("GENE");
ct.setHgvs(variantAnnotation.getHgvs());
ct.setGeneId(null);
variantAnnotation.setConsequenceTypes(Arrays.asList(ct, new ConsequenceType()));
Set<String> xrefs = new VariantAnnotationModelUtils().extractXRefs(variantAnnotation);

assertEquals(7, xrefs.size());
// Default fields
assertTrue(xrefs.contains("id"));
assertTrue(xrefs.contains("xref1"));
assertTrue(xrefs.contains("GENE"));

// Untouched hgvs, not starting with ENST or NM_
assertTrue(xrefs.contains("ENSP00000451720.1:p.Asn134Lys"));

assertTrue(xrefs.contains("ENST00000680783.1(ENSG00000135744):c.776T>C"));
assertTrue(xrefs.contains("ENST00000680783.1:c.776T>C"));
assertTrue(xrefs.contains("GENE:c.776T>C"));

}
}
6 changes: 6 additions & 0 deletions ...ava/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ public void testXRefRs() throws StorageEngineException {
with("Cosmic", EvidenceEntry::getId, is("COSV60260399"))))));
matchers.put("ENST00000341832.11(ENSG00000248333):c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(
is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G")))));
matchers.put("ENST00000341832.11:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(
// The variant annotation may not have the "alternate" hgvs
is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G")))));
matchers.put("ENSG00000248333:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(
// The variant annotation may not have the "alternate" hgvs
is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G")))));
matchers.put("VSP_039324", hasAnnotation(
with("ConsequenceType", VariantAnnotation::getConsequenceTypes,
hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation,
Expand Down
Loading

0 comments on commit 422cfb8

Please sign in to comment.