Fix missing clinical data NA counts which have been filtered out

.github/workflows/integration-test.yml

@@ -43,9 +43,9 @@ jobs:
         run: |
           cd ./data && ./init.sh && rm -rf ./studies/* && cd ../config && \
           cat $PORTAL_SOURCE_DIR/src/main/resources/application.properties | \
-              sed 's|spring.datasource.url=.*|spring.datasource.url=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false|' | \
-              sed 's|spring.datasource.username=.*|spring.datasource.username=cbio_user|' | \
-              sed 's|spring.datasource.password=.*|spring.datasource.password=somepassword|' \
+              sed 's|spring.datasource.mysql.url=.*|spring.datasource.mysql.url=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false|' | \
+              sed 's|spring.datasource.mysql.username=.*|spring.datasource.mysql.username=cbio_user|' | \
+              sed 's|spring.datasource.mysql.password=.*|spring.datasource.mysql.password=somepassword|' \
               > application.properties
       - name: 'Copy cgds.sql file into Docker Compose'
         run: cp ./cbioportal/src/main/resources/db-scripts/cgds.sql ./cbioportal-docker-compose/data/.

.github/workflows/sonarcloud.yml

@@ -34,7 +34,7 @@ jobs:
           cp src/main/resources/application.properties.EXAMPLE src/main/resources/application.properties
       - name: Build and analyze
         env:
-          SONAR_TOKEN:  ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN:  de1b5cc660cd210dde840f492c371da6cc801763
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn clean install verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
       - name: 'Add host.testcontainers.internal to /etc/hosts'
@@ -45,6 +45,6 @@ jobs:
           mvn verify -Pintegration-test
       - name: Code Coverage
         env:
-          SONAR_TOKEN:  ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN:  de1b5cc660cd210dde840f492c371da6cc801763
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn sonar:sonar -Pcoverage

pom.xml

@@ -98,6 +98,7 @@
 		<selenium_chrome_driver.version>3.14.0</selenium_chrome_driver.version>
         <selenium.version>4.17.0</selenium.version>
 		<sentry.version>7.1.0</sentry.version>
+        <clickhouse_testcontainer.version>1.19.7</clickhouse_testcontainer.version>
 
 
 		<!-- No sure what these are for -->
@@ -357,7 +358,32 @@
 			<artifactId>sentry-spring-boot-starter-jakarta</artifactId>
 			<version>${sentry.version}</version>
 		</dependency>
-	</dependencies>
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.6.2</version>
+            <!-- use uber jar with all dependencies included, change classifier
+            to http for smaller jar -->
+            <classifier>all</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>clickhouse</artifactId>
+            <version>${clickhouse_testcontainer.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mybatis.spring.boot</groupId>
+            <artifactId>mybatis-spring-boot-test-autoconfigure</artifactId>
+            <version>3.0.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 
     <dependencyManagement>
         <dependencies>
@@ -421,9 +447,25 @@
 				<includes>
 					<include>cgds.sql</include>
 					<include>migration.sql</include>
+                    <include>clickhouse/clickhouse.sql</include>
+                    <include>clickhouse/materialized_views.sql</include>
 				</includes>
 			</resource>
 		</resources>
+        <testResources>
+            <testResource>
+                <directory>src/main/resources/db-scripts</directory>
+                <filtering>false</filtering>
+                <includes>
+                    <include>clickhouse/clickhouse.sql</include>
+                    <include>clickhouse/materialized_views.sql</include>
+                </includes> 
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <filtering>false</filtering>
+            </testResource>
+        </testResources>
 
 		<plugins>
 			<plugin>

src/main/java/org/cbioportal/WebAppConfig.java

@@ -2,6 +2,7 @@
 
 import java.util.List;
 
+import org.cbioportal.web.ExecuterTimeInterceptor;
 import org.cbioportal.web.util.InvolvedCancerStudyExtractorInterceptor;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
@@ -12,6 +13,7 @@
 import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
 import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+import org.springframework.web.servlet.handler.WebRequestHandlerInterceptorAdapter;
 
 // TODO Consider creating separate DispatcherServlets as in the original web.xml
 // See: https://stackoverflow.com/a/30686733/11651683
@@ -74,6 +76,9 @@ public HandlerInterceptor involvedCancerStudyExtractorInterceptor() {
 	@Override
 	public void addInterceptors(InterceptorRegistry registry) {
 		registry.addInterceptor(involvedCancerStudyExtractorInterceptor());
+        registry.addInterceptor(new WebRequestHandlerInterceptorAdapter(
+            new ExecuterTimeInterceptor()
+        )).addPathPatterns("/**");
 	}
 
 	@Override

src/main/java/org/cbioportal/model/AlterationType.java

@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
 public enum AlterationType {
-    MUTATION,
-    COPY_NUMBER_ALTERATION
+    MUTATION_EXTENDED,
+    COPY_NUMBER_ALTERATION,
+    STRUCTURAL_VARIANT,
+    GENERIC_ASSAY;
 }

src/main/java/org/cbioportal/model/ClinicalViolinPlotBoxData.java

@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class ClinicalViolinPlotBoxData {
+import java.io.Serializable;
+
+public class ClinicalViolinPlotBoxData implements Serializable {
     private double whiskerLower;
     private double whiskerUpper;
     private double median;

src/main/java/org/cbioportal/model/ClinicalViolinPlotData.java

@@ -1,8 +1,9 @@
 package org.cbioportal.model;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ClinicalViolinPlotData {
+public class ClinicalViolinPlotData implements Serializable {
     private List<ClinicalViolinPlotRowData> rows;
     private Double axisStart;
     private Double axisEnd;

src/main/java/org/cbioportal/model/ClinicalViolinPlotIndividualPoint.java

@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class ClinicalViolinPlotIndividualPoint {
+import java.io.Serializable;
+
+public class ClinicalViolinPlotIndividualPoint implements Serializable {
     private String sampleId;
     private String studyId;
     private double value;

src/main/java/org/cbioportal/model/ClinicalViolinPlotRowData.java

@@ -1,8 +1,9 @@
 package org.cbioportal.model;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ClinicalViolinPlotRowData {
+public class ClinicalViolinPlotRowData implements Serializable {
     private String category;
     private Integer numSamples;
     private List<Double> curveData;

src/main/java/org/cbioportal/model/GenomicDataCount.java

@@ -10,6 +10,21 @@ public class GenomicDataCount implements Serializable {
     private Integer count;
     private Integer uniqueCount;
 
+    public GenomicDataCount() {}
+
+    public GenomicDataCount(String label, String value, Integer count) {
+        this.label = label;
+        this.value = value;
+        this.count = count;
+    }
+
+    public GenomicDataCount(String label, String value, Integer count, Integer uniqueCount) {
+        this.label = label;
+        this.value = value;
+        this.count = count;
+        this.uniqueCount = uniqueCount;
+    }
+
     public String getLabel() {
         return label;
     }

src/main/java/org/cbioportal/model/GenomicDataCountItem.java

@@ -9,6 +9,14 @@ public class GenomicDataCountItem implements Serializable {
     private String profileType;
     private List<GenomicDataCount> counts;
 
+    public GenomicDataCountItem() {}
+
+    public GenomicDataCountItem(String hugoGeneSymbol, String profileType, List<GenomicDataCount> counts) {
+        this.hugoGeneSymbol = hugoGeneSymbol;
+        this.profileType = profileType;
+        this.counts = counts;
+    }
+
     public String getHugoGeneSymbol() {
         return hugoGeneSymbol;
     }

src/main/java/org/cbioportal/model/PatientTreatment.java

@@ -0,0 +1,7 @@
+package org.cbioportal.model;
+
+import java.io.Serializable;
+
+public record PatientTreatment (String treatment, int count) implements Serializable {
+
+}

src/main/java/org/cbioportal/model/PatientTreatmentReport.java

@@ -0,0 +1,11 @@
+package org.cbioportal.model;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+
+public record PatientTreatmentReport (int totalPatients, int totalSamples, List<PatientTreatment> patientTreatments) implements Serializable {
+    public PatientTreatmentReport(int totalPatients, int totalSamples) {
+        this(totalPatients, totalSamples, Collections.emptyList());
+    } 
+}

src/main/java/org/cbioportal/model/SampleTreatment.java

@@ -0,0 +1,4 @@
+package org.cbioportal.model;
+
+public record SampleTreatment(String treatment, int preSampleCount, int postSampleCount) {
+}

src/main/java/org/cbioportal/model/SampleTreatmentReport.java

@@ -0,0 +1,6 @@
+package org.cbioportal.model;
+
+import java.util.Collection;
+
+public record SampleTreatmentReport(int totalSamples, Collection<SampleTreatmentRow> treatments) {
+}

src/main/java/org/cbioportal/model/StudyViewFilterContext.java

@@ -0,0 +1,11 @@
+package org.cbioportal.model;
+
+import org.cbioportal.web.parameter.CustomSampleIdentifier;
+import org.cbioportal.web.parameter.StudyViewFilter;
+
+import java.util.List;
+
+public record StudyViewFilterContext( StudyViewFilter studyViewFilter,
+                                     List<CustomSampleIdentifier> customDataFilterSamples) {
+
+}

src/main/java/org/cbioportal/persistence/CacheEnabledConfig.java

@@ -16,8 +16,11 @@ public class CacheEnabledConfig {
 
     @Value("${persistence.cache_type:no-cache}")
     private String cacheType;
+    @Value("${persistence.cache_type_clickhouse:no-cache}")
+    private String cacheTypeClickhouse;
 
     private boolean enabled;
+    private boolean enabledClickhouse;
 
     public static final String EHCACHE_DISK = "ehcache-disk";
     public static final String EHCACHE_HEAP = "ehcache-heap";
@@ -30,6 +33,8 @@ public class CacheEnabledConfig {
     public void init() {
         this.enabled = enableCache(cacheType);
         LOG.info("Cache is enabled: " + this.enabled);
+        this.enabledClickhouse = enableCache(cacheTypeClickhouse);
+        LOG.info("Cache is enabled for clickhouse: " + this.enabledClickhouse);
     }
 
     public static boolean enableCache(String cacheType) {
@@ -53,4 +58,16 @@ public boolean isEnabled() {
         return enabled;
     }
 
+    public String getEnabledClickhouse() {
+        if (enabledClickhouse) {
+            return "true";
+        } else {
+            return "false";
+        }
+    }
+
+    public boolean isEnabledClickhouse() {
+        return enabledClickhouse;
+    }
+
 }

src/main/java/org/cbioportal/persistence/StudyViewRepository.java

@@ -0,0 +1,87 @@
+package org.cbioportal.persistence;
+
+import org.cbioportal.model.AlterationCountByGene;
+import org.cbioportal.model.ClinicalAttribute;
+import org.cbioportal.model.CaseListDataCount;
+import org.cbioportal.model.ClinicalData;
+import org.cbioportal.model.ClinicalDataCount;
+import org.cbioportal.model.ClinicalEventTypeCount;
+import org.cbioportal.model.CopyNumberCountByGene;
+import org.cbioportal.model.GenericAssayDataCountItem;
+import org.cbioportal.model.GenomicDataCountItem;
+import org.cbioportal.model.GenomicDataCount;
+import org.cbioportal.model.MolecularProfile;
+import org.cbioportal.model.PatientTreatment;
+import org.cbioportal.model.Sample;
+import org.cbioportal.model.SampleTreatment;
+import org.cbioportal.model.StudyViewFilterContext;
+import org.cbioportal.web.parameter.ClinicalDataType;
+import org.cbioportal.web.parameter.GenericAssayDataBinFilter;
+import org.cbioportal.web.parameter.GenericAssayDataFilter;
+import org.cbioportal.web.parameter.GenomicDataBinFilter;
+import org.cbioportal.web.parameter.GenomicDataFilter;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public interface StudyViewRepository {
+    List<Sample> getFilteredSamples(StudyViewFilterContext studyViewFilterContext);
+
+    List<String> getFilteredStudyIds(StudyViewFilterContext studyViewFilterContext);
+
+    List<ClinicalData> getSampleClinicalData(StudyViewFilterContext studyViewFilterContext, List<String> attributeIds);
+
+    List<ClinicalData> getPatientClinicalData(StudyViewFilterContext studyViewFilterContext, List<String> attributeIds);
+
+    List<AlterationCountByGene> getMutatedGenes(StudyViewFilterContext studyViewFilterContext);
+
+    List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilterContext studyViewFilterContext);
+    List<CopyNumberCountByGene> getCnaGenes(StudyViewFilterContext studyViewFilterContext);
+
+    List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilterContext studyViewFilterContext, List<String> filteredAttributes);
+
+    List<GenomicDataCount> getMolecularProfileSampleCounts(StudyViewFilterContext studyViewFilterContext);
+
+    List<ClinicalAttribute> getClinicalAttributes();
+
+    Map<String, ClinicalDataType> getClinicalAttributeDatatypeMap();
+
+    List<CaseListDataCount> getCaseListDataCountsPerStudy(StudyViewFilterContext studyViewFilterContext);
+
+    Map<String, Integer> getTotalProfiledCounts(StudyViewFilterContext studyViewFilterContext, String alterationType);
+
+    int getFilteredSamplesCount(StudyViewFilterContext studyViewFilterContext);
+
+    int getFilteredPatientCount(StudyViewFilterContext studyViewFilterContext);
+
+    Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilterContext studyViewFilterContext, String alterationType);
+
+    int getTotalProfiledCountsByAlterationType(StudyViewFilterContext studyViewFilterContext, String alterationType);
+
+    int getSampleProfileCountWithoutPanelData(StudyViewFilterContext studyViewFilterContext, String alterationType);
+
+    List<ClinicalEventTypeCount> getClinicalEventTypeCounts(StudyViewFilterContext studyViewFilterContext);
+
+    List<PatientTreatment> getPatientTreatments(StudyViewFilterContext studyViewFilterContext);
+
+    int getTotalPatientTreatmentCount(StudyViewFilterContext studyViewFilterContext);
+
+    List<SampleTreatment> getSampleTreatments(StudyViewFilterContext studyViewFilterContext);
+
+    int getTotalSampleTreatmentCount(StudyViewFilterContext studyViewFilterContext);
+
+    List<GenomicDataCountItem> getCNACounts(StudyViewFilterContext studyViewFilterContext, List<GenomicDataFilter> genomicDataFilters);
+
+    List<GenericAssayDataCountItem> getGenericAssayDataCounts(StudyViewFilterContext studyViewFilterContext, List<GenericAssayDataFilter> genericAssayDataFilters);
+
+    Map<String, Integer> getMutationCounts(StudyViewFilterContext studyViewFilterContext, GenomicDataFilter genomicDataFilter);
+
+    List<GenomicDataCountItem> getMutationCountsByType(StudyViewFilterContext studyViewFilterContext, List<GenomicDataFilter> genomicDataFilters);
+
+    List<ClinicalDataCount> getGenomicDataBinCounts(StudyViewFilterContext studyViewFilterContext, List<GenomicDataBinFilter> genomicDataBinFilters);
+
+    List<ClinicalDataCount> getGenericAssayDataBinCounts(StudyViewFilterContext studyViewFilterContext, List<GenericAssayDataBinFilter> genericAssayDataBinFilters);
+
+    List<MolecularProfile> getGenericAssayProfiles();
+}