From 0c6551baef2fc628368c4507b2b456f7addf03ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 30 Aug 2024 10:37:39 +0100 Subject: [PATCH] storage: Ensure SearchIndexVariantQueryExecutor is used with skip>500. #TASK-6789 --- .../SearchIndexVariantQueryExecutor.java | 4 +++ .../VariantQueryUsingSearchIndexTest.java | 36 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index bd6b8e6437f..bd7c79cb2e9 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -262,6 +262,10 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { intersect = true; } else { if (options.getBoolean(QueryOptions.COUNT)) { + // The SearchIndex is better than anyone in terms of counting + intersect = true; + } else if (options.getInt(QueryOptions.SKIP, 0) > 500) { + // Large "skip" queries should use SearchIndex when possible intersect = true; } else { // TODO: Improve this heuristic diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java index 6194d25c6d8..cdc96e15377 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java @@ -2,16 +2,22 @@ import com.google.common.base.Throwables; import org.junit.ClassRule; +import org.junit.Test; import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager.SEARCH_ENGINE_ID; + /** * Created on 22/12/17. * @@ -87,4 +93,32 @@ public DataResult rank(int limit, Query query, String field, boolean asc) { throw Throwables.propagate(e); } } + + @Test + public void testQueryExecutor() { + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 0)).getSource(), + not(containsString(SEARCH_ENGINE_ID))); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, true) + .append(QueryOptions.SKIP, 0)).getSource(), + containsString(SEARCH_ENGINE_ID)); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 100)).getSource(), + not(containsString(SEARCH_ENGINE_ID))); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 1000)).getSource(), + containsString(SEARCH_ENGINE_ID)); + + } }