From 7429b5f94aa95c79a35c70d2baa70626a0159722 Mon Sep 17 00:00:00 2001 From: Oleksandr Porunov Date: Wed, 15 May 2024 00:44:53 +0100 Subject: [PATCH] Add batch query support for drop step [tp-tests] - Reuse multi-query optimization for TinkerPop's - Change restriction on eligible multi-query traversals and allow multi-query optimizations to be used for queries with steps - Add release template for JanusGraph 1.1.0 Signed-off-by: Oleksandr Porunov --- docs/changelog.md | 77 ++++++++ docs/configs/janusgraph-cfg.md | 1 + docs/operations/batch-processing.md | 3 +- .../janusgraph/graphdb/JanusGraphTest.java | 164 ++++++++++++++++-- .../CQLMultiQueryDropBenchmark.java | 25 ++- .../core/JanusGraphMultiVertexQuery.java | 6 +- .../janusgraph/core/TransactionBuilder.java | 10 ++ .../GraphDatabaseConfiguration.java | 16 ++ .../MultiVertexCentricQueryBuilder.java | 4 +- .../optimize/step/JanusGraphDropStep.java | 146 ++++++++++++++++ .../JanusGraphNoOpBarrierVertexOnlyStep.java | 64 +++++++ .../step/fetcher/DropStepBatchFetcher.java | 38 ++++ .../MultiQueriableStepBatchFetcher.java | 13 +- .../step/fetcher/TriggeredBatchConsumer.java | 23 +++ ...JanusGraphLocalQueryOptimizerStrategy.java | 19 ++ .../JanusGraphMultiQueryStrategy.java | 10 +- .../MultiQueryDropStepStrategyMode.java | 42 +++++ .../StandardTransactionBuilder.java | 25 ++- .../transaction/TransactionConfiguration.java | 6 + .../graphdb/inmemory/InMemoryGraphTest.java | 10 ++ .../inmemory/InMemoryLazyLoadGraphTest.java | 1 + ...asUniquePropertyOptimizerStrategyTest.java | 17 +- .../JanusGraphMultiQueryStrategyTest.java | 19 +- .../optimize/JanusGraphStepStrategyTest.java | 98 ++++++----- mkdocs.yml | 2 +- pom.xml | 2 +- 26 files changed, 754 insertions(+), 87 deletions(-) create mode 100644 janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphDropStep.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphNoOpBarrierVertexOnlyStep.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/DropStepBatchFetcher.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/TriggeredBatchConsumer.java create mode 100644 janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/MultiQueryDropStepStrategyMode.java diff --git a/docs/changelog.md b/docs/changelog.md index 632e4db3ae..63705a30f2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -28,6 +28,7 @@ All currently supported versions of JanusGraph are listed below. | JanusGraph | Storage Version | Cassandra | HBase | Bigtable | ScyllaDB | Elasticsearch | Solr | TinkerPop | Spark | Scala | | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | | 1.0.z | 2 | 3.11.z, 4.0.z | 2.5.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 5.y | 6.y, 7.y, 8.y | 8.y | 3.7.z | 3.2.z | 2.12.z | +| 1.1.z | 2 | 3.11.z, 4.0.z | 2.5.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 5.y | 6.y, 7.y, 8.y | 8.y | 3.7.z | 3.2.z | 2.12.z | !!! info Even so ScyllaDB is marked as `N/A` prior version 1.0.0 it was actually supported using `cql` storage option. @@ -49,6 +50,82 @@ The versions of JanusGraph listed below are outdated and will no longer receive ## Release Notes +### Version 1.1.0 (Release Date: ???) + +/// tab | Maven +```xml + + org.janusgraph + janusgraph-core + 1.1.0 + +``` +/// + +/// tab | Gradle +```groovy +compile "org.janusgraph:janusgraph-core:1.1.0" +``` +/// + +**Tested Compatibility:** + +* Apache Cassandra 3.11.10, 4.0.6 +* Apache HBase 2.5.0 +* Oracle BerkeleyJE 7.5.11 +* ScyllaDB 5.1.4 +* Elasticsearch 6.0.1, 6.6.0, 7.17.8, 8.10.4 +* Apache Lucene 8.11.1 +* Apache Solr 8.11.1 +* Apache TinkerPop 3.7.3 +* Java 8, 11 + +**Installed versions in the Pre-Packaged Distribution:** + +* Cassandra 4.0.6 +* Elasticsearch 7.14.0 + +#### Changes + +For more information on features and bug fixes in 1.1.0, see the GitHub milestone: + +- + +#### Assets + +* [JavaDoc](https://javadoc.io/doc/org.janusgraph/janusgraph-core/1.1.0) +* [GitHub Release](https://github.com/JanusGraph/janusgraph/releases/tag/v1.1.0) +* [JanusGraph zip](https://github.com/JanusGraph/janusgraph/releases/download/v1.1.0/janusgraph-1.1.0.zip) +* [JanusGraph zip with embedded Cassandra and ElasticSearch](https://github.com/JanusGraph/janusgraph/releases/download/v1.1.0/janusgraph-full-1.1.0.zip) + +#### Upgrade Instructions + +##### Batched Queries Enhancement: Introduction of `JanusGraphNoOpBarrierVertexOnlyStep` + +In previous versions, when a query that could benefit from batch-query optimization (multi-query) was executed without +a user-defined barrier step, JanusGraph would inject a `NoOpBarrierStep` by default. This approach allowed batching +for edges and properties, which do not gain advantages from multi-query optimization. + +Starting with JanusGraph 1.1.0, this behavior has been improved. The system now injects a +`JanusGraphNoOpBarrierVertexOnlyStep` instead of the standard `NoOpBarrierStep` when no barrier steps are detected. +This change ensures that batching is applied exclusively to vertices, which do benefit from batch queries, +while excluding edges and properties from the batching process. + +If a user explicitly defines a `.barrier()` step in the query, the system will continue to use the `NoOpBarrierStep` as expected. + +##### Batch Query Optimizations Now Support Traversals Containing the `drop()` Step + +Starting with JanusGraph 1.1.0, batch optimizations for vertex removal have been introduced in the `drop()` step and +are enabled by default. Previously, any batch optimization would be skipped for queries containing at least one +`drop()` step. However, with this update, such queries are now eligible for batch query optimization (multi-query). + +Please note that the `LazyBarrierStrategy` (a TinkerPop strategy) is disabled for any query that includes at least one `drop()` step. + +To disable the `drop()` step optimization and maintain the previous behavior, users can set the following configuration: +``` +query.batch.drop-step-mode=none +``` + ### Version 1.0.0 (Release Date: October 21, 2023) /// tab | Maven diff --git a/docs/configs/janusgraph-cfg.md b/docs/configs/janusgraph-cfg.md index ff08908267..0d02e3a205 100644 --- a/docs/configs/janusgraph-cfg.md +++ b/docs/configs/janusgraph-cfg.md @@ -365,6 +365,7 @@ Configuration options to configure batch queries optimization behavior | Name | Description | Datatype | Default Value | Mutability | | ---- | ---- | ---- | ---- | ---- | +| query.batch.drop-step-mode | Batching mode for `drop()` step. Used only when `query.batch.enabled` is `true`.
Supported modes:
- `all` - Drops all vertices in a batch.
- `none` - Skips drop batching optimization.
| String | all | MASKABLE | | query.batch.enabled | Whether traversal queries should be batched when executed against the storage backend. This can lead to significant performance improvement if there is a non-trivial latency to the backend. If `false` then all other configuration options under `query.batch` namespace are ignored. | Boolean | true | MASKABLE | | query.batch.has-step-mode | Properties pre-fetching mode for `has` step. Used only when `query.batch.enabled` is `true`.
Supported modes:
- `all_properties` - Pre-fetch all vertex properties on any property access (fetches all vertex properties in a single slice query)
- `required_properties_only` - Pre-fetch necessary vertex properties for the whole chain of foldable `has` steps (uses a separate slice query per each required property)
- `required_and_next_properties` - Prefetch the same properties as with `required_properties_only` mode, but also prefetch
properties which may be needed in the next properties access step like `values`, `properties,` `valueMap`, `elementMap`, or `propertyMap`.
In case the next step is not one of those properties access steps then this mode behaves same as `required_properties_only`.
In case the next step is one of the properties access steps with limited scope of properties, those properties will be
pre-fetched together in the same multi-query.
In case the next step is one of the properties access steps with unspecified scope of property keys then this mode
behaves same as `all_properties`.
- `required_and_next_properties_or_all` - Prefetch the same properties as with `required_and_next_properties`, but in case the next step is not
`values`, `properties,` `valueMap`, `elementMap`, or `propertyMap` then acts like `all_properties`.
- `none` - Skips `has` step batch properties pre-fetch optimization.
| String | required_and_next_properties | MASKABLE | | query.batch.label-step-mode | Labels pre-fetching mode for `label()` step. Used only when `query.batch.enabled` is `true`.
Supported modes:
- `all` - Pre-fetch labels for all vertices in a batch.
- `none` - Skips vertex labels pre-fetching optimization.
| String | all | MASKABLE | diff --git a/docs/operations/batch-processing.md b/docs/operations/batch-processing.md index 29a225f9ab..1602ec8e96 100644 --- a/docs/operations/batch-processing.md +++ b/docs/operations/batch-processing.md @@ -159,7 +159,7 @@ Batched query processing takes into account two types of steps: 1. Batch compatible step. This is the step which will execute batch requests. Currently, the list of such steps is the next: `out()`, `in()`, `both()`, `inE()`, `outE()`, `bothE()`, `has()`, `values()`, `properties()`, `valueMap()`, - `propertyMap()`, `elementMap()`, `label()`. + `propertyMap()`, `elementMap()`, `label()`, `drop()`. 2. Parent step. This is a parent step which has local traversals with the same start. Such parent steps also implement the interface `TraversalParent`. There are many such steps, but as for an example those could be: `and(...)`, `or(...)`, `not(...)`, `order().by(...)`, `project("valueA", "valueB", "valueC").by(...).by(...).by(...)`, `union(..., ..., ...)`, @@ -331,3 +331,4 @@ See configuration option `query.batch.has-step-mode` to control properties pre-f See configuration option `query.batch.properties-mode` to control properties pre-fetching behaviour for `values`, `properties`, `valueMap`, `propertyMap`, and `elementMap` steps. See configuration option `query.batch.label-step-mode` to control labels pre-fetching behaviour for `label` step. +See configuration option `query.batch.drop-step-mode` to control drop batching behaviour for `drop` step. diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java index 717451ab1f..958691f588 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java @@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.WithOptions; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy; @@ -59,6 +60,7 @@ import org.janusgraph.core.PropertyKey; import org.janusgraph.core.RelationType; import org.janusgraph.core.SchemaViolationException; +import org.janusgraph.core.Transaction; import org.janusgraph.core.VertexLabel; import org.janusgraph.core.VertexList; import org.janusgraph.core.attribute.Cmp; @@ -135,10 +137,12 @@ import org.janusgraph.graphdb.relations.StandardVertexProperty; import org.janusgraph.graphdb.serializer.SpecialInt; import org.janusgraph.graphdb.serializer.SpecialIntSerializer; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphDropStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphElementMapStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphHasStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertiesStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertyMapStep; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryLabelStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode; @@ -214,6 +218,7 @@ import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE_CLEAN_WAIT; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE_TIME; +import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DROP_STEP_BATCH_MODE; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.FORCE_INDEX_USAGE; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.HARD_MAX_LIMIT; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.HAS_STEP_BATCH_MODE; @@ -10023,11 +10028,7 @@ public void testMultiQueryDropsVertices() { int verticesAmount = 42; - for (int i = 0; i < verticesAmount; i++) { - Vertex vertex = tx.addVertex("id", i); - vertex.property("name", "name_test"); - vertex.property("details", "details_" + i); - } + addVerticesForDropTest(verticesAmount, tx); clopen(); @@ -10039,20 +10040,161 @@ public void testMultiQueryDropsVertices() { .map(v -> (JanusGraphVertex) v) .collect(Collectors.toList()); - int actualCount = tx.multiQuery(vertices).drop(); + int actualCount = tx.multiQuery(vertices).drop().size(); clopen(); assertEquals(verticesAmount, actualCount); - int afterDropCount = tx.traversal() - .V() - .has("name", "name_test") - .toList() - .size(); + long afterDropCount = getVerticesForDropTestCount(tx.traversal()); assertEquals(0, afterDropCount); } + @Test + public void testMultiQueryDropsStrategyModes() { + + mgmt.makePropertyKey("id").dataType(Integer.class).cardinality(Cardinality.SINGLE).make(); + PropertyKey nameProp = mgmt.makePropertyKey("name").dataType(String.class).cardinality(Cardinality.SINGLE).make(); + mgmt.makePropertyKey("details").dataType(String.class).cardinality(Cardinality.SINGLE).make(); + mgmt.buildIndex("nameIndex", Vertex.class).addKey(nameProp).buildCompositeIndex(); + + finishSchema(); + + long verticesAmount = 42; + + // Mode: NONE + + addVerticesForDropTest(verticesAmount); + graph.tx().commit(); + clopen(option(DROP_STEP_BATCH_MODE), MultiQueryDropStepStrategyMode.NONE.getConfigName()); + assertEquals(verticesAmount, getVerticesForDropTestCount()); + TraversalMetrics profileT = graph.traversal().V().drop().profile().next(); + assertTrue(profileT.getMetrics().stream().anyMatch(metrics -> metrics.getName().equals(DropStep.class.getSimpleName()))); + graph.tx().commit(); + assertEquals(0, getVerticesForDropTestCount()); + + // Mode: ALL + + addVerticesForDropTest(verticesAmount); + graph.tx().commit(); + clopen(option(DROP_STEP_BATCH_MODE), MultiQueryDropStepStrategyMode.ALL.getConfigName()); + assertEquals(verticesAmount, getVerticesForDropTestCount()); + profileT = graph.traversal().V().drop().profile().next(); + assertEquals("true", profileT.getMetrics().stream().filter(metrics -> metrics.getName().equals(JanusGraphDropStep.class.getSimpleName())).findAny().get().getAnnotation("multi")); + graph.tx().commit(); + assertEquals(0, getVerticesForDropTestCount()); + + // `limit` with `drop` step. + + addVerticesForDropTest(verticesAmount); + graph.tx().commit(); + clopen(option(DROP_STEP_BATCH_MODE), MultiQueryDropStepStrategyMode.NONE.getConfigName()); + assertEquals(verticesAmount, getVerticesForDropTestCount()); + int limitSize = 2; + profileT = graph.traversal().V().limit(limitSize).drop().profile().next(); + assertTrue(profileT.getMetrics().stream().anyMatch(metrics -> metrics.getName().equals(DropStep.class.getSimpleName()))); + graph.tx().commit(); + long afterDropCount = getVerticesForDropTestCount(); + assertEquals(verticesAmount-limitSize, afterDropCount); + graph.traversal().V().drop().iterate(); + graph.tx().commit(); + addVerticesForDropTest(verticesAmount); + graph.tx().commit(); + clopen(option(DROP_STEP_BATCH_MODE), MultiQueryDropStepStrategyMode.ALL.getConfigName()); + assertEquals(verticesAmount, getVerticesForDropTestCount()); + profileT = graph.traversal().V().limit(limitSize).drop().profile().next(); + assertEquals("true", profileT.getMetrics().stream().filter(metrics -> metrics.getName().equals(JanusGraphDropStep.class.getSimpleName())).findAny().get().getAnnotation("multi")); + graph.tx().commit(); + afterDropCount = getVerticesForDropTestCount(); + assertEquals(verticesAmount-limitSize, afterDropCount); + } + + @Test + public void testMetaPropertiesDrop(){ + mgmt.makePropertyKey("id").dataType(Integer.class).cardinality(Cardinality.SINGLE).make(); + PropertyKey nameProp = mgmt.makePropertyKey("name").dataType(String.class).cardinality(Cardinality.SINGLE).make(); + mgmt.makePropertyKey("details").dataType(String.class).cardinality(Cardinality.SINGLE).make(); + mgmt.buildIndex("nameIndex", Vertex.class).addKey(nameProp).buildCompositeIndex(); + + finishSchema(); + + long verticesAmount = 42; + + for (int i = 0; i < verticesAmount; i++) { + Vertex vertex = tx.addVertex("id", i); + VertexProperty property = vertex.property("name", "name_test"); + property.property("details", "details_" + i); + } + graph.tx().commit(); + + clopen(option(DROP_STEP_BATCH_MODE), MultiQueryDropStepStrategyMode.ALL.getConfigName()); + + assertEquals(verticesAmount, graph.traversal().V().properties("name").properties("details").count().next()); + + graph.traversal().V().properties("name").properties("details").drop().hasNext(); + + assertEquals(0, graph.traversal().V().properties("name").properties("details").count().next()); + + graph.tx().commit(); + + assertEquals(0, graph.traversal().V().properties("name").properties("details").count().next()); + assertEquals(verticesAmount, graph.traversal().V().has("name").count().next()); + } + + @Test + public void testEdgePropertiesDrop(){ + mgmt.makePropertyKey("id").dataType(Integer.class).cardinality(Cardinality.SINGLE).make(); + mgmt.makePropertyKey("name").dataType(String.class).cardinality(Cardinality.SINGLE).make(); + mgmt.makeEdgeLabel("relate").make(); + + finishSchema(); + + long verticesAmount = 42; + + for (int i = 0; i < verticesAmount; i++) { + Vertex vertex = tx.addVertex("id", i); + Vertex vertex2 = tx.addVertex("id", i+verticesAmount); + vertex.addEdge("relate", vertex2).property("name", "name_"+i); + } + + graph.tx().commit(); + + clopen(option(DROP_STEP_BATCH_MODE), MultiQueryDropStepStrategyMode.ALL.getConfigName()); + + assertEquals(verticesAmount, graph.traversal().E().properties("name").count().next()); + + graph.traversal().E().properties("name").drop().hasNext(); + + assertEquals(0, graph.traversal().E().properties("name").count().next()); + + graph.tx().commit(); + + assertEquals(0, graph.traversal().E().properties("name").count().next()); + assertEquals(verticesAmount, graph.traversal().E().count().next()); + } + + private void addVerticesForDropTest(long verticesAmount){ + addVerticesForDropTest(verticesAmount, graph); + } + + private long getVerticesForDropTestCount(){ + return getVerticesForDropTestCount(graph.traversal()); + } + + private void addVerticesForDropTest(long verticesAmount, Transaction tx){ + for (int i = 0; i < verticesAmount; i++) { + Vertex vertex = tx.addVertex("id", i); + vertex.property("name", "name_test"); + vertex.property("details", "details_" + i); + } + } + + private long getVerticesForDropTestCount(GraphTraversalSource g){ + return g.V() + .has("name", "name_test") + .count().next(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testParallelBackendOps(boolean parallelBackendOpsEnabled) { diff --git a/janusgraph-benchmark/src/main/java/org/janusgraph/CQLMultiQueryDropBenchmark.java b/janusgraph-benchmark/src/main/java/org/janusgraph/CQLMultiQueryDropBenchmark.java index 749e30d48a..9d0f565a98 100644 --- a/janusgraph-benchmark/src/main/java/org/janusgraph/CQLMultiQueryDropBenchmark.java +++ b/janusgraph-benchmark/src/main/java/org/janusgraph/CQLMultiQueryDropBenchmark.java @@ -27,6 +27,7 @@ import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.janusgraph.diskstorage.cql.CQLConfigOptions; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -65,6 +66,7 @@ public WriteConfiguration getConfiguration() { config.set(GraphDatabaseConfiguration.STORAGE_BACKEND,"cql"); config.set(CQLConfigOptions.LOCAL_DATACENTER, "dc1"); config.set(GraphDatabaseConfiguration.USE_MULTIQUERY, true); + config.set(GraphDatabaseConfiguration.DROP_STEP_BATCH_MODE, MultiQueryDropStepStrategyMode.NONE.getConfigName()); return config.getConfiguration(); } @@ -103,7 +105,7 @@ public Integer dropVertices() { .map(v -> (JanusGraphVertex) v) .collect(Collectors.toList()); - dropCount = tx.multiQuery(vertices).drop(); + dropCount = tx.multiQuery(vertices).drop().size(); } else { dropCount = tx.traversal() .V() @@ -117,6 +119,27 @@ public Integer dropVertices() { return dropCount; } + @Benchmark + public Integer dropVerticesGremlinQuery() { + + JanusGraphTransaction tx; + if (isMultiDrop) { + tx = graph.buildTransaction().setDropStepStrategyMode(MultiQueryDropStepStrategyMode.ALL).start(); + } else { + tx = graph.buildTransaction().setDropStepStrategyMode(MultiQueryDropStepStrategyMode.NONE).start(); + } + + Integer dropCount = tx.traversal() + .V() + .has("name", "name_test") + .drop() + .toList() + .size(); + + tx.rollback(); + return dropCount; + } + private void addVertices() { for (int i = 0; i < verticesAmount; i++) { Vertex vertex = graph.addVertex("id", i); diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphMultiVertexQuery.java b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphMultiVertexQuery.java index a6e1994bf6..e3d3d36c0a 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphMultiVertexQuery.java +++ b/janusgraph-core/src/main/java/org/janusgraph/core/JanusGraphMultiVertexQuery.java @@ -60,7 +60,6 @@ public interface JanusGraphMultiVertexQuery vertices); - @Override Q adjacent(Vertex vertex); @@ -156,7 +155,8 @@ public interface JanusGraphMultiVertexQuery> drop(); + } diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java index 6fb041070f..8c6426d5d4 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java @@ -15,6 +15,7 @@ package org.janusgraph.core; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryLabelStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode; @@ -186,6 +187,15 @@ public interface TransactionBuilder { */ TransactionBuilder setLabelsStepStrategyMode(MultiQueryLabelStepStrategyMode labelStepStrategyMode); + /** + * Sets `drop` step strategy mode. + *

+ * Doesn't have any effect if multi-query was disabled via config `query.batch.enabled = false`. + * + * @return Object with the set drop strategy mode settings + */ + TransactionBuilder setDropStepStrategyMode(MultiQueryDropStepStrategyMode dropStepStrategyMode); + /** * Sets the group name for this transaction which provides a way for gathering * reporting on multiple transactions into one group. diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java index 10bf05bb3d..dce5267a2b 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java @@ -58,6 +58,7 @@ import org.janusgraph.graphdb.query.index.BruteForceIndexSelectionStrategy; import org.janusgraph.graphdb.query.index.IndexSelectionStrategy; import org.janusgraph.graphdb.query.index.ThresholdBasedIndexSelectionStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryLabelStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryStrategyRepeatStepMode; @@ -386,6 +387,15 @@ public class GraphDatabaseConfiguration { MultiQueryLabelStepStrategyMode.NONE.getConfigName()), ConfigOption.Type.MASKABLE, MultiQueryLabelStepStrategyMode.ALL.getConfigName()); + public static final ConfigOption DROP_STEP_BATCH_MODE = new ConfigOption<>(QUERY_BATCH_NS,"drop-step-mode", + String.format("Batching mode for `drop()` step. Used only when `"+USE_MULTIQUERY.toStringWithoutRoot()+"` is `true`.
" + + "Supported modes:
" + + "- `%s` - Drops all vertices in a batch.
" + + "- `%s` - Skips drop batching optimization.
", + MultiQueryDropStepStrategyMode.ALL.getConfigName(), + MultiQueryDropStepStrategyMode.NONE.getConfigName()), + ConfigOption.Type.MASKABLE, MultiQueryDropStepStrategyMode.ALL.getConfigName()); + // ################ SCHEMA ####################### // ################################################ @@ -1371,6 +1381,7 @@ public boolean apply(@Nullable String s) { private MultiQueryHasStepStrategyMode hasStepStrategyMode; private MultiQueryPropertiesStrategyMode propertiesStrategyMode; private MultiQueryLabelStepStrategyMode labelStepStrategyMode; + private MultiQueryDropStepStrategyMode dropStepStrategyMode; private StoreFeatures storeFeatures = null; @@ -1497,6 +1508,10 @@ public MultiQueryLabelStepStrategyMode labelStepStrategyMode() { return labelStepStrategyMode; } + public MultiQueryDropStepStrategyMode dropStepStrategyMode() { + return dropStepStrategyMode; + } + public boolean adjustQueryLimit() { return adjustQueryLimit; } @@ -1627,6 +1642,7 @@ private void preLoadConfiguration() { hasStepStrategyMode = selectExactConfig(HAS_STEP_BATCH_MODE, MultiQueryHasStepStrategyMode.values()); propertiesStrategyMode = selectExactConfig(PROPERTIES_BATCH_MODE, MultiQueryPropertiesStrategyMode.values()); labelStepStrategyMode = selectExactConfig(LABEL_STEP_BATCH_MODE, MultiQueryLabelStepStrategyMode.values()); + dropStepStrategyMode = selectExactConfig(DROP_STEP_BATCH_MODE, MultiQueryDropStepStrategyMode.values()); indexSelectionStrategy = Backend.getImplementationClass(configuration, configuration.get(INDEX_SELECT_STRATEGY), REGISTERED_INDEX_SELECTION_STRATEGIES); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java index c461f41852..7e6420a52d 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/query/vertex/MultiVertexCentricQueryBuilder.java @@ -203,9 +203,9 @@ public Map vertexIds() { } @Override - public Integer drop() { + public Map> drop() { Map> allRelations = this.noPartitionRestriction().all().relations(); allRelations.forEach((vertex, relations) -> ((InternalVertex) vertex).remove(relations)); - return allRelations.size(); + return allRelations; } } diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphDropStep.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphDropStep.java new file mode 100644 index 0000000000..f55596cd47 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphDropStep.java @@ -0,0 +1,146 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.tinkerpop.optimize.step; + +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.event.CallbackRegistry; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.event.Event; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.event.EventCallback; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.event.EventUtil; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.core.BaseVertexQuery; +import org.janusgraph.graphdb.query.profile.QueryProfiler; +import org.janusgraph.graphdb.query.vertex.BasicVertexCentricQueryBuilder; +import org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher.DropStepBatchFetcher; +import org.janusgraph.graphdb.tinkerpop.profile.TP3ProfileWrapper; +import org.janusgraph.graphdb.util.CopyStepUtil; +import org.janusgraph.graphdb.util.JanusGraphTraverserUtil; + +import java.util.List; + +/** + * This class extends the default TinkerPop's {@link DropStep} and adds vertices multi-query optimization to this step. + *

+ * Before this step is evaluated it usually receives multiple future vertices which might be processed next with this step. + * This step stores all these vertices which might be needed later for evaluation and whenever this step receives the + * vertex for evaluation which wasn't evaluated previously it sends multi-query for a batch of vertices to drop them. + *

+ * This step optimizes only drop of Vertices and skips optimization for any other Element. + */ +public class JanusGraphDropStep extends DropStep implements Profiling, MultiQueriable { + + private boolean useMultiQuery = false; + private QueryProfiler queryProfiler = QueryProfiler.NO_OP; + private int batchSize = Integer.MAX_VALUE; + private DropStepBatchFetcher dropStepBatchFetcher; + + public JanusGraphDropStep(DropStep originalStep){ + super(originalStep.getTraversal()); + CopyStepUtil.copyAbstractStepModifiableFields(originalStep, this); + + CallbackRegistry callbackRegistry = getMutatingCallbackRegistry(); + for(EventCallback callback : originalStep.getMutatingCallbackRegistry().getCallbacks()){ + callbackRegistry.addCallback(callback); + } + + if (originalStep instanceof JanusGraphDropStep) { + JanusGraphDropStep originalJanusGraphLabelStep = (JanusGraphDropStep) originalStep; + setBatchSize(originalJanusGraphLabelStep.batchSize); + setUseMultiQuery(originalJanusGraphLabelStep.useMultiQuery); + } + } + + @Override + protected boolean filter(final Traverser.Admin traverser) { + if (useMultiQuery && traverser.get() instanceof Vertex) { + dropStepBatchFetcher.fetchData(getTraversal(), (Vertex) traverser.get(), JanusGraphTraverserUtil.getLoops(traverser)); + return false; + } else { + return super.filter(traverser); + } + } + + @Override + public void setUseMultiQuery(boolean useMultiQuery) { + this.useMultiQuery = useMultiQuery; + if(this.useMultiQuery && dropStepBatchFetcher == null){ + dropStepBatchFetcher = new DropStepBatchFetcher(this::makeLabelsQuery, batchSize, (batchVertices, requiredVertex) -> { + List> callbacksForRemovalEvents = getMutatingCallbackRegistry().getCallbacks(); + if(!callbacksForRemovalEvents.isEmpty()){ + final EventStrategy eventStrategy = EventUtil.forceGetEventStrategy(traversal); + produceRemovedEvent(eventStrategy, callbacksForRemovalEvents, requiredVertex); + for(Vertex vertexInBatch : batchVertices){ + if(vertexInBatch != requiredVertex){ + produceRemovedEvent(eventStrategy, callbacksForRemovalEvents, vertexInBatch); + } + } + } + }); + } + } + + private static void produceRemovedEvent(EventStrategy eventStrategy, + List> callbacksForRemovalEvents, + Vertex vertex){ + final Event removeEvent = new Event.VertexRemovedEvent(eventStrategy.detach(vertex)); + for(EventCallback callback : callbacksForRemovalEvents){ + callback.accept(removeEvent); + } + } + + private Q makeLabelsQuery(Q query) { + ((BasicVertexCentricQueryBuilder) query).profiler(queryProfiler); + return query; + } + + @Override + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + if(dropStepBatchFetcher != null){ + dropStepBatchFetcher.setBatchSize(batchSize); + } + } + + @Override + public void registerFirstNewLoopFutureVertexForPrefetching(Vertex futureVertex, int futureVertexTraverserLoop) { + if(useMultiQuery){ + dropStepBatchFetcher.registerFirstNewLoopFutureVertexForPrefetching(futureVertex); + } + } + + @Override + public void registerSameLoopFutureVertexForPrefetching(Vertex futureVertex, int futureVertexTraverserLoop) { + if(useMultiQuery){ + dropStepBatchFetcher.registerCurrentLoopFutureVertexForPrefetching(futureVertex, futureVertexTraverserLoop); + } + } + + @Override + public void registerNextLoopFutureVertexForPrefetching(Vertex futureVertex, int futureVertexTraverserLoop) { + if(useMultiQuery){ + dropStepBatchFetcher.registerNextLoopFutureVertexForPrefetching(futureVertex, futureVertexTraverserLoop); + } + } + + @Override + public void setMetrics(MutableMetrics metrics) { + queryProfiler = new TP3ProfileWrapper(metrics); + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphNoOpBarrierVertexOnlyStep.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphNoOpBarrierVertexOnlyStep.java new file mode 100644 index 0000000000..2810c6f38a --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphNoOpBarrierVertexOnlyStep.java @@ -0,0 +1,64 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.tinkerpop.optimize.step; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; +import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +/** + * This implementation overwrites the standard NoOpBarrierStep with the same logic except that anytime we receive non-Vertex + * element during `processAllStarts` process we stop barrier fetching of more data into the barrier until that non-Vertex + * element is out from this barrier step. + * It means that for non-Vertex elements this barrier acts like it has size of `1`. For Vertex elements it acts + * same as normal `NoOpBarrierStep`. + */ +public class JanusGraphNoOpBarrierVertexOnlyStep extends NoOpBarrierStep { + + public JanusGraphNoOpBarrierVertexOnlyStep(final Traversal.Admin traversal) { + super(traversal); + } + + public JanusGraphNoOpBarrierVertexOnlyStep(final Traversal.Admin traversal, final int maxBarrierSize) { + super(traversal, maxBarrierSize); + } + + public JanusGraphNoOpBarrierVertexOnlyStep(final Traversal.Admin traversal, final int maxBarrierSize, TraverserSet barrier) { + super(traversal, maxBarrierSize, barrier); + } + + @Override + public void processAllStarts() { + if(!this.barrier.isEmpty()){ + return; + } + while ((this.maxBarrierSize == Integer.MAX_VALUE || this.barrier.size() < this.maxBarrierSize) && this.starts.hasNext()) { + final Traverser.Admin traverser = this.starts.next(); + traverser.setStepId(this.getNextStep().getId()); // when barrier is reloaded, the traversers should be at the next step + this.barrier.add(traverser); + if(!(traverser.get() instanceof Vertex)){ + break; + } + } + } + + @Override + public JanusGraphNoOpBarrierVertexOnlyStep clone() { + return (JanusGraphNoOpBarrierVertexOnlyStep) super.clone(); + } + +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/DropStepBatchFetcher.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/DropStepBatchFetcher.java new file mode 100644 index 0000000000..fb9c088a08 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/DropStepBatchFetcher.java @@ -0,0 +1,38 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher; + +import org.janusgraph.core.JanusGraphMultiVertexQuery; +import org.janusgraph.core.JanusGraphRelation; +import org.janusgraph.core.JanusGraphVertex; + +import java.util.Map; + +public class DropStepBatchFetcher extends MultiQueriableStepBatchFetcher>{ + + private final FetchQueryBuildFunction fetchQueryBuildFunction; + + public DropStepBatchFetcher(FetchQueryBuildFunction fetchQueryBuildFunction, int batchSize, + TriggeredBatchConsumer batchConsumerBeforeQuery) { + super(batchSize, batchConsumerBeforeQuery); + this.fetchQueryBuildFunction = fetchQueryBuildFunction; + } + + @Override + protected Map> makeQueryAndExecute(JanusGraphMultiVertexQuery multiQuery) { + multiQuery = fetchQueryBuildFunction.makeQuery(multiQuery); + return multiQuery.drop(); + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/MultiQueriableStepBatchFetcher.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/MultiQueriableStepBatchFetcher.java index 00fd7587d0..e4a25136c5 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/MultiQueriableStepBatchFetcher.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/MultiQueriableStepBatchFetcher.java @@ -31,17 +31,25 @@ */ public abstract class MultiQueriableStepBatchFetcher { + private static final TriggeredBatchConsumer EMPTY_BATCH_CONSUMER = (b, t) -> {}; + private Map multiQueryResults = null; private int batchSize; private int currentLoops = 0; private BatchProcessingQueue firstLoopBatchProcessingQueue; private BatchProcessingQueue currentLoopBatchProcessingQueue; private BatchProcessingQueue nextLoopBatchProcessingQueue; + private final TriggeredBatchConsumer batchConsumerBeforeQuery; public MultiQueriableStepBatchFetcher(int batchSize){ + this(batchSize, EMPTY_BATCH_CONSUMER); + } + + public MultiQueriableStepBatchFetcher(int batchSize, TriggeredBatchConsumer batchConsumerBeforeQuery){ this.batchSize = batchSize; this.currentLoopBatchProcessingQueue = generateNewBatchProcessingQueue(); this.nextLoopBatchProcessingQueue = generateNewBatchProcessingQueue(); + this.batchConsumerBeforeQuery = batchConsumerBeforeQuery; } public void registerCurrentLoopFutureVertexForPrefetching(Vertex forGeneralVertex, int traverserLoops) { @@ -105,9 +113,10 @@ private boolean hasNoFetchedData(Vertex forVertex){ } protected Map prefetchNextBatch(final Traversal.Admin traversal, JanusGraphVertex requiredFetchVertex){ - final JanusGraphMultiVertexQuery multiQuery = JanusGraphTraversalUtil.getTx(traversal) - .multiQuery(nextBatch()); + Collection batch = nextBatch(); + final JanusGraphMultiVertexQuery multiQuery = JanusGraphTraversalUtil.getTx(traversal).multiQuery(batch); multiQuery.addVertex(requiredFetchVertex); + batchConsumerBeforeQuery.consume(batch, requiredFetchVertex); try { return makeQueryAndExecute(multiQuery); } catch (JanusGraphException janusGraphException) { diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/TriggeredBatchConsumer.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/TriggeredBatchConsumer.java new file mode 100644 index 0000000000..7d6bfd2949 --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/fetcher/TriggeredBatchConsumer.java @@ -0,0 +1,23 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher; + +import java.util.Collection; + +public interface TriggeredBatchConsumer { + + void consume(Collection batch, E triggeringElement); + +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphLocalQueryOptimizerStrategy.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphLocalQueryOptimizerStrategy.java index 8e3a56dc03..e1ba04c03a 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphLocalQueryOptimizerStrategy.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphLocalQueryOptimizerStrategy.java @@ -18,6 +18,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.step.branch.LocalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ElementMapStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.LabelStep; @@ -31,6 +32,7 @@ import org.janusgraph.graphdb.query.QueryUtil; import org.janusgraph.graphdb.tinkerpop.optimize.JanusGraphTraversalUtil; import org.janusgraph.graphdb.tinkerpop.optimize.step.HasStepFolder; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphDropStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphElementMapStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphLabelStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertiesStep; @@ -70,6 +72,7 @@ public void apply(final Traversal.Admin traversal) { final Optional tx = JanusGraphTraversalUtil.getJanusGraphTx(traversal); final MultiQueryPropertiesStrategyMode propertiesStrategyMode; final MultiQueryLabelStepStrategyMode labelStepStrategyMode; + final MultiQueryDropStepStrategyMode dropStepStrategyMode; final int txVertexCacheSize; if(tx.isPresent()){ @@ -77,16 +80,19 @@ public void apply(final Traversal.Admin traversal) { txVertexCacheSize = txConfig.getVertexCacheSize(); propertiesStrategyMode = txConfig.getPropertiesStrategyMode(); labelStepStrategyMode = txConfig.getLabelStepStrategyMode(); + dropStepStrategyMode = txConfig.getDropStepStrategyMode(); } else { GraphDatabaseConfiguration graphConfig = janusGraph.getConfiguration(); txVertexCacheSize = graphConfig.getTxVertexCacheSize(); propertiesStrategyMode = graphConfig.propertiesStrategyMode(); labelStepStrategyMode = graphConfig.labelStepStrategyMode(); + dropStepStrategyMode = graphConfig.dropStepStrategyMode(); } applyJanusGraphVertexSteps(traversal); applyJanusGraphPropertiesSteps(traversal, txVertexCacheSize, propertiesStrategyMode); applyJanusGraphLabelSteps(traversal, labelStepStrategyMode); + applyJanusGraphDropSteps(traversal, dropStepStrategyMode); inspectLocalTraversals(traversal, txVertexCacheSize, propertiesStrategyMode); } @@ -203,6 +209,19 @@ private void applyJanusGraphLabelSteps(Traversal.Admin traversal, MultiQue }); } + private void applyJanusGraphDropSteps(Traversal.Admin traversal, MultiQueryDropStepStrategyMode dropStepStrategyMode){ + if(MultiQueryDropStepStrategyMode.NONE.equals(dropStepStrategyMode)){ + return; + } + TraversalHelper.getStepsOfAssignableClass(DropStep.class, traversal).forEach(originalStep -> { + if(originalStep instanceof JanusGraphDropStep){ + return; + } + final JanusGraphDropStep janusGraphDropStep = new JanusGraphDropStep(originalStep); + TraversalHelper.replaceStep(originalStep, janusGraphDropStep, originalStep.getTraversal()); + }); + } + private static void unfoldLocalTraversal(final Traversal.Admin traversal, LocalStep localStep, Traversal.Admin localTraversal, MultiQueriable vertexStep) { diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphMultiQueryStrategy.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphMultiQueryStrategy.java index 85329a6ba4..d8adfcffce 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphMultiQueryStrategy.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphMultiQueryStrategy.java @@ -20,7 +20,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; import org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; @@ -30,6 +29,7 @@ import org.janusgraph.graphdb.tinkerpop.optimize.MultiQueryPositions; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphMultiQueryStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.MultiQueriable; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphNoOpBarrierVertexOnlyStep; import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; import java.util.Arrays; @@ -55,11 +55,7 @@ private JanusGraphMultiQueryStrategy() { @Override public void apply(final Admin traversal) { - if (!traversal.getGraph().isPresent() - || TraversalHelper.onGraphComputer(traversal) - // The LazyBarrierStrategy is not allowed to run on traversals which use drop(). As a precaution, - // this strategy should not run on those traversals either, because it can also insert barrier(). - || !TraversalHelper.getStepsOfAssignableClassRecursively(DropStep.class, traversal).isEmpty()) { + if (!traversal.getGraph().isPresent() || TraversalHelper.onGraphComputer(traversal)) { return; } @@ -129,7 +125,7 @@ private void insertMultiQueryStep(Step position, boolean limitedBatch, int limit multiQueryStep = new JanusGraphMultiQueryStep(traversal, limitedBatch, ((NoOpBarrierStep) position).getMaxBarrierSize()); } else { - NoOpBarrierStep barrier = new NoOpBarrierStep(traversal, limitedBatchSize); + NoOpBarrierStep barrier = new JanusGraphNoOpBarrierVertexOnlyStep(traversal, limitedBatchSize); TraversalHelper.insertBeforeStep(barrier, position, traversal); position = barrier; multiQueryStep = new JanusGraphMultiQueryStep(traversal, limitedBatch, barrier); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/MultiQueryDropStepStrategyMode.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/MultiQueryDropStepStrategyMode.java new file mode 100644 index 0000000000..3ee6a1a90b --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/MultiQueryDropStepStrategyMode.java @@ -0,0 +1,42 @@ +// Copyright 2024 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.tinkerpop.optimize.strategy; + +import org.janusgraph.graphdb.configuration.ConfigName; + +public enum MultiQueryDropStepStrategyMode implements ConfigName { + + /** + * Prefetch labels for all vertices in a batch. + */ + ALL("all"), + + /** + * Skips `label` step pre-fetch optimization. + */ + NONE("none") + ; + + private final String configurationOptionName; + + MultiQueryDropStepStrategyMode(String configurationOptionName){ + this.configurationOptionName = configurationOptionName; + } + + @Override + public String getConfigName() { + return configurationOptionName; + } +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java index 18285ee8d5..8c79cd6784 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardTransactionBuilder.java @@ -28,6 +28,7 @@ import org.janusgraph.diskstorage.util.time.TimestampProvider; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; import org.janusgraph.graphdb.database.StandardJanusGraph; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryLabelStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode; @@ -91,9 +92,9 @@ public class StandardTransactionBuilder implements TransactionConfiguration, Tra private boolean isLazyLoadRelations; private MultiQueryHasStepStrategyMode hasStepStrategyMode; - private MultiQueryPropertiesStrategyMode propertiesStrategyMode; private MultiQueryLabelStepStrategyMode labelStepStrategyMode; + private MultiQueryDropStepStrategyMode dropStepStrategyMode; private final boolean forceIndexUsage; @@ -120,6 +121,7 @@ private StandardTransactionBuilder(GraphDatabaseConfiguration graphConfig, Stand this.hasStepStrategyMode = graphConfig.hasStepStrategyMode(); this.propertiesStrategyMode = graphConfig.propertiesStrategyMode(); this.labelStepStrategyMode = graphConfig.labelStepStrategyMode(); + this.dropStepStrategyMode = graphConfig.dropStepStrategyMode(); this.writableCustomOptions = writableCustomOptions; if(customOptions == null){ this.customOptions = new MergedConfiguration(writableCustomOptions, graphConfig.getConfiguration()); @@ -260,6 +262,12 @@ public TransactionBuilder setLabelsStepStrategyMode(MultiQueryLabelStepStrategyM return this; } + @Override + public TransactionBuilder setDropStepStrategyMode(MultiQueryDropStepStrategyMode dropStepStrategyMode) { + this.dropStepStrategyMode = dropStepStrategyMode; + return this; + } + @Override public void setCommitTime(Instant time) { throw new UnsupportedOperationException("Use setCommitTime(long,TimeUnit)"); @@ -307,7 +315,7 @@ propertyPrefetching, multiQuery, singleThreaded, threadBound, getTimestampProvid indexCacheWeight, getVertexCacheSize(), getDirtyVertexSize(), logIdentifier, restrictedPartitions, groupName, defaultSchemaMaker, hasDisabledSchemaConstraints, skipDBCacheRead, isLazyLoadRelations, hasStepStrategyMode, propertiesStrategyMode, - labelStepStrategyMode, customOptions); + labelStepStrategyMode, dropStepStrategyMode, customOptions); return graph.newTransaction(immutable); } @@ -444,6 +452,11 @@ public MultiQueryLabelStepStrategyMode getLabelStepStrategyMode() { return labelStepStrategyMode; } + @Override + public MultiQueryDropStepStrategyMode getDropStepStrategyMode() { + return dropStepStrategyMode; + } + @Override public String getGroupName() { return groupName; @@ -508,6 +521,7 @@ private static class ImmutableTxCfg implements TransactionConfiguration { private MultiQueryHasStepStrategyMode hasStepStrategyMode; private MultiQueryPropertiesStrategyMode propertiesStrategyMode; private MultiQueryLabelStepStrategyMode labelStepStrategyMode; + private MultiQueryDropStepStrategyMode dropStepStrategyMode; private final BaseTransactionConfig handleConfig; @@ -531,6 +545,7 @@ public ImmutableTxCfg(boolean isReadOnly, MultiQueryHasStepStrategyMode hasStepStrategyMode, MultiQueryPropertiesStrategyMode propertiesStrategyMode, MultiQueryLabelStepStrategyMode labelStepStrategyMode, + MultiQueryDropStepStrategyMode dropStepStrategyMode, Configuration customOptions) { this.isReadOnly = isReadOnly; this.hasEnabledBatchLoading = hasEnabledBatchLoading; @@ -557,6 +572,7 @@ public ImmutableTxCfg(boolean isReadOnly, this.hasStepStrategyMode = hasStepStrategyMode; this.propertiesStrategyMode = propertiesStrategyMode; this.labelStepStrategyMode = labelStepStrategyMode; + this.dropStepStrategyMode = dropStepStrategyMode; this.handleConfig = new StandardBaseTransactionConfig.Builder() .commitTime(commitTime) .timestampProvider(times) @@ -694,6 +710,11 @@ public MultiQueryLabelStepStrategyMode getLabelStepStrategyMode() { return labelStepStrategyMode; } + @Override + public MultiQueryDropStepStrategyMode getDropStepStrategyMode() { + return dropStepStrategyMode; + } + @Override public Instant getCommitTime() { return handleConfig.getCommitTime(); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java index 9f2aed3122..7373765fe7 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/TransactionConfiguration.java @@ -16,6 +16,7 @@ import org.janusgraph.core.schema.DefaultSchemaMaker; import org.janusgraph.diskstorage.BaseTransactionConfig; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryLabelStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode; @@ -220,4 +221,9 @@ public interface TransactionConfiguration extends BaseTransactionConfig { */ MultiQueryLabelStepStrategyMode getLabelStepStrategyMode(); + /** + * @return Drop step strategy mode used for the transaction. Can be configured via config `query.batch.drop-step-mode`. + */ + MultiQueryDropStepStrategyMode getDropStepStrategyMode(); + } diff --git a/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryGraphTest.java b/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryGraphTest.java index a37c4cb9d9..a9c3b8a6d4 100644 --- a/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryGraphTest.java +++ b/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryGraphTest.java @@ -134,4 +134,14 @@ public void testSupportDirectCommitOfSchemaChangesForConnection() {} @Override @Test @Disabled public void testSupportDirectCommitOfSchemaChangesForEdgeProperties() {} + + @Override @Test @Disabled + public void testMultiQueryDropsStrategyModes() {} + + @Override @Test @Disabled + public void testMetaPropertiesDrop() {} + + @Override @Test @Disabled + public void testEdgePropertiesDrop() {} + } diff --git a/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java b/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java index b91b2482ba..acc112fe83 100644 --- a/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java +++ b/janusgraph-inmemory/src/test/java/org/janusgraph/graphdb/inmemory/InMemoryLazyLoadGraphTest.java @@ -50,4 +50,5 @@ public void testPropertyIdAccessInDifferentTransaction() { Exception exception = assertThrows(IllegalStateException.class, p::id); assertEquals(exception.getMessage(), "Any lazy load operation is not supported when transaction is already closed."); } + } diff --git a/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/AdjacentVertexHasUniquePropertyOptimizerStrategyTest.java b/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/AdjacentVertexHasUniquePropertyOptimizerStrategyTest.java index 19b2085aef..d91ca8f67e 100644 --- a/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/AdjacentVertexHasUniquePropertyOptimizerStrategyTest.java +++ b/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/AdjacentVertexHasUniquePropertyOptimizerStrategyTest.java @@ -18,6 +18,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphHasStep; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphNoOpBarrierVertexOnlyStep; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexHasUniquePropertyOptimizerStrategy; import org.junit.jupiter.api.Test; @@ -43,11 +44,11 @@ public void testWithBackendAccess() { // AdjacentVertexHasUniquePropertyOptimizer assertNumStep(1, 0, g.V(sv[0]).out().has("uniqueId", 0), JanusGraphHasStep.class); - assertNumStep(1, 1, g.V(sv[0]).out().barrier(2500).has("uniqueId", 0), NoOpBarrierStep.class); - assertNumStep(1, 1, g.V(sv[0]).outE().inV().has("uniqueId", 0), NoOpBarrierStep.class); + assertNumStep(1, 1, g.V(sv[0]).out().barrier(2500).has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); + assertNumStep(1, 1, g.V(sv[0]).outE().inV().has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); assertNumStep(1, 0, g.V(sv[0]).outE().inV().has("uniqueId", 0), JanusGraphHasStep.class); - assertNumStep(1, 1, g.V(sv[0]).outE().inV().barrier(2500).has("uniqueId", 0), NoOpBarrierStep.class); - assertNumStep(1, 1, g.V(sv[0]).outE().barrier(2500).inV().has("uniqueId", 0), NoOpBarrierStep.class); + assertNumStep(1, 1, g.V(sv[0]).outE().inV().barrier(2500).has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); + assertNumStep(1, 1, g.V(sv[0]).outE().barrier(2500).inV().has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); assertNumStep(1, 1, g.V(sv[0]).barrier(2500).out().barrier(2500).has("uniqueId", 0), NoOpBarrierStep.class); assertNumStep(1, 0, g.V(sv[0]).out().barrier(2500).has("uniqueId", 0), JanusGraphHasStep.class); assertNumStep(1, 1, g.V(sv[0]).out().has("id", 0), JanusGraphHasStep.class); @@ -67,10 +68,12 @@ public void testWithoutBackendAccess() { // AdjacentVertexHasUniquePropertyOptimizer assertNumStep(1, 1, g.V(sv[0]).out().has("uniqueId", 0), JanusGraphHasStep.class); - assertNumStep(1, 2, g.V(sv[0]).outE().inV().has("uniqueId", 0), NoOpBarrierStep.class); + assertNumStep(1, 2, g.V(sv[0]).outE().inV().has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); assertNumStep(1, 1, g.V(sv[0]).outE().inV().has("uniqueId", 0), JanusGraphHasStep.class); - assertNumStep(1, 2, g.V(sv[0]).outE().inV().barrier(2500).has("uniqueId", 0), NoOpBarrierStep.class); - assertNumStep(1, 3, g.V(sv[0]).outE().barrier(2500).inV().has("uniqueId", 0), NoOpBarrierStep.class); + assertNumStep(1, 1, g.V(sv[0]).outE().inV().barrier(2500).has("uniqueId", 0), NoOpBarrierStep.class); + assertNumStep(1, 1, g.V(sv[0]).outE().inV().barrier(2500).has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); + assertNumStep(1, 1, g.V(sv[0]).outE().barrier(2500).inV().has("uniqueId", 0), NoOpBarrierStep.class); + assertNumStep(1, 2, g.V(sv[0]).outE().barrier(2500).inV().has("uniqueId", 0), JanusGraphNoOpBarrierVertexOnlyStep.class); assertNumStep(1, 2, g.V(sv[0]).barrier(2500).out().barrier(2500).has("uniqueId", 0), NoOpBarrierStep.class); assertNumStep(1, 1, g.V(sv[0]).out().barrier(2500).has("uniqueId", 0), JanusGraphHasStep.class); assertNumStep(1, 1, g.V(sv[0]).out().has("id", 0), JanusGraphHasStep.class); diff --git a/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphMultiQueryStrategyTest.java b/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphMultiQueryStrategyTest.java index ad3f389630..104143030e 100644 --- a/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphMultiQueryStrategyTest.java +++ b/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphMultiQueryStrategyTest.java @@ -31,6 +31,7 @@ import org.apache.tinkerpop.gremlin.structure.Edge; import org.janusgraph.graphdb.query.profile.QueryProfiler; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphMultiQueryStep; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphNoOpBarrierVertexOnlyStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertiesStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphVertexStep; @@ -85,7 +86,7 @@ public void testNoOpBarrierStepInsertedIfNotPresentAndLimitBatchSize() { makeSampleGraph(); final GraphTraversal traversalWithoutExplicitBarrier = g.V(sv[0]).outE().inV(); - assertNumStep(numV, 1, traversalWithoutExplicitBarrier, NoOpBarrierStep.class); + assertNumStep(numV, 1, traversalWithoutExplicitBarrier, JanusGraphNoOpBarrierVertexOnlyStep.class); final GraphTraversal traversalWithExplicitBarrier = g.V(sv[0]).barrier(1).outE().inV(); assertNumStep(numV, 1, traversalWithExplicitBarrier, NoOpBarrierStep.class); @@ -96,11 +97,11 @@ public void testNoOpBarrierStepNotInsertedLimitBatchSizeDisabled() { clopen(option(USE_MULTIQUERY), true, option(LIMITED_BATCH), false); makeSampleGraph(); - final GraphTraversal traversalWithoutExplicitBarrier = g.V(sv[0]).outE().inV(); - assertNumStep(numV, 0, traversalWithoutExplicitBarrier, NoOpBarrierStep.class); + assertNumStep(numV, 0, g.V(sv[0]).outE().inV(), JanusGraphNoOpBarrierVertexOnlyStep.class); + assertNumStep(numV, 0, g.V(sv[0]).outE().inV(), NoOpBarrierStep.class); - final GraphTraversal traversalWithExplicitBarrier = g.V(sv[0]).barrier(1).outE().inV(); - assertNumStep(numV, 1, traversalWithExplicitBarrier, NoOpBarrierStep.class); + assertNumStep(numV, 1, g.V(sv[0]).barrier(1).outE().inV(), NoOpBarrierStep.class); + assertNumStep(numV, 0, g.V(sv[0]).barrier(1).outE().inV(), JanusGraphNoOpBarrierVertexOnlyStep.class); } @Test @@ -176,8 +177,8 @@ public void testNonNecessaryMultiQueryUnused(){ makeSampleGraph(); TraversalMetrics traversalMetricsWithStrategy = graph.traversal() - .V().id() - .choose(__.is(P.gte(5000L)), __.constant(12345L), __.identity()) + .V() + .choose(__.id().is(P.gte(5000L)), __.constant(12345L), __.id()) .limit(3).profile().next(); Long graphStepTraversalsCount = traversalMetricsWithStrategy.getMetrics().stream().filter(metrics -> metrics.getName().startsWith(JanusGraphStep.class.getSimpleName())) @@ -188,8 +189,8 @@ public void testNonNecessaryMultiQueryUnused(){ assertTrue(graphStepTraversalsCount < 5L); TraversalMetrics traversalMetricsWithoutStrategy = graph.traversal().withoutStrategies(JanusGraphUnusedMultiQueryRemovalStrategy.class) - .V().id() - .choose(__.is(P.gte(5000L)), __.constant(12345L), __.identity()) + .V() + .choose(__.id().is(P.gte(5000L)), __.constant(12345L), __.id()) .limit(3).profile().next(); graphStepTraversalsCount = traversalMetricsWithoutStrategy.getMetrics().stream().filter(metrics -> metrics.getName().startsWith(JanusGraphStep.class.getSimpleName())) diff --git a/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphStepStrategyTest.java b/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphStepStrategyTest.java index 8316af3fdd..cd4b04a317 100644 --- a/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphStepStrategyTest.java +++ b/janusgraph-test/src/test/java/org/janusgraph/graphdb/tinkerpop/optimize/JanusGraphStepStrategyTest.java @@ -24,6 +24,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.OrStep; @@ -55,10 +56,12 @@ import org.janusgraph.graphdb.predicate.ConnectiveJanusPredicate; import org.janusgraph.graphdb.query.JanusGraphPredicateUtils; import org.janusgraph.graphdb.tinkerpop.optimize.step.HasStepFolder; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphDropStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphElementMapStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphHasStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphLabelStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphMultiQueryStep; +import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphNoOpBarrierVertexOnlyStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertiesStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertyMapStep; import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphStep; @@ -68,6 +71,7 @@ import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMultiQueryStrategy; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphStepStrategy; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphUnusedMultiQueryRemovalStrategy; +import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryDropStepStrategyMode; import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryStrategyRepeatStepMode; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -91,6 +95,8 @@ import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.not; import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.properties; import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.values; +import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DROP_STEP_BATCH_MODE; +import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.LIMITED_BATCH_SIZE; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REPEAT_STEP_BATCH_MODE; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.USE_MULTIQUERY; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -175,6 +181,8 @@ private void applyMultiQueryTraversalSteps(Traversal.Admin traversal) { Object expectedStep = isStep.getPredicate().getValue(); if (expectedStep.equals(JanusGraphMultiQueryStep.class.getSimpleName())) { TraversalHelper.replaceStep(isStep, new JanusGraphMultiQueryStep(isStep.getTraversal(), false), isStep.getTraversal()); + } else if (expectedStep.equals(JanusGraphNoOpBarrierVertexOnlyStep.class.getSimpleName())) { + TraversalHelper.replaceStep(isStep, new JanusGraphNoOpBarrierVertexOnlyStep(isStep.getTraversal(), LIMITED_BATCH_SIZE.getDefaultValue()), isStep.getTraversal()); } }); TraversalHelper.getStepsOfAssignableClassRecursively(HasStep.class, traversal).forEach(hasStep -> { @@ -193,6 +201,10 @@ private void applyMultiQueryTraversalSteps(Traversal.Admin traversal) { JanusGraphLabelStep janusGraphLabelStep = new JanusGraphLabelStep<>(labelStep); TraversalHelper.replaceStep(labelStep, janusGraphLabelStep, labelStep.getTraversal()); }); + TraversalHelper.getStepsOfAssignableClassRecursively(DropStep.class, traversal).forEach(dropStep -> { + JanusGraphDropStep janusGraphDropStep = new JanusGraphDropStep<>(dropStep); + TraversalHelper.replaceStep(dropStep, janusGraphDropStep, dropStep.getTraversal()); + }); } @SuppressWarnings({ "unchecked", "rawtypes" }) @@ -321,7 +333,9 @@ private static Stream generateTestParameters() { } private static Stream generateMultiQueryTestParameters() { - final StandardJanusGraph generalGraph = (StandardJanusGraph) StorageSetup.getInMemoryGraphWithMultiQuery(); + final StandardJanusGraph generalGraph = (StandardJanusGraph) JanusGraphFactory.open( + StorageSetup.getInMemoryConfiguration().set(USE_MULTIQUERY, true) + .set(DROP_STEP_BATCH_MODE, MultiQueryDropStepStrategyMode.ALL.getConfigName()));; final StandardJanusGraph graphWithRepeatClosestParent = (StandardJanusGraph) JanusGraphFactory.open( StorageSetup.getInMemoryConfiguration().set(USE_MULTIQUERY, true) .set(REPEAT_STEP_BATCH_MODE, MultiQueryStrategyRepeatStepMode.CLOSEST_REPEAT_PARENT.getConfigName())); @@ -350,6 +364,7 @@ private static Stream generateMultiQueryTestParameters() { // String constant for expected JanusGraphMultiQueryStep final String MQ_STEP = JanusGraphMultiQueryStep.class.getSimpleName(); + final String NO_OP_B_V_STEP = JanusGraphNoOpBarrierVertexOnlyStep.class.getSimpleName(); List otherStrategies = new ArrayList<>(4); otherStrategies.add(JanusGraphLocalQueryOptimizerStrategy.instance()); @@ -360,120 +375,123 @@ private static Stream generateMultiQueryTestParameters() { return Arrays.stream(new Arguments[]{ arguments(g.V().in("knows").out("knows"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).in("knows").is(MQ_STEP).barrier(defaultBarrierSize).out("knows"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).in("knows").is(MQ_STEP).is(NO_OP_B_V_STEP).out("knows"), otherStrategies), arguments(g.V().in("knows").values("weight"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).in("knows").is(MQ_STEP).barrier(defaultBarrierSize).values("weight"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).in("knows").is(MQ_STEP).is(NO_OP_B_V_STEP).values("weight"), otherStrategies), // Need two JanusGraphMultiQuerySteps, one for each sub query because caches are flushed when queried arguments(g.V().choose(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).choose(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).choose(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)), otherStrategies), arguments(g.V().union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)), otherStrategies), arguments(g.V().outE().optional(__.inE("knows").has("weight", 0)), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).outE().is(MQ_STEP).barrier(defaultBarrierSize).optional(__.inE("knows").has("weight", 0)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).outE().is(MQ_STEP).is(NO_OP_B_V_STEP).optional(__.inE("knows").has("weight", 0)), otherStrategies), arguments(g.V().outE().filter(__.inE("knows").has("weight", 0)), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).outE().is(MQ_STEP).barrier(defaultBarrierSize).filter(__.inE("knows").has("weight", 0)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).outE().is(MQ_STEP).is(NO_OP_B_V_STEP).filter(__.inE("knows").has("weight", 0)), otherStrategies), // An additional JanusGraphMultiQueryStep for repeat goes before the RepeatEndStep allowing it to feed its starts to the next iteration arguments(g.V().outE("knows").inV().repeat(__.outE("knows").inV().has("weight", 0)).times(10), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).outE("knows").inV().is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.outE("knows").inV().is(MQ_STEP).barrier(defaultBarrierSize).has("weight", 0).is(MQ_STEP).barrier(defaultBarrierSize)).times(10), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).outE("knows").inV().is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.outE("knows").inV().is(MQ_STEP).is(NO_OP_B_V_STEP).has("weight", 0).is(MQ_STEP).is(NO_OP_B_V_STEP)).times(10), otherStrategies), // Choose does not have a child traversal of JanusGraphVertexStep so won't benefit from JanusGraphMultiQueryStep(ChooseStep) arguments(g.V().choose(has("weight", lt(3)), __.union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1))), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).choose(has("weight", lt(3)), __.union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1))), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).choose(has("weight", lt(3)), __.union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1))), otherStrategies), // Choose now has a child traversal of JanusGraphVertexStep and so will benefit from JanusGraphMultiQueryStep(ChooseStep) arguments(g.V().choose(__.union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)),__.inE("knows").has("weight", gt(2))), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).choose(__.union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)),__.inE("knows").has("weight", gt(2))), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).choose(__.union(__.inE("knows").has("weight", 0),__.inE("knows").has("weight", 1)),__.inE("knows").has("weight", gt(2))), otherStrategies), // There are 'as' side effect steps preceding the JanusGraphVertexStep arguments(g.V().choose(has("weight", 0),__.as("true").inE("knows"),__.as("false").inE("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).choose(has("weight", 0),__.as("true").inE("knows"),__.as("false").inE("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).choose(has("weight", 0),__.as("true").inE("knows"),__.as("false").inE("knows")), otherStrategies), // There are 'sideEffect' and 'as' steps preceding the JanusGraphVertexStep arguments(g.V().choose(has("weight", 0),__.as("true").sideEffect(i -> {}).inE("knows"),__.as("false").sideEffect(i -> {}).inE("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).choose(has("weight", 0),__.as("true").sideEffect(i -> {}).inE("knows"),__.as("false").sideEffect(i -> {}).inE("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).choose(has("weight", 0),__.as("true").sideEffect(i -> {}).inE("knows"),__.as("false").sideEffect(i -> {}).inE("knows")), otherStrategies), // Only the most outer eligible `JanusGraphMultiQueryStep` should be used for MultiQueryCompatible start steps arguments(g.V().and(__.inE("knows"), __.outE("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).and(__.inE("knows"), __.outE("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).and(__.inE("knows"), __.outE("knows")), otherStrategies), // `JanusGraphMultiQueryStep` should be used for filter step when at least one child is registered as a client of `JanusGraphMultiQueryStep`. arguments(g.V().where(__.out("knows").count().is(P.gte(5))), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).where(__.out("knows").count().is(P.gte(5))), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).where(__.out("knows").count().is(P.gte(5))), otherStrategies), // `JanusGraphMultiQueryStep` should not be used for filter step when none of children is registered as a client of `JanusGraphMultiQueryStep`. arguments(g.V().where(__.count().is(P.gte(5))), g_V().where(__.count().is(P.gte(5))), otherStrategies), // Should include barrier steps with the default configured size by `LIMITED_BATCH_SIZE` option arguments(g.V().out().repeat(__.out()).emit(), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)).emit(), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)).emit(), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when `emit` is used. arguments(g.V().out().emit().repeat(__.out()), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).emit().repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).emit().repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when non-true `emit` is used after `repeat`. arguments(g.V().out().repeat(__.out()).emit(__.in("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)).emit( __.in("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)).emit( __.in("knows")), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when non-true `emit` is used before `repeat`. arguments(g.V().out().emit(__.in("knows")).repeat(__.out()), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).emit( __.in("knows")).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).emit( __.in("knows")).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when `until` is used after `repeat`. arguments(g.V().out().repeat(__.out()).until(__.in("knows").count().is(P.gte(5))), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)).until(__.in("knows").count().is(P.gte(5))), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)).until(__.in("knows").count().is(P.gte(5))), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when `until` is used before `repeat`. arguments(g.V().out().until(__.in("knows").count().is(P.gte(5))).repeat(__.out()), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).until(__.in("knows").count().is(P.gte(5))).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).until(__.in("knows").count().is(P.gte(5))).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when `emit` is used before `repeat` and `until` is used after `repeat`. arguments(g.V().out().emit(__.in("knows")).repeat(__.out()).until(__.in("knows").count().is(P.gte(5))), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).emit(__.in("knows")).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)).until(__.in("knows").count().is(P.gte(5))), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).emit(__.in("knows")).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)).until(__.in("knows").count().is(P.gte(5))), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the repeat child traversals when `emit` is used after `repeat` and `until` is used after `repeat`. arguments(g.V().out().repeat(__.out()).emit(__.in("knows")).until(__.in("knows").count().is(P.gte(5))), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)).emit(__.in("knows")).until(__.in("knows").count().is(P.gte(5))), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)).emit(__.in("knows")).until(__.in("knows").count().is(P.gte(5))), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals when `emit` is used after `repeat` and `until` is used before `repeat`. arguments(g.V().out().until(__.in("knows").count().is(P.gte(5))).repeat(__.out()).emit(__.in("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).until(__.in("knows").count().is(P.gte(5))).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)).emit(__.in("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).until(__.in("knows").count().is(P.gte(5))).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)).emit(__.in("knows")), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the repeat child traversals when `emit` is used before `repeat` and `until` is used before `repeat`. arguments(g.V().out().until(__.in("knows").count().is(P.gte(5))).emit(__.in("knows")).repeat(__.out()), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).until(__.in("knows").count().is(P.gte(5))).emit(__.in("knows")).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize)), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).until(__.in("knows").count().is(P.gte(5))).emit(__.in("knows")).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP)), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the beginning of repeat child traversals, but should use it for any other steps after the fist step. arguments(g.V().out().until(__.in("knows").in("knows").count().is(P.gte(5))).repeat(__.out().out()).emit(__.in("knows").in("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).until(__.in("knows").is(MQ_STEP).barrier(defaultBarrierSize).in("knows").count().is(P.gte(5))).repeat(__.out().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize)).emit(__.in("knows").is(MQ_STEP).barrier(defaultBarrierSize).in("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).until(__.in("knows").is(MQ_STEP).is(NO_OP_B_V_STEP).in("knows").count().is(P.gte(5))).repeat(__.out().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP)).emit(__.in("knows").is(MQ_STEP).is(NO_OP_B_V_STEP).in("knows")), otherStrategies), // Should not use `JanusGraphMultiQueryStep` at the end of repeat child traversal and before repeat step if none of repeat child starts are MultiQueriable steps arguments(g.V().out().until(__.map(Traverser::get).in("knows").count().is(P.gte(5))).repeat(__.map(Traverser::get).out()).emit(__.map(Traverser::get).in("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().until(__.map(Traverser::get).is(MQ_STEP).barrier(defaultBarrierSize).in("knows").count().is(P.gte(5))).repeat(__.map(Traverser::get).is(MQ_STEP).barrier(defaultBarrierSize).out()).emit(__.map(Traverser::get).is(MQ_STEP).barrier(defaultBarrierSize).in("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().until(__.map(Traverser::get).is(MQ_STEP).is(NO_OP_B_V_STEP).in("knows").count().is(P.gte(5))).repeat(__.map(Traverser::get).is(MQ_STEP).is(NO_OP_B_V_STEP).out()).emit(__.map(Traverser::get).is(MQ_STEP).is(NO_OP_B_V_STEP).in("knows")), otherStrategies), // Needs one JanusGraphMultiQueryStep before executing `and` step with `MultiQueriable` steps arguments(g.V().and(__.inE("knows"), __.inE("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).and(__.inE("knows"), __.inE("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).and(__.inE("knows"), __.inE("knows")), otherStrategies), // Needs one JanusGraphMultiQueryStep before executing `or` step with `MultiQueriable` steps arguments(g.V().or(__.inE("knows"), __.inE("knows")), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).or(__.inE("knows"), __.inE("knows")), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).or(__.inE("knows"), __.inE("knows")), otherStrategies), // Should use start of the last outer parent step after the first repeat step as JanusGraphMultiQueryStep. arguments(g.V().union(__.repeat(__.out("knows")).emit()), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).union(__.repeat(__.out("knows").is(MQ_STEP).barrier(defaultBarrierSize)).emit()), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).union(__.repeat(__.out("knows").is(MQ_STEP).is(NO_OP_B_V_STEP)).emit()), otherStrategies), // Repeat step mode: CLOSEST_REPEAT_PARENT. Should use start of the closest outer parent step after the first repeat step as JanusGraphMultiQueryStep, but not the last outer start step of the most outer repeat step. arguments(gRepeatClosestParent.V().repeat(__.union(__.repeat(__.out("knows")).emit())).emit(), - ((GraphTraversal) g_V()).repeat(__.is(MQ_STEP).barrier(defaultBarrierSize).union(__.repeat(__.out("knows").is(MQ_STEP).barrier(defaultBarrierSize)).emit())).emit(), otherStrategies), + ((GraphTraversal) g_V()).repeat(__.is(MQ_STEP).is(NO_OP_B_V_STEP).union(__.repeat(__.out("knows").is(MQ_STEP).is(NO_OP_B_V_STEP)).emit())).emit(), otherStrategies), // Repeat step mode: ALL_REPEAT_PARENTS. Should use starts and ends of all parent repeat steps as JanusGraphMultiQueryStep arguments(gRepeatAllParents.V().repeat(__.union(__.repeat(__.out("knows")).emit())).emit(), - ((GraphTraversal) g_V()).is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.is(MQ_STEP).barrier(defaultBarrierSize).union(__.repeat(__.out("knows").is(MQ_STEP).barrier(defaultBarrierSize)).emit()).is(MQ_STEP).barrier(defaultBarrierSize)).emit(), otherStrategies), + ((GraphTraversal) g_V()).is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.is(MQ_STEP).is(NO_OP_B_V_STEP).union(__.repeat(__.out("knows").is(MQ_STEP).is(NO_OP_B_V_STEP)).emit()).is(MQ_STEP).is(NO_OP_B_V_STEP)).emit(), otherStrategies), // Repeat step mode: STARTS_ONLY_OF_ALL_REPEAT_PARENTS. Should use starts of all parent repeat steps as JanusGraphMultiQueryStep arguments(gRepeatStartsOnlyOfAllParents.V().repeat(__.union(__.repeat(__.out("knows")).emit())).emit(), - ((GraphTraversal) g_V()).is(MQ_STEP).barrier(defaultBarrierSize).repeat(__.is(MQ_STEP).barrier(defaultBarrierSize).union(__.repeat(__.out("knows").is(MQ_STEP).barrier(defaultBarrierSize)).emit())).emit(), otherStrategies), + ((GraphTraversal) g_V()).is(MQ_STEP).is(NO_OP_B_V_STEP).repeat(__.is(MQ_STEP).is(NO_OP_B_V_STEP).union(__.repeat(__.out("knows").is(MQ_STEP).is(NO_OP_B_V_STEP)).emit())).emit(), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `has` step (we use `map` step here just to not fold has step containers into JanusGraphStep arguments(g.V().map(Traverser::get).has("weight", 0), - g_V().map(Traverser::get).is(MQ_STEP).barrier(defaultBarrierSize).has("weight", 0), otherStrategies), + g_V().map(Traverser::get).is(MQ_STEP).is(NO_OP_B_V_STEP).has("weight", 0), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `has` step which is used after other steps arguments(g.V().out().has("weight", 0), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).out().is(MQ_STEP).barrier(defaultBarrierSize).has("weight", 0), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).out().is(MQ_STEP).is(NO_OP_B_V_STEP).has("weight", 0), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `valueMap` step which is used after other steps arguments(g.V().valueMap("weight"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).valueMap("weight"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).valueMap("weight"), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `propertyMap` step which is used after other steps arguments(g.V().propertyMap("weight"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).propertyMap("weight"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).propertyMap("weight"), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `elementMap` step which is used after other steps arguments(g.V().elementMap("weight"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).elementMap("weight"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).elementMap("weight"), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `values` step which is used after other steps arguments(g.V().values("weight"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).values("weight"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).values("weight"), otherStrategies), // Need `JanusGraphMultiQuerySteps` before `properties` step which is used after other steps arguments(g.V().properties("weight"), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).properties("weight"), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).properties("weight"), otherStrategies), // Need `JanusGraphLabelStep` to use instead of `LabelStep` arguments(g.V().label(), - g_V().is(MQ_STEP).barrier(defaultBarrierSize).label(), otherStrategies), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).label(), otherStrategies), + // Need `JanusGraphDropStep` to use instead of `DropStep` + arguments(g.V().drop(), + g_V().is(MQ_STEP).is(NO_OP_B_V_STEP).drop(), otherStrategies), }); } } diff --git a/mkdocs.yml b/mkdocs.yml index acb8a978ae..2e58891588 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -96,7 +96,7 @@ extra: link: https://lists.lfaidata.foundation/g/janusgraph-dev latest_version: 1.0.0 snapshot_version: 1.1.0-SNAPSHOT - tinkerpop_version: 3.7.2 + tinkerpop_version: 3.7.3 hadoop_version: 3.3.5 markdown_extensions: diff --git a/pom.xml b/pom.xml index ca9ea3a1f5..1fa3e754cb 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ 1.0.0,1.1.0-SNAPSHOT - 3.7.2 + 3.7.3-SNAPSHOT 1.11.3 1.10.2 5.10.2