From f892a474de332edf44de4059067bb7b1b393a3d6 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 16:39:39 -0500 Subject: [PATCH 01/18] debug(ci): debug ci failure in metadata-io test --- .github/workflows/metadata-io.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index e5e5f1dae5447..f7cb230afaef0 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -8,6 +8,7 @@ on: - "li-utils/**" - "metadata-models/**" - "metadata-io/**" + - ".github/workflows/metadata-io.yaml" pull_request: branches: - "**" @@ -16,6 +17,7 @@ on: - "li-utils/**" - "metadata-models/**" - "metadata-io/**" + - ".github/workflows/metadata-io.yaml" release: types: [published] @@ -52,6 +54,8 @@ jobs: sudo apt-get remove 'dotnet-*' azure-cli || true sudo rm -rf /usr/local/lib/android/ || true sudo docker image prune -a -f || true + - name: Disk Check + run: df -h . && docker images - uses: acryldata/sane-checkout-action@v3 - name: Set up JDK 17 uses: actions/setup-java@v4 @@ -66,7 +70,7 @@ jobs: cache: "pip" - name: Gradle build (and test) run: | - ./gradlew :metadata-io:test + ./gradlew :metadata-io:test --no-daemon - uses: actions/upload-artifact@v3 if: always() with: From 6db8ab44bc203915870e30444ce05a8b43640ee2 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:09:36 -0500 Subject: [PATCH 02/18] Update metadata-io.yml --- .github/workflows/metadata-io.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index f7cb230afaef0..5cce02f8c14c1 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -8,7 +8,7 @@ on: - "li-utils/**" - "metadata-models/**" - "metadata-io/**" - - ".github/workflows/metadata-io.yaml" + - ".github/workflows/metadata-io.yml" pull_request: branches: - "**" @@ -17,7 +17,7 @@ on: - "li-utils/**" - "metadata-models/**" - "metadata-io/**" - - ".github/workflows/metadata-io.yaml" + - ".github/workflows/metadata-io.yml" release: types: [published] From e7545da19f2c683bec6a43e912821b01764adc5b Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 17:28:44 -0500 Subject: [PATCH 03/18] test if schema fields effects test --- .../linkedin/datahub/graphql/resolvers/search/SearchUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 04777c3fcdb4e..f8c9e6df7fe1b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -73,8 +73,7 @@ private SearchUtils() {} EntityType.DOMAIN, EntityType.DATA_PRODUCT, EntityType.NOTEBOOK, - EntityType.BUSINESS_ATTRIBUTE, - EntityType.SCHEMA_FIELD); + EntityType.BUSINESS_ATTRIBUTE); /** Entities that are part of autocomplete by default in Auto Complete Across Entities */ public static final List AUTO_COMPLETE_ENTITY_TYPES = From 716822f3384aee7f553c9bd8dc787822d09561de Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 17:42:21 -0500 Subject: [PATCH 04/18] Revert "test if schema fields effects test" This reverts commit e7545da19f2c683bec6a43e912821b01764adc5b. --- .../linkedin/datahub/graphql/resolvers/search/SearchUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index f8c9e6df7fe1b..04777c3fcdb4e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -73,7 +73,8 @@ private SearchUtils() {} EntityType.DOMAIN, EntityType.DATA_PRODUCT, EntityType.NOTEBOOK, - EntityType.BUSINESS_ATTRIBUTE); + EntityType.BUSINESS_ATTRIBUTE, + EntityType.SCHEMA_FIELD); /** Entities that are part of autocomplete by default in Auto Complete Across Entities */ public static final List AUTO_COMPLETE_ENTITY_TYPES = From 90bb246e4263ed07a2d8c306c2183f29e68353f3 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 17:42:52 -0500 Subject: [PATCH 05/18] fix test warnings test gradle caching disabled --- .github/workflows/metadata-io.yml | 2 +- gradle.properties | 2 +- .../search/query/request/AggregationQueryBuilderTest.java | 2 +- .../java/com/linkedin/metadata/search/utils/ESUtilsTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 5cce02f8c14c1..e74b97c9ec737 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -70,7 +70,7 @@ jobs: cache: "pip" - name: Gradle build (and test) run: | - ./gradlew :metadata-io:test --no-daemon + ./gradlew :metadata-io:test --rerun-tasks - uses: actions/upload-artifact@v3 if: always() with: diff --git a/gradle.properties b/gradle.properties index e42e18dab677b..00e6e8a3a9d3a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.configureondemand=true org.gradle.parallel=true -org.gradle.caching=true +org.gradle.caching=false # Cycle daemons after 30m org.gradle.daemon.idletimeout=1800000 diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index 0ea2340ae8217..1381e9560b7e5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -41,7 +41,7 @@ public class AggregationQueryBuilderTest { private static AspectRetriever aspectRetrieverV1; @BeforeClass - public static void setup() throws RemoteInvocationException, URISyntaxException { + public void setup() throws RemoteInvocationException, URISyntaxException { Urn helloUrn = Urn.createFromString("urn:li:structuredProperty:hello"); Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index c5f9986284627..03d104b9e7bfb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -38,7 +38,7 @@ public class ESUtilsTest { private static AspectRetriever aspectRetrieverV1; @BeforeClass - public static void setup() throws RemoteInvocationException, URISyntaxException { + public void setup() throws RemoteInvocationException, URISyntaxException { Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); // legacy From 55e7b881f6e2b2b762074cbaf03600d53024321b Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 18:03:16 -0500 Subject: [PATCH 06/18] revert cache test --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 00e6e8a3a9d3a..e42e18dab677b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.configureondemand=true org.gradle.parallel=true -org.gradle.caching=false +org.gradle.caching=true # Cycle daemons after 30m org.gradle.daemon.idletimeout=1800000 From 1ea932bdd638c168e70d564c5428c529e0b9d8b8 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 18:03:45 -0500 Subject: [PATCH 07/18] sync netty versions --- build.gradle | 2 +- datahub-frontend/play.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 302b37281798f..9b6bddec28d03 100644 --- a/build.gradle +++ b/build.gradle @@ -391,7 +391,7 @@ subprojects { implementation externalDependency.annotationApi constraints { implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion") - implementation('io.netty:netty-all:4.1.100.Final') + implementation('io.netty:netty-all:4.1.105.Final') implementation('org.apache.commons:commons-compress:1.26.0') implementation('org.apache.velocity:velocity-engine-core:2.3') implementation('org.hibernate:hibernate-validator:6.0.20.Final') diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index b14962e5900cd..3fe8e48eb48e4 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -20,7 +20,7 @@ dependencies { play('com.nimbusds:nimbus-jose-jwt:8.18') play('com.typesafe.akka:akka-actor_2.12:2.6.20') play(externalDependency.jsonSmart) - play('io.netty:netty-all:4.1.86.Final') + play('io.netty:netty-all:4.1.105.Final') implementation(externalDependency.commonsText) { because("previous versions are vulnerable to CVE-2022-42889") } From 6f01c2b39afb5ce3b8a9cc6bc4b8d3b96db018b3 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 18:04:02 -0500 Subject: [PATCH 08/18] revert rerun-tasks --- .github/workflows/metadata-io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index e74b97c9ec737..7018b42949e89 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -70,7 +70,7 @@ jobs: cache: "pip" - name: Gradle build (and test) run: | - ./gradlew :metadata-io:test --rerun-tasks + ./gradlew :metadata-io:test - uses: actions/upload-artifact@v3 if: always() with: From 9c8c90ad37ab7e1588eab8e1dcbe5c259f6a9b20 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 19:36:51 -0500 Subject: [PATCH 09/18] test fix - search should() handling --- .../graph/elastic/ESGraphQueryDAO.java | 22 +++++++++++++++++-- .../graph/elastic/GraphFilterUtils.java | 11 ++++++---- .../query/filter/BaseQueryFilterRewriter.java | 5 ++++- .../request/AutocompleteRequestHandler.java | 13 ++++++----- .../query/request/SearchQueryBuilder.java | 15 ++++++++++--- .../metadata/search/utils/ESUtils.java | 9 ++++---- .../ContainerExpansionRewriterTest.java | 11 +++++++--- .../filter/DomainExpansionRewriterTest.java | 20 ++++++++++++----- .../metadata/search/utils/ESUtilsTest.java | 8 +++++++ .../test/fixtures/search/LineageExporter.java | 8 ++++--- .../lineage_query_filters_full.json | 6 +++++ ...eage_query_filters_full_empty_filters.json | 2 ++ ...e_query_filters_full_multiple_filters.json | 6 +++++ .../lineage_query_filters_limited.json | 1 + .../lineage_time_query_filters_1.json | 4 ++++ 15 files changed, 109 insertions(+), 32 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index 8c7f0e3256cf8..a801cab81c952 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -128,6 +128,9 @@ private static void addFilterToQueryBuilder( criterion.getValues()))); orQuery.should(andQuery); } + if (!orQuery.should().isEmpty()) { + orQuery.minimumShouldMatch(1); + } rootQuery.filter(orQuery); } @@ -177,21 +180,26 @@ private SearchResponse executeGroupByLineageSearchQuery( // directions for lineage // set up filters for each relationship type in the correct direction to limit buckets BoolQueryBuilder sourceFilterQuery = QueryBuilders.boolQuery(); - sourceFilterQuery.minimumShouldMatch(1); + validEdges.stream() .filter(pair -> RelationshipDirection.OUTGOING.equals(pair.getValue().getDirection())) .forEach( pair -> sourceFilterQuery.should( getAggregationFilter(pair, RelationshipDirection.OUTGOING))); + if (!sourceFilterQuery.should().isEmpty()) { + sourceFilterQuery.minimumShouldMatch(1); + } BoolQueryBuilder destFilterQuery = QueryBuilders.boolQuery(); - destFilterQuery.minimumShouldMatch(1); validEdges.stream() .filter(pair -> RelationshipDirection.INCOMING.equals(pair.getValue().getDirection())) .forEach( pair -> destFilterQuery.should(getAggregationFilter(pair, RelationshipDirection.INCOMING))); + if (!destFilterQuery.should().isEmpty()) { + destFilterQuery.minimumShouldMatch(1); + } FilterAggregationBuilder sourceRelationshipTypeFilters = AggregationBuilders.filter(FILTER_BY_SOURCE_RELATIONSHIP, sourceFilterQuery); @@ -347,6 +355,9 @@ public static BoolQueryBuilder buildQuery( relationshipType -> relationshipQuery.should( QueryBuilders.termQuery(RELATIONSHIP_TYPE, relationshipType))); + if (!relationshipQuery.should().isEmpty()) { + relationshipQuery.minimumShouldMatch(1); + } finalQuery.filter(relationshipQuery); } @@ -697,6 +708,9 @@ public static QueryBuilder getLineageQuery( urns, edgesPerEntityType.get(entityType), graphFilters)); } }); + if (!entityTypeQueries.should().isEmpty()) { + entityTypeQueries.minimumShouldMatch(1); + } BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); @@ -741,6 +755,10 @@ static QueryBuilder getLineageQueryForEntityType( query.should(getIncomingEdgeQuery(urns, incomingEdges, graphFilters)); } + if (!query.should().isEmpty()) { + query.minimumShouldMatch(1); + } + return query; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java index 982bcae9b5fd9..b57b5b0b4b5eb 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java @@ -37,11 +37,14 @@ public static QueryBuilder getUrnStatusQuery( if (removed) { finalQuery.filter(QueryBuilders.termQuery(statusField, removed.toString())); } else { - finalQuery.minimumShouldMatch(1); finalQuery.should(QueryBuilders.termQuery(statusField, removed.toString())); finalQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(statusField))); } + if (!finalQuery.should().isEmpty()) { + finalQuery.minimumShouldMatch(1); + } + return finalQuery; } @@ -102,7 +105,7 @@ public static QueryBuilder getEdgeTimeFilterQuery( * 2. The createdOn and updatedOn window does not exist on the edge at all (support legacy cases) * 3. Special lineage case: The edge is marked as a "manual" edge, meaning that the time filters should NOT be applied. */ - BoolQueryBuilder timeFilterQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder timeFilterQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); timeFilterQuery.should(buildTimeWindowFilter(startTimeMillis, endTimeMillis)); timeFilterQuery.should(buildTimestampsMissingFilter()); timeFilterQuery.should(buildManualLineageFilter()); @@ -158,7 +161,7 @@ public static QueryBuilder getEdgeTimeFilterQuery( */ private static QueryBuilder buildTimeWindowFilter( final long startTimeMillis, final long endTimeMillis) { - final BoolQueryBuilder timeWindowQuery = QueryBuilders.boolQuery(); + final BoolQueryBuilder timeWindowQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); /* * To perform comparison: @@ -198,7 +201,7 @@ private static QueryBuilder buildTimestampsMissingFilter() { private static QueryBuilder buildNotExistsFilter(String fieldName) { // This filter returns 'true' if the field DOES NOT EXIST or it exists but is equal to 0. - final BoolQueryBuilder notExistsFilter = QueryBuilders.boolQuery(); + final BoolQueryBuilder notExistsFilter = QueryBuilders.boolQuery().minimumShouldMatch(1); notExistsFilter.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(fieldName))); notExistsFilter.should(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fieldName, 0L))); return notExistsFilter; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java index 367705d369c7c..d545f60a1ee8f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java @@ -110,9 +110,12 @@ private BoolQueryBuilder handleNestedFilters( mustNotQueryBuilders.forEach(expandedQueryBuilder::mustNot); expandedQueryBuilder.queryName(boolQueryBuilder.queryName()); expandedQueryBuilder.adjustPureNegative(boolQueryBuilder.adjustPureNegative()); - expandedQueryBuilder.minimumShouldMatch(boolQueryBuilder.minimumShouldMatch()); expandedQueryBuilder.boost(boolQueryBuilder.boost()); + if (!expandedQueryBuilder.should().isEmpty()) { + expandedQueryBuilder.minimumShouldMatch(1); + } + return expandedQueryBuilder; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index b7a04f2064d9b..294efb069a904 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -115,8 +115,7 @@ public SearchRequest getSearchRequest( QueryConfiguration customQueryConfig = customizedQueryHandler.lookupQueryConfig(input).orElse(null); - BoolQueryBuilder baseQuery = QueryBuilders.boolQuery(); - baseQuery.minimumShouldMatch(1); + BoolQueryBuilder baseQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); // Initial query with input filters BoolQueryBuilder filterQuery = @@ -176,12 +175,15 @@ public BoolQueryBuilder getQuery( BoolQueryBuilder finalQuery = Optional.ofNullable(customAutocompleteConfig) .flatMap(cac -> CustomizedQueryHandler.boolQueryBuilder(objectMapper, cac, query)) - .orElse(QueryBuilders.boolQuery()) - .minimumShouldMatch(1); + .orElse(QueryBuilders.boolQuery()); getAutocompleteQuery(customAutocompleteConfig, autocompleteFields, query) .ifPresent(finalQuery::should); + if (!finalQuery.should().isEmpty()) { + finalQuery.minimumShouldMatch(1); + } + return finalQuery; } @@ -200,8 +202,7 @@ private Optional getAutocompleteQuery( private static BoolQueryBuilder defaultQuery( List autocompleteFields, @Nonnull String query) { - BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); - finalQuery.minimumShouldMatch(1); + BoolQueryBuilder finalQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); // Search for exact matches with higher boost and ngram matches MultiMatchQueryBuilder autocompleteQueryBuilder = diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 3e76d3600d6a6..529c13c7d71ef 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -103,8 +103,7 @@ private QueryBuilder buildInternalQuery( cqc -> CustomizedQueryHandler.boolQueryBuilder( opContext.getObjectMapper(), cqc, sanitizedQuery)) - .orElse(QueryBuilders.boolQuery()) - .minimumShouldMatch(1); + .orElse(QueryBuilders.boolQuery()); if (fulltext && !query.startsWith(STRUCTURED_QUERY_PREFIX)) { getSimpleQuery(opContext.getEntityRegistry(), customQueryConfig, entitySpecs, sanitizedQuery) @@ -135,6 +134,10 @@ private QueryBuilder buildInternalQuery( } } + if (!finalQuery.should().isEmpty()) { + finalQuery.minimumShouldMatch(1); + } + return finalQuery; } @@ -368,6 +371,10 @@ private Optional getSimpleQuery( simplePerField.should(simpleBuilder); }); + if (!simplePerField.should().isEmpty()) { + simplePerField.minimumShouldMatch(1); + } + result = Optional.of(simplePerField); } @@ -454,7 +461,9 @@ private Optional getPrefixAndExactMatchQuery( } }); - return finalQuery.should().size() > 0 ? Optional.of(finalQuery) : Optional.empty(); + return finalQuery.should().size() > 0 + ? Optional.of(finalQuery.minimumShouldMatch(1)) + : Optional.empty(); } private Optional getStructuredQuery( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index ace7fa2bc197c..3c825640ae3ca 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -166,8 +166,6 @@ public static BoolQueryBuilder buildFilterQuery( searchableFieldTypes, opContext, queryFilterRewriteChain))); - // The default is not always 1 (ensure consistent default) - finalQueryBuilder.minimumShouldMatch(1); } else if (filter.getCriteria() != null) { // Otherwise, build boolean query from the deprecated "criteria" field. log.warn("Received query Filter with a deprecated field 'criteria'. Use 'or' instead."); @@ -187,7 +185,8 @@ public static BoolQueryBuilder buildFilterQuery( } }); finalQueryBuilder.should(andQueryBuilder); - // The default is not always 1 (ensure consistent default) + } + if (!finalQueryBuilder.should().isEmpty()) { finalQueryBuilder.minimumShouldMatch(1); } return finalQueryBuilder; @@ -533,7 +532,7 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand( final Map> searchableFieldTypes, @Nonnull OperationContext opContext, @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { - final BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder(); + final BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder().minimumShouldMatch(1); for (String field : fields) { orQueryBuilder.should( getQueryBuilderFromCriterionForSingleField( @@ -619,7 +618,7 @@ private static QueryBuilder buildWildcardQueryWithMultipleValues( @Nullable String queryName, @Nonnull AspectRetriever aspectRetriever, String wildcardPattern) { - BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); for (String value : criterion.getValues()) { boolQuery.should( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java index f91e3a28f1bd6..5246e4dbe5bf9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java @@ -311,7 +311,7 @@ public void testNestedBoolQueryRewrite() { new RelatedEntities( "IsPartOf", childUrn, parentUrn, RelationshipDirection.OUTGOING, null)))); - BoolQueryBuilder testQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder testQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); testQuery.filter( QueryBuilders.boolQuery() .filter( @@ -319,8 +319,11 @@ public void testNestedBoolQueryRewrite() { testQuery.filter(QueryBuilders.existsQuery("someField")); testQuery.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( - QueryBuilders.boolQuery().should(QueryBuilders.termsQuery(FIELD_NAME, childUrn)))); + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.termsQuery(FIELD_NAME, childUrn)))); testQuery.should(QueryBuilders.existsQuery("someField")); testQuery.must( QueryBuilders.boolQuery() @@ -332,7 +335,7 @@ public void testNestedBoolQueryRewrite() { QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(FIELD_NAME, childUrn)))); testQuery.mustNot(QueryBuilders.existsQuery("someField")); - BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery(); + BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery().minimumShouldMatch(1); expectedRewrite.filter( QueryBuilders.boolQuery() .filter( @@ -341,8 +344,10 @@ public void testNestedBoolQueryRewrite() { expectedRewrite.filter(QueryBuilders.existsQuery("someField")); expectedRewrite.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should(QueryBuilders.termsQuery(FIELD_NAME, childUrn, parentUrn)))); expectedRewrite.should(QueryBuilders.existsQuery("someField")); expectedRewrite.must( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java index 76e650f405456..edc6449438581 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java @@ -312,7 +312,7 @@ public void testNestedBoolQueryRewrite() { new RelatedEntities( "IsPartOf", childUrn, parentUrn, RelationshipDirection.INCOMING, null)))); - BoolQueryBuilder testQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder testQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); testQuery.filter( QueryBuilders.boolQuery() .filter( @@ -320,9 +320,15 @@ public void testNestedBoolQueryRewrite() { testQuery.filter(QueryBuilders.boolQuery().filter(QueryBuilders.existsQuery("someField"))); testQuery.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( - QueryBuilders.boolQuery().should(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); - testQuery.should(QueryBuilders.boolQuery().should(QueryBuilders.existsQuery("someField"))); + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); + testQuery.should( + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.existsQuery("someField"))); testQuery.must( QueryBuilders.boolQuery() .must(QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); @@ -334,7 +340,7 @@ public void testNestedBoolQueryRewrite() { .mustNot(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); testQuery.mustNot(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("someField"))); - BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery(); + BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery().minimumShouldMatch(1); expectedRewrite.filter( QueryBuilders.boolQuery() .filter( @@ -344,11 +350,15 @@ public void testNestedBoolQueryRewrite() { QueryBuilders.boolQuery().filter(QueryBuilders.existsQuery("someField"))); expectedRewrite.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should(QueryBuilders.termsQuery(FIELD_NAME, childUrn, parentUrn)))); expectedRewrite.should( - QueryBuilders.boolQuery().should(QueryBuilders.existsQuery("someField"))); + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.existsQuery("someField"))); expectedRewrite.must( QueryBuilders.boolQuery() .must( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index 03d104b9e7bfb..cd6f0551a68e1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -179,6 +179,7 @@ public void testGetQueryBuilderFromCriterionContain() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -222,6 +223,7 @@ public void testGetQueryBuilderFromCriterionContain() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -258,6 +260,7 @@ public void testWildcardQueryBuilderFromCriterionWhenStartsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -301,6 +304,7 @@ public void testWildcardQueryBuilderFromCriterionWhenStartsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -337,6 +341,7 @@ public void testWildcardQueryBuilderFromCriterionWhenEndsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -379,6 +384,7 @@ public void testWildcardQueryBuilderFromCriterionWhenEndsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -540,6 +546,7 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -582,6 +589,7 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java index 4b7d81aa04416..4a2411138ed67 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java @@ -58,7 +58,7 @@ public void exportGraphIndex( Set urns, Set visitedUrns, Set visitedIds, int hops) { Set nextIds = new HashSet<>(); if (!urns.isEmpty()) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1); boolQueryBuilder.must(QueryBuilders.termQuery("relationshipType", "DownstreamOf")); @@ -70,7 +70,6 @@ public void exportGraphIndex( boolQueryBuilder.should( QueryBuilders.termsQuery("destination.urn", batch.toArray(String[]::new))); }); - boolQueryBuilder.minimumShouldMatch(1); // Exclude visited Lists.partition(Arrays.asList(visitedIds.toArray(String[]::new)), queryStatementSize) @@ -144,7 +143,10 @@ public void exportEntityIndex(Set ids, Set visitedIds, int hops) batch -> boolQueryBuilder.should( QueryBuilders.idsQuery().addIds(batch.toArray(String[]::new)))); - boolQueryBuilder.minimumShouldMatch(1); + + if (!boolQueryBuilder.should().isEmpty()) { + boolQueryBuilder.minimumShouldMatch(1); + } // Exclude visited Lists.partition(Arrays.asList(visitedIds.toArray(String[]::new)), queryStatementSize) diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json index 0a1cee08414a9..1aa1b4b5088a1 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json @@ -33,11 +33,13 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -115,6 +117,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -156,6 +159,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -194,6 +198,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } @@ -212,6 +217,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json index ab2841d6602d8..5ba0e36456889 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json @@ -33,11 +33,13 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json index 39f595e0e8dd2..938d878a9c8d1 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json @@ -36,11 +36,13 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -122,6 +124,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -163,6 +166,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -201,6 +205,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } @@ -219,6 +224,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json index 95d468ec3dac8..24fbb56065ebf 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json @@ -27,6 +27,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } \ No newline at end of file diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json index 327f1d4ff9338..13eb02fb61a4e 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json @@ -56,6 +56,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -97,6 +98,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -135,6 +137,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } @@ -153,6 +156,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } \ No newline at end of file From d50de8534fd9e67772ee311be51a3b715b2998d8 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 20:22:07 -0500 Subject: [PATCH 10/18] test logging --- .../fixtures/SampleDataFixtureTestBase.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index 8cb0678180ccb..939c13c80580e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -971,8 +971,13 @@ public void testSmokeTestQueries() { actualCount, expectedCount, String.format( - "Search term `%s` has %s fulltext results, expected %s results.", - key, actualCount, expectedCount)); + "Search term `%s` has %s fulltext results, expected %s results. Results: %s", + key, + actualCount, + expectedCount, + value.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); }); Map expectedStructuredMinimums = @@ -998,8 +1003,13 @@ public void testSmokeTestQueries() { actualCount, expectedCount, String.format( - "Search term `%s` has %s structured results, expected %s results.", - key, actualCount, expectedCount)); + "Search term `%s` has %s structured results, expected %s results. Results: %s", + key, + actualCount, + expectedCount, + value.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); }); } @@ -1318,6 +1328,7 @@ public void testScrollAcrossEntities() throws IOException { String query = "logging_events"; final int batchSize = 1; int totalResults = 0; + List resultUrns = new ArrayList<>(); String scrollId = null; do { ScrollResult result = @@ -1325,10 +1336,11 @@ public void testScrollAcrossEntities() throws IOException { int numResults = result.hasEntities() ? result.getEntities().size() : 0; assertTrue(numResults <= batchSize); totalResults += numResults; + resultUrns.addAll(result.getEntities().stream().map(SearchEntity::getEntity).toList()); scrollId = result.getScrollId(); } while (scrollId != null); // expect 2 total matching results - assertEquals(totalResults, 2); + assertEquals(totalResults, 2, String.format("query `%s` Results: %s", query, resultUrns)); } @Test From 9cbe5a02118a902921141927914dfbaa65345ac8 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 22:44:40 -0500 Subject: [PATCH 11/18] test configuration refactor --- .../LineageServiceElasticSearchTest.java | 6 +++++- .../SearchServiceElasticSearchTest.java | 6 +++++- .../elasticsearch/TestEntityElasticSearchTest.java | 6 +++++- .../opensearch/LineageServiceOpenSearchTest.java | 6 +++++- .../opensearch/SearchServiceOpenSearchTest.java | 6 +++++- .../search/opensearch/TestEntityOpenSearchTest.java | 6 +++++- .../metadata/search/query/BrowseDAOTest.java | 6 +++++- .../query/request/SearchQueryBuilderTest.java | 4 ++++ .../search/SampleDataFixtureConfiguration.java | 13 ++++--------- .../search/SearchLineageFixtureConfiguration.java | 4 +++- .../config/SearchCommonTestConfiguration.java | 12 ++++++++++-- 11 files changed, 56 insertions(+), 19 deletions(-) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java index 8c4195f9ff534..28fe212734bfa 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java @@ -10,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -25,7 +26,10 @@ public class LineageServiceElasticSearchTest extends LineageServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java index 7133971847f98..5b19200e62b2f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java @@ -10,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -25,7 +26,10 @@ public class SearchServiceElasticSearchTest extends SearchServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java index 5ad7b1218a5bf..701e6118f5fa0 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java @@ -10,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -25,7 +26,10 @@ public class TestEntityElasticSearchTest extends TestEntityTestBase { @Autowired private ESBulkProcessor bulkProcessor; @Autowired private ESIndexBuilder esIndexBuilder; @Autowired private SearchConfiguration searchConfiguration; - @Autowired private CustomSearchConfiguration customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; @NotNull @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java index 26c2cf28cdeca..5759dafffc7b6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java @@ -10,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -25,7 +26,10 @@ public class LineageServiceOpenSearchTest extends LineageServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java index 1127ba2089a91..41f87d52b8847 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java @@ -10,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -25,7 +26,10 @@ public class SearchServiceOpenSearchTest extends SearchServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java index 80db8864014c3..f4b69929d8e1a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java @@ -10,6 +10,7 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.testng.AssertJUnit; import org.testng.annotations.Test; @@ -25,7 +26,10 @@ public class TestEntityOpenSearchTest extends TestEntityTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java index 9c3d515f9322f..e71865921678b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java @@ -27,6 +27,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeMethod; @@ -39,7 +40,10 @@ public class BrowseDAOTest extends AbstractTestNGSpringContextTests { private OperationContext opContext; @Autowired private SearchConfiguration searchConfiguration; - @Autowired private CustomSearchConfiguration customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; @BeforeMethod public void setup() throws RemoteInvocationException, URISyntaxException { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 8d83317449a1e..374a69ee9a553 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -61,6 +61,10 @@ public class SearchQueryBuilderTest extends AbstractTestNGSpringContextTests { @Qualifier("queryOperationContext") private OperationContext operationContext; + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; + public static SearchConfiguration testQueryConfig; static { diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java index 781201f3478f9..df278bd305fd5 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java @@ -6,12 +6,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.config.search.CustomConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; @@ -73,7 +71,9 @@ public class SampleDataFixtureConfiguration { @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + @Autowired + @Qualifier("fixtureCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @Autowired private QueryFilterRewriteChain queryFilterRewriteChain; @@ -188,11 +188,6 @@ protected ElasticSearchService longTailEntitySearchService( protected ElasticSearchService entitySearchServiceHelper(EntityIndexBuilders indexBuilders) throws IOException { - CustomConfiguration customConfiguration = new CustomConfiguration(); - customConfiguration.setEnabled(true); - customConfiguration.setFile("search_config_fixture_test.yml"); - CustomSearchConfiguration customSearchConfiguration = - customConfiguration.resolve(new YAMLMapper()); ESSearchDAO searchDAO = new ESSearchDAO( @@ -200,7 +195,7 @@ protected ElasticSearchService entitySearchServiceHelper(EntityIndexBuilders ind false, ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, - customSearchConfiguration, + _customSearchConfiguration, queryFilterRewriteChain); ESBrowseDAO browseDAO = new ESBrowseDAO( diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index a7603f97792e7..059749e04f6a4 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -67,7 +67,9 @@ public class SearchLineageFixtureConfiguration { @Autowired private SearchConfiguration searchConfiguration; - @Autowired private CustomSearchConfiguration customSearchConfiguration; + @Autowired + @Qualifier("fixtureCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; @Bean(name = "searchLineagePrefix") protected String indexPrefix() { diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java index 547ab1d746dbe..e84ecf677e3a6 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java @@ -44,14 +44,22 @@ public SearchConfiguration searchConfiguration() { return searchConfiguration; } - @Bean - public CustomSearchConfiguration customSearchConfiguration() throws Exception { + @Bean("defaultTestCustomSearchConfig") + public CustomSearchConfiguration defaultTestCustomSearchConfig() throws Exception { CustomConfiguration customConfiguration = new CustomConfiguration(); customConfiguration.setEnabled(true); customConfiguration.setFile("search_config_builder_test.yml"); return customConfiguration.resolve(new YAMLMapper()); } + @Bean("fixtureCustomSearchConfig") + public CustomSearchConfiguration fixtureCustomSearchConfig() throws Exception { + CustomConfiguration customConfiguration = new CustomConfiguration(); + customConfiguration.setEnabled(true); + customConfiguration.setFile("search_config_fixture_test.yml"); + return customConfiguration.resolve(new YAMLMapper()); + } + @Bean(name = "queryOperationContext") public OperationContext queryOperationContext() { return TestOperationContexts.systemContextNoSearchAuthorization(); From 61fe36ebe459cd140f2fb13a6ed0a119a1670ddb Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 06:49:50 -0500 Subject: [PATCH 12/18] fix checkjar --- .../java/datahub-client/scripts/check_jar.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metadata-integration/java/datahub-client/scripts/check_jar.sh b/metadata-integration/java/datahub-client/scripts/check_jar.sh index f76931428e3d6..10299ec714d16 100755 --- a/metadata-integration/java/datahub-client/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-client/scripts/check_jar.sh @@ -36,10 +36,11 @@ jar -tvf $jarFile |\ grep -v "darwin" |\ grep -v "MetadataChangeProposal.avsc" |\ grep -v "aix" |\ - grep -v "com/sun/" - grep -v "mozilla" - grep -v "VersionInfo.java" - grep -v "mime.types" + grep -v "com/sun/" |\ + grep -v "mozilla" |\ + grep -v "VersionInfo.java" |\ + grep -v "mime.types" |\ + grep -v "com/ibm/.*" if [ $? -ne 0 ]; then From 4e5356cbcf84040bed8ce249d22fc3cd6737ee38 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 07:23:26 -0500 Subject: [PATCH 13/18] test only logging --- .../elasticsearch/query/ESSearchDAO.java | 40 +++++++++++++++++++ .../fixtures/SampleDataFixtureTestBase.java | 30 ++++++++++++-- .../SampleDataFixtureConfiguration.java | 3 +- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index cec73de704126..49f593e52d73a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -6,7 +6,9 @@ import com.codahale.metrics.Timer; import com.datahub.util.exception.ESQueryException; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.linkedin.data.template.LongMap; import com.linkedin.metadata.config.search.SearchConfiguration; @@ -78,6 +80,24 @@ public class ESSearchDAO { @Nonnull private final SearchConfiguration searchConfiguration; @Nullable private final CustomSearchConfiguration customSearchConfiguration; @Nonnull private final QueryFilterRewriteChain queryFilterRewriteChain; + private final boolean testLoggingEnabled; + + public ESSearchDAO( + RestHighLevelClient client, + boolean pointInTimeCreationEnabled, + String elasticSearchImplementation, + @Nonnull SearchConfiguration searchConfiguration, + @Nullable CustomSearchConfiguration customSearchConfiguration, + @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { + this( + client, + pointInTimeCreationEnabled, + elasticSearchImplementation, + searchConfiguration, + customSearchConfiguration, + queryFilterRewriteChain, + false); + } public long docCount(@Nonnull OperationContext opContext, @Nonnull String entityName) { return docCount(opContext, entityName, null); @@ -279,6 +299,11 @@ public SearchResult search( searchRequest.indices( entityNames.stream().map(indexConvention::getEntityIndexName).toArray(String[]::new)); searchRequestTimer.stop(); + + if (testLoggingEnabled) { + testLog(opContext.getObjectMapper(), searchRequest); + } + // Step 2: execute the query and extract results, validated against document model as well return executeAndExtract(opContext, entitySpecs, searchRequest, transformedFilters, from, size); } @@ -478,6 +503,11 @@ public ScrollResult scroll( } scrollRequestTimer.stop(); + + if (testLoggingEnabled) { + testLog(opContext.getObjectMapper(), searchRequest); + } + return executeAndExtract( opContext, entitySpecs, searchRequest, transformedFilters, keepAlive, size); } @@ -605,4 +635,14 @@ public ExplainResponse explain( throw new IllegalStateException("Failed to explain query:", e); } } + + private static void testLog(ObjectMapper mapper, SearchRequest searchRequest) { + try { + final String[] indices = searchRequest.indices(); + log.warn(String.format("SearchRequest: %s", mapper.writerWithDefaultPrettyPrinter().writeValueAsString(indices))); + log.warn(String.format("SearchRequest: %s", searchRequest.source().toString())); + } catch (JsonProcessingException e) { + log.warn("Error writing test log"); + } + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index 939c13c80580e..eb849019b2362 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -1715,7 +1715,15 @@ public void testOr() { assertTrue( result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), String.format("%s - Expected search results to include matched fields", query)); - assertEquals(result.getEntities().size(), 2); + assertEquals( + result.getEntities().size(), + 2, + String.format( + "Query: `%s` Results: %s", + query, + result.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); } @Test @@ -1738,7 +1746,15 @@ public void testNegate() { assertTrue( result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), String.format("%s - Expected search results to include matched fields", query)); - assertEquals(result.getEntities().size(), 2); + assertEquals( + result.getEntities().size(), + 2, + String.format( + "Query: `%s` Results: %s", + query, + result.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); } @Test @@ -1908,7 +1924,15 @@ public void testPrefixVsExact() { result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), String.format("%s - Expected search results to include matched fields", query)); - assertEquals(result.getEntities().size(), 2); + assertEquals( + result.getEntities().size(), + 2, + String.format( + "Query: `%s` Results: %s", + query, + result.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); assertEquals( result.getEntities().get(0).getEntity().toString(), "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java index df278bd305fd5..e47cdf80281c9 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java @@ -196,7 +196,8 @@ protected ElasticSearchService entitySearchServiceHelper(EntityIndexBuilders ind ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, _customSearchConfiguration, - queryFilterRewriteChain); + queryFilterRewriteChain, + true); ESBrowseDAO browseDAO = new ESBrowseDAO( _searchClient, From ac7c9ce3317a4b96d913dc559277c7f2266a0c38 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 10:38:17 -0500 Subject: [PATCH 14/18] fix-up queryByDefault --- .../elasticsearch/query/ESSearchDAO.java | 10 +- .../query/request/SearchRequestHandler.java | 3 +- .../request/SearchRequestHandlerTest.java | 115 ++++++++++++++++++ .../common/DocumentationAssociation.pdl | 9 +- .../pegasus/com/linkedin/common/Forms.pdl | 15 ++- .../common/GlossaryTermAssociation.pdl | 9 +- .../com/linkedin/common/IncidentsSummary.pdl | 6 +- .../com/linkedin/common/RoleAssociation.pdl | 3 +- .../pegasus/com/linkedin/common/SubTypes.pdl | 2 +- .../com/linkedin/common/TagAssociation.pdl | 9 +- .../glossary/GlossaryRelatedTerms.pdl | 4 +- .../com/linkedin/identity/CorpUserInfo.pdl | 2 +- .../schema/EditableSchemaFieldInfo.pdl | 12 +- .../com/linkedin/schema/SchemaField.pdl | 18 ++- .../pegasus/com/linkedin/test/TestResults.pdl | 2 + 15 files changed, 187 insertions(+), 32 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index 49f593e52d73a..14e8fb2ddaf80 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -639,8 +639,14 @@ public ExplainResponse explain( private static void testLog(ObjectMapper mapper, SearchRequest searchRequest) { try { final String[] indices = searchRequest.indices(); - log.warn(String.format("SearchRequest: %s", mapper.writerWithDefaultPrettyPrinter().writeValueAsString(indices))); - log.warn(String.format("SearchRequest: %s", searchRequest.source().toString())); + log.warn( + String.format( + "SearchRequest: %s", + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(indices))); + log.warn( + String.format( + "SearchRequest: %s", + mapper.writeValueAsString(mapper.readTree(searchRequest.source().toString())))); } catch (JsonProcessingException e) { log.warn("Error writing test log"); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index c935e6f54742c..cb02fb1c8b2f7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -58,6 +58,7 @@ import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.opensearch.action.search.SearchRequest; @@ -80,7 +81,7 @@ public class SearchRequestHandler { private static final Map, SearchRequestHandler> REQUEST_HANDLER_BY_ENTITY_NAME = new ConcurrentHashMap<>(); private final List entitySpecs; - private final Set defaultQueryFieldNames; + @Getter private final Set defaultQueryFieldNames; @Nonnull private final HighlightBuilder highlights; private final SearchConfiguration configs; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java index a90c0291f53b8..a3ef62760d797 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search.query.request; +import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES; import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion; import static com.linkedin.metadata.utils.CriterionUtils.buildIsNullCriterion; @@ -8,7 +9,10 @@ import static org.testng.Assert.*; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.TestEntitySpecBuilder; import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; @@ -35,6 +39,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.opensearch.action.search.SearchRequest; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.ExistsQueryBuilder; @@ -628,6 +633,116 @@ public void testBrowsePathQueryFilter() { assertEquals(((ExistsQueryBuilder) mustHaveV1.must().get(0)).fieldName(), "browsePaths"); } + @Test + public void testQueryByDefault() { + final Set COMMON = + Set.of( + "container", + "fieldDescriptions", + "description", + "platform", + "fieldPaths", + "editedFieldGlossaryTerms", + "editedFieldDescriptions", + "fieldTags", + "id", + "editedDescription", + "qualifiedName", + "domains", + "platformInstance", + "tags", + "urn", + "customProperties", + "fieldGlossaryTerms", + "editedName", + "name", + "fieldLabels", + "glossaryTerms", + "editedFieldTags", + "displayName", + "title"); + + Map> expectedQueryByDefault = + ImmutableMap.>builder() + .put( + EntityType.DASHBOARD, + Stream.concat(COMMON.stream(), Stream.of("tool")).collect(Collectors.toSet())) + .put( + EntityType.CHART, + Stream.concat(COMMON.stream(), Stream.of("tool")).collect(Collectors.toSet())) + .put( + EntityType.MLMODEL, + Stream.concat(COMMON.stream(), Stream.of("type")).collect(Collectors.toSet())) + .put( + EntityType.MLFEATURE_TABLE, + Stream.concat(COMMON.stream(), Stream.of("features", "primaryKeys")) + .collect(Collectors.toSet())) + .put( + EntityType.MLFEATURE, + Stream.concat(COMMON.stream(), Stream.of("featureNamespace")) + .collect(Collectors.toSet())) + .put( + EntityType.MLPRIMARY_KEY, + Stream.concat(COMMON.stream(), Stream.of("featureNamespace")) + .collect(Collectors.toSet())) + .put( + EntityType.DATA_FLOW, + Stream.concat(COMMON.stream(), Stream.of("cluster", "orchestrator", "flowId")) + .collect(Collectors.toSet())) + .put( + EntityType.DATA_JOB, + Stream.concat(COMMON.stream(), Stream.of("jobId")).collect(Collectors.toSet())) + .put( + EntityType.GLOSSARY_TERM, + Stream.concat( + COMMON.stream(), + Stream.of("values", "parentNode", "relatedTerms", "definition")) + .collect(Collectors.toSet())) + .put( + EntityType.GLOSSARY_NODE, + Stream.concat(COMMON.stream(), Stream.of("definition", "parentNode")) + .collect(Collectors.toSet())) + .put( + EntityType.CORP_USER, + Stream.concat( + COMMON.stream(), Stream.of("skills", "teams", "ldap", "fullName", "email")) + .collect(Collectors.toSet())) + .put( + EntityType.DOMAIN, + Stream.concat(COMMON.stream(), Stream.of("parentDomain")) + .collect(Collectors.toSet())) + .put( + EntityType.SCHEMA_FIELD, + Stream.concat(COMMON.stream(), Stream.of("schemaFieldAliases", "parent")) + .collect(Collectors.toSet())) + .build(); + + for (EntityType entityType : SEARCHABLE_ENTITY_TYPES) { + Set expectedEntityQueryByDefault = + expectedQueryByDefault.getOrDefault(entityType, COMMON); + assertFalse(expectedEntityQueryByDefault.isEmpty()); + + EntitySpec entitySpec = + operationContext.getEntityRegistry().getEntitySpec(EntityTypeMapper.getName(entityType)); + SearchRequestHandler handler = + SearchRequestHandler.getBuilder( + operationContext.getEntityRegistry(), + entitySpec, + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); + + Set unexpected = new HashSet<>(handler.getDefaultQueryFieldNames()); + unexpected.removeAll(expectedEntityQueryByDefault); + + assertTrue( + unexpected.isEmpty(), + String.format( + "Consider whether these field(s) for entity %s should be included for general search. Fields: %s If yes, please update the test expectations. If no, please annotate the PDL model with \"queryByDefault\": false", + entityType, unexpected)); + } + } + private BoolQueryBuilder getQuery(final Criterion filterCriterion) { final Filter filter = new Filter() diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl index 19404346797bb..bee331f15b679 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl @@ -15,15 +15,18 @@ record DocumentationAssociation { @Searchable = { "/time": { "fieldName": "documentationAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/actor": { "fieldName": "documentationAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/source": { "fieldName": "documentationAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } attribution: optional MetadataAttribution diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl index 0a97c7d5099ed..3c05c00fd6fb9 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl @@ -13,19 +13,23 @@ record Forms { @Searchable = { "/*/urn": { "fieldType": "URN", - "fieldName": "incompleteForms" + "fieldName": "incompleteForms", + "queryByDefault": false, }, "/*/completedPrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "incompleteFormsCompletedPromptIds", + "queryByDefault": false, }, "/*/incompletePrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "incompleteFormsIncompletePromptIds", + "queryByDefault": false, }, "/*/completedPrompts/*/lastModified/time" : { "fieldType": "DATETIME", "fieldName": "incompleteFormsCompletedPromptResponseTimes", + "queryByDefault": false, } } incompleteForms: array[FormAssociation] @@ -36,19 +40,23 @@ record Forms { @Searchable = { "/*/urn": { "fieldType": "URN", - "fieldName": "completedForms" + "fieldName": "completedForms", + "queryByDefault": false }, "/*/completedPrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "completedFormsCompletedPromptIds", + "queryByDefault": false, }, "/*/incompletePrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "completedFormsIncompletePromptIds", + "queryByDefault": false, }, "/*/completedPrompts/*/lastModified/time" : { "fieldType": "DATETIME", "fieldName": "completedFormsCompletedPromptResponseTimes", + "queryByDefault": false, } } completedForms: array[FormAssociation] @@ -59,7 +67,8 @@ record Forms { @Searchable = { "/*/form": { "fieldType": "URN", - "fieldName": "verifiedForms" + "fieldName": "verifiedForms", + "queryByDefault": false, } } verifications: array[FormVerificationAssociation] = [] diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl index a5267bbc635e4..58423ccc2228d 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl @@ -36,15 +36,18 @@ record GlossaryTermAssociation { @Searchable = { "/time": { "fieldName": "termAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/actor": { "fieldName": "termAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/source": { "fieldName": "termAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } attribution: optional MetadataAttribution diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl index e1367a326e24b..9e4e81656f7c9 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl @@ -29,7 +29,8 @@ record IncidentsSummary { "fieldType": "URN", "fieldName": "resolvedIncidents", "hasValuesFieldName": "hasResolvedIncidents", - "numValuesFieldName": "numResolvedIncidents" + "numValuesFieldName": "numResolvedIncidents", + "queryByDefault": false, }, "/*/type" : { "fieldType": "KEYWORD", @@ -65,7 +66,8 @@ record IncidentsSummary { "fieldName": "activeIncidents", "hasValuesFieldName": "hasActiveIncidents", "numValuesFieldName": "numActiveIncidents", - "addHasValuesToFilters": true + "addHasValuesToFilters": true, + "queryByDefault": false, }, "/*/type" : { "fieldType": "KEYWORD", diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl index ddd63ed64014b..05c46dfdf69d7 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl @@ -18,7 +18,8 @@ record RoleAssociation { "fieldType": "URN", "hasValuesFieldName": "hasRoles", "addToFilters": true, - "filterNameOverride": "Role" + "filterNameOverride": "Role", + "queryByDefault": false, } urn: Urn } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl index a1063afe1eae9..1f2ff275c7ae9 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl @@ -17,7 +17,7 @@ record SubTypes { "fieldType": "KEYWORD", "addToFilters": true, "filterNameOverride": "Sub Type", - "queryByDefault": true + "queryByDefault": false, } } typeNames: array[string] diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl index 8a58ca97de195..caed4961272de 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl @@ -21,15 +21,18 @@ record TagAssociation { @Searchable = { "/time": { "fieldName": "tagAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/actor": { "fieldName": "tagAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/source": { "fieldName": "tagAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } attribution: optional MetadataAttribution diff --git a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl index 5e10219235347..9f5f312a6bcce 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl @@ -24,7 +24,7 @@ record GlossaryRelatedTerms { "/*": { "fieldName": "isRelatedTerms", "fieldType": "URN", - "boostScore": 2.0 + "queryByDefault": false, } } isRelatedTerms: optional array[GlossaryTermUrn] @@ -42,7 +42,7 @@ record GlossaryRelatedTerms { "/*": { "fieldName": "hasRelatedTerms", "fieldType": "URN", - "boostScore": 2.0 + "queryByDefault": false, } } hasRelatedTerms: optional array[GlossaryTermUrn] diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl index 382b120fa942a..53c31daeca437 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl @@ -62,7 +62,7 @@ record CorpUserInfo includes CustomProperties { @Searchable = { "fieldName": "managerLdap", "fieldType": "URN", - "queryByDefault": true + "queryByDefault": false, } managerUrn: optional CorpuserUrn diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl index 816277bd1e0c9..048c2dcd9f58f 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl @@ -43,11 +43,13 @@ record EditableSchemaFieldInfo { }, "/tags/*/attribution/actor": { "fieldName": "editedFieldTagAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/tags/*/attribution/source": { "fieldName": "editedFieldTagAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } globalTags: optional GlobalTags @@ -73,11 +75,13 @@ record EditableSchemaFieldInfo { }, "/terms/*/attribution/actor": { "fieldName": "editedFieldTermAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/terms/*/attribution/source": { "fieldName": "editedFieldTermAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } glossaryTerms: optional GlossaryTerms diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl index f91e2004401cf..0b72d376b0be4 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl @@ -99,15 +99,18 @@ record SchemaField { }, "/tags/*/attribution/time": { "fieldName": "fieldTagAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/tags/*/attribution/actor": { "fieldName": "fieldTagAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/tags/*/attribution/source": { "fieldName": "fieldTagAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } globalTags: optional GlobalTags @@ -129,15 +132,18 @@ record SchemaField { }, "/terms/*/attribution/time": { "fieldName": "fieldTermAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/terms/*/attribution/actor": { "fieldName": "fieldTermAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/terms/*/attribution/source": { "fieldName": "fieldTermAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } glossaryTerms: optional GlossaryTerms diff --git a/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl b/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl index 6f210abf0597f..c8a99faef88a0 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl @@ -15,6 +15,7 @@ record TestResults { "fieldType": "URN", "fieldName": "failingTests" "hasValuesFieldName": "hasFailingTests", + "queryByDefault": false, } } @Relationship = { @@ -33,6 +34,7 @@ record TestResults { "fieldType": "URN", "fieldName": "passingTests", "hasValuesFieldName": "hasPassingTests", + "queryByDefault": false, } } @Relationship = { From 0cd67098853ada703b060d6dc8d98f83abdb4e1a Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 13:32:37 -0500 Subject: [PATCH 15/18] test updates --- .../elasticsearch/ElasticSearchService.java | 4 +- .../elasticsearch/query/ESSearchDAO.java | 7 +- .../entity/DeleteEntityServiceTest.java | 4 +- .../entity/ebean/EbeanAspectDaoTest.java | 7 +- .../extractor/AspectExtractorTest.java | 2 +- .../SearchGraphServiceElasticSearchTest.java | 5 +- .../SearchGraphServiceOpenSearchTest.java | 5 +- .../LineageSearchResultCacheKeyTest.java | 4 +- .../search/LineageServiceTestBase.java | 9 +-- .../search/SearchServiceTestBase.java | 9 +-- .../metadata/search/TestEntityTestBase.java | 9 +-- .../GoldenElasticSearchTest.java | 2 +- .../IndexBuilderElasticSearchTest.java | 2 +- .../LineageDataFixtureElasticSearchTest.java | 5 +- .../LineageServiceElasticSearchTest.java | 11 +-- .../SampleDataFixtureElasticSearchTest.java | 8 +- .../SearchDAOElasticSearchTest.java | 18 ++++- .../SearchServiceElasticSearchTest.java | 11 +-- ...ystemMetadataServiceElasticSearchTest.java | 5 +- .../TestEntityElasticSearchTest.java | 11 +-- ...eseriesAspectServiceElasticSearchTest.java | 5 +- .../search/fixtures/GoldenTestBase.java | 1 - .../fixtures/SampleDataFixtureSetupTest.java | 47 ----------- .../fixtures/SampleDataFixtureTestBase.java | 42 +++++++++- .../opensearch/GoldenOpenSearchTest.java | 2 +- .../IndexBuilderOpenSearchTest.java | 2 +- .../LineageDataFixtureOpenSearchTest.java | 5 +- .../LineageServiceOpenSearchTest.java | 11 +-- .../SampleDataFixtureOpenSearchTest.java | 8 +- .../opensearch/SearchDAOOpenSearchTest.java | 19 ++++- .../SearchServiceOpenSearchTest.java | 11 +-- .../SystemMetadataServiceOpenSearchTest.java | 5 +- .../opensearch/TestEntityOpenSearchTest.java | 11 +-- ...TimeseriesAspectServiceOpenSearchTest.java | 5 +- .../search/query/SearchDAOTestBase.java | 78 +++++++++++++------ .../PropertyDefinitionValidatorTest.java | 2 +- ...chemaMetadataChangeEventGeneratorTest.java | 2 +- .../SearchLineageFixtureConfiguration.java | 2 +- .../src/main/resources/application.yaml | 4 +- 39 files changed, 213 insertions(+), 187 deletions(-) delete mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index e66b12db891df..6001e2f6e660f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -2,6 +2,7 @@ import static com.linkedin.metadata.search.utils.SearchUtils.applyDefaultSearchFlags; +import com.google.common.annotations.VisibleForTesting; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.browse.BrowseResultV2; @@ -30,6 +31,7 @@ import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.opensearch.action.explain.ExplainResponse; @@ -51,7 +53,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI private static final int MAX_RUN_IDS_INDEXED = 25; // Save the previous 25 run ids in the index. private final EntityIndexBuilders indexBuilders; - private final ESSearchDAO esSearchDAO; + @VisibleForTesting @Getter private final ESSearchDAO esSearchDAO; private final ESBrowseDAO esBrowseDAO; private final ESWriteDAO esWriteDAO; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index 14e8fb2ddaf80..f09a81c0c8b89 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -636,16 +636,17 @@ public ExplainResponse explain( } } - private static void testLog(ObjectMapper mapper, SearchRequest searchRequest) { + private void testLog(ObjectMapper mapper, SearchRequest searchRequest) { try { + log.warn("SearchRequest(custom): {}", mapper.writeValueAsString(customSearchConfiguration)); final String[] indices = searchRequest.indices(); log.warn( String.format( - "SearchRequest: %s", + "SearchRequest(indices): %s", mapper.writerWithDefaultPrettyPrinter().writeValueAsString(indices))); log.warn( String.format( - "SearchRequest: %s", + "SearchRequest(query): %s", mapper.writeValueAsString(mapper.readTree(searchRequest.source().toString())))); } catch (JsonProcessingException e) { log.warn("Error writing test log"); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java index d585ff1ce8383..0e8ee08e60739 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java @@ -2,7 +2,9 @@ import static com.linkedin.metadata.search.utils.QueryUtils.*; import static org.mockito.Mockito.*; -import static org.testng.AssertJUnit.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableList; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java index 43123fb9872a0..4c87797ca1120 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java @@ -4,6 +4,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.config.EbeanConfiguration; @@ -49,7 +51,7 @@ public void testGetNextVersionForUpdate() { } @Test - public void testGetLatestAspectsForUpdate() { + public void testGetLatestAspectsForUpdate() throws JsonProcessingException { LoggedSql.start(); testDao.runInTransactionWithRetryUnlocked( @@ -65,7 +67,8 @@ public void testGetLatestAspectsForUpdate() { LoggedSql.stop().stream() .filter(str -> !str.contains("INFORMATION_SCHEMA.TABLES")) .toList(); - assertEquals(sql.size(), 1, String.format("Found: %s", sql)); + assertEquals( + sql.size(), 1, String.format("Found: %s", new ObjectMapper().writeValueAsString(sql))); assertTrue( sql.get(0).contains("for update;"), String.format("Did not find `for update` in %s ", sql)); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java index a98386f6f871b..f8bf2376e6bc1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.extractor; -import static org.testng.AssertJUnit.assertEquals; +import static org.testng.Assert.assertEquals; import com.datahub.test.TestEntityAspect; import com.datahub.test.TestEntityAspectArray; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java index b2c49857cb0b9..8a0dfcbe34a69 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.graph.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.graph.search.SearchGraphServiceTestBase; import com.linkedin.metadata.search.elasticsearch.ElasticSearchSuite; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -9,7 +11,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -39,6 +40,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java index 28b545f817539..08a6ea4ef2c9c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.graph.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.graph.search.SearchGraphServiceTestBase; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; @@ -9,7 +11,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -39,6 +40,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java index 1d4a545fc06a2..d2d27bc2de27c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search; -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index d9268c1b50efe..39fb6001eeb95 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -35,7 +35,6 @@ import com.linkedin.metadata.config.cache.SearchCacheConfiguration; import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.LineageDirection; @@ -103,9 +102,6 @@ public abstract class LineageServiceTestBase extends AbstractTestNGSpringContext @Nonnull protected abstract SearchConfiguration getSearchConfiguration(); - @Nonnull - protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); - private SettingsBuilder settingsBuilder; private ElasticSearchService elasticSearchService; private GraphService graphService; @@ -211,10 +207,7 @@ private ElasticSearchService buildEntitySearchService() { QueryFilterRewriteChain.EMPTY); ESBrowseDAO browseDAO = new ESBrowseDAO( - searchClientSpy, - getSearchConfiguration(), - getCustomSearchConfiguration(), - QueryFilterRewriteChain.EMPTY); + searchClientSpy, getSearchConfiguration(), null, QueryFilterRewriteChain.EMPTY); ESWriteDAO writeDAO = new ESWriteDAO(searchClientSpy, getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java index ba83a381916c2..b20326deeb945 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java @@ -14,7 +14,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -64,9 +63,6 @@ public abstract class SearchServiceTestBase extends AbstractTestNGSpringContextT @Nonnull protected abstract SearchConfiguration getSearchConfiguration(); - @Nonnull - protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); - protected OperationContext operationContext; private SettingsBuilder settingsBuilder; private ElasticSearchService elasticSearchService; @@ -136,10 +132,7 @@ private ElasticSearchService buildEntitySearchService() { QueryFilterRewriteChain.EMPTY); ESBrowseDAO browseDAO = new ESBrowseDAO( - getSearchClient(), - getSearchConfiguration(), - getCustomSearchConfiguration(), - QueryFilterRewriteChain.EMPTY); + getSearchClient(), getSearchConfiguration(), null, QueryFilterRewriteChain.EMPTY); ESWriteDAO writeDAO = new ESWriteDAO(getSearchClient(), getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java index 7b6fcd46333d2..206b97ce6c104 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java @@ -13,7 +13,6 @@ import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.browse.BrowseResultV2; import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -53,9 +52,6 @@ public abstract class TestEntityTestBase extends AbstractTestNGSpringContextTest @Nonnull protected abstract SearchConfiguration getSearchConfiguration(); - @Nonnull - protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); - private SettingsBuilder settingsBuilder; private ElasticSearchService elasticSearchService; private OperationContext opContext; @@ -102,10 +98,7 @@ private ElasticSearchService buildService() { QueryFilterRewriteChain.EMPTY); ESBrowseDAO browseDAO = new ESBrowseDAO( - getSearchClient(), - getSearchConfiguration(), - getCustomSearchConfiguration(), - QueryFilterRewriteChain.EMPTY); + getSearchClient(), getSearchConfiguration(), null, QueryFilterRewriteChain.EMPTY); ESWriteDAO writeDAO = new ESWriteDAO(getSearchClient(), getBulkProcessor(), 1); ElasticSearchService searchService = new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java index 29f5964c853f1..ad30c9d0229aa 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.GoldenTestBase; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java index 911a21767bdea..af0b7003bd1d1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.indexbuilder.IndexBuilderTestBase; import io.datahubproject.test.search.config.SearchTestContainerConfiguration; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java index 143ae80abc52d..fe992f61d311d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.LineageDataFixtureTestBase; @@ -10,7 +12,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Getter @@ -35,6 +36,6 @@ public class LineageDataFixtureElasticSearchTest extends LineageDataFixtureTestB @Test public void initTest() { - AssertJUnit.assertNotNull(lineageService); + assertNotNull(lineageService); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java index 28fe212734bfa..7ccf7605432ee 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.LineageServiceTestBase; @@ -12,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -55,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java index e256f75242a42..68b68b289dd2c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java @@ -1,8 +1,9 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase; import io.datahubproject.metadata.context.OperationContext; @@ -37,6 +38,11 @@ public class SampleDataFixtureElasticSearchTest extends SampleDataFixtureTestBas @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java index a6a8279fe86de..3fc49a4d624fa 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java @@ -1,8 +1,10 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.query.SearchDAOTestBase; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; @@ -28,6 +30,20 @@ public class SearchDAOElasticSearchTest extends SearchDAOTestBase { @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Autowired + @Qualifier("sampleDataEntitySearchService") + protected ElasticSearchService entitySearchService; + + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + + @Override + protected ESSearchDAO getESSearchDao() { + return entitySearchService.getEsSearchDAO(); + } + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java index 5b19200e62b2f..92dfa18d4feeb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchServiceTestBase; @@ -12,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -55,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java index a23cd5b051ecb..b4093459ab3f1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.systemmetadata.SystemMetadataServiceTestBase; @@ -8,7 +10,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -38,6 +39,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java index 701e6118f5fa0..ec6f2e2b3f07a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.TestEntityTestBase; @@ -12,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -55,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(searchClient); + assertNotNull(searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java index 1f51d463a2963..8a5f5d673aa7b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.timeseries.search.TimeseriesAspectServiceTestBase; @@ -9,7 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -42,6 +43,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java index 1ebcc03eb690b..052daeece8cd0 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java @@ -4,7 +4,6 @@ import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static io.datahubproject.test.search.SearchTestUtils.searchAcrossEntities; import static org.testng.Assert.*; -import static org.testng.AssertJUnit.assertNotNull; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java deleted file mode 100644 index b908933fcc8e3..0000000000000 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.linkedin.metadata.search.fixtures; - -import static org.testng.AssertJUnit.assertEquals; - -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import org.springframework.core.io.ClassPathResource; -import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; -import org.testng.annotations.Test; - -public class SampleDataFixtureSetupTest extends AbstractTestNGSpringContextTests { - private static final String DEFAULT_CONFIG = "search_config.yaml"; - private static final String TEST_FIXTURE_CONFIG = "search_config_fixture_test.yml"; - private static final YAMLMapper MAPPER = new YAMLMapper(); - - /** - * Ensure default search configuration matches the test fixture configuration (allowing for some - * differences) - */ - @Test - public void testConfig() throws IOException { - final CustomSearchConfiguration defaultConfig; - final CustomSearchConfiguration fixtureConfig; - - try (InputStream stream = new ClassPathResource(DEFAULT_CONFIG).getInputStream()) { - defaultConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); - } - try (InputStream stream = new ClassPathResource(TEST_FIXTURE_CONFIG).getInputStream()) { - fixtureConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); - - // test specifics - ((List>) - fixtureConfig.getQueryConfigurations().get(1).getFunctionScore().get("functions")) - .remove(1); - - ((List>) - fixtureConfig.getQueryConfigurations().get(2).getFunctionScore().get("functions")) - .remove(1); - } - - assertEquals(fixtureConfig, defaultConfig); - } -} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index eb849019b2362..bc3c892e07b1b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -12,6 +12,7 @@ import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; @@ -22,6 +23,7 @@ import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; import com.linkedin.datahub.graphql.types.dataset.DatasetType; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -43,6 +45,7 @@ import com.linkedin.r2.RemoteInvocationException; import io.datahubproject.metadata.context.OperationContext; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -63,11 +66,13 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortBuilder; +import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; -import org.testng.AssertJUnit; import org.testng.annotations.Test; public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringContextTests { + public static final String DEFAULT_CONFIG = "search_config.yaml"; + public static final YAMLMapper MAPPER = new YAMLMapper(); @Nonnull protected abstract SearchService getSearchService(); @@ -81,6 +86,9 @@ public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringCont @Nonnull protected abstract OperationContext getOperationContext(); + @Nonnull + protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); + @Test public void testSearchFieldConfig() throws IOException { /* @@ -2024,7 +2032,7 @@ public void testSortOrdering() { @Test public void testFilterOnHasValuesField() { - AssertJUnit.assertNotNull(getSearchService()); + assertNotNull(getSearchService()); Filter filter = new Filter() .setOr( @@ -2046,7 +2054,7 @@ public void testFilterOnHasValuesField() { @Test public void testFilterOnNumValuesField() { - AssertJUnit.assertNotNull(getSearchService()); + assertNotNull(getSearchService()); Filter filter = new Filter() .setOr( @@ -2066,6 +2074,34 @@ public void testFilterOnNumValuesField() { assertEquals(searchResult.getEntities().size(), 4); } + /** + * Ensure default search configuration matches the test fixture configuration (allowing for some + * differences) + */ + @Test + public void testConfig() throws IOException { + final CustomSearchConfiguration defaultConfig; + try (InputStream stream = new ClassPathResource(DEFAULT_CONFIG).getInputStream()) { + defaultConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); + } + + final CustomSearchConfiguration fixtureConfig = + MAPPER.readValue( + MAPPER.writeValueAsBytes(getCustomSearchConfiguration()), + CustomSearchConfiguration.class); + + // test specifics + ((List>) + fixtureConfig.getQueryConfigurations().get(1).getFunctionScore().get("functions")) + .remove(1); + + ((List>) + fixtureConfig.getQueryConfigurations().get(2).getFunctionScore().get("functions")) + .remove(1); + + assertEquals(fixtureConfig, defaultConfig); + } + private Stream getTokens(AnalyzeRequest request) throws IOException { return getSearchClient() diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java index db39531bba08c..9e105a69de5d1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.GoldenTestBase; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java index ef1ed51eb4799..01a9fc84c83a6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.indexbuilder.IndexBuilderTestBase; import io.datahubproject.test.search.config.SearchTestContainerConfiguration; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java index 98ac401344352..ec1c485cc0f55 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.LineageDataFixtureTestBase; @@ -10,7 +12,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Getter @@ -35,6 +36,6 @@ public class LineageDataFixtureOpenSearchTest extends LineageDataFixtureTestBase @Test public void initTest() { - AssertJUnit.assertNotNull(lineageService); + assertNotNull(lineageService); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java index 5759dafffc7b6..d24501d118925 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.LineageServiceTestBase; @@ -12,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -55,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java index 5d47e6ffd6fa5..4bf8465b14ac9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java @@ -1,8 +1,9 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase; import io.datahubproject.metadata.context.OperationContext; @@ -37,6 +38,11 @@ public class SampleDataFixtureOpenSearchTest extends SampleDataFixtureTestBase { @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java index a3a767807d7b9..1512e146948ac 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java @@ -1,8 +1,11 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.query.SearchDAOTestBase; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; @@ -28,6 +31,20 @@ public class SearchDAOOpenSearchTest extends SearchDAOTestBase { @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Autowired + @Qualifier("sampleDataEntitySearchService") + protected ElasticSearchService entitySearchService; + + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + + @Override + protected ESSearchDAO getESSearchDao() { + return entitySearchService.getEsSearchDAO(); + } + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java index 41f87d52b8847..ab1137c94f2f4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchServiceTestBase; @@ -12,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -55,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java index 7ba90319cf1d3..46bf30fcedafb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.systemmetadata.SystemMetadataServiceTestBase; @@ -8,7 +10,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -38,6 +39,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java index f4b69929d8e1a..96adf052cde45 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.TestEntityTestBase; @@ -12,7 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -55,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java index 16ac03415ee5c..b60ba08d9785b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.timeseries.search.TimeseriesAspectServiceTestBase; @@ -9,7 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -42,6 +43,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java index eafe5c7b5c310..6779b8f3d825c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.search.query; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase.DEFAULT_CONFIG; +import static com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase.MAPPER; import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static com.linkedin.metadata.utils.SearchUtil.AGGREGATION_SEPARATOR_CHAR; import static com.linkedin.metadata.utils.SearchUtil.ES_INDEX_FIELD; @@ -12,6 +14,7 @@ import com.linkedin.data.template.LongMap; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -27,15 +30,17 @@ import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain; -import com.linkedin.metadata.search.opensearch.SearchDAOOpenSearchTest; import com.linkedin.metadata.utils.SearchUtil; import io.datahubproject.metadata.context.OperationContext; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import org.opensearch.action.explain.ExplainResponse; import org.opensearch.client.RestHighLevelClient; +import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; @@ -47,6 +52,10 @@ public abstract class SearchDAOTestBase extends AbstractTestNGSpringContextTests protected abstract OperationContext getOperationContext(); + protected abstract ESSearchDAO getESSearchDao(); + + protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); + @Test public void testTransformFilterForEntitiesNoChange() { Criterion c = @@ -413,30 +422,21 @@ public void testTransformIndexIntoEntityNameNested() { @Test public void testExplain() { - ESSearchDAO searchDAO = - new ESSearchDAO( - getSearchClient(), - false, - this instanceof SearchDAOOpenSearchTest - ? ELASTICSEARCH_IMPLEMENTATION_OPENSEARCH - : ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, - getSearchConfiguration(), - null, - QueryFilterRewriteChain.EMPTY); ExplainResponse explainResponse = - searchDAO.explain( - getOperationContext() - .withSearchFlags(flags -> ElasticSearchService.DEFAULT_SERVICE_SEARCH_FLAGS), - "*", - "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact." - + "ca_border_wait_times,PROD)", - DATASET_ENTITY_NAME, - null, - null, - null, - null, - 10, - null); + getESSearchDao() + .explain( + getOperationContext() + .withSearchFlags(flags -> ElasticSearchService.DEFAULT_SERVICE_SEARCH_FLAGS), + "*", + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact." + + "ca_border_wait_times,PROD)", + DATASET_ENTITY_NAME, + null, + null, + null, + null, + 10, + null); assertNotNull(explainResponse); assertEquals(explainResponse.getIndex(), "smpldat_datasetindex_v2"); @@ -444,6 +444,34 @@ public void testExplain() { explainResponse.getId(), "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.ca_border_wait_times,PROD)"); assertTrue(explainResponse.isExists()); - assertEquals(explainResponse.getExplanation().getValue(), 18.0f); + assertEquals(explainResponse.getExplanation().getValue(), 1.25f); + } + + /** + * Ensure default search configuration matches the test fixture configuration (allowing for some + * differences) + */ + @Test + public void testConfig() throws IOException { + final CustomSearchConfiguration defaultConfig; + try (InputStream stream = new ClassPathResource(DEFAULT_CONFIG).getInputStream()) { + defaultConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); + } + + final CustomSearchConfiguration fixtureConfig = + MAPPER.readValue( + MAPPER.writeValueAsBytes(getCustomSearchConfiguration()), + CustomSearchConfiguration.class); + + // test specifics + ((List>) + fixtureConfig.getQueryConfigurations().get(1).getFunctionScore().get("functions")) + .remove(1); + + ((List>) + fixtureConfig.getQueryConfigurations().get(2).getFunctionScore().get("functions")) + .remove(1); + + assertEquals(fixtureConfig, defaultConfig); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java index 22224f16f2210..2af731a51145e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java @@ -2,7 +2,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.testng.AssertJUnit.assertEquals; +import static org.testng.Assert.assertEquals; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java index d8d33f4c356bb..772ef374af18b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.timeline.eventgenerator; -import static org.testng.AssertJUnit.assertEquals; +import static org.testng.Assert.assertEquals; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index 059749e04f6a4..889473d32d1a3 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -143,7 +143,7 @@ protected ElasticSearchService entitySearchService( false, ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, searchConfiguration, - null, + customSearchConfiguration, queryFilterRewriteChain); ESBrowseDAO browseDAO = new ESBrowseDAO( diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 5e07bfc479e93..aaeb8b5368282 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -176,8 +176,8 @@ cassandra: elasticsearch: host: ${ELASTICSEARCH_HOST:localhost} port: ${ELASTICSEARCH_PORT:9200} - threadCount: ${ELASTICSEARCH_THREAD_COUNT:1} - connectionRequestTimeout: ${ELASTICSEARCH_CONNECTION_REQUEST_TIMEOUT:0} + threadCount: ${ELASTICSEARCH_THREAD_COUNT:2} + connectionRequestTimeout: ${ELASTICSEARCH_CONNECTION_REQUEST_TIMEOUT:5000} username: ${ELASTICSEARCH_USERNAME:#{null}} password: ${ELASTICSEARCH_PASSWORD:#{null}} pathPrefix: ${ELASTICSEARCH_PATH_PREFIX:#{null}} From 41c0ed1d5d5c4e8e218f2a91976b52371c28f393 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 15:00:48 -0500 Subject: [PATCH 16/18] fix ebean test --- .../linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java index 4c87797ca1120..109c9b5c44efb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java @@ -43,9 +43,9 @@ public void testGetNextVersionForUpdate() { // Get the captured SQL statements List sql = LoggedSql.stop().stream() - .filter(str -> !str.contains("INFORMATION_SCHEMA.TABLES")) + .filter(str -> str.contains("(t0.urn,t0.aspect,t0.version)")) .toList(); - assertEquals(sql.size(), 2, String.format("Found: %s", sql)); + assertEquals(sql.size(), 1, String.format("Found: %s", sql)); assertTrue( sql.get(0).contains("for update;"), String.format("Did not find `for update` in %s ", sql)); } @@ -65,7 +65,7 @@ public void testGetLatestAspectsForUpdate() throws JsonProcessingException { // Get the captured SQL statements List sql = LoggedSql.stop().stream() - .filter(str -> !str.contains("INFORMATION_SCHEMA.TABLES")) + .filter(str -> str.contains("(t0.urn,t0.aspect,t0.version)")) .toList(); assertEquals( sql.size(), 1, String.format("Found: %s", new ObjectMapper().writeValueAsString(sql))); From cf73b169ee17c86017de4a663f94f7bd42234be3 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 16:35:36 -0500 Subject: [PATCH 17/18] fix spark smoke-test --- docker/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/build.gradle b/docker/build.gradle index 20608bd857827..e4ee7f91723fa 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -108,6 +108,9 @@ dockerCompose { environment.put "ACTIONS_EXTRA_PACKAGES", 'acryl-datahub-actions[executor] acryl-datahub-actions' environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml' environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally + // disabled for spark-lineage smoke-test + environment.put 'METADATA_SERVICE_AUTH_ENABLED', 'false' + environment.put 'REST_API_AUTHORIZATION_ENABLED', 'false' useComposeFiles = ['profiles/docker-compose.yml'] projectName = 'datahub' From a57831af4b671f7b28bff14b897f50c2cbf1862d Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 18:14:01 -0500 Subject: [PATCH 18/18] correct spark-test env --- docker/build.gradle | 3 +-- .../java/spark-lineage-legacy/spark-smoke-test/smoke-gms.env | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 metadata-integration/java/spark-lineage-legacy/spark-smoke-test/smoke-gms.env diff --git a/docker/build.gradle b/docker/build.gradle index e4ee7f91723fa..c09bf16d1d724 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -109,8 +109,7 @@ dockerCompose { environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml' environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally // disabled for spark-lineage smoke-test - environment.put 'METADATA_SERVICE_AUTH_ENABLED', 'false' - environment.put 'REST_API_AUTHORIZATION_ENABLED', 'false' + environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env" useComposeFiles = ['profiles/docker-compose.yml'] projectName = 'datahub' diff --git a/metadata-integration/java/spark-lineage-legacy/spark-smoke-test/smoke-gms.env b/metadata-integration/java/spark-lineage-legacy/spark-smoke-test/smoke-gms.env new file mode 100644 index 0000000000000..7b437a98089ce --- /dev/null +++ b/metadata-integration/java/spark-lineage-legacy/spark-smoke-test/smoke-gms.env @@ -0,0 +1,2 @@ +REST_API_AUTHORIZATION_ENABLED=false +METADATA_SERVICE_AUTH_ENABLED=false \ No newline at end of file