From 9e0f68cc94a65b6dacd289590a0bb5468f2e7d81 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:42:21 -0500 Subject: [PATCH] fix(tests): fix metadata-io tests (#11530) --- .../elasticsearch/ElasticSearchService.java | 4 +- .../elasticsearch/query/ESSearchDAO.java | 47 +++++++ .../query/request/SearchRequestHandler.java | 3 +- .../entity/DeleteEntityServiceTest.java | 4 +- .../entity/ebean/EbeanAspectDaoTest.java | 13 +- .../extractor/AspectExtractorTest.java | 2 +- .../SearchGraphServiceElasticSearchTest.java | 5 +- .../SearchGraphServiceOpenSearchTest.java | 5 +- .../LineageSearchResultCacheKeyTest.java | 4 +- .../search/LineageServiceTestBase.java | 9 +- .../search/SearchServiceTestBase.java | 9 +- .../metadata/search/TestEntityTestBase.java | 9 +- .../GoldenElasticSearchTest.java | 2 +- .../IndexBuilderElasticSearchTest.java | 2 +- .../LineageDataFixtureElasticSearchTest.java | 5 +- .../LineageServiceElasticSearchTest.java | 17 ++- .../SampleDataFixtureElasticSearchTest.java | 8 +- .../SearchDAOElasticSearchTest.java | 18 ++- .../SearchServiceElasticSearchTest.java | 17 ++- ...ystemMetadataServiceElasticSearchTest.java | 5 +- .../TestEntityElasticSearchTest.java | 17 ++- ...eseriesAspectServiceElasticSearchTest.java | 5 +- .../fixtures/SampleDataFixtureSetupTest.java | 47 ------- .../fixtures/SampleDataFixtureTestBase.java | 94 ++++++++++++-- .../opensearch/GoldenOpenSearchTest.java | 2 +- .../IndexBuilderOpenSearchTest.java | 2 +- .../LineageDataFixtureOpenSearchTest.java | 5 +- .../LineageServiceOpenSearchTest.java | 17 ++- .../SampleDataFixtureOpenSearchTest.java | 8 +- .../opensearch/SearchDAOOpenSearchTest.java | 19 ++- .../SearchServiceOpenSearchTest.java | 17 ++- .../SystemMetadataServiceOpenSearchTest.java | 5 +- .../opensearch/TestEntityOpenSearchTest.java | 17 ++- ...TimeseriesAspectServiceOpenSearchTest.java | 5 +- .../metadata/search/query/BrowseDAOTest.java | 6 +- .../search/query/SearchDAOTestBase.java | 78 ++++++++---- .../request/AggregationQueryBuilderTest.java | 2 +- .../query/request/SearchQueryBuilderTest.java | 4 + .../request/SearchRequestHandlerTest.java | 115 ++++++++++++++++++ .../metadata/search/utils/ESUtilsTest.java | 2 +- .../PropertyDefinitionValidatorTest.java | 2 +- ...chemaMetadataChangeEventGeneratorTest.java | 2 +- .../SampleDataFixtureConfiguration.java | 16 +-- .../SearchLineageFixtureConfiguration.java | 6 +- .../config/SearchCommonTestConfiguration.java | 12 +- .../common/DocumentationAssociation.pdl | 9 +- .../pegasus/com/linkedin/common/Forms.pdl | 15 ++- .../common/GlossaryTermAssociation.pdl | 9 +- .../com/linkedin/common/IncidentsSummary.pdl | 6 +- .../com/linkedin/common/RoleAssociation.pdl | 3 +- .../pegasus/com/linkedin/common/SubTypes.pdl | 2 +- .../com/linkedin/common/TagAssociation.pdl | 9 +- .../glossary/GlossaryRelatedTerms.pdl | 4 +- .../com/linkedin/identity/CorpUserInfo.pdl | 2 +- .../schema/EditableSchemaFieldInfo.pdl | 12 +- .../com/linkedin/schema/SchemaField.pdl | 18 ++- .../pegasus/com/linkedin/test/TestResults.pdl | 2 + 57 files changed, 540 insertions(+), 244 deletions(-) delete mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index e66b12db891df8..6001e2f6e660fa 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -2,6 +2,7 @@ import static com.linkedin.metadata.search.utils.SearchUtils.applyDefaultSearchFlags; +import com.google.common.annotations.VisibleForTesting; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.browse.BrowseResultV2; @@ -30,6 +31,7 @@ import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.opensearch.action.explain.ExplainResponse; @@ -51,7 +53,7 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI private static final int MAX_RUN_IDS_INDEXED = 25; // Save the previous 25 run ids in the index. private final EntityIndexBuilders indexBuilders; - private final ESSearchDAO esSearchDAO; + @VisibleForTesting @Getter private final ESSearchDAO esSearchDAO; private final ESBrowseDAO esBrowseDAO; private final ESWriteDAO esWriteDAO; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index cec73de7041263..f09a81c0c8b891 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -6,7 +6,9 @@ import com.codahale.metrics.Timer; import com.datahub.util.exception.ESQueryException; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.linkedin.data.template.LongMap; import com.linkedin.metadata.config.search.SearchConfiguration; @@ -78,6 +80,24 @@ public class ESSearchDAO { @Nonnull private final SearchConfiguration searchConfiguration; @Nullable private final CustomSearchConfiguration customSearchConfiguration; @Nonnull private final QueryFilterRewriteChain queryFilterRewriteChain; + private final boolean testLoggingEnabled; + + public ESSearchDAO( + RestHighLevelClient client, + boolean pointInTimeCreationEnabled, + String elasticSearchImplementation, + @Nonnull SearchConfiguration searchConfiguration, + @Nullable CustomSearchConfiguration customSearchConfiguration, + @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { + this( + client, + pointInTimeCreationEnabled, + elasticSearchImplementation, + searchConfiguration, + customSearchConfiguration, + queryFilterRewriteChain, + false); + } public long docCount(@Nonnull OperationContext opContext, @Nonnull String entityName) { return docCount(opContext, entityName, null); @@ -279,6 +299,11 @@ public SearchResult search( searchRequest.indices( entityNames.stream().map(indexConvention::getEntityIndexName).toArray(String[]::new)); searchRequestTimer.stop(); + + if (testLoggingEnabled) { + testLog(opContext.getObjectMapper(), searchRequest); + } + // Step 2: execute the query and extract results, validated against document model as well return executeAndExtract(opContext, entitySpecs, searchRequest, transformedFilters, from, size); } @@ -478,6 +503,11 @@ public ScrollResult scroll( } scrollRequestTimer.stop(); + + if (testLoggingEnabled) { + testLog(opContext.getObjectMapper(), searchRequest); + } + return executeAndExtract( opContext, entitySpecs, searchRequest, transformedFilters, keepAlive, size); } @@ -605,4 +635,21 @@ public ExplainResponse explain( throw new IllegalStateException("Failed to explain query:", e); } } + + private void testLog(ObjectMapper mapper, SearchRequest searchRequest) { + try { + log.warn("SearchRequest(custom): {}", mapper.writeValueAsString(customSearchConfiguration)); + final String[] indices = searchRequest.indices(); + log.warn( + String.format( + "SearchRequest(indices): %s", + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(indices))); + log.warn( + String.format( + "SearchRequest(query): %s", + mapper.writeValueAsString(mapper.readTree(searchRequest.source().toString())))); + } catch (JsonProcessingException e) { + log.warn("Error writing test log"); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index c935e6f54742c3..cb02fb1c8b2f76 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -58,6 +58,7 @@ import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.opensearch.action.search.SearchRequest; @@ -80,7 +81,7 @@ public class SearchRequestHandler { private static final Map, SearchRequestHandler> REQUEST_HANDLER_BY_ENTITY_NAME = new ConcurrentHashMap<>(); private final List entitySpecs; - private final Set defaultQueryFieldNames; + @Getter private final Set defaultQueryFieldNames; @Nonnull private final HighlightBuilder highlights; private final SearchConfiguration configs; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java index d585ff1ce8383f..0e8ee08e60739f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java @@ -2,7 +2,9 @@ import static com.linkedin.metadata.search.utils.QueryUtils.*; import static org.mockito.Mockito.*; -import static org.testng.AssertJUnit.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableList; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java index 43123fb9872a0f..109c9b5c44efb9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/EbeanAspectDaoTest.java @@ -4,6 +4,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.config.EbeanConfiguration; @@ -41,15 +43,15 @@ public void testGetNextVersionForUpdate() { // Get the captured SQL statements List sql = LoggedSql.stop().stream() - .filter(str -> !str.contains("INFORMATION_SCHEMA.TABLES")) + .filter(str -> str.contains("(t0.urn,t0.aspect,t0.version)")) .toList(); - assertEquals(sql.size(), 2, String.format("Found: %s", sql)); + assertEquals(sql.size(), 1, String.format("Found: %s", sql)); assertTrue( sql.get(0).contains("for update;"), String.format("Did not find `for update` in %s ", sql)); } @Test - public void testGetLatestAspectsForUpdate() { + public void testGetLatestAspectsForUpdate() throws JsonProcessingException { LoggedSql.start(); testDao.runInTransactionWithRetryUnlocked( @@ -63,9 +65,10 @@ public void testGetLatestAspectsForUpdate() { // Get the captured SQL statements List sql = LoggedSql.stop().stream() - .filter(str -> !str.contains("INFORMATION_SCHEMA.TABLES")) + .filter(str -> str.contains("(t0.urn,t0.aspect,t0.version)")) .toList(); - assertEquals(sql.size(), 1, String.format("Found: %s", sql)); + assertEquals( + sql.size(), 1, String.format("Found: %s", new ObjectMapper().writeValueAsString(sql))); assertTrue( sql.get(0).contains("for update;"), String.format("Did not find `for update` in %s ", sql)); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java index a98386f6f871b0..f8bf2376e6bc12 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/extractor/AspectExtractorTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.extractor; -import static org.testng.AssertJUnit.assertEquals; +import static org.testng.Assert.assertEquals; import com.datahub.test.TestEntityAspect; import com.datahub.test.TestEntityAspectArray; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java index b2c49857cb0b96..8a0dfcbe34a69e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/elasticsearch/SearchGraphServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.graph.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.graph.search.SearchGraphServiceTestBase; import com.linkedin.metadata.search.elasticsearch.ElasticSearchSuite; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -9,7 +11,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -39,6 +40,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java index 28b545f8175391..08a6ea4ef2c9c7 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/opensearch/SearchGraphServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.graph.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.graph.search.SearchGraphServiceTestBase; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; @@ -9,7 +11,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -39,6 +40,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java index 1d4a545fc06a2b..d2d27bc2de27c5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search; -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index d9268c1b50efeb..39fb6001eeb952 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -35,7 +35,6 @@ import com.linkedin.metadata.config.cache.SearchCacheConfiguration; import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.LineageDirection; @@ -103,9 +102,6 @@ public abstract class LineageServiceTestBase extends AbstractTestNGSpringContext @Nonnull protected abstract SearchConfiguration getSearchConfiguration(); - @Nonnull - protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); - private SettingsBuilder settingsBuilder; private ElasticSearchService elasticSearchService; private GraphService graphService; @@ -211,10 +207,7 @@ private ElasticSearchService buildEntitySearchService() { QueryFilterRewriteChain.EMPTY); ESBrowseDAO browseDAO = new ESBrowseDAO( - searchClientSpy, - getSearchConfiguration(), - getCustomSearchConfiguration(), - QueryFilterRewriteChain.EMPTY); + searchClientSpy, getSearchConfiguration(), null, QueryFilterRewriteChain.EMPTY); ESWriteDAO writeDAO = new ESWriteDAO(searchClientSpy, getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java index ba83a381916c29..b20326deeb9458 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java @@ -14,7 +14,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -64,9 +63,6 @@ public abstract class SearchServiceTestBase extends AbstractTestNGSpringContextT @Nonnull protected abstract SearchConfiguration getSearchConfiguration(); - @Nonnull - protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); - protected OperationContext operationContext; private SettingsBuilder settingsBuilder; private ElasticSearchService elasticSearchService; @@ -136,10 +132,7 @@ private ElasticSearchService buildEntitySearchService() { QueryFilterRewriteChain.EMPTY); ESBrowseDAO browseDAO = new ESBrowseDAO( - getSearchClient(), - getSearchConfiguration(), - getCustomSearchConfiguration(), - QueryFilterRewriteChain.EMPTY); + getSearchClient(), getSearchConfiguration(), null, QueryFilterRewriteChain.EMPTY); ESWriteDAO writeDAO = new ESWriteDAO(getSearchClient(), getBulkProcessor(), 1); return new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java index 7b6fcd46333d2d..206b97ce6c1045 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java @@ -13,7 +13,6 @@ import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.browse.BrowseResultV2; import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -53,9 +52,6 @@ public abstract class TestEntityTestBase extends AbstractTestNGSpringContextTest @Nonnull protected abstract SearchConfiguration getSearchConfiguration(); - @Nonnull - protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); - private SettingsBuilder settingsBuilder; private ElasticSearchService elasticSearchService; private OperationContext opContext; @@ -102,10 +98,7 @@ private ElasticSearchService buildService() { QueryFilterRewriteChain.EMPTY); ESBrowseDAO browseDAO = new ESBrowseDAO( - getSearchClient(), - getSearchConfiguration(), - getCustomSearchConfiguration(), - QueryFilterRewriteChain.EMPTY); + getSearchClient(), getSearchConfiguration(), null, QueryFilterRewriteChain.EMPTY); ESWriteDAO writeDAO = new ESWriteDAO(getSearchClient(), getBulkProcessor(), 1); ElasticSearchService searchService = new ElasticSearchService(indexBuilders, searchDAO, browseDAO, writeDAO); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java index 29f5964c853f13..ad30c9d0229aa6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/GoldenElasticSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.GoldenTestBase; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java index 911a21767bdeaf..af0b7003bd1d15 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/IndexBuilderElasticSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.indexbuilder.IndexBuilderTestBase; import io.datahubproject.test.search.config.SearchTestContainerConfiguration; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java index 143ae80abc52d3..fe992f61d311d6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageDataFixtureElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.LineageDataFixtureTestBase; @@ -10,7 +12,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Getter @@ -35,6 +36,6 @@ public class LineageDataFixtureElasticSearchTest extends LineageDataFixtureTestB @Test public void initTest() { - AssertJUnit.assertNotNull(lineageService); + assertNotNull(lineageService); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java index 8c4195f9ff5343..7ccf7605432eeb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/LineageServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.LineageServiceTestBase; @@ -10,8 +12,8 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -25,7 +27,10 @@ public class LineageServiceElasticSearchTest extends LineageServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override @@ -51,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java index e256f75242a42f..68b68b289dd2c0 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SampleDataFixtureElasticSearchTest.java @@ -1,8 +1,9 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase; import io.datahubproject.metadata.context.OperationContext; @@ -37,6 +38,11 @@ public class SampleDataFixtureElasticSearchTest extends SampleDataFixtureTestBas @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java index a6a8279fe86de2..3fc49a4d624fa5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchDAOElasticSearchTest.java @@ -1,8 +1,10 @@ package com.linkedin.metadata.search.elasticsearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.query.SearchDAOTestBase; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; @@ -28,6 +30,20 @@ public class SearchDAOElasticSearchTest extends SearchDAOTestBase { @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Autowired + @Qualifier("sampleDataEntitySearchService") + protected ElasticSearchService entitySearchService; + + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + + @Override + protected ESSearchDAO getESSearchDao() { + return entitySearchService.getEsSearchDAO(); + } + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java index 7133971847f982..92dfa18d4feeba 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SearchServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchServiceTestBase; @@ -10,8 +12,8 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -25,7 +27,10 @@ public class SearchServiceElasticSearchTest extends SearchServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override @@ -51,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java index a23cd5b051ecbb..b4093459ab3f1b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/SystemMetadataServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.systemmetadata.SystemMetadataServiceTestBase; @@ -8,7 +10,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -38,6 +39,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java index 5ad7b1218a5bf4..ec6f2e2b3f07a5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TestEntityElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.TestEntityTestBase; @@ -10,8 +12,8 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -25,7 +27,10 @@ public class TestEntityElasticSearchTest extends TestEntityTestBase { @Autowired private ESBulkProcessor bulkProcessor; @Autowired private ESIndexBuilder esIndexBuilder; @Autowired private SearchConfiguration searchConfiguration; - @Autowired private CustomSearchConfiguration customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; @NotNull @Override @@ -51,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(searchClient); + assertNotNull(searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java index 1f51d463a2963a..8a5f5d673aa7b9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/TimeseriesAspectServiceElasticSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.elasticsearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.timeseries.search.TimeseriesAspectServiceTestBase; @@ -9,7 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ElasticSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -42,6 +43,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java deleted file mode 100644 index b908933fcc8e37..00000000000000 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureSetupTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.linkedin.metadata.search.fixtures; - -import static org.testng.AssertJUnit.assertEquals; - -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import org.springframework.core.io.ClassPathResource; -import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; -import org.testng.annotations.Test; - -public class SampleDataFixtureSetupTest extends AbstractTestNGSpringContextTests { - private static final String DEFAULT_CONFIG = "search_config.yaml"; - private static final String TEST_FIXTURE_CONFIG = "search_config_fixture_test.yml"; - private static final YAMLMapper MAPPER = new YAMLMapper(); - - /** - * Ensure default search configuration matches the test fixture configuration (allowing for some - * differences) - */ - @Test - public void testConfig() throws IOException { - final CustomSearchConfiguration defaultConfig; - final CustomSearchConfiguration fixtureConfig; - - try (InputStream stream = new ClassPathResource(DEFAULT_CONFIG).getInputStream()) { - defaultConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); - } - try (InputStream stream = new ClassPathResource(TEST_FIXTURE_CONFIG).getInputStream()) { - fixtureConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); - - // test specifics - ((List>) - fixtureConfig.getQueryConfigurations().get(1).getFunctionScore().get("functions")) - .remove(1); - - ((List>) - fixtureConfig.getQueryConfigurations().get(2).getFunctionScore().get("functions")) - .remove(1); - } - - assertEquals(fixtureConfig, defaultConfig); - } -} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index 8cb0678180ccbf..bc3c892e07b1bb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -12,6 +12,7 @@ import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; @@ -22,6 +23,7 @@ import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; import com.linkedin.datahub.graphql.types.dataset.DatasetType; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -43,6 +45,7 @@ import com.linkedin.r2.RemoteInvocationException; import io.datahubproject.metadata.context.OperationContext; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -63,11 +66,13 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortBuilder; +import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; -import org.testng.AssertJUnit; import org.testng.annotations.Test; public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringContextTests { + public static final String DEFAULT_CONFIG = "search_config.yaml"; + public static final YAMLMapper MAPPER = new YAMLMapper(); @Nonnull protected abstract SearchService getSearchService(); @@ -81,6 +86,9 @@ public abstract class SampleDataFixtureTestBase extends AbstractTestNGSpringCont @Nonnull protected abstract OperationContext getOperationContext(); + @Nonnull + protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); + @Test public void testSearchFieldConfig() throws IOException { /* @@ -971,8 +979,13 @@ public void testSmokeTestQueries() { actualCount, expectedCount, String.format( - "Search term `%s` has %s fulltext results, expected %s results.", - key, actualCount, expectedCount)); + "Search term `%s` has %s fulltext results, expected %s results. Results: %s", + key, + actualCount, + expectedCount, + value.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); }); Map expectedStructuredMinimums = @@ -998,8 +1011,13 @@ public void testSmokeTestQueries() { actualCount, expectedCount, String.format( - "Search term `%s` has %s structured results, expected %s results.", - key, actualCount, expectedCount)); + "Search term `%s` has %s structured results, expected %s results. Results: %s", + key, + actualCount, + expectedCount, + value.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); }); } @@ -1318,6 +1336,7 @@ public void testScrollAcrossEntities() throws IOException { String query = "logging_events"; final int batchSize = 1; int totalResults = 0; + List resultUrns = new ArrayList<>(); String scrollId = null; do { ScrollResult result = @@ -1325,10 +1344,11 @@ public void testScrollAcrossEntities() throws IOException { int numResults = result.hasEntities() ? result.getEntities().size() : 0; assertTrue(numResults <= batchSize); totalResults += numResults; + resultUrns.addAll(result.getEntities().stream().map(SearchEntity::getEntity).toList()); scrollId = result.getScrollId(); } while (scrollId != null); // expect 2 total matching results - assertEquals(totalResults, 2); + assertEquals(totalResults, 2, String.format("query `%s` Results: %s", query, resultUrns)); } @Test @@ -1703,7 +1723,15 @@ public void testOr() { assertTrue( result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), String.format("%s - Expected search results to include matched fields", query)); - assertEquals(result.getEntities().size(), 2); + assertEquals( + result.getEntities().size(), + 2, + String.format( + "Query: `%s` Results: %s", + query, + result.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); } @Test @@ -1726,7 +1754,15 @@ public void testNegate() { assertTrue( result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), String.format("%s - Expected search results to include matched fields", query)); - assertEquals(result.getEntities().size(), 2); + assertEquals( + result.getEntities().size(), + 2, + String.format( + "Query: `%s` Results: %s", + query, + result.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); } @Test @@ -1896,7 +1932,15 @@ public void testPrefixVsExact() { result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), String.format("%s - Expected search results to include matched fields", query)); - assertEquals(result.getEntities().size(), 2); + assertEquals( + result.getEntities().size(), + 2, + String.format( + "Query: `%s` Results: %s", + query, + result.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); assertEquals( result.getEntities().get(0).getEntity().toString(), "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", @@ -1988,7 +2032,7 @@ public void testSortOrdering() { @Test public void testFilterOnHasValuesField() { - AssertJUnit.assertNotNull(getSearchService()); + assertNotNull(getSearchService()); Filter filter = new Filter() .setOr( @@ -2010,7 +2054,7 @@ public void testFilterOnHasValuesField() { @Test public void testFilterOnNumValuesField() { - AssertJUnit.assertNotNull(getSearchService()); + assertNotNull(getSearchService()); Filter filter = new Filter() .setOr( @@ -2030,6 +2074,34 @@ public void testFilterOnNumValuesField() { assertEquals(searchResult.getEntities().size(), 4); } + /** + * Ensure default search configuration matches the test fixture configuration (allowing for some + * differences) + */ + @Test + public void testConfig() throws IOException { + final CustomSearchConfiguration defaultConfig; + try (InputStream stream = new ClassPathResource(DEFAULT_CONFIG).getInputStream()) { + defaultConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); + } + + final CustomSearchConfiguration fixtureConfig = + MAPPER.readValue( + MAPPER.writeValueAsBytes(getCustomSearchConfiguration()), + CustomSearchConfiguration.class); + + // test specifics + ((List>) + fixtureConfig.getQueryConfigurations().get(1).getFunctionScore().get("functions")) + .remove(1); + + ((List>) + fixtureConfig.getQueryConfigurations().get(2).getFunctionScore().get("functions")) + .remove(1); + + assertEquals(fixtureConfig, defaultConfig); + } + private Stream getTokens(AnalyzeRequest request) throws IOException { return getSearchClient() diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java index db39531bba08c6..9e105a69de5d14 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/GoldenOpenSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.GoldenTestBase; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java index ef1ed51eb47991..01a9fc84c83a6f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/IndexBuilderOpenSearchTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.search.indexbuilder.IndexBuilderTestBase; import io.datahubproject.test.search.config.SearchTestContainerConfiguration; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java index 98ac4013443524..ec1c485cc0f551 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageDataFixtureOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.LineageDataFixtureTestBase; @@ -10,7 +12,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Getter @@ -35,6 +36,6 @@ public class LineageDataFixtureOpenSearchTest extends LineageDataFixtureTestBase @Test public void initTest() { - AssertJUnit.assertNotNull(lineageService); + assertNotNull(lineageService); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java index 26c2cf28cdecad..d24501d118925e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/LineageServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.LineageServiceTestBase; @@ -10,8 +12,8 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -25,7 +27,10 @@ public class LineageServiceOpenSearchTest extends LineageServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override @@ -51,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java index 5d47e6ffd6fa5c..4bf8465b14ac9d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SampleDataFixtureOpenSearchTest.java @@ -1,8 +1,9 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase; import io.datahubproject.metadata.context.OperationContext; @@ -37,6 +38,11 @@ public class SampleDataFixtureOpenSearchTest extends SampleDataFixtureTestBase { @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java index a3a767807d7b9d..1512e146948ac7 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchDAOOpenSearchTest.java @@ -1,8 +1,11 @@ package com.linkedin.metadata.search.opensearch; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertNotNull; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.query.SearchDAOTestBase; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.fixtures.search.SampleDataFixtureConfiguration; @@ -28,6 +31,20 @@ public class SearchDAOOpenSearchTest extends SearchDAOTestBase { @Qualifier("sampleDataOperationContext") protected OperationContext operationContext; + @Autowired + @Qualifier("sampleDataEntitySearchService") + protected ElasticSearchService entitySearchService; + + @Getter + @Autowired + @Qualifier("fixtureCustomSearchConfig") + protected CustomSearchConfiguration customSearchConfiguration; + + @Override + protected ESSearchDAO getESSearchDao() { + return entitySearchService.getEsSearchDAO(); + } + @Test public void initTest() { assertNotNull(searchClient); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java index 1127ba2089a91b..ab1137c94f2f43 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SearchServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.SearchServiceTestBase; @@ -10,8 +12,8 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -25,7 +27,10 @@ public class SearchServiceOpenSearchTest extends SearchServiceTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override @@ -51,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java index 7ba90319cf1d3e..46bf30fcedafb8 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/SystemMetadataServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.systemmetadata.SystemMetadataServiceTestBase; @@ -8,7 +10,6 @@ import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -38,6 +39,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java index 80db8864014c32..96adf052cde456 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TestEntityOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.search.TestEntityTestBase; @@ -10,8 +12,8 @@ import org.jetbrains.annotations.NotNull; import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({ @@ -25,7 +27,10 @@ public class TestEntityOpenSearchTest extends TestEntityTestBase { @Autowired private ESBulkProcessor _bulkProcessor; @Autowired private ESIndexBuilder _esIndexBuilder; @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @NotNull @Override @@ -51,14 +56,8 @@ protected SearchConfiguration getSearchConfiguration() { return _searchConfiguration; } - @NotNull - @Override - protected CustomSearchConfiguration getCustomSearchConfiguration() { - return _customSearchConfiguration; - } - @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java index 16ac03415ee5c2..b60ba08d9785bf 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/opensearch/TimeseriesAspectServiceOpenSearchTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.opensearch; +import static org.testng.Assert.assertNotNull; + import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.timeseries.search.TimeseriesAspectServiceTestBase; @@ -9,7 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; -import org.testng.AssertJUnit; import org.testng.annotations.Test; @Import({OpenSearchSuite.class, SearchTestContainerConfiguration.class}) @@ -42,6 +43,6 @@ protected ESIndexBuilder getIndexBuilder() { @Test public void initTest() { - AssertJUnit.assertNotNull(_searchClient); + assertNotNull(_searchClient); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java index 9c3d515f9322fb..e71865921678bb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java @@ -27,6 +27,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Import; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeMethod; @@ -39,7 +40,10 @@ public class BrowseDAOTest extends AbstractTestNGSpringContextTests { private OperationContext opContext; @Autowired private SearchConfiguration searchConfiguration; - @Autowired private CustomSearchConfiguration customSearchConfiguration; + + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; @BeforeMethod public void setup() throws RemoteInvocationException, URISyntaxException { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java index eafe5c7b5c3103..6779b8f3d825c4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/SearchDAOTestBase.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.search.query; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase.DEFAULT_CONFIG; +import static com.linkedin.metadata.search.fixtures.SampleDataFixtureTestBase.MAPPER; import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static com.linkedin.metadata.utils.SearchUtil.AGGREGATION_SEPARATOR_CHAR; import static com.linkedin.metadata.utils.SearchUtil.ES_INDEX_FIELD; @@ -12,6 +14,7 @@ import com.linkedin.data.template.LongMap; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -27,15 +30,17 @@ import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain; -import com.linkedin.metadata.search.opensearch.SearchDAOOpenSearchTest; import com.linkedin.metadata.utils.SearchUtil; import io.datahubproject.metadata.context.OperationContext; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import org.opensearch.action.explain.ExplainResponse; import org.opensearch.client.RestHighLevelClient; +import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; @@ -47,6 +52,10 @@ public abstract class SearchDAOTestBase extends AbstractTestNGSpringContextTests protected abstract OperationContext getOperationContext(); + protected abstract ESSearchDAO getESSearchDao(); + + protected abstract CustomSearchConfiguration getCustomSearchConfiguration(); + @Test public void testTransformFilterForEntitiesNoChange() { Criterion c = @@ -413,30 +422,21 @@ public void testTransformIndexIntoEntityNameNested() { @Test public void testExplain() { - ESSearchDAO searchDAO = - new ESSearchDAO( - getSearchClient(), - false, - this instanceof SearchDAOOpenSearchTest - ? ELASTICSEARCH_IMPLEMENTATION_OPENSEARCH - : ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, - getSearchConfiguration(), - null, - QueryFilterRewriteChain.EMPTY); ExplainResponse explainResponse = - searchDAO.explain( - getOperationContext() - .withSearchFlags(flags -> ElasticSearchService.DEFAULT_SERVICE_SEARCH_FLAGS), - "*", - "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact." - + "ca_border_wait_times,PROD)", - DATASET_ENTITY_NAME, - null, - null, - null, - null, - 10, - null); + getESSearchDao() + .explain( + getOperationContext() + .withSearchFlags(flags -> ElasticSearchService.DEFAULT_SERVICE_SEARCH_FLAGS), + "*", + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact." + + "ca_border_wait_times,PROD)", + DATASET_ENTITY_NAME, + null, + null, + null, + null, + 10, + null); assertNotNull(explainResponse); assertEquals(explainResponse.getIndex(), "smpldat_datasetindex_v2"); @@ -444,6 +444,34 @@ public void testExplain() { explainResponse.getId(), "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.ca_border_wait_times,PROD)"); assertTrue(explainResponse.isExists()); - assertEquals(explainResponse.getExplanation().getValue(), 18.0f); + assertEquals(explainResponse.getExplanation().getValue(), 1.25f); + } + + /** + * Ensure default search configuration matches the test fixture configuration (allowing for some + * differences) + */ + @Test + public void testConfig() throws IOException { + final CustomSearchConfiguration defaultConfig; + try (InputStream stream = new ClassPathResource(DEFAULT_CONFIG).getInputStream()) { + defaultConfig = MAPPER.readValue(stream, CustomSearchConfiguration.class); + } + + final CustomSearchConfiguration fixtureConfig = + MAPPER.readValue( + MAPPER.writeValueAsBytes(getCustomSearchConfiguration()), + CustomSearchConfiguration.class); + + // test specifics + ((List>) + fixtureConfig.getQueryConfigurations().get(1).getFunctionScore().get("functions")) + .remove(1); + + ((List>) + fixtureConfig.getQueryConfigurations().get(2).getFunctionScore().get("functions")) + .remove(1); + + assertEquals(fixtureConfig, defaultConfig); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index 0ea2340ae82173..1381e9560b7e53 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -41,7 +41,7 @@ public class AggregationQueryBuilderTest { private static AspectRetriever aspectRetrieverV1; @BeforeClass - public static void setup() throws RemoteInvocationException, URISyntaxException { + public void setup() throws RemoteInvocationException, URISyntaxException { Urn helloUrn = Urn.createFromString("urn:li:structuredProperty:hello"); Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 8d83317449a1ea..374a69ee9a5536 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -61,6 +61,10 @@ public class SearchQueryBuilderTest extends AbstractTestNGSpringContextTests { @Qualifier("queryOperationContext") private OperationContext operationContext; + @Autowired + @Qualifier("defaultTestCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; + public static SearchConfiguration testQueryConfig; static { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java index a90c0291f53b8f..a3ef62760d7972 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search.query.request; +import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES; import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion; import static com.linkedin.metadata.utils.CriterionUtils.buildIsNullCriterion; @@ -8,7 +9,10 @@ import static org.testng.Assert.*; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.TestEntitySpecBuilder; import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; @@ -35,6 +39,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.opensearch.action.search.SearchRequest; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.ExistsQueryBuilder; @@ -628,6 +633,116 @@ public void testBrowsePathQueryFilter() { assertEquals(((ExistsQueryBuilder) mustHaveV1.must().get(0)).fieldName(), "browsePaths"); } + @Test + public void testQueryByDefault() { + final Set COMMON = + Set.of( + "container", + "fieldDescriptions", + "description", + "platform", + "fieldPaths", + "editedFieldGlossaryTerms", + "editedFieldDescriptions", + "fieldTags", + "id", + "editedDescription", + "qualifiedName", + "domains", + "platformInstance", + "tags", + "urn", + "customProperties", + "fieldGlossaryTerms", + "editedName", + "name", + "fieldLabels", + "glossaryTerms", + "editedFieldTags", + "displayName", + "title"); + + Map> expectedQueryByDefault = + ImmutableMap.>builder() + .put( + EntityType.DASHBOARD, + Stream.concat(COMMON.stream(), Stream.of("tool")).collect(Collectors.toSet())) + .put( + EntityType.CHART, + Stream.concat(COMMON.stream(), Stream.of("tool")).collect(Collectors.toSet())) + .put( + EntityType.MLMODEL, + Stream.concat(COMMON.stream(), Stream.of("type")).collect(Collectors.toSet())) + .put( + EntityType.MLFEATURE_TABLE, + Stream.concat(COMMON.stream(), Stream.of("features", "primaryKeys")) + .collect(Collectors.toSet())) + .put( + EntityType.MLFEATURE, + Stream.concat(COMMON.stream(), Stream.of("featureNamespace")) + .collect(Collectors.toSet())) + .put( + EntityType.MLPRIMARY_KEY, + Stream.concat(COMMON.stream(), Stream.of("featureNamespace")) + .collect(Collectors.toSet())) + .put( + EntityType.DATA_FLOW, + Stream.concat(COMMON.stream(), Stream.of("cluster", "orchestrator", "flowId")) + .collect(Collectors.toSet())) + .put( + EntityType.DATA_JOB, + Stream.concat(COMMON.stream(), Stream.of("jobId")).collect(Collectors.toSet())) + .put( + EntityType.GLOSSARY_TERM, + Stream.concat( + COMMON.stream(), + Stream.of("values", "parentNode", "relatedTerms", "definition")) + .collect(Collectors.toSet())) + .put( + EntityType.GLOSSARY_NODE, + Stream.concat(COMMON.stream(), Stream.of("definition", "parentNode")) + .collect(Collectors.toSet())) + .put( + EntityType.CORP_USER, + Stream.concat( + COMMON.stream(), Stream.of("skills", "teams", "ldap", "fullName", "email")) + .collect(Collectors.toSet())) + .put( + EntityType.DOMAIN, + Stream.concat(COMMON.stream(), Stream.of("parentDomain")) + .collect(Collectors.toSet())) + .put( + EntityType.SCHEMA_FIELD, + Stream.concat(COMMON.stream(), Stream.of("schemaFieldAliases", "parent")) + .collect(Collectors.toSet())) + .build(); + + for (EntityType entityType : SEARCHABLE_ENTITY_TYPES) { + Set expectedEntityQueryByDefault = + expectedQueryByDefault.getOrDefault(entityType, COMMON); + assertFalse(expectedEntityQueryByDefault.isEmpty()); + + EntitySpec entitySpec = + operationContext.getEntityRegistry().getEntitySpec(EntityTypeMapper.getName(entityType)); + SearchRequestHandler handler = + SearchRequestHandler.getBuilder( + operationContext.getEntityRegistry(), + entitySpec, + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); + + Set unexpected = new HashSet<>(handler.getDefaultQueryFieldNames()); + unexpected.removeAll(expectedEntityQueryByDefault); + + assertTrue( + unexpected.isEmpty(), + String.format( + "Consider whether these field(s) for entity %s should be included for general search. Fields: %s If yes, please update the test expectations. If no, please annotate the PDL model with \"queryByDefault\": false", + entityType, unexpected)); + } + } + private BoolQueryBuilder getQuery(final Criterion filterCriterion) { final Filter filter = new Filter() diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index c5f9986284627d..03d104b9e7bfb3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -38,7 +38,7 @@ public class ESUtilsTest { private static AspectRetriever aspectRetrieverV1; @BeforeClass - public static void setup() throws RemoteInvocationException, URISyntaxException { + public void setup() throws RemoteInvocationException, URISyntaxException { Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); // legacy diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java index 22224f16f2210b..2af731a51145e3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java @@ -2,7 +2,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.testng.AssertJUnit.assertEquals; +import static org.testng.Assert.assertEquals; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java index d8d33f4c356bb8..772ef374af18b0 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.timeline.eventgenerator; -import static org.testng.AssertJUnit.assertEquals; +import static org.testng.Assert.assertEquals; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java index 781201f3478f98..e47cdf80281c9a 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java @@ -6,12 +6,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.config.search.CustomConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; @@ -73,7 +71,9 @@ public class SampleDataFixtureConfiguration { @Autowired private SearchConfiguration _searchConfiguration; - @Autowired private CustomSearchConfiguration _customSearchConfiguration; + @Autowired + @Qualifier("fixtureCustomSearchConfig") + private CustomSearchConfiguration _customSearchConfiguration; @Autowired private QueryFilterRewriteChain queryFilterRewriteChain; @@ -188,11 +188,6 @@ protected ElasticSearchService longTailEntitySearchService( protected ElasticSearchService entitySearchServiceHelper(EntityIndexBuilders indexBuilders) throws IOException { - CustomConfiguration customConfiguration = new CustomConfiguration(); - customConfiguration.setEnabled(true); - customConfiguration.setFile("search_config_fixture_test.yml"); - CustomSearchConfiguration customSearchConfiguration = - customConfiguration.resolve(new YAMLMapper()); ESSearchDAO searchDAO = new ESSearchDAO( @@ -200,8 +195,9 @@ protected ElasticSearchService entitySearchServiceHelper(EntityIndexBuilders ind false, ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, _searchConfiguration, - customSearchConfiguration, - queryFilterRewriteChain); + _customSearchConfiguration, + queryFilterRewriteChain, + true); ESBrowseDAO browseDAO = new ESBrowseDAO( _searchClient, diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index a7603f97792e70..889473d32d1a35 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -67,7 +67,9 @@ public class SearchLineageFixtureConfiguration { @Autowired private SearchConfiguration searchConfiguration; - @Autowired private CustomSearchConfiguration customSearchConfiguration; + @Autowired + @Qualifier("fixtureCustomSearchConfig") + private CustomSearchConfiguration customSearchConfiguration; @Bean(name = "searchLineagePrefix") protected String indexPrefix() { @@ -141,7 +143,7 @@ protected ElasticSearchService entitySearchService( false, ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH, searchConfiguration, - null, + customSearchConfiguration, queryFilterRewriteChain); ESBrowseDAO browseDAO = new ESBrowseDAO( diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java index 547ab1d746dbe7..e84ecf677e3a6d 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchCommonTestConfiguration.java @@ -44,14 +44,22 @@ public SearchConfiguration searchConfiguration() { return searchConfiguration; } - @Bean - public CustomSearchConfiguration customSearchConfiguration() throws Exception { + @Bean("defaultTestCustomSearchConfig") + public CustomSearchConfiguration defaultTestCustomSearchConfig() throws Exception { CustomConfiguration customConfiguration = new CustomConfiguration(); customConfiguration.setEnabled(true); customConfiguration.setFile("search_config_builder_test.yml"); return customConfiguration.resolve(new YAMLMapper()); } + @Bean("fixtureCustomSearchConfig") + public CustomSearchConfiguration fixtureCustomSearchConfig() throws Exception { + CustomConfiguration customConfiguration = new CustomConfiguration(); + customConfiguration.setEnabled(true); + customConfiguration.setFile("search_config_fixture_test.yml"); + return customConfiguration.resolve(new YAMLMapper()); + } + @Bean(name = "queryOperationContext") public OperationContext queryOperationContext() { return TestOperationContexts.systemContextNoSearchAuthorization(); diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl index 19404346797bb0..bee331f15b6797 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl @@ -15,15 +15,18 @@ record DocumentationAssociation { @Searchable = { "/time": { "fieldName": "documentationAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/actor": { "fieldName": "documentationAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/source": { "fieldName": "documentationAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } attribution: optional MetadataAttribution diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl index 0a97c7d5099ed8..3c05c00fd6fb97 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl @@ -13,19 +13,23 @@ record Forms { @Searchable = { "/*/urn": { "fieldType": "URN", - "fieldName": "incompleteForms" + "fieldName": "incompleteForms", + "queryByDefault": false, }, "/*/completedPrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "incompleteFormsCompletedPromptIds", + "queryByDefault": false, }, "/*/incompletePrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "incompleteFormsIncompletePromptIds", + "queryByDefault": false, }, "/*/completedPrompts/*/lastModified/time" : { "fieldType": "DATETIME", "fieldName": "incompleteFormsCompletedPromptResponseTimes", + "queryByDefault": false, } } incompleteForms: array[FormAssociation] @@ -36,19 +40,23 @@ record Forms { @Searchable = { "/*/urn": { "fieldType": "URN", - "fieldName": "completedForms" + "fieldName": "completedForms", + "queryByDefault": false }, "/*/completedPrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "completedFormsCompletedPromptIds", + "queryByDefault": false, }, "/*/incompletePrompts/*/id" : { "fieldType": "KEYWORD", "fieldName": "completedFormsIncompletePromptIds", + "queryByDefault": false, }, "/*/completedPrompts/*/lastModified/time" : { "fieldType": "DATETIME", "fieldName": "completedFormsCompletedPromptResponseTimes", + "queryByDefault": false, } } completedForms: array[FormAssociation] @@ -59,7 +67,8 @@ record Forms { @Searchable = { "/*/form": { "fieldType": "URN", - "fieldName": "verifiedForms" + "fieldName": "verifiedForms", + "queryByDefault": false, } } verifications: array[FormVerificationAssociation] = [] diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl index a5267bbc635e43..58423ccc2228db 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl @@ -36,15 +36,18 @@ record GlossaryTermAssociation { @Searchable = { "/time": { "fieldName": "termAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/actor": { "fieldName": "termAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/source": { "fieldName": "termAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } attribution: optional MetadataAttribution diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl index e1367a326e24bb..9e4e81656f7c9a 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/IncidentsSummary.pdl @@ -29,7 +29,8 @@ record IncidentsSummary { "fieldType": "URN", "fieldName": "resolvedIncidents", "hasValuesFieldName": "hasResolvedIncidents", - "numValuesFieldName": "numResolvedIncidents" + "numValuesFieldName": "numResolvedIncidents", + "queryByDefault": false, }, "/*/type" : { "fieldType": "KEYWORD", @@ -65,7 +66,8 @@ record IncidentsSummary { "fieldName": "activeIncidents", "hasValuesFieldName": "hasActiveIncidents", "numValuesFieldName": "numActiveIncidents", - "addHasValuesToFilters": true + "addHasValuesToFilters": true, + "queryByDefault": false, }, "/*/type" : { "fieldType": "KEYWORD", diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl index ddd63ed64014b0..05c46dfdf69d76 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl @@ -18,7 +18,8 @@ record RoleAssociation { "fieldType": "URN", "hasValuesFieldName": "hasRoles", "addToFilters": true, - "filterNameOverride": "Role" + "filterNameOverride": "Role", + "queryByDefault": false, } urn: Urn } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl index a1063afe1eae94..1f2ff275c7ae9f 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl @@ -17,7 +17,7 @@ record SubTypes { "fieldType": "KEYWORD", "addToFilters": true, "filterNameOverride": "Sub Type", - "queryByDefault": true + "queryByDefault": false, } } typeNames: array[string] diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl index 8a58ca97de1956..caed4961272ded 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl @@ -21,15 +21,18 @@ record TagAssociation { @Searchable = { "/time": { "fieldName": "tagAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/actor": { "fieldName": "tagAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/source": { "fieldName": "tagAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } attribution: optional MetadataAttribution diff --git a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl index 5e10219235347a..9f5f312a6bccec 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryRelatedTerms.pdl @@ -24,7 +24,7 @@ record GlossaryRelatedTerms { "/*": { "fieldName": "isRelatedTerms", "fieldType": "URN", - "boostScore": 2.0 + "queryByDefault": false, } } isRelatedTerms: optional array[GlossaryTermUrn] @@ -42,7 +42,7 @@ record GlossaryRelatedTerms { "/*": { "fieldName": "hasRelatedTerms", "fieldType": "URN", - "boostScore": 2.0 + "queryByDefault": false, } } hasRelatedTerms: optional array[GlossaryTermUrn] diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl index 382b120fa942a7..53c31daeca4373 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl @@ -62,7 +62,7 @@ record CorpUserInfo includes CustomProperties { @Searchable = { "fieldName": "managerLdap", "fieldType": "URN", - "queryByDefault": true + "queryByDefault": false, } managerUrn: optional CorpuserUrn diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl index 816277bd1e0c96..048c2dcd9f58f6 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl @@ -43,11 +43,13 @@ record EditableSchemaFieldInfo { }, "/tags/*/attribution/actor": { "fieldName": "editedFieldTagAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/tags/*/attribution/source": { "fieldName": "editedFieldTagAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } globalTags: optional GlobalTags @@ -73,11 +75,13 @@ record EditableSchemaFieldInfo { }, "/terms/*/attribution/actor": { "fieldName": "editedFieldTermAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/terms/*/attribution/source": { "fieldName": "editedFieldTermAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } glossaryTerms: optional GlossaryTerms diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl index f91e2004401cf9..0b72d376b0be49 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl @@ -99,15 +99,18 @@ record SchemaField { }, "/tags/*/attribution/time": { "fieldName": "fieldTagAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/tags/*/attribution/actor": { "fieldName": "fieldTagAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/tags/*/attribution/source": { "fieldName": "fieldTagAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } globalTags: optional GlobalTags @@ -129,15 +132,18 @@ record SchemaField { }, "/terms/*/attribution/time": { "fieldName": "fieldTermAttributionDates", - "fieldType": "DATETIME" + "fieldType": "DATETIME", + "queryByDefault": false, }, "/terms/*/attribution/actor": { "fieldName": "fieldTermAttributionActors", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, "/terms/*/attribution/source": { "fieldName": "fieldTermAttributionSources", - "fieldType": "URN" + "fieldType": "URN", + "queryByDefault": false, }, } glossaryTerms: optional GlossaryTerms diff --git a/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl b/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl index 6f210abf0597f4..c8a99faef88a0d 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/test/TestResults.pdl @@ -15,6 +15,7 @@ record TestResults { "fieldType": "URN", "fieldName": "failingTests" "hasValuesFieldName": "hasFailingTests", + "queryByDefault": false, } } @Relationship = { @@ -33,6 +34,7 @@ record TestResults { "fieldType": "URN", "fieldName": "passingTests", "hasValuesFieldName": "hasPassingTests", + "queryByDefault": false, } } @Relationship = {