From 7922694ed1206a97b4df3c3fed1555fc0e825723 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Mon, 28 Apr 2025 13:36:05 +0300 Subject: [PATCH 1/6] Add test that captures the different behaviour between `::data` and `::failures`. --- .../FailureStoreSecurityRestIT.java | 241 +++++++++++++++++- 1 file changed, 240 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java index d1ae6abc70125..d8725105b8846 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java @@ -105,7 +105,7 @@ public void setup() throws IOException { upsertRole(Strings.format(""" { "cluster": ["all"], - "indices": [{"names": ["test*"], "privileges": ["write", "auto_configure"]}] + "indices": [{"names": ["test*", "other*"], "privileges": ["write", "auto_configure"]}] }"""), WRITE_ACCESS); } @@ -1921,6 +1921,245 @@ public void testFailureStoreAccess() throws Exception { } } + public void testAliasBasedAccess() throws Exception { + List docIds = setupDataStream(); + assertThat(docIds.size(), equalTo(2)); + assertThat(docIds, hasItem("1")); + String dataDocId = "1"; + String failuresDocId = docIds.stream().filter(id -> false == id.equals(dataDocId)).findFirst().get(); + + List otherDocIds = setupOtherDataStream(); + assertThat(otherDocIds.size(), equalTo(2)); + assertThat(otherDocIds, hasItem("3")); + String otherDataDocId = "3"; + String otherFailuresDocId = otherDocIds.stream().filter(id -> false == id.equals(otherDataDocId)).findFirst().get(); + + final Tuple backingIndices = getSingleDataAndFailureIndices("test1"); + final String dataIndexName = backingIndices.v1(); + final String failureIndexName = backingIndices.v2(); + + final String aliasName = "my-alias"; + final String username = "user"; + final String roleName = "role"; + + createUser(username, PASSWORD, roleName); + // manage is required to add the alias to the data stream + createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["test1", "%s", "other1"], + "privileges": ["manage"] + } + ] + } + """, aliasName)); + + addAlias(username, "test1", aliasName, ""); + addAlias(username, "other1", aliasName, ""); + assertThat(fetchAliases(username, "test1"), containsInAnyOrder(aliasName)); + expectSearchThrows(username, new Search(randomFrom(aliasName + "::data", aliasName)), 403); + expectSearchThrows(username, new Search(randomFrom(aliasName + "::failures")), 403); + + createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["%s"], + "privileges": ["read_failure_store"] + } + ] + } + """, aliasName)); + expectSearch(username, new Search(aliasName + "::failures"), failuresDocId, otherFailuresDocId); + expectSearchThrows( + username, + new Search(randomFrom(aliasName + "::data", "my-alias::failures", dataIndexName, failureIndexName)), + 403 + ); + + createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["%s"], + "privileges": ["read"] + } + ] + } + """, aliasName)); + expectSearch(username, new Search(randomFrom(aliasName + "::data")), dataDocId, otherDataDocId); + expectSearchThrows(username, new Search(aliasName + "::failures"), 403); + + expectThrows(() -> removeAlias(username, "test1", aliasName), 403); + createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["test1", "%s", "other1"], + "privileges": ["manage"] + } + ] + } + """, aliasName)); + removeAlias(username, "test1", aliasName); + removeAlias(username, "other1", aliasName); + + final String filteredAliasName = "my-filtered-alias"; + createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["test1", "%s", "other1"], + "privileges": ["manage"] + } + ] + } + """, filteredAliasName)); + addAlias(username, "test1", filteredAliasName, """ + { + "term": { + "document.source.name": "jack" + } + } + """); + addAlias(username, "other1", filteredAliasName, """ + { + "term": { + "document.source.name": "jack" + } + } + """); + assertThat(fetchAliases(username, "test1"), containsInAnyOrder(filteredAliasName)); + assertThat(fetchAliases(username, "other1"), containsInAnyOrder(filteredAliasName)); + + createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["%s"], + "privileges": ["read", "read_failure_store"] + } + ] + } + """, filteredAliasName)); + + expectSearch(username, new Search(randomFrom(filteredAliasName + "::data", filteredAliasName))); + // the alias filter is not applied to the failure store + expectSearch(username, new Search(filteredAliasName + "::failures"), failuresDocId, otherFailuresDocId); + } + + private void createOrUpdateRoleAndApiKey(String username, String roleName, String roleDescriptor) throws IOException { + upsertRole(roleDescriptor, roleName); + createOrUpdateApiKey(username, randomBoolean() ? null : Strings.format(""" + { + "%s": %s + } + """, roleName, roleDescriptor)); + } + + private void addAlias(String user, String dataStream, String alias, String filter) throws IOException { + aliasAction(user, "add", dataStream, alias, filter); + } + + private void removeAlias(String user, String dataStream, String alias) throws IOException { + aliasAction(user, "remove", dataStream, alias, ""); + } + + private void aliasAction(String user, String action, String dataStream, String alias, String filter) throws IOException { + Request request = new Request("POST", "/_aliases"); + if (filter == null || filter.isEmpty()) { + request.setJsonEntity(Strings.format(""" + { + "actions": [ + { + "%s": { + "index": "%s", + "alias": "%s" + } + } + ] + } + """, action, dataStream, alias)); + } else { + request.setJsonEntity(Strings.format(""" + { + "actions": [ + { + "%s": { + "index": "%s", + "alias": "%s", + "filter": %s + } + } + ] + } + """, action, dataStream, alias, filter)); + } + Response response = performRequestMaybeUsingApiKey(user, request); + var path = assertOKAndCreateObjectPath(response); + assertThat(path.evaluate("acknowledged"), is(true)); + assertThat(path.evaluate("errors"), is(false)); + + } + + private Set fetchAliases(String user, String dataStream) throws IOException { + Response response = performRequestMaybeUsingApiKey(user, new Request("GET", dataStream + "/_alias")); + ObjectPath path = assertOKAndCreateObjectPath(response); + Map aliases = path.evaluate(dataStream + ".aliases"); + return aliases.keySet(); + } + + public void testPatternExclusions() throws Exception { + List docIds = setupDataStream(); + assertThat(docIds.size(), equalTo(2)); + assertThat(docIds, hasItem("1")); + String dataDocId = "1"; + String failuresDocId = docIds.stream().filter(id -> false == id.equals(dataDocId)).findFirst().get(); + + List otherDocIds = setupOtherDataStream(); + assertThat(otherDocIds.size(), equalTo(2)); + assertThat(otherDocIds, hasItem("3")); + String otherDataDocId = "3"; + String otherFailuresDocId = otherDocIds.stream().filter(id -> false == id.equals(otherDataDocId)).findFirst().get(); + + createUser("user", PASSWORD, "role"); + upsertRole(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["test*", "other*"], + "privileges": ["read", "read_failure_store"] + } + ] + } + """, "role"); + createAndStoreApiKey("user", randomBoolean() ? null : """ + { + "role": { + "cluster": ["all"], + "indices": [ + { + "names": ["*"], + "privileges": ["read", "read_failure_store"] + } + ] + } + } + """); + + // no exclusion -> should return two failure docs + expectSearch("user", new Search("*::failures"), failuresDocId, otherFailuresDocId); + expectSearch("user", new Search("*::failures,-other*::failures"), failuresDocId); + } + @SuppressWarnings("unchecked") private void expectEsql(String user, Search search, String... docIds) throws Exception { var response = performRequestMaybeUsingApiKey(user, search.toEsqlRequest()); From 7c016222726f678b7c4e1aa648845a21b6a96770 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Mon, 28 Apr 2025 14:10:53 +0300 Subject: [PATCH 2/6] Retrieve the already resolved alias also for failure indices --- .../TransportClusterSearchShardsAction.java | 10 +- .../action/search/TransportSearchAction.java | 10 +- .../metadata/IndexNameExpressionResolver.java | 56 +++++--- .../IndexNameExpressionResolverTests.java | 129 +++++++++++++----- 4 files changed, 133 insertions(+), 72 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 6b9315cd40157..95936f3ee9caf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.core.Predicates; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; @@ -105,14 +104,7 @@ protected void masterOperation( Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(project.metadata(), request.indices()); for (String index : concreteIndices) { final AliasFilter aliasFilter = indicesService.buildAliasFilter(project, index, indicesAndAliases); - final String[] aliases = indexNameExpressionResolver.indexAliases( - project.metadata(), - index, - Predicates.always(), - Predicates.always(), - true, - indicesAndAliases - ); + final String[] aliases = indexNameExpressionResolver.allIndexAliases(project.metadata(), index, indicesAndAliases); indicesAndFilters.put(index, AliasFilter.of(aliasFilter.getQueryBuilder(), aliases)); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 2b13ac7bd2ae0..21eeaedb7ea54 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -62,7 +62,6 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.Predicates; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -232,14 +231,7 @@ private Map buildPerIndexOriginalIndices( blocks.indexBlockedRaiseException(projectState.projectId(), ClusterBlockLevel.READ, index); } - String[] aliases = indexNameExpressionResolver.indexAliases( - projectState.metadata(), - index, - Predicates.always(), - Predicates.always(), - true, - indicesAndAliases - ); + String[] aliases = indexNameExpressionResolver.allIndexAliases(projectState.metadata(), index, indicesAndAliases); String[] finalIndices = Strings.EMPTY_ARRAY; if (aliases == null || aliases.length == 0 diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 5b6a5dc08adad..267b6bec14dce 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -59,6 +59,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.function.BiFunction; +import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Predicate; @@ -80,6 +81,13 @@ public class IndexNameExpressionResolver { public static final String EXCLUDED_DATA_STREAMS_KEY = "es.excluded_ds"; public static final IndexVersion SYSTEM_INDEX_ENFORCEMENT_INDEX_VERSION = IndexVersions.V_8_0_0; + private static final BiPredicate ALL_DATA_STREAM_ALIASES = (ignoredAlias, ignoredIsData) -> true; + // Alias filters are not applied against indices in an abstraction's failure component. + // They do not match the mapping of the data stream nor are the documents mapped for searching. + private static final BiPredicate ONLY_FILTERING_DATA_STREAM_ALIASES = ( + dataStreamAlias, + isData) -> dataStreamAlias.filteringRequired() && isData; + private final ThreadContext threadContext; private final SystemIndices systemIndices; private final ProjectResolver projectResolver; @@ -1054,12 +1062,21 @@ public String[] filteringAliases(ProjectMetadata project, String index, SetNOTE: The provided expressions must have been resolved already via {@link #resolveExpressions}. + */ + public String[] allIndexAliases(ProjectMetadata project, String index, Set resolvedExpressions) { + return indexAliases(project, index, Predicates.always(), ALL_DATA_STREAM_ALIASES, true, resolvedExpressions); + } + /** * Whether to generate the candidate set from index aliases, or from the set of resolved expressions. * @param indexAliasesSize the number of aliases of the index @@ -1080,7 +1097,7 @@ public String[] indexAliases( ProjectMetadata project, String index, Predicate requiredAlias, - Predicate requiredDataStreamAlias, + BiPredicate requiredDataStreamAlias, boolean skipIdentity, Set resolvedExpressions ) { @@ -1107,13 +1124,8 @@ public String[] indexAliases( IndexAbstraction ia = project.getIndicesLookup().get(index); DataStream dataStream = ia.getParentDataStream(); if (dataStream != null) { - if (dataStream.getFailureComponent().containsIndex(index)) { - // Alias filters are not applied against indices in an abstraction's failure component. - // They do not match the mapping of the data stream nor are the documents mapped for searching. - return null; - } - - if (skipIdentity == false && resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStream.getName())) { + boolean isData = dataStream.isFailureStoreIndex(index) == false; + if (skipIdentity == false && resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStream.getName(), isData)) { // skip the filters when the request targets the data stream name + selector directly return null; } @@ -1122,12 +1134,14 @@ public String[] indexAliases( if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) { aliasesForDataStream = dataStreamAliases.values() .stream() - .filter(dataStreamAlias -> resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStreamAlias.getName())) + .filter( + dataStreamAlias -> resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStreamAlias.getName(), isData) + ) .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); } else { aliasesForDataStream = resolvedExpressions.stream() - .filter(expression -> expression.selector() == null || expression.selector().shouldIncludeData()) + .filter(expression -> (expression.selector() == null || expression.selector().shouldIncludeData()) == isData) .map(ResolvedExpression::resource) .map(dataStreamAliases::get) .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName())) @@ -1136,11 +1150,12 @@ public String[] indexAliases( List requiredAliases = null; for (DataStreamAlias dataStreamAlias : aliasesForDataStream) { - if (requiredDataStreamAlias.test(dataStreamAlias)) { + if (requiredDataStreamAlias.test(dataStreamAlias, isData)) { if (requiredAliases == null) { requiredAliases = new ArrayList<>(aliasesForDataStream.size()); } - requiredAliases.add(dataStreamAlias.getName()); + String alias = isData ? dataStreamAlias.getName() : dataStreamAlias.getName() + "::failures"; + requiredAliases.add(alias); } else { // we have a non-required alias for this data stream so no need to check further return null; @@ -1162,7 +1177,7 @@ public String[] indexAliases( // Indices can only be referenced with a data selector, or a null selector if selectors are disabled for (AliasMetadata aliasMetadata : indexAliases.values()) { var alias = aliasMetadata.alias(); - if (resolvedExpressionsContainsAbstraction(resolvedExpressions, alias)) { + if (resolvedExpressionsContainsAbstraction(resolvedExpressions, alias, true)) { if (requiredAlias.test(aliasMetadata) == false) { return null; } @@ -1185,9 +1200,16 @@ public String[] indexAliases( } } - private static boolean resolvedExpressionsContainsAbstraction(Set resolvedExpressions, String abstractionName) { - return resolvedExpressions.contains(new ResolvedExpression(abstractionName)) - || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA)); + private static boolean resolvedExpressionsContainsAbstraction( + Set resolvedExpressions, + String abstractionName, + boolean isData + ) { + if (isData) { + return resolvedExpressions.contains(new ResolvedExpression(abstractionName)) + || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA)); + } + return resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.FAILURES)); } /** diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 8ef2205f2c127..90efac0cc80a5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -1668,7 +1668,7 @@ public void testIndexAliases() { .build(); Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "test-*"); - String[] strings = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, x -> true, true, resolvedExpressions); + String[] strings = indexNameExpressionResolver.allIndexAliases(project, "test-0", resolvedExpressions); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias-0", "test-alias-1", "test-alias-non-filtering" }, strings); @@ -1676,7 +1676,7 @@ public void testIndexAliases() { project, "test-0", x -> x.alias().equals("test-alias-1"), - x -> false, + (x, y) -> randomBoolean(), true, resolvedExpressions ); @@ -1700,52 +1700,52 @@ public void testIndexAliasesDataStreamAliases() { projectBuilder.put("logs_baz2", dataStreamName2, null, null); ProjectMetadata project = projectBuilder.build(); { - // Only resolve aliases with with that refer to dataStreamName1 + // Only resolve aliases that refer to dataStreamName1 Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); String index = backingIndex1.getIndex().getName(); - String[] result = indexNameExpressionResolver.indexAliases(project, index, x -> true, x -> true, true, resolvedExpressions); + String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_foo", "logs", "logs_bar")); } { - // Only resolve aliases with with that refer to dataStreamName2 + // Only resolve aliases that refer to dataStreamName2 Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); String index = backingIndex2.getIndex().getName(); - String[] result = indexNameExpressionResolver.indexAliases(project, index, x -> true, x -> true, true, resolvedExpressions); + String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_baz", "logs_baz2")); } { // Null is returned, because skipping identity check and resolvedExpressions contains the backing index name Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); String index = backingIndex2.getIndex().getName(); - String[] result = indexNameExpressionResolver.indexAliases(project, index, x -> true, x -> true, false, resolvedExpressions); + String[] result = indexNameExpressionResolver.indexAliases( + project, + index, + x -> randomBoolean(), + (x, y) -> true, + false, + resolvedExpressions + ); assertThat(result, nullValue()); } { // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1 Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); String index = backingIndex1.getIndex().getName(); - String[] result = indexNameExpressionResolver.indexAliases( - project, - index, - x -> true, - DataStreamAlias::filteringRequired, - true, - resolvedExpressions - ); + String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { // Null is returned, because an unfiltered alias is targeting the same data stream Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "logs_bar", "logs"); String index = backingIndex1.getIndex().getName(); - String[] result = indexNameExpressionResolver.indexAliases( - project, - index, - x -> true, - DataStreamAlias::filteringRequired, - true, - resolvedExpressions - ); + String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); + assertThat(result, nullValue()); + } + { + // Null is returned because we target the data stream name and skipIdentity is false + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, dataStreamName1, "logs"); + String index = backingIndex1.getIndex().getName(); + String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { @@ -1756,26 +1756,74 @@ public void testIndexAliasesDataStreamAliases() { project, index, x -> true, - DataStreamAlias::filteringRequired, + (alias, isData) -> alias.filteringRequired() && isData, true, resolvedExpressions ); assertThat(result, arrayContainingInAnyOrder("logs")); } + } + + public void testIndexAliasesDataStreamFailureStoreAndAliases() { + final String dataStreamName1 = "logs-foobar"; + final String dataStreamName2 = "logs-barbaz"; + IndexMetadata backingIndex1 = createBackingIndex(dataStreamName1, 1).build(); + IndexMetadata failureIndex1 = createFailureStore(dataStreamName1, 2).build(); + IndexMetadata backingIndex2 = createBackingIndex(dataStreamName2, 1).build(); + ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID) + .put(backingIndex1, false) + .put(backingIndex2, false) + .put(failureIndex1, false) + .put(newInstance(dataStreamName1, List.of(backingIndex1.getIndex()), List.of(failureIndex1.getIndex()))) + .put(newInstance(dataStreamName2, List.of(backingIndex2.getIndex()))); + projectBuilder.put("logs_foo", dataStreamName1, null, "{ \"term\": \"foo\"}"); + projectBuilder.put("logs", dataStreamName1, null, "{ \"term\": \"logs\"}"); + projectBuilder.put("logs_bar", dataStreamName1, null, null); + projectBuilder.put("logs_baz", dataStreamName2, null, "{ \"term\": \"logs\"}"); + projectBuilder.put("logs_baz2", dataStreamName2, null, null); + ProjectMetadata project = projectBuilder.build(); { - // Null is returned because we target the data stream name and skipIdentity is false - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, dataStreamName1, "logs"); - String index = backingIndex1.getIndex().getName(); + // Resolving the failure component with a backing index should return null + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + String index = randomBoolean() ? backingIndex1.getIndex().getName() : backingIndex2.getIndex().getName(); + String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); + assertThat(result, nullValue()); + } + { + // Only resolve aliases that refer to dataStreamName1 failure store + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + String index = failureIndex1.getIndex().getName(); + String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); + assertThat(result, arrayContainingInAnyOrder("logs_foo::failures", "logs::failures", "logs_bar::failures")); + } + { + // Null is returned, because we perform the identity check and resolvedExpressions contains the failure index name + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + String index = failureIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( project, index, x -> true, - DataStreamAlias::filteringRequired, + (x, y) -> true, false, resolvedExpressions ); assertThat(result, nullValue()); } + { + // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1 + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + String index = failureIndex1.getIndex().getName(); + String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); + assertThat(result, nullValue()); + } + { + // Null is returned because we target the failure store of the data stream + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "logs::failures"); + String index = failureIndex1.getIndex().getName(); + String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); + assertThat(result, nullValue()); + } } public void testIndexAliasesSkipIdentity() { @@ -1788,15 +1836,22 @@ public void testIndexAliasesSkipIdentity() { .build(); Set resolvedExpressions = resolvedExpressionsSet("test-0", "test-alias"); - String[] aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, x -> true, false, resolvedExpressions); + String[] aliases = indexNameExpressionResolver.indexAliases( + project, + "test-0", + x -> true, + (x, y) -> true, + false, + resolvedExpressions + ); assertNull(aliases); - aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, x -> true, true, resolvedExpressions); + aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, (x, y) -> true, true, resolvedExpressions); assertArrayEquals(new String[] { "test-alias" }, aliases); resolvedExpressions = Collections.singleton(new ResolvedExpression("other-alias")); - aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, x -> true, false, resolvedExpressions); + aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, (x, y) -> true, false, resolvedExpressions); assertArrayEquals(new String[] { "other-alias" }, aliases); - aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, x -> true, true, resolvedExpressions); + aliases = indexNameExpressionResolver.indexAliases(project, "test-0", x -> true, (x, y) -> true, true, resolvedExpressions); assertArrayEquals(new String[] { "other-alias" }, aliases); } @@ -1813,7 +1868,7 @@ public void testConcreteWriteIndexSuccessful() { project, "test-0", x -> true, - x -> true, + (x, y) -> true, true, resolvedExpressionsSet("test-0", "test-alias") ); @@ -1892,7 +1947,7 @@ public void testConcreteWriteIndexWithWildcardExpansion() { project, "test-0", x -> true, - x -> true, + (x, y) -> true, true, resolvedExpressionsSet("test-0", "test-1", "test-alias") ); @@ -1930,7 +1985,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { project, "test-0", x -> true, - x -> true, + (x, y) -> true, true, resolvedExpressionsSet("test-0", "test-alias") ); @@ -1964,7 +2019,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { project, "test-0", x -> true, - x -> true, + (x, y) -> true, true, Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-1"), new ResolvedExpression("test-alias")) ); @@ -2005,7 +2060,7 @@ public void testAliasResolutionNotAllowingMultipleIndices() { project, "test-0", x -> true, - x -> true, + (x, y) -> true, true, resolvedExpressionsSet("test-0", "test-1", "test-alias") ); From 76aa1d55577c75d2d4c14b530f58d5c57ef36e98 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Mon, 28 Apr 2025 14:32:33 +0300 Subject: [PATCH 3/6] Use new method in other test cases --- .../IndexNameExpressionResolverTests.java | 33 +++---------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 90efac0cc80a5..4915f55b0405e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -1864,14 +1864,7 @@ public void testConcreteWriteIndexSuccessful() { ) .build(); - String[] strings = indexNameExpressionResolver.indexAliases( - project, - "test-0", - x -> true, - (x, y) -> true, - true, - resolvedExpressionsSet("test-0", "test-alias") - ); + String[] strings = indexNameExpressionResolver.allIndexAliases(project, "test-0", resolvedExpressionsSet("test-0", "test-alias")); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); IndicesRequest request = new IndicesRequest() { @@ -1943,12 +1936,9 @@ public void testConcreteWriteIndexWithWildcardExpansion() { .putAlias(AliasMetadata.builder("test-alias").writeIndex(testZeroWriteIndex ? randomFrom(false, null) : true)) ) .build(); - String[] strings = indexNameExpressionResolver.indexAliases( + String[] strings = indexNameExpressionResolver.allIndexAliases( project, "test-0", - x -> true, - (x, y) -> true, - true, resolvedExpressionsSet("test-0", "test-1", "test-alias") ); Arrays.sort(strings); @@ -1981,14 +1971,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { ProjectMetadata project = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID) .put(indexBuilder("test-0").state(State.OPEN).putAlias(AliasMetadata.builder("test-alias").writeIndex(false))) .build(); - String[] strings = indexNameExpressionResolver.indexAliases( - project, - "test-0", - x -> true, - (x, y) -> true, - true, - resolvedExpressionsSet("test-0", "test-alias") - ); + String[] strings = indexNameExpressionResolver.allIndexAliases(project, "test-0", resolvedExpressionsSet("test-0", "test-alias")); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); DocWriteRequest request = randomFrom( @@ -2015,12 +1998,9 @@ public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { .put(indexBuilder("test-0").state(State.OPEN).putAlias(AliasMetadata.builder("test-alias").writeIndex(randomFrom(false, null)))) .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("test-alias").writeIndex(randomFrom(false, null)))) .build(); - String[] strings = indexNameExpressionResolver.indexAliases( + String[] strings = indexNameExpressionResolver.allIndexAliases( project, "test-0", - x -> true, - (x, y) -> true, - true, Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-1"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); @@ -2056,12 +2036,9 @@ public void testAliasResolutionNotAllowingMultipleIndices() { .putAlias(AliasMetadata.builder("test-alias").writeIndex(randomFrom(test0WriteIndex == false, null))) ) .build(); - String[] strings = indexNameExpressionResolver.indexAliases( + String[] strings = indexNameExpressionResolver.allIndexAliases( project, "test-0", - x -> true, - (x, y) -> true, - true, resolvedExpressionsSet("test-0", "test-1", "test-alias") ); Arrays.sort(strings); From 8c45441bc983ccf82d0a53a8b52f2e5ba6fa3594 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Mon, 28 Apr 2025 14:52:27 +0300 Subject: [PATCH 4/6] Commit the rest of the test code --- .../FailureStoreSecurityRestIT.java | 84 ++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java index d8725105b8846..39b08ddea69bf 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java @@ -2973,9 +2973,13 @@ private static void expectSearch(Response response, String... docIds) throws IOE final SearchResponse searchResponse = SearchResponseUtils.parseSearchResponse(responseAsParser(response)); try { SearchHit[] hits = searchResponse.getHits().getHits(); - assertThat(hits.length, equalTo(docIds.length)); - List actualDocIds = Arrays.stream(hits).map(SearchHit::getId).toList(); - assertThat(actualDocIds, containsInAnyOrder(docIds)); + if (docIds != null) { + assertThat(Arrays.toString(hits), hits.length, equalTo(docIds.length)); + List actualDocIds = Arrays.stream(hits).map(SearchHit::getId).toList(); + assertThat(actualDocIds, containsInAnyOrder(docIds)); + } else { + assertThat(hits.length, equalTo(0)); + } } finally { searchResponse.decRef(); } @@ -3019,6 +3023,32 @@ private List setupDataStream() throws IOException { return randomBoolean() ? populateDataStreamWithBulkRequest() : populateDataStreamWithDocRequests(); } + @SuppressWarnings("unchecked") + private List setupOtherDataStream() throws IOException { + createOtherTemplates(); + + var bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "create" : { "_index" : "other1", "_id" : "3" } } + { "@timestamp": 3, "age" : 1, "name" : "jane", "email" : "jane@example.com" } + { "create" : { "_index" : "other1", "_id" : "4" } } + { "@timestamp": 4, "age" : "this should be an int", "name" : "jane", "email" : "jane@example.com" } + """); + Response response = performRequest(WRITE_ACCESS, bulkRequest); + assertOK(response); + // we need this dance because the ID for the failed document is random, **not** 4 + Map stringObjectMap = responseAsMap(response); + List items = (List) stringObjectMap.get("items"); + List ids = new ArrayList<>(); + for (Object item : items) { + Map itemMap = (Map) item; + Map create = (Map) itemMap.get("create"); + assertThat(create.get("status"), equalTo(201)); + ids.add((String) create.get("_id")); + } + return ids; + } + private void createTemplates() throws IOException { var componentTemplateRequest = new Request("PUT", "/_component_template/component1"); componentTemplateRequest.setJsonEntity(""" @@ -3062,6 +3092,49 @@ private void createTemplates() throws IOException { assertOK(adminClient().performRequest(indexTemplateRequest)); } + private void createOtherTemplates() throws IOException { + var componentTemplateRequest = new Request("PUT", "/_component_template/component2"); + componentTemplateRequest.setJsonEntity(""" + { + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "age": { + "type": "integer" + }, + "email": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } + } + } + } + """); + assertOK(adminClient().performRequest(componentTemplateRequest)); + + var indexTemplateRequest = new Request("PUT", "/_index_template/template2"); + indexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["other*"], + "data_stream": {}, + "priority": 500, + "composed_of": ["component1"] + } + """); + assertOK(adminClient().performRequest(indexTemplateRequest)); + } + private List populateDataStreamWithDocRequests() throws IOException { List ids = new ArrayList<>(); @@ -3181,6 +3254,11 @@ protected String createAndStoreApiKey(String username, @Nullable String roleDesc return apiKeys.get(username); } + protected String createOrUpdateApiKey(String username, @Nullable String roleDescriptors) throws IOException { + apiKeys.put(username, createApiKey(username, roleDescriptors)); + return apiKeys.get(username); + } + private String createApiKey(String username, String roleDescriptors) throws IOException { var request = new Request("POST", "/_security/api_key"); if (roleDescriptors == null) { From 28c51731560d886fbfa69f1cb448ba804c7068d8 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Tue, 29 Apr 2025 09:21:09 +0300 Subject: [PATCH 5/6] Fix test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Slobodan Adamović --- .../xpack/security/failurestore/FailureStoreSecurityRestIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java index 39b08ddea69bf..5836e7b3c9878 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java @@ -1976,7 +1976,7 @@ public void testAliasBasedAccess() throws Exception { expectSearch(username, new Search(aliasName + "::failures"), failuresDocId, otherFailuresDocId); expectSearchThrows( username, - new Search(randomFrom(aliasName + "::data", "my-alias::failures", dataIndexName, failureIndexName)), + new Search(randomFrom(aliasName + "::data", aliasName, dataIndexName, failureIndexName)), 403 ); From 2d5392036bf63384be0e9da3255ee51b17ccfb7a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 29 Apr 2025 06:29:33 +0000 Subject: [PATCH 6/6] [CI] Auto commit changes from spotless --- .../security/failurestore/FailureStoreSecurityRestIT.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java index 5836e7b3c9878..23ea15483ce6c 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/failurestore/FailureStoreSecurityRestIT.java @@ -1974,11 +1974,7 @@ public void testAliasBasedAccess() throws Exception { } """, aliasName)); expectSearch(username, new Search(aliasName + "::failures"), failuresDocId, otherFailuresDocId); - expectSearchThrows( - username, - new Search(randomFrom(aliasName + "::data", aliasName, dataIndexName, failureIndexName)), - 403 - ); + expectSearchThrows(username, new Search(randomFrom(aliasName + "::data", aliasName, dataIndexName, failureIndexName)), 403); createOrUpdateRoleAndApiKey(username, roleName, Strings.format(""" {