From 3bba5438bbded26033f99069db8614c9352fb420 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 11:56:25 +0200 Subject: [PATCH 001/171] Service and request draft --- .../action/apikey/UpdateApiKeyRequest.java | 69 ++++++++ .../xpack/security/authc/ApiKeyService.java | 166 ++++++++++++++++-- 2 files changed, 224 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java new file mode 100644 index 0000000000000..2d44729bc3fb7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.action.apikey; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.core.security.action.role.RoleDescriptorRequestValidator; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.support.MetadataUtils; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public final class UpdateApiKeyRequest extends ActionRequest { + + private final String id; + private final Map metadata; + private final List roleDescriptors; + + public UpdateApiKeyRequest(String id, List roleDescriptors, Map metadata) { + this.id = Objects.requireNonNull(id, "api key id must not be null"); + this.metadata = metadata; + this.roleDescriptors = (roleDescriptors == null) ? List.of() : roleDescriptors; + } + + public UpdateApiKeyRequest(StreamInput in) throws IOException { + super(in); + this.id = in.readString(); + this.roleDescriptors = List.copyOf(in.readList(RoleDescriptor::new)); + this.metadata = in.readMap(); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (metadata != null && MetadataUtils.containsReservedMetadata(metadata)) { + validationException = addValidationError( + "API key metadata keys may not start with [" + MetadataUtils.RESERVED_PREFIX + "]", + validationException + ); + } + for (RoleDescriptor roleDescriptor : roleDescriptors) { + validationException = RoleDescriptorRequestValidator.validate(roleDescriptor, validationException); + } + return validationException; + } + + public String getId() { + return id; + } + + public Map getMetadata() { + return metadata; + } + + public List getRoleDescriptors() { + return roleDescriptors; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 447ff9191ab6f..7b2f7bb75a6a2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; @@ -73,6 +74,7 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentLocation; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.ScrollHelper; @@ -85,6 +87,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; @@ -124,6 +127,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -254,6 +258,10 @@ public void invalidateAll() { } } + private static AcknowledgedResponse toUpdateApiKeyResponse(UpdateResponse updateResponse) { + return AcknowledgedResponse.TRUE; + } + /** * Asynchronously creates a new API key based off of the request and authentication * @param authentication the authentication that this api key should be based off of @@ -332,6 +340,93 @@ private void createApiKeyAndIndexIt( })); } + public void updateApiKey( + Authentication authentication, + UpdateApiKeyRequest request, + Set userRoles, + ActionListener listener + ) { + ensureEnabled(); + if (authentication == null) { + listener.onFailure(new IllegalArgumentException("authentication must be provided")); + return; + } + logger.info("Updating api key [{}]", request.getId()); + // TODO check same user + // TODO check status; possibly no if we filter + // TODO check version + final var version = clusterService.state().nodes().getMinNodeVersion(); + findActiveApiKeyDocs(new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { + if (apiKeys.isEmpty()) { + listener.onResponse(AcknowledgedResponse.TRUE); + return; + } + // TODO what happens if `toBulkUpdateRequest` throws? + final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); + doBulkUpdate(bulkRequest, listener); + }, listener::onFailure)); + } + + private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { + securityIndex.prepareIndexIfNeededThenExecute( + listener::onFailure, + () -> executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + SECURITY_ORIGIN, + bulkUpdateRequest.request(), + ActionListener.wrap(bulkResponse -> { + logger.info("Updated API key"); + listener.onResponse(AcknowledgedResponse.TRUE); + }, listener::onFailure), + client::bulk + ) + ); + } + + private BulkRequestBuilder toBulkUpdateRequest( + Authentication authentication, + UpdateApiKeyRequest request, + Set userRoles, + Version version, + Collection versionedApiKeyDocs + ) throws IOException { + final var bulkRequestBuilder = client.prepareBulk(); + for (VersionedApiKeyDoc versionedApiKeyDoc : versionedApiKeyDocs) { + bulkRequestBuilder.add(singleIndexRequest(authentication, request, userRoles, version, versionedApiKeyDoc)); + } + bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + return bulkRequestBuilder; + } + + private IndexRequest singleIndexRequest( + Authentication authentication, + UpdateApiKeyRequest request, + Set userRoles, + Version version, + VersionedApiKeyDoc versionedApiKeyDoc + ) throws IOException { + final var apiKeyDoc = versionedApiKeyDoc.apiKey(); + return client.prepareIndex(SECURITY_MAIN_ALIAS) + .setId(request.getId()) + .setSource( + newDocument( + // TODO fill array? + apiKeyDoc.hash.toCharArray(), + apiKeyDoc.name, + authentication, + userRoles, + Instant.ofEpochMilli(apiKeyDoc.creationTime), + apiKeyDoc.expirationTime == -1 ? null : Instant.ofEpochMilli(apiKeyDoc.expirationTime), + request.getRoleDescriptors(), + version, + request.getMetadata() + ) + ) + .setIfSeqNo(versionedApiKeyDoc.seqNo()) + .setIfPrimaryTerm(versionedApiKeyDoc.primaryTerm()) + .request(); + } + /** * package-private for testing */ @@ -919,10 +1014,7 @@ public void invalidateApiKeys( ); invalidateListener.onResponse(InvalidateApiKeyResponse.emptyResponse()); } else { - invalidateAllApiKeys( - apiKeys.stream().map(apiKey -> apiKey.getId()).collect(Collectors.toSet()), - invalidateListener - ); + invalidateAllApiKeys(apiKeys.stream().map(ApiKey::getId).collect(Collectors.toSet()), invalidateListener); } }, invalidateListener::onFailure) ); @@ -938,6 +1030,16 @@ private void findApiKeys( boolean filterOutInvalidatedKeys, boolean filterOutExpiredKeys, ActionListener> listener + ) { + find(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener, ApiKeyService::convertSearchHitToApiKeyInfo); + } + + private void find( + final BoolQueryBuilder boolQuery, + boolean filterOutInvalidatedKeys, + boolean filterOutExpiredKeys, + ActionListener> listener, + Function hitParser ) { if (filterOutInvalidatedKeys) { boolQuery.filter(QueryBuilders.termQuery("api_key_invalidated", false)); @@ -959,12 +1061,7 @@ private void findApiKeys( .request(); securityIndex.checkIndexVersionThenExecute( listener::onFailure, - () -> ScrollHelper.fetchAllByEntity( - client, - request, - new ContextPreservingActionListener<>(supplier, listener), - ApiKeyService::convertSearchHitToApiKeyInfo - ) + () -> ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier, listener), hitParser) ); } } @@ -985,6 +1082,19 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } } + private void findActiveApiKeyDocs(String[] apiKeyIds, ActionListener> listener) { + findApiKeysForUserRealmApiKeyIdAndNameCombination( + null, + null, + null, + apiKeyIds, + true, + true, + listener, + ApiKeyService::convertSearchHitToVersionedApiKeyDoc + ); + } + private void findApiKeysForUserRealmApiKeyIdAndNameCombination( String[] realmNames, String userName, @@ -993,6 +1103,28 @@ private void findApiKeysForUserRealmApiKeyIdAndNameCombination( boolean filterOutInvalidatedKeys, boolean filterOutExpiredKeys, ActionListener> listener + ) { + findApiKeysForUserRealmApiKeyIdAndNameCombination( + realmNames, + userName, + apiKeyName, + apiKeyIds, + filterOutInvalidatedKeys, + filterOutExpiredKeys, + listener, + ApiKeyService::convertSearchHitToApiKeyInfo + ); + } + + private void findApiKeysForUserRealmApiKeyIdAndNameCombination( + String[] realmNames, + String userName, + String apiKeyName, + String[] apiKeyIds, + boolean filterOutInvalidatedKeys, + boolean filterOutExpiredKeys, + ActionListener> listener, + Function hitParser ) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); if (frozenSecurityIndex.indexExists() == false) { @@ -1019,7 +1151,7 @@ private void findApiKeysForUserRealmApiKeyIdAndNameCombination( boolQuery.filter(QueryBuilders.idsQuery().addIds(apiKeyIds)); } - findApiKeys(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener); + find(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener, hitParser); } } @@ -1274,6 +1406,18 @@ private static ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) { ); } + private static VersionedApiKeyDoc convertSearchHitToVersionedApiKeyDoc(SearchHit hit) { + try ( + XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, hit.getSourceRef(), XContentType.JSON) + ) { + return new VersionedApiKeyDoc(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + private record VersionedApiKeyDoc(ApiKeyDoc apiKey, long seqNo, long primaryTerm) {} + private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { return notification -> { if (RemovalReason.EVICTED == notification.getRemovalReason() && getApiKeyAuthCache().count() >= maximumWeight) { From 3a709a92c965cadabf3619bba6952cb8f5c9fe0c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 12:08:33 +0200 Subject: [PATCH 002/171] Update response --- .../action/apikey/UpdateApiKeyResponse.java | 59 +++++++++++++++++++ .../xpack/security/authc/ApiKeyService.java | 17 +++--- 2 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java new file mode 100644 index 0000000000000..db8f5b6c1b12d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.action.apikey; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public final class UpdateApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { + + private final boolean updated; + + public UpdateApiKeyResponse(boolean updated) { + this.updated = updated; + } + + public UpdateApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.updated = in.readBoolean(); + } + + public boolean isUpdated() { + return updated; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject().field("updated", updated).endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(updated); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UpdateApiKeyResponse that = (UpdateApiKeyResponse) o; + return isUpdated() == that.isUpdated(); + } + + @Override + public int hashCode() { + return Objects.hash(isUpdated()); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 7b2f7bb75a6a2..1f22671940105 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -88,6 +88,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; @@ -344,7 +345,7 @@ public void updateApiKey( Authentication authentication, UpdateApiKeyRequest request, Set userRoles, - ActionListener listener + ActionListener listener ) { ensureEnabled(); if (authentication == null) { @@ -352,13 +353,13 @@ public void updateApiKey( return; } logger.info("Updating api key [{}]", request.getId()); - // TODO check same user // TODO check status; possibly no if we filter // TODO check version final var version = clusterService.state().nodes().getMinNodeVersion(); findActiveApiKeyDocs(new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { if (apiKeys.isEmpty()) { - listener.onResponse(AcknowledgedResponse.TRUE); + // TODO 404 + listener.onResponse(new UpdateApiKeyResponse(false)); return; } // TODO what happens if `toBulkUpdateRequest` throws? @@ -367,17 +368,17 @@ public void updateApiKey( }, listener::onFailure)); } - private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { + private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute( listener::onFailure, () -> executeAsyncWithOrigin( client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkUpdateRequest.request(), - ActionListener.wrap(bulkResponse -> { - logger.info("Updated API key"); - listener.onResponse(AcknowledgedResponse.TRUE); - }, listener::onFailure), + ActionListener.wrap( + bulkResponse -> { listener.onResponse(new UpdateApiKeyResponse(true)); }, + listener::onFailure + ), client::bulk ) ); From fc53cf3b3d75212fd0a76df237c266eb3a87d9c5 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 12:20:22 +0200 Subject: [PATCH 003/171] Get api key docs for name --- .../xpack/security/authc/ApiKeyService.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 1f22671940105..74583f039ef9f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -356,16 +356,21 @@ public void updateApiKey( // TODO check status; possibly no if we filter // TODO check version final var version = clusterService.state().nodes().getMinNodeVersion(); - findActiveApiKeyDocs(new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { - if (apiKeys.isEmpty()) { - // TODO 404 - listener.onResponse(new UpdateApiKeyResponse(false)); - return; - } - // TODO what happens if `toBulkUpdateRequest` throws? - final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); - doBulkUpdate(bulkRequest, listener); - }, listener::onFailure)); + findApiKeyDocs( + new String[] { authentication.getEffectiveSubject().getRealm().getName() }, + authentication.getEffectiveSubject().getUser().principal(), + new String[] { request.getId() }, + ActionListener.wrap((apiKeys) -> { + if (apiKeys.isEmpty()) { + // TODO 404 + listener.onResponse(new UpdateApiKeyResponse(false)); + return; + } + // TODO what happens if `toBulkUpdateRequest` throws? + final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); + doBulkUpdate(bulkRequest, listener); + }, listener::onFailure) + ); } private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { @@ -1083,14 +1088,19 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } } - private void findActiveApiKeyDocs(String[] apiKeyIds, ActionListener> listener) { + private void findApiKeyDocs( + String[] realmNames, + String userName, + String[] apiKeyIds, + ActionListener> listener + ) { findApiKeysForUserRealmApiKeyIdAndNameCombination( - null, - null, + realmNames, + userName, null, apiKeyIds, - true, - true, + false, + false, listener, ApiKeyService::convertSearchHitToVersionedApiKeyDoc ); From 74d264cacce5565a9247105a9ed594d79e768351 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 12:48:08 +0200 Subject: [PATCH 004/171] More --- .../xpack/security/authc/ApiKeyService.java | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 74583f039ef9f..15a122e1390e8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -12,6 +12,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; @@ -94,6 +95,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authc.RealmDomain; +import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; @@ -353,24 +355,35 @@ public void updateApiKey( return; } logger.info("Updating api key [{}]", request.getId()); - // TODO check status; possibly no if we filter - // TODO check version - final var version = clusterService.state().nodes().getMinNodeVersion(); - findApiKeyDocs( - new String[] { authentication.getEffectiveSubject().getRealm().getName() }, - authentication.getEffectiveSubject().getUser().principal(), - new String[] { request.getId() }, - ActionListener.wrap((apiKeys) -> { - if (apiKeys.isEmpty()) { - // TODO 404 - listener.onResponse(new UpdateApiKeyResponse(false)); - return; - } - // TODO what happens if `toBulkUpdateRequest` throws? - final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); - doBulkUpdate(bulkRequest, listener); - }, listener::onFailure) - ); + findApiKeyDocsForSubject(authentication.getEffectiveSubject(), new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { + if (apiKeys.isEmpty()) { + listener.onFailure(apiKeyNotFound(request.getId())); + return; + } else if (apiKeys.size() != 1) { + listener.onFailure(new IllegalStateException("more than one api key found for single api key update")); + return; + } + final var apiKey = apiKeys.stream().iterator().next().apiKey(); + if (isActive(apiKey) == false) { + // TODO should be 400 + listener.onFailure(new IllegalStateException("api key must be active")); + return; + } + + final var version = clusterService.state().nodes().getMinNodeVersion(); + // TODO what happens if `toBulkUpdateRequest` throws? + final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); + doBulkUpdate(bulkRequest, listener); + }, listener::onFailure)); + } + + private boolean isActive(ApiKeyDoc apiKeyDoc) { + // TODO check if expired + return apiKeyDoc.invalidated == false; + } + + private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { + return new ResourceNotFoundException("api key " + apiKeyId + " not found"); } private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { @@ -400,6 +413,7 @@ private BulkRequestBuilder toBulkUpdateRequest( for (VersionedApiKeyDoc versionedApiKeyDoc : versionedApiKeyDocs) { bulkRequestBuilder.add(singleIndexRequest(authentication, request, userRoles, version, versionedApiKeyDoc)); } + // TODO should this be immediate? bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder; } @@ -1088,15 +1102,10 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } } - private void findApiKeyDocs( - String[] realmNames, - String userName, - String[] apiKeyIds, - ActionListener> listener - ) { + private void findApiKeyDocsForSubject(Subject subject, String[] apiKeyIds, ActionListener> listener) { findApiKeysForUserRealmApiKeyIdAndNameCombination( - realmNames, - userName, + new String[] { subject.getRealm().getName() }, + subject.getUser().principal(), null, apiKeyIds, false, From 0bc2ec5c408706dd083ed4f93d21cb81b9acd52e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 12:55:31 +0200 Subject: [PATCH 005/171] More --- .../xpack/security/authc/ApiKeyService.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 15a122e1390e8..a44d1b5bba93b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -354,7 +354,6 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } - logger.info("Updating api key [{}]", request.getId()); findApiKeyDocsForSubject(authentication.getEffectiveSubject(), new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { if (apiKeys.isEmpty()) { listener.onFailure(apiKeyNotFound(request.getId())); @@ -363,12 +362,13 @@ public void updateApiKey( listener.onFailure(new IllegalStateException("more than one api key found for single api key update")); return; } - final var apiKey = apiKeys.stream().iterator().next().apiKey(); - if (isActive(apiKey) == false) { + final var apiKeyDoc = apiKeys.stream().iterator().next().apiKey(); + if (isActive(apiKeyDoc) == false) { // TODO should be 400 listener.onFailure(new IllegalStateException("api key must be active")); return; } + // TODO assert on creator final var version = clusterService.state().nodes().getMinNodeVersion(); // TODO what happens if `toBulkUpdateRequest` throws? @@ -1045,16 +1045,7 @@ private void invalidateAllApiKeys(Collection apiKeyIds, ActionListener> listener - ) { - find(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener, ApiKeyService::convertSearchHitToApiKeyInfo); - } - - private void find( + private void findApiKeys( final BoolQueryBuilder boolQuery, boolean filterOutInvalidatedKeys, boolean filterOutExpiredKeys, @@ -1171,7 +1162,7 @@ private void findApiKeysForUserRealmApiKeyIdAndNameCombination( boolQuery.filter(QueryBuilders.idsQuery().addIds(apiKeyIds)); } - find(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener, hitParser); + findApiKeys(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener, hitParser); } } From 648ae529c9d8e0bdfdc3c183260ddf6f747c6861 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 13:35:10 +0200 Subject: [PATCH 006/171] Check expiration --- .../xpack/security/authc/ApiKeyService.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index a44d1b5bba93b..62cd9456b3b88 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -358,7 +358,10 @@ public void updateApiKey( if (apiKeys.isEmpty()) { listener.onFailure(apiKeyNotFound(request.getId())); return; - } else if (apiKeys.size() != 1) { + } + + // Validation + if (apiKeys.size() != 1) { listener.onFailure(new IllegalStateException("more than one api key found for single api key update")); return; } @@ -378,8 +381,8 @@ public void updateApiKey( } private boolean isActive(ApiKeyDoc apiKeyDoc) { - // TODO check if expired - return apiKeyDoc.invalidated == false; + return apiKeyDoc.invalidated == false + && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); } private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { From 2d43dda12a7f2cf0b725b09641700a5548329b23 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 14:22:23 +0200 Subject: [PATCH 007/171] Integ test --- .../security/authc/ApiKeyIntegTests.java | 31 +++++++++++++++++++ .../xpack/security/authc/ApiKeyService.java | 1 + 2 files changed, 32 insertions(+) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 4a3e964e189c9..7735f4b8ba856 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -52,6 +52,8 @@ import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequestBuilder; import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse; @@ -59,6 +61,8 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; +import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.transport.filter.IPFilter; @@ -89,6 +93,7 @@ import java.util.stream.Stream; import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER; +import static org.elasticsearch.test.SecuritySettingsSourceField.ES_TEST_ROOT_ROLE; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -331,6 +336,32 @@ public void testInvalidateApiKeysForApiKeyName() throws InterruptedException, Ex verifyInvalidateResponse(1, responses, invalidateResponse); } + public void testUpdateApiKey() throws ExecutionException, InterruptedException { + CreateApiKeyResponse createResponse = createApiKeys(1, null).v1().get(0); + String[] nodeNames = internalCluster().getNodeNames(); + final List services = Arrays.stream(nodeNames) + .map(n -> internalCluster().getInstance(ApiKeyService.class, n)) + .toList(); + ApiKeyService service = services.get(0); + + final RoleDescriptor descriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); + + PlainActionFuture listener = new PlainActionFuture<>(); + service.updateApiKey( + Authentication.newRealmAuthentication( + new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new Authentication.RealmRef("file", FileRealmSettings.TYPE, nodeNames[0]) + ), + new UpdateApiKeyRequest(createResponse.getId(), List.of(descriptor), ApiKeyTests.randomMetadata()), + Set.of(descriptor), + listener + ); + + UpdateApiKeyResponse response = listener.get(); + assertNotNull(response); + assertTrue(response.isUpdated()); + } + public void testInvalidateApiKeyWillClearApiKeyCache() throws IOException, ExecutionException, InterruptedException { final List services = Arrays.stream(internalCluster().getNodeNames()) .map(n -> internalCluster().getInstance(ApiKeyService.class, n)) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 62cd9456b3b88..512f95fd5fe12 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -382,6 +382,7 @@ public void updateApiKey( private boolean isActive(ApiKeyDoc apiKeyDoc) { return apiKeyDoc.invalidated == false + // TODO shared && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); } From 4384e563714c4d3fc7bf162749b23c985ccd75db Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 15:02:13 +0200 Subject: [PATCH 008/171] More --- .../security/authc/ApiKeyIntegTests.java | 64 +++++++++++-------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 7735f4b8ba856..50e3aa68ebd9a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -61,7 +61,6 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; -import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; @@ -336,32 +335,6 @@ public void testInvalidateApiKeysForApiKeyName() throws InterruptedException, Ex verifyInvalidateResponse(1, responses, invalidateResponse); } - public void testUpdateApiKey() throws ExecutionException, InterruptedException { - CreateApiKeyResponse createResponse = createApiKeys(1, null).v1().get(0); - String[] nodeNames = internalCluster().getNodeNames(); - final List services = Arrays.stream(nodeNames) - .map(n -> internalCluster().getInstance(ApiKeyService.class, n)) - .toList(); - ApiKeyService service = services.get(0); - - final RoleDescriptor descriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - - PlainActionFuture listener = new PlainActionFuture<>(); - service.updateApiKey( - Authentication.newRealmAuthentication( - new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new Authentication.RealmRef("file", FileRealmSettings.TYPE, nodeNames[0]) - ), - new UpdateApiKeyRequest(createResponse.getId(), List.of(descriptor), ApiKeyTests.randomMetadata()), - Set.of(descriptor), - listener - ); - - UpdateApiKeyResponse response = listener.get(); - assertNotNull(response); - assertTrue(response.isUpdated()); - } - public void testInvalidateApiKeyWillClearApiKeyCache() throws IOException, ExecutionException, InterruptedException { final List services = Arrays.stream(internalCluster().getNodeNames()) .map(n -> internalCluster().getInstance(ApiKeyService.class, n)) @@ -1436,6 +1409,38 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce }); } + public void testUpdateApiKey() throws ExecutionException, InterruptedException { + final var createdApiKey = createApiKey(null); + + final var role = new RoleDescriptor("role", new String[] { "monitor" }, null, null); + final var request = new UpdateApiKeyRequest(createdApiKey.v1().getId(), List.of(role), ApiKeyTests.randomMetadata()); + + final PlainActionFuture listener = new PlainActionFuture<>(); + final var serviceWithNodeName = getServiceWithNodeName(); + serviceWithNodeName.service() + .updateApiKey( + Authentication.newRealmAuthentication( + new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new Authentication.RealmRef("file", FileRealmSettings.TYPE, serviceWithNodeName.nodeName()) + ), + request, + Set.of(role), + listener + ); + UpdateApiKeyResponse response = listener.get(); + + assertNotNull(response); + assertTrue(response.isUpdated()); + } + + private ServiceWithNodeName getServiceWithNodeName() { + final var nodeName = internalCluster().getNodeNames()[0]; + final var service = internalCluster().getInstance(ApiKeyService.class, nodeName); + return new ServiceWithNodeName(service, nodeName); + } + + private record ServiceWithNodeName(ApiKeyService service, String nodeName) {} + private Tuple createApiKeyAndAuthenticateWithIt() throws IOException { Client client = client().filterWithHeader( Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) @@ -1566,6 +1571,11 @@ private void verifyGetResponse( } } + private Tuple> createApiKey(TimeValue expiration) { + final var res = createApiKeys(ES_TEST_ROOT_USER, 1, expiration, "monitor"); + return new Tuple<>(res.v1().get(0), res.v2().get(0)); + } + private Tuple, List>> createApiKeys(int noOfApiKeys, TimeValue expiration) { return createApiKeys(ES_TEST_ROOT_USER, noOfApiKeys, expiration, "monitor"); } From 3d73b99262e4ad3940c4676d7b7cdbda4bd96a6d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 15:25:56 +0200 Subject: [PATCH 009/171] Assert role descriptors as expected --- .../security/authc/ApiKeyIntegTests.java | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 50e3aa68ebd9a..1566082add282 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -16,6 +16,9 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -36,8 +39,10 @@ import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.test.TestSecurityClient; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheAction; import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheRequest; @@ -64,6 +69,7 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.authc.service.ServiceAccountService; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.junit.After; import org.junit.Before; @@ -104,6 +110,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -1409,11 +1416,11 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce }); } - public void testUpdateApiKey() throws ExecutionException, InterruptedException { + public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final var createdApiKey = createApiKey(null); - - final var role = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - final var request = new UpdateApiKeyRequest(createdApiKey.v1().getId(), List.of(role), ApiKeyTests.randomMetadata()); + final var apiKeyId = createdApiKey.v1().getId(); + final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); + final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); final PlainActionFuture listener = new PlainActionFuture<>(); final var serviceWithNodeName = getServiceWithNodeName(); @@ -1424,13 +1431,38 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException { new Authentication.RealmRef("file", FileRealmSettings.TYPE, serviceWithNodeName.nodeName()) ), request, - Set.of(role), + Set.of(expectedRoleDescriptor), listener ); UpdateApiKeyResponse response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); + + // Assert that we can authenticate with the updated API key + final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); + assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); + + final var updatedDocument = getApiKeyDocument(apiKeyId); + @SuppressWarnings("unchecked") + final var limitedRoleDescriptor = (Map) updatedDocument.get("limited_by_role_descriptors"); + assertThat(limitedRoleDescriptor.size(), equalTo(1)); + assertThat(limitedRoleDescriptor, hasKey(expectedRoleDescriptor.getName())); + + @SuppressWarnings("unchecked") + final var descriptor = (Map) limitedRoleDescriptor.get(expectedRoleDescriptor.getName()); + final var roleDescriptor = RoleDescriptor.parse( + expectedRoleDescriptor.getName(), + XContentTestUtils.convertToXContent(descriptor, XContentType.JSON), + false, + XContentType.JSON + ); + assertEquals(roleDescriptor, expectedRoleDescriptor); + } + + private Map getApiKeyDocument(String apiKeyId) { + final GetResponse getResponse = client().execute(GetAction.INSTANCE, new GetRequest(SECURITY_MAIN_ALIAS, apiKeyId)).actionGet(); + return getResponse.getSource(); } private ServiceWithNodeName getServiceWithNodeName() { From ff54fec919a7b3b38e5d0d71cdd1ec61ea2c353a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 15:30:49 +0200 Subject: [PATCH 010/171] Clean up role descriptor checks --- .../xpack/security/authc/ApiKeyIntegTests.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 1566082add282..273e4cc787095 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -69,7 +69,6 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.authc.service.ServiceAccountService; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.junit.After; import org.junit.Before; @@ -1419,7 +1418,7 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final var createdApiKey = createApiKey(null); final var apiKeyId = createdApiKey.v1().getId(); - final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); + final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); final PlainActionFuture listener = new PlainActionFuture<>(); @@ -1443,14 +1442,21 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); + assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "role_descriptors"); + assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "limited_by_role_descriptors"); + } + + private void assertExpectedRoleDescriptorForApiKey(String apiKeyId, RoleDescriptor expectedRoleDescriptor, String roleDescriptorType) + throws IOException { + assertThat(roleDescriptorType, in(new String[] { "role_descriptors", "limited_by_role_descriptors" })); final var updatedDocument = getApiKeyDocument(apiKeyId); @SuppressWarnings("unchecked") - final var limitedRoleDescriptor = (Map) updatedDocument.get("limited_by_role_descriptors"); - assertThat(limitedRoleDescriptor.size(), equalTo(1)); - assertThat(limitedRoleDescriptor, hasKey(expectedRoleDescriptor.getName())); + final var rawRoleDescriptor = (Map) updatedDocument.get(roleDescriptorType); + assertThat(rawRoleDescriptor.size(), equalTo(1)); + assertThat(rawRoleDescriptor, hasKey(expectedRoleDescriptor.getName())); @SuppressWarnings("unchecked") - final var descriptor = (Map) limitedRoleDescriptor.get(expectedRoleDescriptor.getName()); + final var descriptor = (Map) rawRoleDescriptor.get(expectedRoleDescriptor.getName()); final var roleDescriptor = RoleDescriptor.parse( expectedRoleDescriptor.getName(), XContentTestUtils.convertToXContent(descriptor, XContentType.JSON), From e4849239e8e52b22a34f3c2845b3b3fa39963dfc Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 15:56:54 +0200 Subject: [PATCH 011/171] Test not found --- .../security/authc/ApiKeyIntegTests.java | 40 +++++++++++++------ .../xpack/security/authc/ApiKeyService.java | 2 +- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 273e4cc787095..93da52e8601fc 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security.authc; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; @@ -105,12 +106,14 @@ import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -1424,26 +1427,37 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final PlainActionFuture listener = new PlainActionFuture<>(); final var serviceWithNodeName = getServiceWithNodeName(); serviceWithNodeName.service() - .updateApiKey( - Authentication.newRealmAuthentication( - new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new Authentication.RealmRef("file", FileRealmSettings.TYPE, serviceWithNodeName.nodeName()) - ), - request, - Set.of(expectedRoleDescriptor), - listener - ); + .updateApiKey(rootUserFileRealmAuth(serviceWithNodeName), request, Set.of(expectedRoleDescriptor), listener); UpdateApiKeyResponse response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); + assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "role_descriptors"); + assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "limited_by_role_descriptors"); - // Assert that we can authenticate with the updated API key final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); - assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "role_descriptors"); - assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "limited_by_role_descriptors"); + // Test not found exception on non-existent API key + final var otherApiKeyId = randomValueOtherThan(apiKeyId, () -> randomAlphaOfLength(20)); + final PlainActionFuture listener2 = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey( + rootUserFileRealmAuth(serviceWithNodeName), + new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()), + Set.of(expectedRoleDescriptor), + listener2 + ); + ExecutionException ex = expectThrows(ExecutionException.class, listener2::get); + assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); + assertThat(ex.getMessage(), containsString("api key [" + otherApiKeyId + "] not found")); + } + + private Authentication rootUserFileRealmAuth(ServiceWithNodeName serviceWithNodeName) { + return Authentication.newRealmAuthentication( + new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new Authentication.RealmRef("file", FileRealmSettings.TYPE, serviceWithNodeName.nodeName()) + ); } private void assertExpectedRoleDescriptorForApiKey(String apiKeyId, RoleDescriptor expectedRoleDescriptor, String roleDescriptorType) @@ -1463,7 +1477,7 @@ private void assertExpectedRoleDescriptorForApiKey(String apiKeyId, RoleDescript false, XContentType.JSON ); - assertEquals(roleDescriptor, expectedRoleDescriptor); + assertThat(roleDescriptor, equalTo(expectedRoleDescriptor)); } private Map getApiKeyDocument(String apiKeyId) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 512f95fd5fe12..8802cd8261f7c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -387,7 +387,7 @@ private boolean isActive(ApiKeyDoc apiKeyDoc) { } private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { - return new ResourceNotFoundException("api key " + apiKeyId + " not found"); + return new ResourceNotFoundException("api key [" + apiKeyId + "] not found"); } private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { From a9c8caa06e20c7b14ef5e2ff22c04cdbd5fce681 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 16:11:26 +0200 Subject: [PATCH 012/171] WIP expected metadata --- .../security/authc/ApiKeyIntegTests.java | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 93da52e8601fc..29e387f148d89 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -106,7 +106,6 @@ import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayWithSize; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -1422,7 +1421,8 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var createdApiKey = createApiKey(null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); - final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); + final var expectedMetadata = ApiKeyTests.randomMetadata(); + final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), expectedMetadata); final PlainActionFuture listener = new PlainActionFuture<>(); final var serviceWithNodeName = getServiceWithNodeName(); @@ -1432,9 +1432,12 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertNotNull(response); assertTrue(response.isUpdated()); - assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "role_descriptors"); - assertExpectedRoleDescriptorForApiKey(apiKeyId, expectedRoleDescriptor, "limited_by_role_descriptors"); + final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); + expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + // Test authenticate works with updated API key final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); @@ -1460,16 +1463,25 @@ private Authentication rootUserFileRealmAuth(ServiceWithNodeName serviceWithNode ); } - private void assertExpectedRoleDescriptorForApiKey(String apiKeyId, RoleDescriptor expectedRoleDescriptor, String roleDescriptorType) - throws IOException { - assertThat(roleDescriptorType, in(new String[] { "role_descriptors", "limited_by_role_descriptors" })); - final var updatedDocument = getApiKeyDocument(apiKeyId); + private void expectMetadataForApiKey(Map expectedMetadata, Map actualRawApiKeyDoc) { + assertNotNull(actualRawApiKeyDoc); @SuppressWarnings("unchecked") - final var rawRoleDescriptor = (Map) updatedDocument.get(roleDescriptorType); + final var actualMetadata = (Map) actualRawApiKeyDoc.get("flattened_metadata"); + assertThat(actualMetadata, equalTo(expectedMetadata)); + } + + @SuppressWarnings("unchecked") + private void expectRoleDescriptorForApiKey( + String roleDescriptorType, + RoleDescriptor expectedRoleDescriptor, + Map actualRawApiKeyDoc + ) throws IOException { + assertNotNull(actualRawApiKeyDoc); + assertThat(roleDescriptorType, in(new String[] { "role_descriptors", "limited_by_role_descriptors" })); + final var rawRoleDescriptor = (Map) actualRawApiKeyDoc.get(roleDescriptorType); assertThat(rawRoleDescriptor.size(), equalTo(1)); assertThat(rawRoleDescriptor, hasKey(expectedRoleDescriptor.getName())); - @SuppressWarnings("unchecked") final var descriptor = (Map) rawRoleDescriptor.get(expectedRoleDescriptor.getName()); final var roleDescriptor = RoleDescriptor.parse( expectedRoleDescriptor.getName(), From d638a45730b5e796973d893a9c0a89e0c455ef47 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 16:19:53 +0200 Subject: [PATCH 013/171] Fix metadata --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 29e387f148d89..97c84e3c0a729 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1466,8 +1466,8 @@ private Authentication rootUserFileRealmAuth(ServiceWithNodeName serviceWithNode private void expectMetadataForApiKey(Map expectedMetadata, Map actualRawApiKeyDoc) { assertNotNull(actualRawApiKeyDoc); @SuppressWarnings("unchecked") - final var actualMetadata = (Map) actualRawApiKeyDoc.get("flattened_metadata"); - assertThat(actualMetadata, equalTo(expectedMetadata)); + final var actualMetadata = (Map) actualRawApiKeyDoc.get("metadata_flattened"); + assertThat("for api key doc " + actualRawApiKeyDoc, actualMetadata, equalTo(expectedMetadata)); } @SuppressWarnings("unchecked") From 0c819e4c2eff2b933a7781adaf1545ca3f44060e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 16:21:00 +0200 Subject: [PATCH 014/171] Nit --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 97c84e3c0a729..76d508aa0b14b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1451,7 +1451,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, Set.of(expectedRoleDescriptor), listener2 ); - ExecutionException ex = expectThrows(ExecutionException.class, listener2::get); + final var ex = expectThrows(ExecutionException.class, listener2::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); assertThat(ex.getMessage(), containsString("api key [" + otherApiKeyId + "] not found")); } From abe677bb175a55290bf5d71489a0f1bdedab5a88 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 17:24:05 +0200 Subject: [PATCH 015/171] Fix setting metadata --- .../action/apikey/UpdateApiKeyRequest.java | 2 +- .../security/authc/ApiKeyIntegTests.java | 25 +++++++++++++------ .../xpack/security/authc/ApiKeyService.java | 17 +++++++++++-- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 2d44729bc3fb7..ba058fd127db9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -29,8 +29,8 @@ public final class UpdateApiKeyRequest extends ActionRequest { public UpdateApiKeyRequest(String id, List roleDescriptors, Map metadata) { this.id = Objects.requireNonNull(id, "api key id must not be null"); - this.metadata = metadata; this.roleDescriptors = (roleDescriptors == null) ? List.of() : roleDescriptors; + this.metadata = metadata; } public UpdateApiKeyRequest(StreamInput in) throws IOException { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 76d508aa0b14b..345cf47da9fb1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1421,19 +1421,28 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var createdApiKey = createApiKey(null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); - final var expectedMetadata = ApiKeyTests.randomMetadata(); - final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), expectedMetadata); + final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); - final PlainActionFuture listener = new PlainActionFuture<>(); final var serviceWithNodeName = getServiceWithNodeName(); + final PlainActionFuture listener = new PlainActionFuture<>(); serviceWithNodeName.service() - .updateApiKey(rootUserFileRealmAuth(serviceWithNodeName), request, Set.of(expectedRoleDescriptor), listener); + .updateApiKey( + fileRealmAuthForEsTestRootUser(serviceWithNodeName.nodeName()), + request, + Set.of(expectedRoleDescriptor), + listener + ); UpdateApiKeyResponse response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); - expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); + if (request.getMetadata() != null) { + expectMetadataForApiKey(request.getMetadata(), updatedApiKeyDoc); + } else { + // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it + expectMetadataForApiKey(createdApiKey.v2(), updatedApiKeyDoc); + } expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); @@ -1446,7 +1455,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final PlainActionFuture listener2 = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( - rootUserFileRealmAuth(serviceWithNodeName), + fileRealmAuthForEsTestRootUser(serviceWithNodeName.nodeName()), new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()), Set.of(expectedRoleDescriptor), listener2 @@ -1456,10 +1465,10 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertThat(ex.getMessage(), containsString("api key [" + otherApiKeyId + "] not found")); } - private Authentication rootUserFileRealmAuth(ServiceWithNodeName serviceWithNodeName) { + private Authentication fileRealmAuthForEsTestRootUser(String nodeName) { return Authentication.newRealmAuthentication( new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new Authentication.RealmRef("file", FileRealmSettings.TYPE, serviceWithNodeName.nodeName()) + new Authentication.RealmRef("file", FileRealmSettings.TYPE, nodeName) ); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 8802cd8261f7c..25bfbc1427d88 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -417,7 +417,6 @@ private BulkRequestBuilder toBulkUpdateRequest( for (VersionedApiKeyDoc versionedApiKeyDoc : versionedApiKeyDocs) { bulkRequestBuilder.add(singleIndexRequest(authentication, request, userRoles, version, versionedApiKeyDoc)); } - // TODO should this be immediate? bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder; } @@ -430,6 +429,19 @@ private IndexRequest singleIndexRequest( VersionedApiKeyDoc versionedApiKeyDoc ) throws IOException { final var apiKeyDoc = versionedApiKeyDoc.apiKey(); + + // TODO gnarly + // Only write new metadata if it's present, i.e., not null. Otherwise, use existing old metadata + final Map metadata; + try ( + XContentParser parser = XContentHelper.createParser( + XContentParserConfiguration.EMPTY, + apiKeyDoc.metadataFlattened, + XContentType.JSON + ) + ) { + metadata = request.getMetadata() != null ? request.getMetadata() : parser.map(); + } return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) .setSource( @@ -441,9 +453,10 @@ private IndexRequest singleIndexRequest( userRoles, Instant.ofEpochMilli(apiKeyDoc.creationTime), apiKeyDoc.expirationTime == -1 ? null : Instant.ofEpochMilli(apiKeyDoc.expirationTime), + // TODO also fix this request.getRoleDescriptors(), version, - request.getMetadata() + metadata ) ) .setIfSeqNo(versionedApiKeyDoc.seqNo()) From 3c7b2c1d78f92f6b4f35f6844aaf1ed5f4f90fe8 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 17:25:45 +0200 Subject: [PATCH 016/171] Nit --- .../xpack/security/authc/ApiKeyIntegTests.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 345cf47da9fb1..5dd1c24699cf0 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1437,12 +1437,8 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertNotNull(response); assertTrue(response.isUpdated()); final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); - if (request.getMetadata() != null) { - expectMetadataForApiKey(request.getMetadata(), updatedApiKeyDoc); - } else { - // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it - expectMetadataForApiKey(createdApiKey.v2(), updatedApiKeyDoc); - } + // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it + expectMetadataForApiKey(request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), updatedApiKeyDoc); expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); From 29f0320a94596c369e41701cb1349b1440f0efad Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 18:41:46 +0200 Subject: [PATCH 017/171] WIP use old role descriptors --- .../action/apikey/UpdateApiKeyRequest.java | 4 ++- .../security/authc/ApiKeyIntegTests.java | 27 +++++++++++++++ .../xpack/security/authc/ApiKeyService.java | 33 +++++++++++++++++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index ba058fd127db9..5ab8946062b40 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -29,18 +29,20 @@ public final class UpdateApiKeyRequest extends ActionRequest { public UpdateApiKeyRequest(String id, List roleDescriptors, Map metadata) { this.id = Objects.requireNonNull(id, "api key id must not be null"); - this.roleDescriptors = (roleDescriptors == null) ? List.of() : roleDescriptors; + this.roleDescriptors = roleDescriptors; this.metadata = metadata; } public UpdateApiKeyRequest(StreamInput in) throws IOException { super(in); this.id = in.readString(); + // TODO handle null this.roleDescriptors = List.copyOf(in.readList(RoleDescriptor::new)); this.metadata = in.readMap(); } @Override + // TODO test me public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (metadata != null && MetadataUtils.containsReservedMetadata(metadata)) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 5dd1c24699cf0..945ff095189b2 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1461,6 +1461,33 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertThat(ex.getMessage(), containsString("api key [" + otherApiKeyId + "] not found")); } + public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExistingRoleDescriptors() throws ExecutionException, + InterruptedException, IOException { + final var createdApiKey = createApiKey(null); + final var apiKeyId = createdApiKey.v1().getId(); + + final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); + final var expectedLimitedByRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var request = new UpdateApiKeyRequest(apiKeyId, null, ApiKeyTests.randomMetadata()); + + final var serviceWithNodeName = getServiceWithNodeName(); + final PlainActionFuture listener = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey( + fileRealmAuthForEsTestRootUser(serviceWithNodeName.nodeName()), + request, + Set.of(expectedLimitedByRoleDescriptor), + listener + ); + UpdateApiKeyResponse response = listener.get(); + + assertNotNull(response); + assertTrue(response.isUpdated()); + final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); + expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); + } + private Authentication fileRealmAuthForEsTestRootUser(String nodeName) { return Authentication.newRealmAuthentication( new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 25bfbc1427d88..664a7f5030d55 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -58,6 +58,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.ObjectParserHelper; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.CharArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -431,8 +432,35 @@ private IndexRequest singleIndexRequest( final var apiKeyDoc = versionedApiKeyDoc.apiKey(); // TODO gnarly + + final List keyRoles; + // TODO need to account for legacy versions here, potentially + if (request.getRoleDescriptors() != null) { + keyRoles = request.getRoleDescriptors(); + } else { + try ( + XContentParser parser = XContentHelper.createParser( + XContentParserConfiguration.EMPTY, + apiKeyDoc.roleDescriptorsBytes, + XContentType.JSON + ) + ) { + // TODO gnarly + keyRoles = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String roleName = parser.currentName(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + final RoleDescriptor role = RoleDescriptor.parse(roleName, parser, false); + keyRoles.add(role); + } + } + } + // Only write new metadata if it's present, i.e., not null. Otherwise, use existing old metadata final Map metadata; + // TODO need to account for legacy versions here, potentially try ( XContentParser parser = XContentHelper.createParser( XContentParserConfiguration.EMPTY, @@ -453,8 +481,7 @@ private IndexRequest singleIndexRequest( userRoles, Instant.ofEpochMilli(apiKeyDoc.creationTime), apiKeyDoc.expirationTime == -1 ? null : Instant.ofEpochMilli(apiKeyDoc.expirationTime), - // TODO also fix this - request.getRoleDescriptors(), + keyRoles, version, metadata ) @@ -1119,6 +1146,7 @@ private void findApiKeyDocsForSubject(Subject subject, String[] apiKeyIds, Actio false, false, listener, + // TODO what if this throws? ApiKeyService::convertSearchHitToVersionedApiKeyDoc ); } @@ -1444,6 +1472,7 @@ private static VersionedApiKeyDoc convertSearchHitToVersionedApiKeyDoc(SearchHit } } + // TODO rename private record VersionedApiKeyDoc(ApiKeyDoc apiKey, long seqNo, long primaryTerm) {} private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { From eef2b38be9b764d5fb2b8ac0138f40a7ffdec214 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 18:52:52 +0200 Subject: [PATCH 018/171] Clean up --- .../security/authc/ApiKeyIntegTests.java | 1 + .../xpack/security/authc/ApiKeyService.java | 86 ++++++++++--------- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 945ff095189b2..6b05e96928de3 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1503,6 +1503,7 @@ private void expectMetadataForApiKey(Map expectedMetadata, Map keyRoles; - // TODO need to account for legacy versions here, potentially - if (request.getRoleDescriptors() != null) { - keyRoles = request.getRoleDescriptors(); - } else { - try ( - XContentParser parser = XContentHelper.createParser( - XContentParserConfiguration.EMPTY, - apiKeyDoc.roleDescriptorsBytes, - XContentType.JSON - ) - ) { - // TODO gnarly - keyRoles = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); - final String roleName = parser.currentName(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); - final RoleDescriptor role = RoleDescriptor.parse(roleName, parser, false); - keyRoles.add(role); - } - } - } - - // Only write new metadata if it's present, i.e., not null. Otherwise, use existing old metadata - final Map metadata; - // TODO need to account for legacy versions here, potentially - try ( - XContentParser parser = XContentHelper.createParser( - XContentParserConfiguration.EMPTY, - apiKeyDoc.metadataFlattened, - XContentType.JSON - ) - ) { - metadata = request.getMetadata() != null ? request.getMetadata() : parser.map(); - } return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) .setSource( @@ -491,6 +454,51 @@ private IndexRequest singleIndexRequest( .request(); } + private Map getMetadata(Map newMetadata, ApiKeyDoc apiKeyDoc) throws IOException { + if (newMetadata != null) { + return newMetadata; + } + + try ( + XContentParser parser = XContentHelper.createParser( + XContentParserConfiguration.EMPTY, + apiKeyDoc.metadataFlattened, + XContentType.JSON + ) + ) { + // TODO order? + return parser.map(); + } + } + + private List getRoleDescriptors(List newRoleDescriptors, ApiKeyDoc apiKeyDoc) throws IOException { + // TODO gnarly + // TODO need to account for legacy versions here, potentially + if (newRoleDescriptors != null) { + return newRoleDescriptors; + } + + try ( + XContentParser parser = XContentHelper.createParser( + XContentParserConfiguration.EMPTY, + apiKeyDoc.roleDescriptorsBytes, + XContentType.JSON + ) + ) { + // TODO order? + final List keyRoles = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String roleName = parser.currentName(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + final RoleDescriptor role = RoleDescriptor.parse(roleName, parser, false); + keyRoles.add(role); + } + return keyRoles; + } + } + /** * package-private for testing */ From e5b86507b24b19bd3186fb48ba5d2eec5096951a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 19:12:05 +0200 Subject: [PATCH 019/171] More todos --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index ea340a1e90a8b..6696aedbd4774 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -375,6 +375,8 @@ public void updateApiKey( // TODO assert on creator final var version = clusterService.state().nodes().getMinNodeVersion(); + // TODO assert on version for now + // TODO what happens if `toBulkUpdateRequest` throws? final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); doBulkUpdate(bulkRequest, listener); @@ -437,7 +439,7 @@ private IndexRequest singleIndexRequest( .setId(request.getId()) .setSource( newDocument( - // TODO fill array? + // TODO fill array in finally block? apiKeyDoc.hash.toCharArray(), apiKeyDoc.name, authentication, From 30cfc39245be52647ae611429645a4ad9387399e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 19:12:43 +0200 Subject: [PATCH 020/171] One more --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 6696aedbd4774..f0f55a084860c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -401,6 +401,7 @@ private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListenerwrap( + // TODO translate here bulkResponse -> { listener.onResponse(new UpdateApiKeyResponse(true)); }, listener::onFailure ), From b14271acb4f5de38979dfe80412ad8b03f59cd12 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 19:52:29 +0200 Subject: [PATCH 021/171] Other users API key not found --- .../security/authc/ApiKeyIntegTests.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 6b05e96928de3..ff99bd4644d51 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1418,7 +1418,7 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce } public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { - final var createdApiKey = createApiKey(null); + final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); @@ -1427,7 +1427,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final PlainActionFuture listener = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( - fileRealmAuthForEsTestRootUser(serviceWithNodeName.nodeName()), + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), request, Set.of(expectedRoleDescriptor), listener @@ -1451,19 +1451,34 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final PlainActionFuture listener2 = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( - fileRealmAuthForEsTestRootUser(serviceWithNodeName.nodeName()), + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()), Set.of(expectedRoleDescriptor), listener2 ); - final var ex = expectThrows(ExecutionException.class, listener2::get); + var ex = expectThrows(ExecutionException.class, listener2::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); assertThat(ex.getMessage(), containsString("api key [" + otherApiKeyId + "] not found")); + + // Test not found exception on other user's API key + final var otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); + final var otherUsersApiKeyId = otherUsersApiKey.v1().getId(); + final PlainActionFuture listener3 = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey( + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new UpdateApiKeyRequest(otherUsersApiKeyId, request.getRoleDescriptors(), request.getMetadata()), + Set.of(expectedRoleDescriptor), + listener3 + ); + ex = expectThrows(ExecutionException.class, listener3::get); + assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); + assertThat(ex.getMessage(), containsString("api key [" + otherUsersApiKeyId + "] not found")); } public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExistingRoleDescriptors() throws ExecutionException, InterruptedException, IOException { - final var createdApiKey = createApiKey(null); + final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); @@ -1474,7 +1489,7 @@ public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExisti final PlainActionFuture listener = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( - fileRealmAuthForEsTestRootUser(serviceWithNodeName.nodeName()), + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), request, Set.of(expectedLimitedByRoleDescriptor), listener @@ -1488,9 +1503,9 @@ public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExisti expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); } - private Authentication fileRealmAuthForEsTestRootUser(String nodeName) { + private Authentication fileRealmAuth(String nodeName, String userName, String roleName) { return Authentication.newRealmAuthentication( - new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new User(userName, roleName), new Authentication.RealmRef("file", FileRealmSettings.TYPE, nodeName) ); } @@ -1668,8 +1683,8 @@ private void verifyGetResponse( } } - private Tuple> createApiKey(TimeValue expiration) { - final var res = createApiKeys(ES_TEST_ROOT_USER, 1, expiration, "monitor"); + private Tuple> createApiKey(String user, TimeValue expiration) { + final var res = createApiKeys(user, 1, expiration, "monitor"); return new Tuple<>(res.v1().get(0), res.v2().get(0)); } From cd5d58f564b1e9e69eb3b7db84f8a2565304d075 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 19:54:02 +0200 Subject: [PATCH 022/171] Nit --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index ff99bd4644d51..1e5d1945b46e0 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1462,18 +1462,17 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // Test not found exception on other user's API key final var otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); - final var otherUsersApiKeyId = otherUsersApiKey.v1().getId(); final PlainActionFuture listener3 = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new UpdateApiKeyRequest(otherUsersApiKeyId, request.getRoleDescriptors(), request.getMetadata()), + new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()), Set.of(expectedRoleDescriptor), listener3 ); ex = expectThrows(ExecutionException.class, listener3::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("api key [" + otherUsersApiKeyId + "] not found")); + assertThat(ex.getMessage(), containsString("api key [" + otherUsersApiKey.v1().getId() + "] not found")); } public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExistingRoleDescriptors() throws ExecutionException, From eeca38a0b41449db55c66be9e56638ebd06b498c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 22:50:36 +0200 Subject: [PATCH 023/171] Tweaks --- .../xpack/security/authc/ApiKeyService.java | 64 ++++++------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index f0f55a084860c..d66f6701692a4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -58,7 +58,6 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.ObjectParserHelper; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.CharArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -433,11 +432,12 @@ private IndexRequest singleIndexRequest( VersionedApiKeyDoc versionedApiKeyDoc ) throws IOException { final var apiKeyDoc = versionedApiKeyDoc.apiKey(); - final var keyRoles = getRoleDescriptors(request.getRoleDescriptors(), apiKeyDoc); - final var metadata = getMetadata(request.getMetadata(), apiKeyDoc); + final var keyRoles = getRoleDescriptors(request.getId(), request.getRoleDescriptors(), apiKeyDoc); + final var metadata = request.getMetadata() != null ? request.getMetadata() : apiKeyDoc.metadataAsMap(); return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) + .setSource() .setSource( newDocument( // TODO fill array in finally block? @@ -457,49 +457,16 @@ private IndexRequest singleIndexRequest( .request(); } - private Map getMetadata(Map newMetadata, ApiKeyDoc apiKeyDoc) throws IOException { - if (newMetadata != null) { - return newMetadata; - } - - try ( - XContentParser parser = XContentHelper.createParser( - XContentParserConfiguration.EMPTY, - apiKeyDoc.metadataFlattened, - XContentType.JSON - ) - ) { - // TODO order? - return parser.map(); - } - } - - private List getRoleDescriptors(List newRoleDescriptors, ApiKeyDoc apiKeyDoc) throws IOException { - // TODO gnarly + private List getRoleDescriptors(String apiKeyId, List newRoleDescriptors, ApiKeyDoc apiKeyDoc) { // TODO need to account for legacy versions here, potentially - if (newRoleDescriptors != null) { - return newRoleDescriptors; - } - - try ( - XContentParser parser = XContentHelper.createParser( - XContentParserConfiguration.EMPTY, - apiKeyDoc.roleDescriptorsBytes, - XContentType.JSON - ) - ) { - // TODO order? - final List keyRoles = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); - final String roleName = parser.currentName(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); - final RoleDescriptor role = RoleDescriptor.parse(roleName, parser, false); - keyRoles.add(role); - } - return keyRoles; - } + // TODO feels backwards that we deserialize just to serialize again + return newRoleDescriptors != null + ? newRoleDescriptors + : parseRoleDescriptors( + apiKeyId, + XContentHelper.convertToMap(apiKeyDoc.roleDescriptorsBytes, true, XContentType.JSON).v2(), + RoleReference.ApiKeyRoleType.ASSIGNED + ); } /** @@ -1661,7 +1628,6 @@ public ApiKeyDoc( Map creator, @Nullable BytesReference metadataFlattened ) { - this.docType = docType; this.creationTime = creationTime; this.expirationTime = expirationTime; @@ -1697,8 +1663,14 @@ public CachedApiKeyDoc toCachedApiKeyDoc() { } static ApiKeyDoc fromXContent(XContentParser parser) { + assert parser.contentType() == XContentType.JSON; return PARSER.apply(parser, null); } + + Map metadataAsMap() { + // TODO is json safe here? + return XContentHelper.convertToMap(metadataFlattened, true, XContentType.JSON).v2(); + } } /** From 7c3d3f0a5a455bcabd506ca9537b0aa5dda3beac Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 22:53:50 +0200 Subject: [PATCH 024/171] Null bytes --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index d66f6701692a4..1b3d1fe3dac12 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1669,7 +1669,7 @@ static ApiKeyDoc fromXContent(XContentParser parser) { Map metadataAsMap() { // TODO is json safe here? - return XContentHelper.convertToMap(metadataFlattened, true, XContentType.JSON).v2(); + return metadataFlattened == NULL_BYTES ? null : XContentHelper.convertToMap(metadataFlattened, true, XContentType.JSON).v2(); } } From f6d1783bba1792c71b7ae632dfcfb5f526c687c1 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 21 Jun 2022 22:56:28 +0200 Subject: [PATCH 025/171] Other --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 1b3d1fe3dac12..f1bea3cc97318 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1669,7 +1669,7 @@ static ApiKeyDoc fromXContent(XContentParser parser) { Map metadataAsMap() { // TODO is json safe here? - return metadataFlattened == NULL_BYTES ? null : XContentHelper.convertToMap(metadataFlattened, true, XContentType.JSON).v2(); + return metadataFlattened == null ? null : XContentHelper.convertToMap(metadataFlattened, true, XContentType.JSON).v2(); } } From 0e44586af012895a2f9cddc40d4326c98f65d626 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 10:56:57 +0200 Subject: [PATCH 026/171] Not found test clean up --- .../security/authc/ApiKeyIntegTests.java | 71 ++++++++++++++----- .../xpack/security/authc/ApiKeyService.java | 1 + 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 1e5d1945b46e0..0ed80a369a7f8 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -67,9 +67,11 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.authc.esnative.NativeRealm; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.junit.After; import org.junit.Before; @@ -1420,6 +1422,7 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); + // TODO randomize more final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); @@ -1445,34 +1448,66 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // Test authenticate works with updated API key final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); + } - // Test not found exception on non-existent API key - final var otherApiKeyId = randomValueOtherThan(apiKeyId, () -> randomAlphaOfLength(20)); - final PlainActionFuture listener2 = new PlainActionFuture<>(); + public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException, IOException { + final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final var apiKeyId = createdApiKey.v1().getId(); + final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); + + // Validate can update own API key + final var serviceWithNodeName = getServiceWithNodeName(); + final PlainActionFuture listener = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()), + request, Set.of(expectedRoleDescriptor), - listener2 + listener ); - var ex = expectThrows(ExecutionException.class, listener2::get); - assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("api key [" + otherApiKeyId + "] not found")); + UpdateApiKeyResponse response = listener.get(); + assertNotNull(response); + assertTrue(response.isUpdated()); + + // Test not found exception on non-existent API key + final var otherApiKeyId = randomValueOtherThan(apiKeyId, () -> randomAlphaOfLength(20)); + testUpdateApiKeyNotFound( + serviceWithNodeName, + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()) + ); // Test not found exception on other user's API key final var otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); - final PlainActionFuture listener3 = new PlainActionFuture<>(); - serviceWithNodeName.service() - .updateApiKey( - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()), - Set.of(expectedRoleDescriptor), - listener3 - ); - ex = expectThrows(ExecutionException.class, listener3::get); + testUpdateApiKeyNotFound( + serviceWithNodeName, + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) + ); + + // Test not found exception on API key of user from other realm + testUpdateApiKeyNotFound( + serviceWithNodeName, + Authentication.newRealmAuthentication( + new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + // Use native realm; no need to actually create user since we are injecting the authentication object directly + new Authentication.RealmRef(NativeRealmSettings.DEFAULT_NAME, NativeRealmSettings.TYPE, serviceWithNodeName.nodeName()) + ), + new UpdateApiKeyRequest(apiKeyId, request.getRoleDescriptors(), request.getMetadata()) + ); + } + + private void testUpdateApiKeyNotFound( + ServiceWithNodeName serviceWithNodeName, + Authentication authentication, + UpdateApiKeyRequest request + ) { + final PlainActionFuture listener = new PlainActionFuture<>(); + serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); + final var ex = expectThrows(ExecutionException.class, listener::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("api key [" + otherUsersApiKey.v1().getId() + "] not found")); + assertThat(ex.getMessage(), containsString("api key [" + request.getId() + "] not found")); } public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExistingRoleDescriptors() throws ExecutionException, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index f1bea3cc97318..9f6db0b06393c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -350,6 +350,7 @@ public void updateApiKey( ActionListener listener ) { ensureEnabled(); + if (authentication == null) { listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; From 3c8374337363ee686d70308d84b50e6926c0e570 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 11:05:17 +0200 Subject: [PATCH 027/171] Tweaks --- .../security/authc/ApiKeyIntegTests.java | 1 - .../xpack/security/authc/ApiKeyService.java | 29 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 0ed80a369a7f8..a57ea5ac542f5 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -71,7 +71,6 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.authc.esnative.NativeRealm; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.junit.After; import org.junit.Before; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 9f6db0b06393c..2f2f558e4f9b3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -415,11 +415,11 @@ private BulkRequestBuilder toBulkUpdateRequest( UpdateApiKeyRequest request, Set userRoles, Version version, - Collection versionedApiKeyDocs + Collection apiKeyDocWithSeqNoAndPrimaryTerms ) throws IOException { final var bulkRequestBuilder = client.prepareBulk(); - for (VersionedApiKeyDoc versionedApiKeyDoc : versionedApiKeyDocs) { - bulkRequestBuilder.add(singleIndexRequest(authentication, request, userRoles, version, versionedApiKeyDoc)); + for (ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDocWithSeqNoAndPrimaryTerm : apiKeyDocWithSeqNoAndPrimaryTerms) { + bulkRequestBuilder.add(singleIndexRequest(authentication, request, userRoles, version, apiKeyDocWithSeqNoAndPrimaryTerm)); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder; @@ -430,9 +430,9 @@ private IndexRequest singleIndexRequest( UpdateApiKeyRequest request, Set userRoles, Version version, - VersionedApiKeyDoc versionedApiKeyDoc + ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDocWithSeqNoAndPrimaryTerm ) throws IOException { - final var apiKeyDoc = versionedApiKeyDoc.apiKey(); + final var apiKeyDoc = apiKeyDocWithSeqNoAndPrimaryTerm.apiKey(); final var keyRoles = getRoleDescriptors(request.getId(), request.getRoleDescriptors(), apiKeyDoc); final var metadata = request.getMetadata() != null ? request.getMetadata() : apiKeyDoc.metadataAsMap(); @@ -453,8 +453,8 @@ private IndexRequest singleIndexRequest( metadata ) ) - .setIfSeqNo(versionedApiKeyDoc.seqNo()) - .setIfPrimaryTerm(versionedApiKeyDoc.primaryTerm()) + .setIfSeqNo(apiKeyDocWithSeqNoAndPrimaryTerm.seqNo()) + .setIfPrimaryTerm(apiKeyDocWithSeqNoAndPrimaryTerm.primaryTerm()) .request(); } @@ -1116,7 +1116,11 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } } - private void findApiKeyDocsForSubject(Subject subject, String[] apiKeyIds, ActionListener> listener) { + private void findApiKeyDocsForSubject( + Subject subject, + String[] apiKeyIds, + ActionListener> listener + ) { findApiKeysForUserRealmApiKeyIdAndNameCombination( new String[] { subject.getRealm().getName() }, subject.getUser().principal(), @@ -1126,7 +1130,7 @@ private void findApiKeyDocsForSubject(Subject subject, String[] apiKeyIds, Actio false, listener, // TODO what if this throws? - ApiKeyService::convertSearchHitToVersionedApiKeyDoc + ApiKeyService::convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm ); } @@ -1441,18 +1445,17 @@ private static ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) { ); } - private static VersionedApiKeyDoc convertSearchHitToVersionedApiKeyDoc(SearchHit hit) { + private static ApiKeyDocWithSeqNoAndPrimaryTerm convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm(SearchHit hit) { try ( XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, hit.getSourceRef(), XContentType.JSON) ) { - return new VersionedApiKeyDoc(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); + return new ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); } catch (IOException ex) { throw new UncheckedIOException(ex); } } - // TODO rename - private record VersionedApiKeyDoc(ApiKeyDoc apiKey, long seqNo, long primaryTerm) {} + private record ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc apiKey, long seqNo, long primaryTerm) {} private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { return notification -> { From b88cdc8541f1c27944efd299a04cf98435193f1e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 11:27:26 +0200 Subject: [PATCH 028/171] Inactive tests --- .../security/authc/ApiKeyIntegTests.java | 32 ++++++++++++++++++- .../xpack/security/authc/ApiKeyService.java | 16 ++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index a57ea5ac542f5..75e6954e485df 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -107,8 +108,10 @@ import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; @@ -1449,7 +1452,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); } - public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException, IOException { + public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); @@ -1497,6 +1500,33 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter ); } + public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, InterruptedException { + final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final var apiKeyId = createdApiKey.v1().getId(); + + PlainActionFuture listener = new PlainActionFuture<>(); + client().execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingRealmName("file"), listener); + InvalidateApiKeyResponse invalidateResponse = listener.get(); + assertThat(invalidateResponse.getErrors(), empty()); + assertThat(invalidateResponse.getInvalidatedApiKeys(), contains(apiKeyId)); + + final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); + + final var serviceWithNodeName = getServiceWithNodeName(); + final PlainActionFuture updateListener = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey( + fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + request, + Set.of(roleDescriptor), + updateListener + ); + final var ex = expectThrows(ExecutionException.class, updateListener::get); + assertThat(ex.getCause(), instanceOf(ValidationException.class)); + assertThat(ex.getMessage(), containsString("cannot update inactive api key")); + } + private void testUpdateApiKeyNotFound( ServiceWithNodeName serviceWithNodeName, Authentication authentication, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 2f2f558e4f9b3..fad4afe2ea73e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; @@ -361,28 +362,29 @@ public void updateApiKey( return; } - // Validation if (apiKeys.size() != 1) { listener.onFailure(new IllegalStateException("more than one api key found for single api key update")); return; } + final var apiKeyDoc = apiKeys.stream().iterator().next().apiKey(); if (isActive(apiKeyDoc) == false) { - // TODO should be 400 - listener.onFailure(new IllegalStateException("api key must be active")); + listener.onFailure(cannotUpdateInactiveApiKey()); return; } - // TODO assert on creator - - final var version = clusterService.state().nodes().getMinNodeVersion(); - // TODO assert on version for now + // TODO assert on creator match, on version compatibility match // TODO what happens if `toBulkUpdateRequest` throws? + final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); doBulkUpdate(bulkRequest, listener); }, listener::onFailure)); } + private ValidationException cannotUpdateInactiveApiKey() { + return new ValidationException().addValidationError("cannot update inactive api key"); + } + private boolean isActive(ApiKeyDoc apiKeyDoc) { return apiKeyDoc.invalidated == false // TODO shared From 68eaa9149b3ce57beb0ee3eff365971630e670ba Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 11:34:25 +0200 Subject: [PATCH 029/171] Expiration test --- .../security/authc/ApiKeyIntegTests.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 75e6954e485df..267a263902d88 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1504,11 +1504,23 @@ public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, Inter final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); - PlainActionFuture listener = new PlainActionFuture<>(); - client().execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingRealmName("file"), listener); - InvalidateApiKeyResponse invalidateResponse = listener.get(); - assertThat(invalidateResponse.getErrors(), empty()); - assertThat(invalidateResponse.getInvalidatedApiKeys(), contains(apiKeyId)); + boolean invalidated = randomBoolean(); + if (invalidated) { + PlainActionFuture listener = new PlainActionFuture<>(); + client().execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingRealmName("file"), listener); + InvalidateApiKeyResponse invalidateResponse = listener.get(); + assertThat(invalidateResponse.getErrors(), empty()); + assertThat(invalidateResponse.getInvalidatedApiKeys(), contains(apiKeyId)); + } + if (invalidated == false || randomBoolean()) { + Instant dayBefore = Instant.now().minus(1L, ChronoUnit.DAYS); + assertTrue(Instant.now().isAfter(dayBefore)); + UpdateResponse expirationDateUpdatedResponse = client().prepareUpdate(SECURITY_MAIN_ALIAS, apiKeyId) + .setDoc("expiration_time", dayBefore.toEpochMilli()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); + } final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); From 36bb431173ea2f1322941051202b0cf080fccda2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 11:40:52 +0200 Subject: [PATCH 030/171] Friendlier exception message --- .../xpack/security/authc/ApiKeyIntegTests.java | 13 +++++++------ .../xpack/security/authc/ApiKeyService.java | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 267a263902d88..1c8196299f552 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1504,18 +1504,18 @@ public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, Inter final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); - boolean invalidated = randomBoolean(); + final boolean invalidated = randomBoolean(); if (invalidated) { - PlainActionFuture listener = new PlainActionFuture<>(); + final PlainActionFuture listener = new PlainActionFuture<>(); client().execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingRealmName("file"), listener); - InvalidateApiKeyResponse invalidateResponse = listener.get(); + final var invalidateResponse = listener.get(); assertThat(invalidateResponse.getErrors(), empty()); assertThat(invalidateResponse.getInvalidatedApiKeys(), contains(apiKeyId)); } if (invalidated == false || randomBoolean()) { - Instant dayBefore = Instant.now().minus(1L, ChronoUnit.DAYS); + final var dayBefore = Instant.now().minus(1L, ChronoUnit.DAYS); assertTrue(Instant.now().isAfter(dayBefore)); - UpdateResponse expirationDateUpdatedResponse = client().prepareUpdate(SECURITY_MAIN_ALIAS, apiKeyId) + final var expirationDateUpdatedResponse = client().prepareUpdate(SECURITY_MAIN_ALIAS, apiKeyId) .setDoc("expiration_time", dayBefore.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -1535,8 +1535,9 @@ public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, Inter updateListener ); final var ex = expectThrows(ExecutionException.class, updateListener::get); + assertThat(ex.getCause(), instanceOf(ValidationException.class)); - assertThat(ex.getMessage(), containsString("cannot update inactive api key")); + assertThat(ex.getMessage(), containsString("cannot update inactive api key [" + apiKeyId + "]")); } private void testUpdateApiKeyNotFound( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index fad4afe2ea73e..5aba1935676de 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -369,7 +369,7 @@ public void updateApiKey( final var apiKeyDoc = apiKeys.stream().iterator().next().apiKey(); if (isActive(apiKeyDoc) == false) { - listener.onFailure(cannotUpdateInactiveApiKey()); + listener.onFailure(cannotUpdateInactiveApiKey(request.getId())); return; } // TODO assert on creator match, on version compatibility match @@ -381,8 +381,8 @@ public void updateApiKey( }, listener::onFailure)); } - private ValidationException cannotUpdateInactiveApiKey() { - return new ValidationException().addValidationError("cannot update inactive api key"); + private ValidationException cannotUpdateInactiveApiKey(String apiKeyId) { + return new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); } private boolean isActive(ApiKeyDoc apiKeyDoc) { From c545e203ebaeca535fd8bf8d673403f4798e96bb Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 11:54:32 +0200 Subject: [PATCH 031/171] Translate response --- .../xpack/security/authc/ApiKeyService.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 5aba1935676de..6b5ba1f4fd184 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -356,6 +356,7 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } + findApiKeyDocsForSubject(authentication.getEffectiveSubject(), new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { if (apiKeys.isEmpty()) { listener.onFailure(apiKeyNotFound(request.getId())); @@ -377,7 +378,18 @@ public void updateApiKey( // TODO what happens if `toBulkUpdateRequest` throws? final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); - doBulkUpdate(bulkRequest, listener); + doBulkUpdate(bulkRequest, ActionListener.wrap(bulkResponse -> { + assert bulkResponse.getItems().length == 1; + final var bulkItemResponse = bulkResponse.getItems()[0]; + if (bulkItemResponse.isFailed()) { + Throwable cause = bulkItemResponse.getFailure().getCause(); + listener.onFailure(new ElasticsearchException("Error updating api key", cause)); + return; + } + final var result = bulkItemResponse.getResponse().getResult(); + assert result == DocWriteResponse.Result.UPDATED || result == DocWriteResponse.Result.NOOP; + listener.onResponse(new UpdateApiKeyResponse(result == DocWriteResponse.Result.UPDATED)); + }, listener::onFailure)); }, listener::onFailure)); } @@ -395,18 +407,14 @@ private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { return new ResourceNotFoundException("api key [" + apiKeyId + "] not found"); } - private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { + private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute( listener::onFailure, () -> executeAsyncWithOrigin( client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkUpdateRequest.request(), - ActionListener.wrap( - // TODO translate here - bulkResponse -> { listener.onResponse(new UpdateApiKeyResponse(true)); }, - listener::onFailure - ), + listener, client::bulk ) ); From 63c0729aac5fc715035a9cef83547923084ebd14 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 14:01:57 +0200 Subject: [PATCH 032/171] Noop not possible --- .../security/authc/ApiKeyIntegTests.java | 5 +- .../xpack/security/authc/ApiKeyService.java | 47 ++++++++++++------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 1c8196299f552..50052769b412e 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1426,6 +1426,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var apiKeyId = createdApiKey.v1().getId(); // TODO randomize more final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var expectedLimitedByRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); final var serviceWithNodeName = getServiceWithNodeName(); @@ -1434,7 +1435,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, .updateApiKey( fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), request, - Set.of(expectedRoleDescriptor), + Set.of(expectedLimitedByRoleDescriptor), listener ); UpdateApiKeyResponse response = listener.get(); @@ -1445,7 +1446,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it expectMetadataForApiKey(request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), updatedApiKeyDoc); expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); - expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); // Test authenticate works with updated API key final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 6b5ba1f4fd184..1f5c88f3f3ad7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -363,36 +363,47 @@ public void updateApiKey( return; } - if (apiKeys.size() != 1) { - listener.onFailure(new IllegalStateException("more than one api key found for single api key update")); - return; - } - - final var apiKeyDoc = apiKeys.stream().iterator().next().apiKey(); - if (isActive(apiKeyDoc) == false) { - listener.onFailure(cannotUpdateInactiveApiKey(request.getId())); + try { + final var apiKeyDoc = single(apiKeys).apiKey(); + if (isActive(apiKeyDoc) == false) { + throw cannotUpdateInactiveApiKey(request.getId()); + } + // TODO assert on creator match, on version compatibility match + } catch (IllegalStateException | ValidationException ex) { + listener.onFailure(ex); return; } - // TODO assert on creator match, on version compatibility match - // TODO what happens if `toBulkUpdateRequest` throws? final var version = clusterService.state().nodes().getMinNodeVersion(); + // TODO what happens if this throws? final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); doBulkUpdate(bulkRequest, ActionListener.wrap(bulkResponse -> { - assert bulkResponse.getItems().length == 1; - final var bulkItemResponse = bulkResponse.getItems()[0]; + // TODO what happens if this throws? + final var bulkItemResponse = single(bulkResponse.getItems()); if (bulkItemResponse.isFailed()) { - Throwable cause = bulkItemResponse.getFailure().getCause(); - listener.onFailure(new ElasticsearchException("Error updating api key", cause)); - return; + listener.onFailure(new ElasticsearchException("error updating api key", bulkItemResponse.getFailure().getCause())); + } else { + assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; + listener.onResponse(new UpdateApiKeyResponse(true)); } - final var result = bulkItemResponse.getResponse().getResult(); - assert result == DocWriteResponse.Result.UPDATED || result == DocWriteResponse.Result.NOOP; - listener.onResponse(new UpdateApiKeyResponse(result == DocWriteResponse.Result.UPDATED)); }, listener::onFailure)); }, listener::onFailure)); } + private static T single(Collection elements) { + if (elements.size() != 1) { + throw new IllegalStateException("collection must contain single element"); + } + return elements.iterator().next(); + } + + private static T single(T[] elements) { + if (elements.length != 1) { + throw new IllegalStateException("array must contain single element"); + } + return elements[0]; + } + private ValidationException cannotUpdateInactiveApiKey(String apiKeyId) { return new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); } From 89d7229fa0b8869c2671577be07a56ba6b9dc901 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 15:38:08 +0200 Subject: [PATCH 033/171] Much clean up --- .../xpack/security/authc/ApiKeyService.java | 93 +++++++++---------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 1f5c88f3f3ad7..e2b8898713dba 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -358,48 +358,51 @@ public void updateApiKey( } findApiKeyDocsForSubject(authentication.getEffectiveSubject(), new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { + final var apiKeyId = request.getId(); + if (apiKeys.isEmpty()) { - listener.onFailure(apiKeyNotFound(request.getId())); - return; + throw apiKeyNotFound(apiKeyId); } - try { - final var apiKeyDoc = single(apiKeys).apiKey(); - if (isActive(apiKeyDoc) == false) { - throw cannotUpdateInactiveApiKey(request.getId()); - } - // TODO assert on creator match, on version compatibility match - } catch (IllegalStateException | ValidationException ex) { - listener.onFailure(ex); - return; - } + validateApiKeyForUpdate(apiKeyId, single(apiKeys).apiKey()); - final var version = clusterService.state().nodes().getMinNodeVersion(); - // TODO what happens if this throws? - final var bulkRequest = toBulkUpdateRequest(authentication, request, userRoles, version, apiKeys); - doBulkUpdate(bulkRequest, ActionListener.wrap(bulkResponse -> { - // TODO what happens if this throws? - final var bulkItemResponse = single(bulkResponse.getItems()); - if (bulkItemResponse.isFailed()) { - listener.onFailure(new ElasticsearchException("error updating api key", bulkItemResponse.getFailure().getCause())); - } else { - assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; - listener.onResponse(new UpdateApiKeyResponse(true)); - } - }, listener::onFailure)); + doBulkUpdate( + buildBulkUpdateRequest(authentication, request, userRoles, apiKeys), + ActionListener.wrap(bulkResponse -> toUpdateApiKeyResponse(apiKeyId, bulkResponse, listener), listener::onFailure) + ); }, listener::onFailure)); } + private void toUpdateApiKeyResponse(String apiKeyId, BulkResponse bulkResponse, ActionListener listener) { + final var bulkItemResponse = single(bulkResponse.getItems()); + if (bulkItemResponse.isFailed()) { + listener.onFailure(bulkItemResponse.getFailure().getCause()); + } else { + assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; + assert bulkItemResponse.getResponse().getId().equals(apiKeyId); + listener.onResponse(new UpdateApiKeyResponse(true)); + } + } + + private void validateApiKeyForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { + if (isActive(apiKeyDoc) == false) { + throw cannotUpdateInactiveApiKey(apiKeyId); + } + if (Strings.isNullOrEmpty(apiKeyDoc.name)) { + throw new ValidationException().addValidationError("cannot update legacy api key [" + apiKeyId + "] without name"); + } + } + private static T single(Collection elements) { if (elements.size() != 1) { - throw new IllegalStateException("collection must contain single element"); + throw new IllegalStateException("collection must contain exactly one element"); } return elements.iterator().next(); } private static T single(T[] elements) { if (elements.length != 1) { - throw new IllegalStateException("array must contain single element"); + throw new IllegalStateException("array must contain exactly one element"); } return elements[0]; } @@ -410,7 +413,6 @@ private ValidationException cannotUpdateInactiveApiKey(String apiKeyId) { private boolean isActive(ApiKeyDoc apiKeyDoc) { return apiKeyDoc.invalidated == false - // TODO shared && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); } @@ -431,22 +433,22 @@ private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener userRoles, - Version version, - Collection apiKeyDocWithSeqNoAndPrimaryTerms + Collection apiKeyDocs ) throws IOException { + final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); - for (ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDocWithSeqNoAndPrimaryTerm : apiKeyDocWithSeqNoAndPrimaryTerms) { - bulkRequestBuilder.add(singleIndexRequest(authentication, request, userRoles, version, apiKeyDocWithSeqNoAndPrimaryTerm)); + for (ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDoc : apiKeyDocs) { + bulkRequestBuilder.add(buildIndexRequestForUpdate(authentication, request, userRoles, version, apiKeyDoc)); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder; } - private IndexRequest singleIndexRequest( + private IndexRequest buildIndexRequestForUpdate( Authentication authentication, UpdateApiKeyRequest request, Set userRoles, @@ -454,7 +456,15 @@ private IndexRequest singleIndexRequest( ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDocWithSeqNoAndPrimaryTerm ) throws IOException { final var apiKeyDoc = apiKeyDocWithSeqNoAndPrimaryTerm.apiKey(); - final var keyRoles = getRoleDescriptors(request.getId(), request.getRoleDescriptors(), apiKeyDoc); + final var newRoleDescriptors = request.getRoleDescriptors(); + // TODO feels backwards that we deserialize just to serialize again + final var keyRoles = newRoleDescriptors != null + ? newRoleDescriptors + : parseRoleDescriptors( + request.getId(), + XContentHelper.convertToMap(apiKeyDoc.roleDescriptorsBytes, true, XContentType.JSON).v2(), + RoleReference.ApiKeyRoleType.ASSIGNED + ); final var metadata = request.getMetadata() != null ? request.getMetadata() : apiKeyDoc.metadataAsMap(); return client.prepareIndex(SECURITY_MAIN_ALIAS) @@ -479,18 +489,6 @@ private IndexRequest singleIndexRequest( .request(); } - private List getRoleDescriptors(String apiKeyId, List newRoleDescriptors, ApiKeyDoc apiKeyDoc) { - // TODO need to account for legacy versions here, potentially - // TODO feels backwards that we deserialize just to serialize again - return newRoleDescriptors != null - ? newRoleDescriptors - : parseRoleDescriptors( - apiKeyId, - XContentHelper.convertToMap(apiKeyDoc.roleDescriptorsBytes, true, XContentType.JSON).v2(), - RoleReference.ApiKeyRoleType.ASSIGNED - ); - } - /** * package-private for testing */ @@ -1476,6 +1474,7 @@ private static ApiKeyDocWithSeqNoAndPrimaryTerm convertSearchHitToApiKeyDocWithS } } + // TODO this a very long name -- maybe `ApiKeyWithDocVersioning`? private record ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc apiKey, long seqNo, long primaryTerm) {} private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { From 003f9695526acc34ed7e132492de4d04f085f444 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 15:51:46 +0200 Subject: [PATCH 034/171] Test validate --- .../xpack/security/authc/ApiKeyService.java | 9 +++------ .../security/authc/ApiKeyServiceTests.java | 20 ++++++++++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index e2b8898713dba..304bf5db194c0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -384,9 +384,10 @@ private void toUpdateApiKeyResponse(String apiKeyId, BulkResponse bulkResponse, } } - private void validateApiKeyForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { + // package-private for testing + void validateApiKeyForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { if (isActive(apiKeyDoc) == false) { - throw cannotUpdateInactiveApiKey(apiKeyId); + throw new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); } if (Strings.isNullOrEmpty(apiKeyDoc.name)) { throw new ValidationException().addValidationError("cannot update legacy api key [" + apiKeyId + "] without name"); @@ -407,10 +408,6 @@ private static T single(T[] elements) { return elements[0]; } - private ValidationException cannotUpdateInactiveApiKey(String apiKeyId) { - return new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); - } - private boolean isActive(ApiKeyDoc apiKeyDoc) { return apiKeyDoc.invalidated == false && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 338dea27b9936..8eb5920905aaa 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; @@ -78,6 +79,7 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; import org.elasticsearch.xpack.core.security.authz.store.RoleReference; +import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc; @@ -1639,6 +1641,18 @@ public void testApiKeyDocDeserialization() throws IOException { assertEquals("bar", ((Map) creator.get("metadata")).get("foo")); } + public void testValidateApiKeyDocBeforeUpdate() throws IOException { + final String apiKeyId = randomAlphaOfLength(12); + final String apiKey = randomAlphaOfLength(16); + Hasher hasher = getFastStoredHashAlgoForTests(); + final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); + + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, null); + ApiKeyService apiKeyService = createApiKeyService(); + ValidationException ex = expectThrows(ValidationException.class, () -> apiKeyService.validateApiKeyForUpdate(apiKeyId, apiKeyDoc)); + assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); + } + public void testApiKeyDocDeserializationWithNullValues() throws IOException { final String apiKeyDocumentSource = """ { @@ -1857,6 +1871,10 @@ private void mockSourceDocument(String id, Map sourceMap) throws } private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated) throws IOException { + return buildApiKeyDoc(hash, expirationTime, invalidated, randomAlphaOfLength(12)); + } + + private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, String name) throws IOException { final BytesReference metadataBytes = XContentTestUtils.convertToXContent(ApiKeyTests.randomMetadata(), XContentType.JSON); return new ApiKeyDoc( "api_key", @@ -1864,7 +1882,7 @@ private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean inval expirationTime, invalidated, new String(hash), - randomAlphaOfLength(12), + name, 0, new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"), new BytesArray("{\"limited role\": {\"cluster\": [\"all\"]}}"), From 8937d94ea3b12a938ff8aaa9c8b1659fc55a994a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 15:52:28 +0200 Subject: [PATCH 035/171] Spotless --- .../elasticsearch/xpack/security/authc/ApiKeyServiceTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 8eb5920905aaa..272f0d5271554 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -79,7 +79,6 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; import org.elasticsearch.xpack.core.security.authz.store.RoleReference; -import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc; From bdd4dfb5220f2907be479ce1bfb77c1613b7da45 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 15:55:00 +0200 Subject: [PATCH 036/171] Also test empty name --- .../xpack/security/authc/ApiKeyServiceTests.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 272f0d5271554..e0360d2941bae 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1641,14 +1641,18 @@ public void testApiKeyDocDeserialization() throws IOException { } public void testValidateApiKeyDocBeforeUpdate() throws IOException { - final String apiKeyId = randomAlphaOfLength(12); - final String apiKey = randomAlphaOfLength(16); - Hasher hasher = getFastStoredHashAlgoForTests(); + final var apiKeyId = randomAlphaOfLength(12); + final var apiKey = randomAlphaOfLength(16); + final var hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, null); - ApiKeyService apiKeyService = createApiKeyService(); - ValidationException ex = expectThrows(ValidationException.class, () -> apiKeyService.validateApiKeyForUpdate(apiKeyId, apiKeyDoc)); + final var apiKeyService = createApiKeyService(); + final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null); + var ex = expectThrows(ValidationException.class, () -> apiKeyService.validateApiKeyForUpdate(apiKeyId, apiKeyDocWithNullName)); + assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); + + final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, ""); + ex = expectThrows(ValidationException.class, () -> apiKeyService.validateApiKeyForUpdate(apiKeyId, apiKeyDocWithEmptyName)); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); } From 4bd5cade80a0c3664435dc9ef5ea6d02b4500619 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 15:57:18 +0200 Subject: [PATCH 037/171] More clean up --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 ++-- .../xpack/security/authc/ApiKeyServiceTests.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 304bf5db194c0..ff1b68af043e3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -364,7 +364,7 @@ public void updateApiKey( throw apiKeyNotFound(apiKeyId); } - validateApiKeyForUpdate(apiKeyId, single(apiKeys).apiKey()); + validateCurrentApiKeyDocForUpdate(apiKeyId, single(apiKeys).apiKey()); doBulkUpdate( buildBulkUpdateRequest(authentication, request, userRoles, apiKeys), @@ -385,7 +385,7 @@ private void toUpdateApiKeyResponse(String apiKeyId, BulkResponse bulkResponse, } // package-private for testing - void validateApiKeyForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { + void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { if (isActive(apiKeyDoc) == false) { throw new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index e0360d2941bae..d55dbe44b0e6b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1648,11 +1648,11 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { final var apiKeyService = createApiKeyService(); final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null); - var ex = expectThrows(ValidationException.class, () -> apiKeyService.validateApiKeyForUpdate(apiKeyId, apiKeyDocWithNullName)); + var ex = expectThrows(ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithNullName)); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, ""); - ex = expectThrows(ValidationException.class, () -> apiKeyService.validateApiKeyForUpdate(apiKeyId, apiKeyDocWithEmptyName)); + ex = expectThrows(ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithEmptyName)); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); } From d88f9491531275eeecd169734cc1875b694ad3ac Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 16:00:27 +0200 Subject: [PATCH 038/171] Skip todo --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index ff1b68af043e3..e7e538e429eaf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1145,7 +1145,6 @@ private void findApiKeyDocsForSubject( false, false, listener, - // TODO what if this throws? ApiKeyService::convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm ); } From 421e04f2af98e6645c130a0788d58a350557b92e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 16:04:01 +0200 Subject: [PATCH 039/171] Remove unused --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index e7e538e429eaf..ec4b51588d7b0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -262,10 +262,6 @@ public void invalidateAll() { } } - private static AcknowledgedResponse toUpdateApiKeyResponse(UpdateResponse updateResponse) { - return AcknowledgedResponse.TRUE; - } - /** * Asynchronously creates a new API key based off of the request and authentication * @param authentication the authentication that this api key should be based off of From 719e4babb1ca5d29f507ea910fe567971a223563 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 16:38:49 +0200 Subject: [PATCH 040/171] Update request tests and serialization --- .../action/apikey/UpdateApiKeyRequest.java | 32 +++++++++-- .../apikey/UpdateApiKeyRequestTests.java | 57 +++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 5ab8946062b40..e3645cfdc6e8f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.core.security.action.role.RoleDescriptorRequestValidator; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.support.MetadataUtils; @@ -36,13 +37,11 @@ public UpdateApiKeyRequest(String id, List roleDescriptors, Map< public UpdateApiKeyRequest(StreamInput in) throws IOException { super(in); this.id = in.readString(); - // TODO handle null - this.roleDescriptors = List.copyOf(in.readList(RoleDescriptor::new)); + this.roleDescriptors = readOptionalList(in); this.metadata = in.readMap(); } @Override - // TODO test me public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (metadata != null && MetadataUtils.containsReservedMetadata(metadata)) { @@ -51,12 +50,35 @@ public ActionRequestValidationException validate() { validationException ); } - for (RoleDescriptor roleDescriptor : roleDescriptors) { - validationException = RoleDescriptorRequestValidator.validate(roleDescriptor, validationException); + if (roleDescriptors != null) { + for (RoleDescriptor roleDescriptor : roleDescriptors) { + validationException = RoleDescriptorRequestValidator.validate(roleDescriptor, validationException); + } } return validationException; } + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + writeOptionalList(out); + out.writeGenericMap(metadata); + } + + private List readOptionalList(StreamInput in) throws IOException { + return in.readBoolean() ? in.readList(RoleDescriptor::new) : null; + } + + private void writeOptionalList(StreamOutput out) throws IOException { + if (roleDescriptors == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeList(roleDescriptors); + } + } + public String getId() { return id; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java new file mode 100644 index 0000000000000..d613014eee711 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.action.apikey; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class UpdateApiKeyRequestTests extends ESTestCase { + + public void testNullValuesValid() { + final var request = new UpdateApiKeyRequest("id", null, null); + assertNull(request.validate()); + } + + public void testSerialization() throws IOException { + final boolean roleDescriptorsPresent = randomBoolean(); + final List descriptorList; + if (roleDescriptorsPresent == false) { + descriptorList = null; + } else { + final int numDescriptors = randomIntBetween(0, 4); + descriptorList = new ArrayList<>(); + for (int i = 0; i < numDescriptors; i++) { + descriptorList.add(new RoleDescriptor("role_" + i, new String[] { "all" }, null, null)); + } + } + + final var id = randomAlphaOfLength(10); + final var metadata = ApiKeyTests.randomMetadata(); + final var request = new UpdateApiKeyRequest(id, descriptorList, metadata); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + request.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + final var serialized = new UpdateApiKeyRequest(in); + assertEquals(id, serialized.getId()); + if (roleDescriptorsPresent) { + assertEquals(descriptorList, serialized.getRoleDescriptors()); + } else { + assertNull(serialized.getRoleDescriptors()); + } + assertEquals(metadata, request.getMetadata()); + } + } + } +} From c53653428c88017512338fbe4c49521a0661fd01 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 16:41:05 +0200 Subject: [PATCH 041/171] Lint --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index ec4b51588d7b0..b6a1ac4c7b3ee 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -31,7 +31,6 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; From c46ab340e7db36ccabd192c785d01253f88be6bd Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 16:49:58 +0200 Subject: [PATCH 042/171] Clean up --- .../xpack/security/authc/ApiKeyService.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index b6a1ac4c7b3ee..7f48c4bc8add6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -359,6 +359,8 @@ public void updateApiKey( throw apiKeyNotFound(apiKeyId); } + // TODO could make idempotency check here + validateCurrentApiKeyDocForUpdate(apiKeyId, single(apiKeys).apiKey()); doBulkUpdate( @@ -373,6 +375,7 @@ private void toUpdateApiKeyResponse(String apiKeyId, BulkResponse bulkResponse, if (bulkItemResponse.isFailed()) { listener.onFailure(bulkItemResponse.getFailure().getCause()); } else { + // Since we made an index request against an existing document, we can't get a NOOP or CREATED here assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; assert bulkItemResponse.getResponse().getId().equals(apiKeyId); listener.onResponse(new UpdateApiKeyResponse(true)); @@ -387,18 +390,19 @@ void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { if (Strings.isNullOrEmpty(apiKeyDoc.name)) { throw new ValidationException().addValidationError("cannot update legacy api key [" + apiKeyId + "] without name"); } + // TODO also assert that authentication subject matches creator on apiKeyDoc } private static T single(Collection elements) { if (elements.size() != 1) { - throw new IllegalStateException("collection must contain exactly one element"); + throw new IllegalStateException("collection must have exactly one element but had [" + elements.size() + "]"); } return elements.iterator().next(); } private static T single(T[] elements) { if (elements.length != 1) { - throw new IllegalStateException("array must contain exactly one element"); + throw new IllegalStateException("array must contain exactly one element but had [" + elements.length + "]"); } return elements[0]; } @@ -412,20 +416,14 @@ private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { return new ResourceNotFoundException("api key [" + apiKeyId + "] not found"); } - private void doBulkUpdate(BulkRequestBuilder bulkUpdateRequest, ActionListener listener) { + private void doBulkUpdate(BulkRequest bulkRequest, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute( listener::onFailure, - () -> executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - SECURITY_ORIGIN, - bulkUpdateRequest.request(), - listener, - client::bulk - ) + () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) ); } - private BulkRequestBuilder buildBulkUpdateRequest( + private BulkRequest buildBulkUpdateRequest( Authentication authentication, UpdateApiKeyRequest request, Set userRoles, @@ -437,7 +435,7 @@ private BulkRequestBuilder buildBulkUpdateRequest( bulkRequestBuilder.add(buildIndexRequestForUpdate(authentication, request, userRoles, version, apiKeyDoc)); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - return bulkRequestBuilder; + return bulkRequestBuilder.request(); } private IndexRequest buildIndexRequestForUpdate( From 7931e001a2b026e1fef3755bde51ab727ac802f9 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 17:18:52 +0200 Subject: [PATCH 043/171] Nits --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 7f48c4bc8add6..1d0be161dfe09 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -429,6 +429,7 @@ private BulkRequest buildBulkUpdateRequest( Set userRoles, Collection apiKeyDocs ) throws IOException { + assert apiKeyDocs.isEmpty() == false; final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); for (ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDoc : apiKeyDocs) { @@ -460,6 +461,7 @@ private IndexRequest buildIndexRequestForUpdate( return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) .setSource() + .setSource() .setSource( newDocument( // TODO fill array in finally block? From 3f8b2188eb0d1d367ab5ef1fe3a03ddf0bd9f6bb Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 17:42:47 +0200 Subject: [PATCH 044/171] Checkstyle --- .../xpack/security/authc/ApiKeyServiceTests.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index d55dbe44b0e6b..48f1f176dba9a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1648,11 +1648,17 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { final var apiKeyService = createApiKeyService(); final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null); - var ex = expectThrows(ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithNullName)); + var ex = expectThrows( + ValidationException.class, + () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithNullName) + ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, ""); - ex = expectThrows(ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithEmptyName)); + ex = expectThrows( + ValidationException.class, + () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithEmptyName) + ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); } From 716621721c905749e191b6e1cafd8109f52385c3 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 17:46:19 +0200 Subject: [PATCH 045/171] Fix --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 1d0be161dfe09..12c833d59e18e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -460,8 +460,6 @@ private IndexRequest buildIndexRequestForUpdate( return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) - .setSource() - .setSource() .setSource( newDocument( // TODO fill array in finally block? From 0f48e8ca825867b9f84706e062c595e75bf0d4d6 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 18:12:30 +0200 Subject: [PATCH 046/171] Owner realms --- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 12c833d59e18e..23bb2b90607be 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -352,7 +352,7 @@ public void updateApiKey( return; } - findApiKeyDocsForSubject(authentication.getEffectiveSubject(), new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { + findApiKeyDocsForSubject(authentication, new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { final var apiKeyId = request.getId(); if (apiKeys.isEmpty()) { @@ -1126,13 +1126,13 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } private void findApiKeyDocsForSubject( - Subject subject, + Authentication authentication, String[] apiKeyIds, ActionListener> listener ) { findApiKeysForUserRealmApiKeyIdAndNameCombination( - new String[] { subject.getRealm().getName() }, - subject.getUser().principal(), + getOwnersRealmNames(authentication), + authentication.getEffectiveSubject().getUser().principal(), null, apiKeyIds, false, From ae573caaf34a51725d563221728a885460f281b2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 22 Jun 2022 18:35:16 +0200 Subject: [PATCH 047/171] Checkstyle --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 23bb2b90607be..df8ae4222c13d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -95,7 +95,6 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authc.RealmDomain; -import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; From d29c1e254ec8b2fb499429179caf0a41418a4130 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 12:46:51 +0200 Subject: [PATCH 048/171] Use raw field --- .../xpack/security/authc/ApiKeyService.java | 115 +++++++++++++----- 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index df8ae4222c13d..b452bb3101e06 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -360,7 +360,7 @@ public void updateApiKey( // TODO could make idempotency check here - validateCurrentApiKeyDocForUpdate(apiKeyId, single(apiKeys).apiKey()); + validateCurrentApiKeyDocForUpdate(apiKeyId, single(apiKeys).doc()); doBulkUpdate( buildBulkUpdateRequest(authentication, request, userRoles, apiKeys), @@ -432,49 +432,33 @@ private BulkRequest buildBulkUpdateRequest( final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); for (ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDoc : apiKeyDocs) { - bulkRequestBuilder.add(buildIndexRequestForUpdate(authentication, request, userRoles, version, apiKeyDoc)); + bulkRequestBuilder.add(buildIndexRequestForUpdate(apiKeyDoc, authentication, request, userRoles, version)); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder.request(); } private IndexRequest buildIndexRequestForUpdate( + ApiKeyDocWithSeqNoAndPrimaryTerm currentApiKeyDoc, Authentication authentication, UpdateApiKeyRequest request, Set userRoles, - Version version, - ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDocWithSeqNoAndPrimaryTerm + Version version ) throws IOException { - final var apiKeyDoc = apiKeyDocWithSeqNoAndPrimaryTerm.apiKey(); - final var newRoleDescriptors = request.getRoleDescriptors(); - // TODO feels backwards that we deserialize just to serialize again - final var keyRoles = newRoleDescriptors != null - ? newRoleDescriptors - : parseRoleDescriptors( - request.getId(), - XContentHelper.convertToMap(apiKeyDoc.roleDescriptorsBytes, true, XContentType.JSON).v2(), - RoleReference.ApiKeyRoleType.ASSIGNED - ); - final var metadata = request.getMetadata() != null ? request.getMetadata() : apiKeyDoc.metadataAsMap(); - return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) .setSource( - newDocument( - // TODO fill array in finally block? - apiKeyDoc.hash.toCharArray(), - apiKeyDoc.name, + mergedDocument( + currentApiKeyDoc.doc(), authentication, userRoles, - Instant.ofEpochMilli(apiKeyDoc.creationTime), - apiKeyDoc.expirationTime == -1 ? null : Instant.ofEpochMilli(apiKeyDoc.expirationTime), - keyRoles, + request.getRoleDescriptors(), version, - metadata + request.getMetadata() ) ) - .setIfSeqNo(apiKeyDocWithSeqNoAndPrimaryTerm.seqNo()) - .setIfPrimaryTerm(apiKeyDocWithSeqNoAndPrimaryTerm.primaryTerm()) + .setIfSeqNo(currentApiKeyDoc.seqNo()) + .setIfPrimaryTerm(currentApiKeyDoc.primaryTerm()) .request(); } @@ -544,6 +528,77 @@ static XContentBuilder newDocument( return builder; } + static XContentBuilder mergedDocument( + ApiKeyDoc currentApiKeyDoc, + Authentication authentication, + Set userRoles, + List keyRoles, + Version version, + Map metadata + ) throws IOException { + final var created = currentApiKeyDoc.creationTime; + final var expiration = currentApiKeyDoc.expirationTime; + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject() + .field("doc_type", "api_key") + .field("creation_time", created) + .field("expiration_time", expiration == -1 ? null : expiration) + .field("api_key_invalidated", false); + + byte[] utf8Bytes = null; + try { + utf8Bytes = CharArrays.toUtf8Bytes(currentApiKeyDoc.hash.toCharArray()); + builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); + } finally { + if (utf8Bytes != null) { + Arrays.fill(utf8Bytes, (byte) 0); + } + } + + if (keyRoles != null) { + builder.startObject("role_descriptors"); + if (keyRoles.isEmpty() == false) { + for (RoleDescriptor descriptor : keyRoles) { + builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); + } + } + builder.endObject(); + } else { + builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); + } + + builder.startObject("limited_by_role_descriptors"); + for (RoleDescriptor descriptor : userRoles) { + builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); + } + builder.endObject(); + + builder.field("name", currentApiKeyDoc.name).field("version", version.id); + if (metadata != null) { + builder.field("metadata_flattened", metadata); + } else { + builder.rawField("metadata_flattened", currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON); + } + + { + builder.startObject("creator") + .field("principal", authentication.getUser().principal()) + .field("full_name", authentication.getUser().fullName()) + .field("email", authentication.getUser().email()) + .field("metadata", authentication.getUser().metadata()) + .field("realm", authentication.getSourceRealm().getName()) + .field("realm_type", authentication.getSourceRealm().getType()); + if (authentication.getSourceRealm().getDomain() != null) { + builder.field("realm_domain", authentication.getSourceRealm().getDomain()); + } + builder.endObject(); + } + builder.endObject(); + + return builder; + } + void tryAuthenticate(ThreadContext ctx, ApiKeyCredentials credentials, ActionListener> listener) { if (false == isEnabled()) { listener.onResponse(AuthenticationResult.notHandled()); @@ -1463,7 +1518,7 @@ private static ApiKeyDocWithSeqNoAndPrimaryTerm convertSearchHitToApiKeyDocWithS } // TODO this a very long name -- maybe `ApiKeyWithDocVersioning`? - private record ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc apiKey, long seqNo, long primaryTerm) {} + private record ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc doc, long seqNo, long primaryTerm) {} private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { return notification -> { @@ -1675,14 +1730,10 @@ public CachedApiKeyDoc toCachedApiKeyDoc() { } static ApiKeyDoc fromXContent(XContentParser parser) { + // TODO remove? assert parser.contentType() == XContentType.JSON; return PARSER.apply(parser, null); } - - Map metadataAsMap() { - // TODO is json safe here? - return metadataFlattened == null ? null : XContentHelper.convertToMap(metadataFlattened, true, XContentType.JSON).v2(); - } } /** From 147adb0fd9266982d28b4ad37c906396b656664f Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 13:34:14 +0200 Subject: [PATCH 049/171] Clean up new and merged doc --- .../xpack/security/authc/ApiKeyService.java | 129 +++++++----------- 1 file changed, 53 insertions(+), 76 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index b452bb3101e06..88a6237ae2ded 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -363,7 +363,7 @@ public void updateApiKey( validateCurrentApiKeyDocForUpdate(apiKeyId, single(apiKeys).doc()); doBulkUpdate( - buildBulkUpdateRequest(authentication, request, userRoles, apiKeys), + buildBulkUpdateRequest(apiKeys, authentication, request, userRoles), ActionListener.wrap(bulkResponse -> toUpdateApiKeyResponse(apiKeyId, bulkResponse, listener), listener::onFailure) ); }, listener::onFailure)); @@ -423,10 +423,10 @@ private void doBulkUpdate(BulkRequest bulkRequest, ActionListener } private BulkRequest buildBulkUpdateRequest( + Collection apiKeyDocs, Authentication authentication, UpdateApiKeyRequest request, - Set userRoles, - Collection apiKeyDocs + Set userRoles ) throws IOException { assert apiKeyDocs.isEmpty() == false; final var version = clusterService.state().nodes().getMinNodeVersion(); @@ -483,49 +483,34 @@ static XContentBuilder newDocument( .field("expiration_time", expiration == null ? null : expiration.toEpochMilli()) .field("api_key_invalidated", false); - byte[] utf8Bytes = null; - try { - utf8Bytes = CharArrays.toUtf8Bytes(apiKeyHashChars); - builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); - } finally { - if (utf8Bytes != null) { - Arrays.fill(utf8Bytes, (byte) 0); - } - } + addApiKeyHash(builder, apiKeyHashChars); + addRoleDescriptors(builder, keyRoles); + addLimitedByRoleDescriptors(builder, userRoles); - // Save role_descriptors - builder.startObject("role_descriptors"); - if (keyRoles != null && keyRoles.isEmpty() == false) { - for (RoleDescriptor descriptor : keyRoles) { - builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); - } - } - builder.endObject(); + builder.field("name", name).field("version", version.id).field("metadata_flattened", metadata); + addCreator(builder, authentication); + + return builder.endObject(); + } - // Save limited_by_role_descriptors + private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { builder.startObject("limited_by_role_descriptors"); for (RoleDescriptor descriptor : userRoles) { builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); } builder.endObject(); + } - builder.field("name", name).field("version", version.id).field("metadata_flattened", metadata); - { - builder.startObject("creator") - .field("principal", authentication.getUser().principal()) - .field("full_name", authentication.getUser().fullName()) - .field("email", authentication.getUser().email()) - .field("metadata", authentication.getUser().metadata()) - .field("realm", authentication.getSourceRealm().getName()) - .field("realm_type", authentication.getSourceRealm().getType()); - if (authentication.getSourceRealm().getDomain() != null) { - builder.field("realm_domain", authentication.getSourceRealm().getDomain()); + private static void addApiKeyHash(XContentBuilder builder, char[] apiKeyHashChars) throws IOException { + byte[] utf8Bytes = null; + try { + utf8Bytes = CharArrays.toUtf8Bytes(apiKeyHashChars); + builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); + } finally { + if (utf8Bytes != null) { + Arrays.fill(utf8Bytes, (byte) 0); } - builder.endObject(); } - builder.endObject(); - - return builder; } static XContentBuilder mergedDocument( @@ -536,43 +521,22 @@ static XContentBuilder mergedDocument( Version version, Map metadata ) throws IOException { - final var created = currentApiKeyDoc.creationTime; - final var expiration = currentApiKeyDoc.expirationTime; - - XContentBuilder builder = XContentFactory.jsonBuilder(); + final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") - .field("creation_time", created) - .field("expiration_time", expiration == -1 ? null : expiration) + .field("creation_time", currentApiKeyDoc.creationTime) + .field("expiration_time", currentApiKeyDoc.expirationTime == -1 ? null : currentApiKeyDoc.expirationTime) .field("api_key_invalidated", false); - byte[] utf8Bytes = null; - try { - utf8Bytes = CharArrays.toUtf8Bytes(currentApiKeyDoc.hash.toCharArray()); - builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); - } finally { - if (utf8Bytes != null) { - Arrays.fill(utf8Bytes, (byte) 0); - } - } + addApiKeyHash(builder, currentApiKeyDoc.hash.toCharArray()); if (keyRoles != null) { - builder.startObject("role_descriptors"); - if (keyRoles.isEmpty() == false) { - for (RoleDescriptor descriptor : keyRoles) { - builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); - } - } - builder.endObject(); + addRoleDescriptors(builder, keyRoles); } else { builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } - builder.startObject("limited_by_role_descriptors"); - for (RoleDescriptor descriptor : userRoles) { - builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); - } - builder.endObject(); + addLimitedByRoleDescriptors(builder, userRoles); builder.field("name", currentApiKeyDoc.name).field("version", version.id); if (metadata != null) { @@ -581,22 +545,35 @@ static XContentBuilder mergedDocument( builder.rawField("metadata_flattened", currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON); } - { - builder.startObject("creator") - .field("principal", authentication.getUser().principal()) - .field("full_name", authentication.getUser().fullName()) - .field("email", authentication.getUser().email()) - .field("metadata", authentication.getUser().metadata()) - .field("realm", authentication.getSourceRealm().getName()) - .field("realm_type", authentication.getSourceRealm().getType()); - if (authentication.getSourceRealm().getDomain() != null) { - builder.field("realm_domain", authentication.getSourceRealm().getDomain()); - } - builder.endObject(); + addCreator(builder, authentication); + + return builder.endObject(); + } + + private static void addCreator(XContentBuilder builder, Authentication authentication) throws IOException { + final var user = authentication.getEffectiveSubject().getUser(); + final var sourceRealm = authentication.getEffectiveSubject().getRealm(); + builder.startObject("creator") + .field("principal", user.principal()) + .field("full_name", user.fullName()) + .field("email", user.email()) + .field("metadata", user.metadata()) + .field("realm", sourceRealm.getName()) + .field("realm_type", sourceRealm.getType()); + if (sourceRealm.getDomain() != null) { + builder.field("realm_domain", sourceRealm.getDomain()); } builder.endObject(); + } - return builder; + private static void addRoleDescriptors(XContentBuilder builder, List keyRoles) throws IOException { + builder.startObject("role_descriptors"); + if (keyRoles != null && keyRoles.isEmpty() == false) { + for (RoleDescriptor descriptor : keyRoles) { + builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); + } + } + builder.endObject(); } void tryAuthenticate(ThreadContext ctx, ApiKeyCredentials credentials, ActionListener> listener) { From 8cffe22e1b3802b56118075c5ebf23233c585e34 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 14:35:40 +0200 Subject: [PATCH 050/171] With versioning --- .../xpack/security/authc/ApiKeyService.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 88a6237ae2ded..35c6d33583239 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -364,19 +364,26 @@ public void updateApiKey( doBulkUpdate( buildBulkUpdateRequest(apiKeys, authentication, request, userRoles), - ActionListener.wrap(bulkResponse -> toUpdateApiKeyResponse(apiKeyId, bulkResponse, listener), listener::onFailure) + ActionListener.wrap( + bulkResponse -> handleBulkResponseForSingleKeyUpdate(apiKeyId, bulkResponse, listener), + listener::onFailure + ) ); }, listener::onFailure)); } - private void toUpdateApiKeyResponse(String apiKeyId, BulkResponse bulkResponse, ActionListener listener) { + private void handleBulkResponseForSingleKeyUpdate( + String apiKeyId, + BulkResponse bulkResponse, + ActionListener listener + ) { final var bulkItemResponse = single(bulkResponse.getItems()); if (bulkItemResponse.isFailed()) { listener.onFailure(bulkItemResponse.getFailure().getCause()); } else { + assert bulkItemResponse.getResponse().getId().equals(apiKeyId); // Since we made an index request against an existing document, we can't get a NOOP or CREATED here assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; - assert bulkItemResponse.getResponse().getId().equals(apiKeyId); listener.onResponse(new UpdateApiKeyResponse(true)); } } @@ -423,7 +430,7 @@ private void doBulkUpdate(BulkRequest bulkRequest, ActionListener } private BulkRequest buildBulkUpdateRequest( - Collection apiKeyDocs, + Collection apiKeyDocs, Authentication authentication, UpdateApiKeyRequest request, Set userRoles @@ -431,7 +438,7 @@ private BulkRequest buildBulkUpdateRequest( assert apiKeyDocs.isEmpty() == false; final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); - for (ApiKeyDocWithSeqNoAndPrimaryTerm apiKeyDoc : apiKeyDocs) { + for (ApiKeyDocWithVersioning apiKeyDoc : apiKeyDocs) { bulkRequestBuilder.add(buildIndexRequestForUpdate(apiKeyDoc, authentication, request, userRoles, version)); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); @@ -439,7 +446,7 @@ private BulkRequest buildBulkUpdateRequest( } private IndexRequest buildIndexRequestForUpdate( - ApiKeyDocWithSeqNoAndPrimaryTerm currentApiKeyDoc, + ApiKeyDocWithVersioning currentApiKeyDoc, Authentication authentication, UpdateApiKeyRequest request, Set userRoles, @@ -1159,7 +1166,7 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { private void findApiKeyDocsForSubject( Authentication authentication, String[] apiKeyIds, - ActionListener> listener + ActionListener> listener ) { findApiKeysForUserRealmApiKeyIdAndNameCombination( getOwnersRealmNames(authentication), @@ -1484,18 +1491,17 @@ private static ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) { ); } - private static ApiKeyDocWithSeqNoAndPrimaryTerm convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm(SearchHit hit) { + private static ApiKeyDocWithVersioning convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm(SearchHit hit) { try ( XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, hit.getSourceRef(), XContentType.JSON) ) { - return new ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); + return new ApiKeyDocWithVersioning(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); } catch (IOException ex) { throw new UncheckedIOException(ex); } } - // TODO this a very long name -- maybe `ApiKeyWithDocVersioning`? - private record ApiKeyDocWithSeqNoAndPrimaryTerm(ApiKeyDoc doc, long seqNo, long primaryTerm) {} + private record ApiKeyDocWithVersioning(ApiKeyDoc doc, long seqNo, long primaryTerm) {} private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { return notification -> { @@ -1707,8 +1713,6 @@ public CachedApiKeyDoc toCachedApiKeyDoc() { } static ApiKeyDoc fromXContent(XContentParser parser) { - // TODO remove? - assert parser.contentType() == XContentType.JSON; return PARSER.apply(parser, null); } } From 0cac5e5453bcd32c328078fa79b4e80d2930d993 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 15:06:15 +0200 Subject: [PATCH 051/171] Clean up --- .../xpack/security/authc/ApiKeyService.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 35c6d33583239..dc64d79c1c316 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -351,19 +351,19 @@ public void updateApiKey( return; } - findApiKeyDocsForSubject(authentication, new String[] { request.getId() }, ActionListener.wrap((apiKeys) -> { + findVersionedApiKeyDocsForSubject(authentication, new String[] { request.getId() }, ActionListener.wrap((versionedDocs) -> { final var apiKeyId = request.getId(); - if (apiKeys.isEmpty()) { + if (versionedDocs.isEmpty()) { throw apiKeyNotFound(apiKeyId); } // TODO could make idempotency check here - validateCurrentApiKeyDocForUpdate(apiKeyId, single(apiKeys).doc()); + validateCurrentApiKeyDocForUpdate(apiKeyId, single(versionedDocs).doc()); doBulkUpdate( - buildBulkUpdateRequest(apiKeys, authentication, request, userRoles), + buildBulkUpdateRequest(versionedDocs, authentication, request, userRoles), ActionListener.wrap( bulkResponse -> handleBulkResponseForSingleKeyUpdate(apiKeyId, bulkResponse, listener), listener::onFailure @@ -430,15 +430,15 @@ private void doBulkUpdate(BulkRequest bulkRequest, ActionListener } private BulkRequest buildBulkUpdateRequest( - Collection apiKeyDocs, + Collection currentVersionedDocs, Authentication authentication, UpdateApiKeyRequest request, Set userRoles ) throws IOException { - assert apiKeyDocs.isEmpty() == false; + assert currentVersionedDocs.isEmpty() == false; final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); - for (ApiKeyDocWithVersioning apiKeyDoc : apiKeyDocs) { + for (VersionedApiKeyDoc apiKeyDoc : currentVersionedDocs) { bulkRequestBuilder.add(buildIndexRequestForUpdate(apiKeyDoc, authentication, request, userRoles, version)); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); @@ -446,7 +446,7 @@ private BulkRequest buildBulkUpdateRequest( } private IndexRequest buildIndexRequestForUpdate( - ApiKeyDocWithVersioning currentApiKeyDoc, + VersionedApiKeyDoc currentVersionedDoc, Authentication authentication, UpdateApiKeyRequest request, Set userRoles, @@ -456,7 +456,7 @@ private IndexRequest buildIndexRequestForUpdate( .setId(request.getId()) .setSource( mergedDocument( - currentApiKeyDoc.doc(), + currentVersionedDoc.doc(), authentication, userRoles, request.getRoleDescriptors(), @@ -464,8 +464,8 @@ private IndexRequest buildIndexRequestForUpdate( request.getMetadata() ) ) - .setIfSeqNo(currentApiKeyDoc.seqNo()) - .setIfPrimaryTerm(currentApiKeyDoc.primaryTerm()) + .setIfSeqNo(currentVersionedDoc.seqNo()) + .setIfPrimaryTerm(currentVersionedDoc.primaryTerm()) .request(); } @@ -1163,10 +1163,10 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } } - private void findApiKeyDocsForSubject( + private void findVersionedApiKeyDocsForSubject( Authentication authentication, String[] apiKeyIds, - ActionListener> listener + ActionListener> listener ) { findApiKeysForUserRealmApiKeyIdAndNameCombination( getOwnersRealmNames(authentication), @@ -1491,17 +1491,17 @@ private static ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) { ); } - private static ApiKeyDocWithVersioning convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm(SearchHit hit) { + private static VersionedApiKeyDoc convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm(SearchHit hit) { try ( XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, hit.getSourceRef(), XContentType.JSON) ) { - return new ApiKeyDocWithVersioning(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); + return new VersionedApiKeyDoc(ApiKeyDoc.fromXContent(parser), hit.getSeqNo(), hit.getPrimaryTerm()); } catch (IOException ex) { throw new UncheckedIOException(ex); } } - private record ApiKeyDocWithVersioning(ApiKeyDoc doc, long seqNo, long primaryTerm) {} + private record VersionedApiKeyDoc(ApiKeyDoc doc, long seqNo, long primaryTerm) {} private RemovalListener> getAuthCacheRemovalListener(int maximumWeight) { return notification -> { From 7d9956e507406bca18a6b4414eb54f6f280cb24f Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 16:02:51 +0200 Subject: [PATCH 052/171] Invalidate doc cache --- .../action/apikey/UpdateApiKeyResponse.java | 17 +++++++--- .../xpack/security/authc/ApiKeyService.java | 34 ++++++++++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java index db8f5b6c1b12d..6f3070459e05d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java @@ -18,29 +18,36 @@ import java.util.Objects; public final class UpdateApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { - + private final String id; private final boolean updated; - public UpdateApiKeyResponse(boolean updated) { + public UpdateApiKeyResponse(String id, boolean updated) { + this.id = id; this.updated = updated; } public UpdateApiKeyResponse(StreamInput in) throws IOException { super(in); + this.id = in.readString(); this.updated = in.readBoolean(); } + public String getId() { + return id; + } + public boolean isUpdated() { return updated; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject().field("updated", updated).endObject(); + return builder.startObject().field("id", id).field("updated", updated).endObject(); } @Override public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); out.writeBoolean(updated); } @@ -49,11 +56,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; UpdateApiKeyResponse that = (UpdateApiKeyResponse) o; - return isUpdated() == that.isUpdated(); + return updated == that.updated && id.equals(that.id); } @Override public int hashCode() { - return Objects.hash(isUpdated()); + return Objects.hash(id, updated); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index dc64d79c1c316..b1348a8b05c3b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -254,6 +254,21 @@ public void invalidateAll() { apiKeyAuthCache.invalidateAll(); } }); + cacheInvalidatorRegistry.registerCacheInvalidator("api_key_doc", new CacheInvalidatorRegistry.CacheInvalidator() { + @Override + public void invalidate(Collection keys) { + if (apiKeyDocCache != null) { + apiKeyDocCache.invalidate(keys); + } + } + + @Override + public void invalidateAll() { + if (apiKeyDocCache != null) { + apiKeyDocCache.invalidateAll(); + } + } + }); } else { this.apiKeyAuthCache = null; this.apiKeyDocCache = null; @@ -384,19 +399,19 @@ private void handleBulkResponseForSingleKeyUpdate( assert bulkItemResponse.getResponse().getId().equals(apiKeyId); // Since we made an index request against an existing document, we can't get a NOOP or CREATED here assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; - listener.onResponse(new UpdateApiKeyResponse(true)); + clearApiKeyDocCache(new UpdateApiKeyResponse(apiKeyId, true), listener); } } // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { + // TODO also assert that authentication subject matches creator on apiKeyDoc if (isActive(apiKeyDoc) == false) { throw new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); } if (Strings.isNullOrEmpty(apiKeyDoc.name)) { throw new ValidationException().addValidationError("cannot update legacy api key [" + apiKeyId + "] without name"); } - // TODO also assert that authentication subject matches creator on apiKeyDoc } private static T single(Collection elements) { @@ -1314,8 +1329,18 @@ private void indexInvalidation( } private void clearCache(InvalidateApiKeyResponse result, ActionListener listener) { - final ClearSecurityCacheRequest clearApiKeyCacheRequest = new ClearSecurityCacheRequest().cacheName("api_key") - .keys(result.getInvalidatedApiKeys().toArray(String[]::new)); + executeClearCacheRequest( + result, + listener, + new ClearSecurityCacheRequest().cacheName("api_key").keys(result.getInvalidatedApiKeys().toArray(String[]::new)) + ); + } + + private void clearApiKeyDocCache(UpdateApiKeyResponse result, ActionListener listener) { + executeClearCacheRequest(result, listener, new ClearSecurityCacheRequest().cacheName("api_key_doc").keys(result.getId())); + } + + private void executeClearCacheRequest(T result, ActionListener listener, ClearSecurityCacheRequest clearApiKeyCacheRequest) { executeAsyncWithOrigin(client, SECURITY_ORIGIN, ClearSecurityCacheAction.INSTANCE, clearApiKeyCacheRequest, new ActionListener<>() { @Override public void onResponse(ClearSecurityCacheResponse nodes) { @@ -1324,6 +1349,7 @@ public void onResponse(ClearSecurityCacheResponse nodes) { @Override public void onFailure(Exception e) { + // TODO logger.error("unable to clear API key cache", e); listener.onFailure(new ElasticsearchException("clearing the API key cache failed; please clear the caches manually", e)); } From f5d089f6b196ed77764ec260f3d5e43291505d55 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 16:06:46 +0200 Subject: [PATCH 053/171] More clean up --- .../xpack/security/authc/ApiKeyService.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index b1348a8b05c3b..05cfac5fa9d52 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -515,26 +515,6 @@ static XContentBuilder newDocument( return builder.endObject(); } - private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { - builder.startObject("limited_by_role_descriptors"); - for (RoleDescriptor descriptor : userRoles) { - builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); - } - builder.endObject(); - } - - private static void addApiKeyHash(XContentBuilder builder, char[] apiKeyHashChars) throws IOException { - byte[] utf8Bytes = null; - try { - utf8Bytes = CharArrays.toUtf8Bytes(apiKeyHashChars); - builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); - } finally { - if (utf8Bytes != null) { - Arrays.fill(utf8Bytes, (byte) 0); - } - } - } - static XContentBuilder mergedDocument( ApiKeyDoc currentApiKeyDoc, Authentication authentication, @@ -572,6 +552,26 @@ static XContentBuilder mergedDocument( return builder.endObject(); } + private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { + builder.startObject("limited_by_role_descriptors"); + for (RoleDescriptor descriptor : userRoles) { + builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); + } + builder.endObject(); + } + + private static void addApiKeyHash(XContentBuilder builder, char[] apiKeyHashChars) throws IOException { + byte[] utf8Bytes = null; + try { + utf8Bytes = CharArrays.toUtf8Bytes(apiKeyHashChars); + builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); + } finally { + if (utf8Bytes != null) { + Arrays.fill(utf8Bytes, (byte) 0); + } + } + } + private static void addCreator(XContentBuilder builder, Authentication authentication) throws IOException { final var user = authentication.getEffectiveSubject().getUser(); final var sourceRealm = authentication.getEffectiveSubject().getRealm(); From 0b34c7b22f8d38da4d9bdb9d24cbbdf6a289bf5b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 20:12:37 +0200 Subject: [PATCH 054/171] Test cache --- .../security/authc/ApiKeyIntegTests.java | 52 +++++++++++++++++ .../xpack/security/authc/ApiKeyService.java | 57 ++++++++----------- 2 files changed, 77 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 50052769b412e..787f0238131f8 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1580,6 +1580,58 @@ public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExisti expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); } + public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { + final List services = Arrays.stream(internalCluster().getNodeNames()) + .map(n -> new ServiceWithNodeName(internalCluster().getInstance(ApiKeyService.class, n), n)) + .toList(); + + // Create two API keys and authenticate with them + final var apiKey1 = createApiKeyAndAuthenticateWithIt(); + final var apiKey2 = createApiKeyAndAuthenticateWithIt(); + + // Find out which nodes handled the above authentication requests + final var serviceForDoc1 = services.stream() + .filter(s -> s.service().getDocCache().get(apiKey1.v1()) != null) + .findFirst() + .orElseThrow(); + final var serviceForDoc2 = services.stream() + .filter(s -> s.service().getDocCache().get(apiKey2.v1()) != null) + .findFirst() + .orElseThrow(); + assertNotNull(serviceForDoc1.service().getFromCache(apiKey1.v1())); + assertNotNull(serviceForDoc2.service().getFromCache(apiKey2.v1())); + final boolean sameServiceNode = serviceForDoc1 == serviceForDoc2; + if (sameServiceNode) { + assertEquals(2, serviceForDoc1.service().getDocCache().count()); + } else { + assertEquals(1, serviceForDoc1.service().getDocCache().count()); + assertEquals(1, serviceForDoc2.service().getDocCache().count()); + } + + // Update the first key + final PlainActionFuture listener = new PlainActionFuture<>(); + serviceForDoc1.service() + .updateApiKey( + fileRealmAuth(serviceForDoc1.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), + Set.of(), + listener + ); + final var response = listener.get(); + assertNotNull(response); + assertTrue(response.isUpdated()); + + // The cache entry should be gone for the first key + if (sameServiceNode) { + assertEquals(1, serviceForDoc1.service().getDocCache().count()); + assertNull(serviceForDoc1.service().getDocCache().get(apiKey1.v1())); + assertNotNull(serviceForDoc1.service().getDocCache().get(apiKey2.v1())); + } else { + assertEquals(0, serviceForDoc1.service().getDocCache().count()); + assertEquals(1, serviceForDoc2.service().getDocCache().count()); + } + } + private Authentication fileRealmAuth(String nodeName, String userName, String roleName) { return Authentication.newRealmAuthentication( new User(userName, roleName), diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 05cfac5fa9d52..5a0ce70233871 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -379,30 +379,11 @@ public void updateApiKey( doBulkUpdate( buildBulkUpdateRequest(versionedDocs, authentication, request, userRoles), - ActionListener.wrap( - bulkResponse -> handleBulkResponseForSingleKeyUpdate(apiKeyId, bulkResponse, listener), - listener::onFailure - ) + ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) ); }, listener::onFailure)); } - private void handleBulkResponseForSingleKeyUpdate( - String apiKeyId, - BulkResponse bulkResponse, - ActionListener listener - ) { - final var bulkItemResponse = single(bulkResponse.getItems()); - if (bulkItemResponse.isFailed()) { - listener.onFailure(bulkItemResponse.getFailure().getCause()); - } else { - assert bulkItemResponse.getResponse().getId().equals(apiKeyId); - // Since we made an index request against an existing document, we can't get a NOOP or CREATED here - assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; - clearApiKeyDocCache(new UpdateApiKeyResponse(apiKeyId, true), listener); - } - } - // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { // TODO also assert that authentication subject matches creator on apiKeyDoc @@ -414,6 +395,23 @@ void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { } } + private boolean isActive(ApiKeyDoc apiKeyDoc) { + return apiKeyDoc.invalidated == false + && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); + } + + private void translateResponseAndClearCache(String apiKeyId, BulkResponse bulkResponse, ActionListener listener) { + final var bulkItemResponse = single(bulkResponse.getItems()); + if (bulkItemResponse.isFailed()) { + listener.onFailure(bulkItemResponse.getFailure().getCause()); + } else { + assert bulkItemResponse.getResponse().getId().equals(apiKeyId); + // Since we made an index request against an existing document, we can't get a NOOP or CREATED here + assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; + clearApiKeyDocCache(new UpdateApiKeyResponse(apiKeyId, true), listener); + } + } + private static T single(Collection elements) { if (elements.size() != 1) { throw new IllegalStateException("collection must have exactly one element but had [" + elements.size() + "]"); @@ -428,22 +426,10 @@ private static T single(T[] elements) { return elements[0]; } - private boolean isActive(ApiKeyDoc apiKeyDoc) { - return apiKeyDoc.invalidated == false - && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); - } - private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { return new ResourceNotFoundException("api key [" + apiKeyId + "] not found"); } - private void doBulkUpdate(BulkRequest bulkRequest, ActionListener listener) { - securityIndex.prepareIndexIfNeededThenExecute( - listener::onFailure, - () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) - ); - } - private BulkRequest buildBulkUpdateRequest( Collection currentVersionedDocs, Authentication authentication, @@ -484,6 +470,13 @@ private IndexRequest buildIndexRequestForUpdate( .request(); } + private void doBulkUpdate(BulkRequest bulkRequest, ActionListener listener) { + securityIndex.prepareIndexIfNeededThenExecute( + listener::onFailure, + () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) + ); + } + /** * package-private for testing */ From 65c8d46b061ca1068f0fc025f2bac757a72c4a31 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 20:36:53 +0200 Subject: [PATCH 055/171] Assert auth cache not cleared --- .../security/authc/ApiKeyIntegTests.java | 49 +++++++++++-------- .../xpack/security/authc/ApiKeyService.java | 6 ++- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 787f0238131f8..0085513afb1c1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1590,46 +1590,55 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution final var apiKey2 = createApiKeyAndAuthenticateWithIt(); // Find out which nodes handled the above authentication requests - final var serviceForDoc1 = services.stream() + final var serviceWithNameForDoc1 = services.stream() .filter(s -> s.service().getDocCache().get(apiKey1.v1()) != null) .findFirst() .orElseThrow(); - final var serviceForDoc2 = services.stream() + final var serviceWithNameForDoc2 = services.stream() .filter(s -> s.service().getDocCache().get(apiKey2.v1()) != null) .findFirst() .orElseThrow(); - assertNotNull(serviceForDoc1.service().getFromCache(apiKey1.v1())); - assertNotNull(serviceForDoc2.service().getFromCache(apiKey2.v1())); - final boolean sameServiceNode = serviceForDoc1 == serviceForDoc2; + final var serviceForDoc1 = serviceWithNameForDoc1.service(); + final var serviceForDoc2 = serviceWithNameForDoc2.service(); + assertNotNull(serviceForDoc1.getFromCache(apiKey1.v1())); + assertNotNull(serviceForDoc2.getFromCache(apiKey2.v1())); + + final boolean sameServiceNode = serviceWithNameForDoc1 == serviceWithNameForDoc2; if (sameServiceNode) { - assertEquals(2, serviceForDoc1.service().getDocCache().count()); + assertEquals(2, serviceForDoc1.getDocCache().count()); } else { - assertEquals(1, serviceForDoc1.service().getDocCache().count()); - assertEquals(1, serviceForDoc2.service().getDocCache().count()); + assertEquals(1, serviceForDoc1.getDocCache().count()); + assertEquals(1, serviceForDoc2.getDocCache().count()); } + final int serviceForDoc1AuthCacheCount = serviceForDoc1.getApiKeyAuthCache().count(); + final int serviceForDoc2AuthCacheCount = serviceForDoc2.getApiKeyAuthCache().count(); + // Update the first key final PlainActionFuture listener = new PlainActionFuture<>(); - serviceForDoc1.service() - .updateApiKey( - fileRealmAuth(serviceForDoc1.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), - Set.of(), - listener - ); + serviceForDoc1.updateApiKey( + fileRealmAuth(serviceWithNameForDoc1.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), + Set.of(), + listener + ); final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); // The cache entry should be gone for the first key if (sameServiceNode) { - assertEquals(1, serviceForDoc1.service().getDocCache().count()); - assertNull(serviceForDoc1.service().getDocCache().get(apiKey1.v1())); - assertNotNull(serviceForDoc1.service().getDocCache().get(apiKey2.v1())); + assertEquals(1, serviceForDoc1.getDocCache().count()); + assertNull(serviceForDoc1.getDocCache().get(apiKey1.v1())); + assertNotNull(serviceForDoc1.getDocCache().get(apiKey2.v1())); } else { - assertEquals(0, serviceForDoc1.service().getDocCache().count()); - assertEquals(1, serviceForDoc2.service().getDocCache().count()); + assertEquals(0, serviceForDoc1.getDocCache().count()); + assertEquals(1, serviceForDoc2.getDocCache().count()); } + + // Auth cache has not been affected + assertEquals(serviceForDoc1AuthCacheCount, serviceForDoc1.getApiKeyAuthCache().count()); + assertEquals(serviceForDoc2AuthCacheCount, serviceForDoc2.getApiKeyAuthCache().count()); } private Authentication fileRealmAuth(String nodeName, String userName, String roleName) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 5a0ce70233871..cceb2817b5d8f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -528,6 +528,7 @@ static XContentBuilder mergedDocument( if (keyRoles != null) { addRoleDescriptors(builder, keyRoles); } else { + // TODO can this be null? builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } @@ -537,7 +538,10 @@ static XContentBuilder mergedDocument( if (metadata != null) { builder.field("metadata_flattened", metadata); } else { - builder.rawField("metadata_flattened", currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON); + // TODO revisit this + if (currentApiKeyDoc.metadataFlattened != null) { + builder.rawField("metadata_flattened", currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON); + } } addCreator(builder, authentication); From 2e3b6238ba0f9c0e7fd4f54544e6c8b209bdb1cf Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 20:37:03 +0200 Subject: [PATCH 056/171] Null tweaks --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index cceb2817b5d8f..378dd39e65e7e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -541,6 +541,8 @@ static XContentBuilder mergedDocument( // TODO revisit this if (currentApiKeyDoc.metadataFlattened != null) { builder.rawField("metadata_flattened", currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON); + } else { + builder.nullField("metadata_flattened"); } } From 2f914300d6d4ee682826c4c43f9310ca2c635369 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 23 Jun 2022 20:39:52 +0200 Subject: [PATCH 057/171] Null input stream --- .../xpack/security/authc/ApiKeyService.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 378dd39e65e7e..7502a00ddeed0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -109,6 +109,7 @@ import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.security.MessageDigest; import java.time.Clock; @@ -539,11 +540,13 @@ static XContentBuilder mergedDocument( builder.field("metadata_flattened", metadata); } else { // TODO revisit this - if (currentApiKeyDoc.metadataFlattened != null) { - builder.rawField("metadata_flattened", currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON); - } else { - builder.nullField("metadata_flattened"); - } + builder.rawField( + "metadata_flattened", + currentApiKeyDoc.metadataFlattened == null + ? InputStream.nullInputStream() + : currentApiKeyDoc.metadataFlattened.streamInput(), + XContentType.JSON + ); } addCreator(builder, authentication); From ef492924f6dbdd7f882d52334756fa723be7bf0d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 11:16:04 +0200 Subject: [PATCH 058/171] Updated doc test --- .../xpack/security/authc/ApiKeyService.java | 11 ++- .../security/authc/ApiKeyServiceTests.java | 74 +++++++++++++++++++ .../security/authz/RoleDescriptorTests.java | 2 +- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 7502a00ddeed0..e3f448ab77265 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -457,7 +457,7 @@ private IndexRequest buildIndexRequestForUpdate( return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) .setSource( - mergedDocument( + updatedDocument( currentVersionedDoc.doc(), authentication, userRoles, @@ -509,7 +509,7 @@ static XContentBuilder newDocument( return builder.endObject(); } - static XContentBuilder mergedDocument( + static XContentBuilder updatedDocument( ApiKeyDoc currentApiKeyDoc, Authentication authentication, Set userRoles, @@ -521,6 +521,7 @@ static XContentBuilder mergedDocument( builder.startObject() .field("doc_type", "api_key") .field("creation_time", currentApiKeyDoc.creationTime) + // TODO double-check .field("expiration_time", currentApiKeyDoc.expirationTime == -1 ? null : currentApiKeyDoc.expirationTime) .field("api_key_invalidated", false); @@ -543,7 +544,7 @@ static XContentBuilder mergedDocument( builder.rawField( "metadata_flattened", currentApiKeyDoc.metadataFlattened == null - ? InputStream.nullInputStream() + ? ApiKeyDoc.NULL_BYTES.streamInput() : currentApiKeyDoc.metadataFlattened.streamInput(), XContentType.JSON ); @@ -683,7 +684,9 @@ void loadApiKeyAndValidateCredentials( }), client::get); } - public List parseRoleDescriptors( + public + List + parseRoleDescriptors( final String apiKeyId, final Map roleDescriptorsMap, RoleReference.ApiKeyRoleType roleType diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 48f1f176dba9a..5f3d61952d1b9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.XPackSettings; @@ -74,6 +75,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.AuthenticationTests; +import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -83,6 +85,7 @@ import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc; import org.elasticsearch.xpack.security.authc.ApiKeyService.CachedApiKeyHashResult; +import org.elasticsearch.xpack.security.authz.RoleDescriptorTests; import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry; import org.elasticsearch.xpack.security.support.FeatureNotEnabledException; @@ -103,6 +106,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -1662,6 +1666,76 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); } + public void testUpdatedDocument() throws IOException { + final var apiKey = randomAlphaOfLength(16); + final var hasher = getFastStoredHashAlgoForTests(); + final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); + + final var oldApiKeyDoc = buildApiKeyDoc(hash, -1, false); + + final Set newUserRoles = randomBoolean() ? Set.of() : Set.of(RoleDescriptorTests.randomRoleDescriptor()); + + final boolean nullKeyRoles = randomBoolean(); + final List newKeyRoles; + if (nullKeyRoles) { + newKeyRoles = null; + } else { + newKeyRoles = List.of(RoleDescriptorTests.randomRoleDescriptor()); + } + + final var metadata = ApiKeyTests.randomMetadata(); + final var version = Version.CURRENT; + final var keyDocSource = ApiKeyService.updatedDocument( + oldApiKeyDoc, + Authentication.newRealmAuthentication( + new User("user", "role"), + new Authentication.RealmRef("file", FileRealmSettings.TYPE, "node") + ), + newUserRoles, + newKeyRoles, + version, + metadata + ); + final var updatedApiKeyDoc = ApiKeyDoc.fromXContent( + XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(keyDocSource), XContentType.JSON) + ); + + assertEquals(oldApiKeyDoc.docType, updatedApiKeyDoc.docType); + assertEquals(oldApiKeyDoc.name, updatedApiKeyDoc.name); + assertEquals(oldApiKeyDoc.hash, updatedApiKeyDoc.hash); + assertEquals(oldApiKeyDoc.expirationTime, updatedApiKeyDoc.expirationTime); + assertEquals(oldApiKeyDoc.creationTime, updatedApiKeyDoc.creationTime); + + final var service = createApiKeyService(Settings.EMPTY); + final var actualUserRoles = service.parseRoleDescriptorsBytes( + "", + updatedApiKeyDoc.limitedByRoleDescriptorsBytes, + RoleReference.ApiKeyRoleType.LIMITED_BY + ); + assertEquals(newUserRoles.size(), actualUserRoles.size()); + assertEquals(new HashSet<>(newUserRoles), new HashSet<>(actualUserRoles)); + + final var actualKeyRoles = service.parseRoleDescriptorsBytes( + "", + updatedApiKeyDoc.roleDescriptorsBytes, + RoleReference.ApiKeyRoleType.ASSIGNED + ); + if (nullKeyRoles) { + assertEquals( + service.parseRoleDescriptorsBytes("", oldApiKeyDoc.roleDescriptorsBytes, RoleReference.ApiKeyRoleType.ASSIGNED), + actualKeyRoles + ); + } else { + assertEquals(newKeyRoles.size(), actualKeyRoles.size()); + assertEquals(new HashSet<>(newKeyRoles), new HashSet<>(actualKeyRoles)); + } + if (metadata == null) { + assertEquals(oldApiKeyDoc.metadataFlattened, updatedApiKeyDoc.metadataFlattened); + } else { + assertEquals(metadata, XContentHelper.convertToMap(updatedApiKeyDoc.metadataFlattened, true, XContentType.JSON).v2()); + } + } + public void testApiKeyDocDeserializationWithNullValues() throws IOException { final String apiKeyDocumentSource = """ { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java index 7ad3bad40fa46..3c7d936fa114a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java @@ -626,7 +626,7 @@ public void testIsEmpty() { } } - private RoleDescriptor randomRoleDescriptor() { + public static RoleDescriptor randomRoleDescriptor() { final RoleDescriptor.IndicesPrivileges[] indexPrivileges = new RoleDescriptor.IndicesPrivileges[randomIntBetween(0, 3)]; for (int i = 0; i < indexPrivileges.length; i++) { final RoleDescriptor.IndicesPrivileges.Builder builder = RoleDescriptor.IndicesPrivileges.builder() From 3f099c444619970f308743177ffe98d8a9f9d1d7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 11:16:40 +0200 Subject: [PATCH 059/171] Lint --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index e3f448ab77265..bdb4f7d267686 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -684,9 +684,7 @@ void loadApiKeyAndValidateCredentials( }), client::get); } - public - List - parseRoleDescriptors( + public List parseRoleDescriptors( final String apiKeyId, final Map roleDescriptorsMap, RoleReference.ApiKeyRoleType roleType From 9426e5653b423a9e5f2b8bce112bc1bdb2c57be2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 11:23:19 +0200 Subject: [PATCH 060/171] More lint --- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index bdb4f7d267686..72d2ef07f80df 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -109,7 +109,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.io.UncheckedIOException; import java.security.MessageDigest; import java.time.Clock; @@ -374,8 +373,6 @@ public void updateApiKey( throw apiKeyNotFound(apiKeyId); } - // TODO could make idempotency check here - validateCurrentApiKeyDocForUpdate(apiKeyId, single(versionedDocs).doc()); doBulkUpdate( @@ -1352,8 +1349,7 @@ public void onResponse(ClearSecurityCacheResponse nodes) { @Override public void onFailure(Exception e) { - // TODO - logger.error("unable to clear API key cache", e); + logger.error("unable to clear API key cache [{}]", clearApiKeyCacheRequest.cacheName(), e); listener.onFailure(new ElasticsearchException("clearing the API key cache failed; please clear the caches manually", e)); } }); From 325d4d0bfe482976ca0706dd8b93b9bdcc7b42a4 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 11:41:33 +0200 Subject: [PATCH 061/171] More clean up --- .../xpack/security/authc/ApiKeyIntegTests.java | 7 ++++--- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 7 ++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 0085513afb1c1..aac360f8ef809 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1438,7 +1438,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, Set.of(expectedLimitedByRoleDescriptor), listener ); - UpdateApiKeyResponse response = listener.get(); + final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); @@ -1469,7 +1469,8 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter Set.of(expectedRoleDescriptor), listener ); - UpdateApiKeyResponse response = listener.get(); + final var response = listener.get(); + assertNotNull(response); assertTrue(response.isUpdated()); @@ -1571,7 +1572,7 @@ public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExisti Set.of(expectedLimitedByRoleDescriptor), listener ); - UpdateApiKeyResponse response = listener.get(); + final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 72d2ef07f80df..8e59587f92fec 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -489,7 +489,7 @@ static XContentBuilder newDocument( Version version, @Nullable Map metadata ) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder(); + final var builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") .field("creation_time", created.toEpochMilli()) @@ -514,11 +514,10 @@ static XContentBuilder updatedDocument( Version version, Map metadata ) throws IOException { - final XContentBuilder builder = XContentFactory.jsonBuilder(); + final var builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") .field("creation_time", currentApiKeyDoc.creationTime) - // TODO double-check .field("expiration_time", currentApiKeyDoc.expirationTime == -1 ? null : currentApiKeyDoc.expirationTime) .field("api_key_invalidated", false); @@ -527,7 +526,6 @@ static XContentBuilder updatedDocument( if (keyRoles != null) { addRoleDescriptors(builder, keyRoles); } else { - // TODO can this be null? builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } @@ -537,7 +535,6 @@ static XContentBuilder updatedDocument( if (metadata != null) { builder.field("metadata_flattened", metadata); } else { - // TODO revisit this builder.rawField( "metadata_flattened", currentApiKeyDoc.metadataFlattened == null From 044831f9d42aa915e5c3d581e9cafe67679d6bc9 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 11:52:23 +0200 Subject: [PATCH 062/171] Put in version stop gap --- .../xpack/security/authc/ApiKeyService.java | 5 +++++ .../xpack/security/authc/ApiKeyServiceTests.java | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 8e59587f92fec..db7b77168baad 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -385,6 +385,11 @@ public void updateApiKey( // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { // TODO also assert that authentication subject matches creator on apiKeyDoc + if (Version.fromId(apiKeyDoc.version).before(Version.V_8_0_0)) { + throw new ValidationException().addValidationError( + "cannot update legacy api key [" + apiKeyId + "] with version [" + Version.fromId(apiKeyDoc.version) + "]" + ); + } if (isActive(apiKeyDoc) == false) { throw new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 5f3d61952d1b9..1dd182d416b1d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1651,19 +1651,23 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); final var apiKeyService = createApiKeyService(); - final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null); + final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null, Version.V_8_0_0.id); var ex = expectThrows( ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithNullName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); - final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, ""); + final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_0_0.id); ex = expectThrows( ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithEmptyName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); + + final var legacyApiKeyDoc = buildApiKeyDoc(hash, -1, false, "", 0); + ex = expectThrows(ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, legacyApiKeyDoc)); + assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] with version")); } public void testUpdatedDocument() throws IOException { @@ -1958,6 +1962,10 @@ private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean inval } private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, String name) throws IOException { + return buildApiKeyDoc(hash, expirationTime, invalidated, name, 0); + } + + private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, String name, int version) throws IOException { final BytesReference metadataBytes = XContentTestUtils.convertToXContent(ApiKeyTests.randomMetadata(), XContentType.JSON); return new ApiKeyDoc( "api_key", @@ -1966,7 +1974,7 @@ private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean inval invalidated, new String(hash), name, - 0, + version, new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"), new BytesArray("{\"limited role\": {\"cluster\": [\"all\"]}}"), Map.of( From 1d3d8a27371ad7f058bf288791709fb6812fc150 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 12:24:54 +0200 Subject: [PATCH 063/171] Combine tests --- .../security/authc/ApiKeyIntegTests.java | 67 ++++++++++--------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index aac360f8ef809..afa35a0ff262b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; @@ -1424,10 +1425,15 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); - // TODO randomize more - final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + + final boolean nullRoleDescriptors = randomBoolean(); + final var newRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "none" }, null, null); final var expectedLimitedByRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); - final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); + final var request = new UpdateApiKeyRequest( + apiKeyId, + nullRoleDescriptors ? null : List.of(newRoleDescriptor), + ApiKeyTests.randomMetadata() + ); final var serviceWithNodeName = getServiceWithNodeName(); final PlainActionFuture listener = new PlainActionFuture<>(); @@ -1445,8 +1451,34 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it expectMetadataForApiKey(request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), updatedApiKeyDoc); - expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); + if (nullRoleDescriptors) { + // Default role descriptor assigned to api key in `createApiKey` + final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); + expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + + // Test authorized because we didn't update key role descriptor + final var authorizationHeaders = Collections.singletonMap( + "Authorization", + "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) + ); + assertNotNull(client().filterWithHeader(authorizationHeaders).admin().cluster().health(new ClusterHealthRequest()).get()); + } else { + expectRoleDescriptorForApiKey("role_descriptors", newRoleDescriptor, updatedApiKeyDoc); + + // Test authorized because we updated key role descriptor to cluster priv none + final var authorizationHeaders = Collections.singletonMap( + "Authorization", + "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) + ); + ExecutionException e = expectThrows( + ExecutionException.class, + () -> client().filterWithHeader(authorizationHeaders).admin().cluster().health(new ClusterHealthRequest()).get() + ); + assertThat(e.getMessage(), containsString("unauthorized")); + assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); + } // Test authenticate works with updated API key final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); @@ -1554,33 +1586,6 @@ private void testUpdateApiKeyNotFound( assertThat(ex.getMessage(), containsString("api key [" + request.getId() + "] not found")); } - public void testUpdateApiKeyRequestWithNullRoleDescriptorsDoesNotOverwriteExistingRoleDescriptors() throws ExecutionException, - InterruptedException, IOException { - final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); - final var apiKeyId = createdApiKey.v1().getId(); - - final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - final var expectedLimitedByRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); - final var request = new UpdateApiKeyRequest(apiKeyId, null, ApiKeyTests.randomMetadata()); - - final var serviceWithNodeName = getServiceWithNodeName(); - final PlainActionFuture listener = new PlainActionFuture<>(); - serviceWithNodeName.service() - .updateApiKey( - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - request, - Set.of(expectedLimitedByRoleDescriptor), - listener - ); - final var response = listener.get(); - - assertNotNull(response); - assertTrue(response.isUpdated()); - final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); - expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); - expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); - } - public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { final List services = Arrays.stream(internalCluster().getNodeNames()) .map(n -> new ServiceWithNodeName(internalCluster().getInstance(ApiKeyService.class, n), n)) From e377155caa4d8644a2d33a6b6620dbac5379a226 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 12:34:04 +0200 Subject: [PATCH 064/171] Include domain --- .../xpack/security/authc/ApiKeyIntegTests.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index afa35a0ff262b..debadf4f7639f 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.test.TestSecurityClient; @@ -69,6 +70,8 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.RealmConfig; +import org.elasticsearch.xpack.core.security.authc.RealmDomain; import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -1648,9 +1651,22 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution } private Authentication fileRealmAuth(String nodeName, String userName, String roleName) { + boolean includeDomain = randomBoolean(); + final var realmName = "file"; + final var realmType = FileRealmSettings.TYPE; return Authentication.newRealmAuthentication( new User(userName, roleName), - new Authentication.RealmRef("file", FileRealmSettings.TYPE, nodeName) + new Authentication.RealmRef( + realmName, + realmType, + nodeName, + includeDomain + ? new RealmDomain( + ESTestCase.randomAlphaOfLengthBetween(3, 8), + Set.of(new RealmConfig.RealmIdentifier(realmType, realmName)) + ) + : null + ) ); } From e95fd340b29807d46a1d7f3ebe40b4fb30e08057 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 12:40:14 +0200 Subject: [PATCH 065/171] Use param message --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index db7b77168baad..df3529dc7eaf5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ExceptionsHelper; @@ -1351,7 +1352,7 @@ public void onResponse(ClearSecurityCacheResponse nodes) { @Override public void onFailure(Exception e) { - logger.error("unable to clear API key cache [{}]", clearApiKeyCacheRequest.cacheName(), e); + logger.error(new ParameterizedMessage("unable to clear API key cache [{}]", clearApiKeyCacheRequest.cacheName()), e); listener.onFailure(new ElasticsearchException("clearing the API key cache failed; please clear the caches manually", e)); } }); From c53681153f27007f30386664aa613217a11ed563 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 12:48:05 +0200 Subject: [PATCH 066/171] Bump version --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- .../xpack/security/authc/ApiKeyServiceTests.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index df3529dc7eaf5..b65be65dcf2f4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -386,7 +386,7 @@ public void updateApiKey( // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { // TODO also assert that authentication subject matches creator on apiKeyDoc - if (Version.fromId(apiKeyDoc.version).before(Version.V_8_0_0)) { + if (Version.fromId(apiKeyDoc.version).before(Version.V_8_2_0)) { throw new ValidationException().addValidationError( "cannot update legacy api key [" + apiKeyId + "] with version [" + Version.fromId(apiKeyDoc.version) + "]" ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 1dd182d416b1d..eb49c73d02d4c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1651,14 +1651,14 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); final var apiKeyService = createApiKeyService(); - final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null, Version.V_8_0_0.id); + final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null, Version.V_8_2_0.id); var ex = expectThrows( ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithNullName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); - final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_0_0.id); + final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_2_0.id); ex = expectThrows( ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithEmptyName) From b7b4c3685d265063a9443e7e7d604710892e8999 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 12:53:05 +0200 Subject: [PATCH 067/171] Check get api correct metadata --- .../xpack/security/authc/ApiKeyIntegTests.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index debadf4f7639f..93a9c768686f1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1451,6 +1451,19 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertNotNull(response); assertTrue(response.isUpdated()); + + Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) + ); + final PlainActionFuture getListener = new PlainActionFuture<>(); + client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener); + GetApiKeyResponse getResponse = getListener.get(); + assertEquals(1, getResponse.getApiKeyInfos().length); + assertEquals( + request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), + getResponse.getApiKeyInfos()[0].getMetadata() + ); + final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it expectMetadataForApiKey(request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), updatedApiKeyDoc); From 30b0267d538b4a570520516bae578413cc45946c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 12:58:41 +0200 Subject: [PATCH 068/171] Rename --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index b65be65dcf2f4..f5d90b7d8902d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1194,7 +1194,7 @@ private void findVersionedApiKeyDocsForSubject( false, false, listener, - ApiKeyService::convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm + ApiKeyService::convertSearchHitToVersionedApiKeyDoc ); } @@ -1519,7 +1519,7 @@ private static ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) { ); } - private static VersionedApiKeyDoc convertSearchHitToApiKeyDocWithSeqNoAndPrimaryTerm(SearchHit hit) { + private static VersionedApiKeyDoc convertSearchHitToVersionedApiKeyDoc(SearchHit hit) { try ( XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, hit.getSourceRef(), XContentType.JSON) ) { From 17375b3de9c9fd48abac3f4687e99846f225a82b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 13:44:42 +0200 Subject: [PATCH 069/171] Still the logging --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index f5d90b7d8902d..5a64c05b74af7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1352,7 +1352,7 @@ public void onResponse(ClearSecurityCacheResponse nodes) { @Override public void onFailure(Exception e) { - logger.error(new ParameterizedMessage("unable to clear API key cache [{}]", clearApiKeyCacheRequest.cacheName()), e); + logger.error(() -> format("unable to clear API key cache [{}]", clearApiKeyCacheRequest.cacheName()), e); listener.onFailure(new ElasticsearchException("clearing the API key cache failed; please clear the caches manually", e)); } }); From 0481c01b4d7242f5c44f0284f7d61aa21e0d65ca Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 13:45:08 +0200 Subject: [PATCH 070/171] Imports --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 5a64c05b74af7..25bf616ba8f67 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ExceptionsHelper; From 1a29f8bea80c8e7218254585824d279237bc57c5 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 14:04:17 +0200 Subject: [PATCH 071/171] Test creator --- .../security/authc/ApiKeyIntegTests.java | 4 ++- .../security/authc/ApiKeyServiceTests.java | 27 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 93a9c768686f1..f220386d3f640 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1463,6 +1463,8 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), getResponse.getApiKeyInfos()[0].getMetadata() ); + assertEquals(ES_TEST_ROOT_USER, getResponse.getApiKeyInfos()[0].getUsername()); + assertEquals("file", getResponse.getApiKeyInfos()[0].getRealm()); final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it @@ -1663,7 +1665,7 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution assertEquals(serviceForDoc2AuthCacheCount, serviceForDoc2.getApiKeyAuthCache().count()); } - private Authentication fileRealmAuth(String nodeName, String userName, String roleName) { + private static Authentication fileRealmAuth(String nodeName, String userName, String roleName) { boolean includeDomain = randomBoolean(); final var realmName = "file"; final var realmType = FileRealmSettings.TYPE; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index eb49c73d02d4c..62e3a08c44224 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1689,17 +1689,9 @@ public void testUpdatedDocument() throws IOException { final var metadata = ApiKeyTests.randomMetadata(); final var version = Version.CURRENT; - final var keyDocSource = ApiKeyService.updatedDocument( - oldApiKeyDoc, - Authentication.newRealmAuthentication( - new User("user", "role"), - new Authentication.RealmRef("file", FileRealmSettings.TYPE, "node") - ), - newUserRoles, - newKeyRoles, - version, - metadata - ); + final var authentication = AuthenticationTestHelper.builder().user(new User("user", "role")).build(false); + + final var keyDocSource = ApiKeyService.updatedDocument(oldApiKeyDoc, authentication, newUserRoles, newKeyRoles, version, metadata); final var updatedApiKeyDoc = ApiKeyDoc.fromXContent( XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(keyDocSource), XContentType.JSON) ); @@ -1738,6 +1730,19 @@ public void testUpdatedDocument() throws IOException { } else { assertEquals(metadata, XContentHelper.convertToMap(updatedApiKeyDoc.metadataFlattened, true, XContentType.JSON).v2()); } + + assertEquals(authentication.getEffectiveSubject().getUser().principal(), updatedApiKeyDoc.creator.getOrDefault("principal", null)); + assertEquals(authentication.getEffectiveSubject().getUser().fullName(), updatedApiKeyDoc.creator.getOrDefault("fullName", null)); + assertEquals(authentication.getEffectiveSubject().getUser().email(), updatedApiKeyDoc.creator.getOrDefault("email", null)); + assertEquals(authentication.getEffectiveSubject().getUser().metadata(), updatedApiKeyDoc.creator.getOrDefault("metadata", null)); + RealmRef realm = authentication.getEffectiveSubject().getRealm(); + assertEquals(realm.getName(), updatedApiKeyDoc.creator.getOrDefault("realm", null)); + assertEquals(realm.getType(), updatedApiKeyDoc.creator.getOrDefault("realm_type", null)); + if (realm.getDomain() != null) { + assertEquals(realm.getDomain(), updatedApiKeyDoc.creator.getOrDefault("realm_domain", null)); + } else { + assertFalse(updatedApiKeyDoc.creator.containsKey("realm_domain")); + } } public void testApiKeyDocDeserializationWithNullValues() throws IOException { From 6d7a341e03dc4be19f992a0ad05ab2eb9c438f7d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 14:18:30 +0200 Subject: [PATCH 072/171] Add creator assertions --- .../xpack/security/authc/ApiKeyService.java | 10 +++++++--- .../xpack/security/authc/ApiKeyServiceTests.java | 15 +++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 25bf616ba8f67..d330683dce05a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -373,7 +373,7 @@ public void updateApiKey( throw apiKeyNotFound(apiKeyId); } - validateCurrentApiKeyDocForUpdate(apiKeyId, single(versionedDocs).doc()); + validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); doBulkUpdate( buildBulkUpdateRequest(versionedDocs, authentication, request, userRoles), @@ -383,8 +383,10 @@ public void updateApiKey( } // package-private for testing - void validateCurrentApiKeyDocForUpdate(String apiKeyId, ApiKeyDoc apiKeyDoc) { - // TODO also assert that authentication subject matches creator on apiKeyDoc + void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentication, ApiKeyDoc apiKeyDoc) { + assert authentication.getEffectiveSubject().getUser().principal() == apiKeyDoc.creator.getOrDefault("principal", null); + assert authentication.getEffectiveSubject().getRealm().getName() == apiKeyDoc.creator.getOrDefault("realm", null); + if (Version.fromId(apiKeyDoc.version).before(Version.V_8_2_0)) { throw new ValidationException().addValidationError( "cannot update legacy api key [" + apiKeyId + "] with version [" + Version.fromId(apiKeyDoc.version) + "]" @@ -531,6 +533,7 @@ static XContentBuilder updatedDocument( if (keyRoles != null) { addRoleDescriptors(builder, keyRoles); } else { + assert currentApiKeyDoc.roleDescriptorsBytes != null; builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } @@ -555,6 +558,7 @@ static XContentBuilder updatedDocument( } private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { + assert userRoles != null; builder.startObject("limited_by_role_descriptors"); for (RoleDescriptor descriptor : userRoles) { builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 62e3a08c44224..77a5e518c6a3e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -75,7 +75,6 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.AuthenticationTests; -import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -1652,21 +1651,29 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { final var apiKeyService = createApiKeyService(); final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null, Version.V_8_2_0.id); + final var auth = Authentication.newRealmAuthentication( + new User("test_user", "role"), + new Authentication.RealmRef("realm1", "realm_type1", "node") + ); + var ex = expectThrows( ValidationException.class, - () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithNullName) + () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, apiKeyDocWithNullName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_2_0.id); ex = expectThrows( ValidationException.class, - () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, apiKeyDocWithEmptyName) + () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, apiKeyDocWithEmptyName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); final var legacyApiKeyDoc = buildApiKeyDoc(hash, -1, false, "", 0); - ex = expectThrows(ValidationException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, legacyApiKeyDoc)); + ex = expectThrows( + ValidationException.class, + () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, legacyApiKeyDoc) + ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] with version")); } From 99d5d0626af4a9ca2503f2ad37965c8bbcd7f5b8 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 14:22:00 +0200 Subject: [PATCH 073/171] Randomize expiration --- .../elasticsearch/xpack/security/authc/ApiKeyServiceTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 77a5e518c6a3e..97ad0de314205 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1682,7 +1682,7 @@ public void testUpdatedDocument() throws IOException { final var hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - final var oldApiKeyDoc = buildApiKeyDoc(hash, -1, false); + final var oldApiKeyDoc = buildApiKeyDoc(hash, randomBoolean() ? -1 : Instant.now().toEpochMilli(), false); final Set newUserRoles = randomBoolean() ? Set.of() : Set.of(RoleDescriptorTests.randomRoleDescriptor()); @@ -1708,6 +1708,7 @@ public void testUpdatedDocument() throws IOException { assertEquals(oldApiKeyDoc.hash, updatedApiKeyDoc.hash); assertEquals(oldApiKeyDoc.expirationTime, updatedApiKeyDoc.expirationTime); assertEquals(oldApiKeyDoc.creationTime, updatedApiKeyDoc.creationTime); + assertEquals(oldApiKeyDoc.invalidated, updatedApiKeyDoc.invalidated); final var service = createApiKeyService(Settings.EMPTY); final var actualUserRoles = service.parseRoleDescriptorsBytes( From 6d7013c4d337c790620684dc44a86c7b3cb7b0f0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 15:09:27 +0200 Subject: [PATCH 074/171] Fix assertion --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index d330683dce05a..4f763360cd443 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -384,8 +384,8 @@ public void updateApiKey( // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentication, ApiKeyDoc apiKeyDoc) { - assert authentication.getEffectiveSubject().getUser().principal() == apiKeyDoc.creator.getOrDefault("principal", null); - assert authentication.getEffectiveSubject().getRealm().getName() == apiKeyDoc.creator.getOrDefault("realm", null); + assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.getOrDefault("principal", null)); + assert authentication.getEffectiveSubject().getRealm().getName().equals(apiKeyDoc.creator.getOrDefault("realm", null)); if (Version.fromId(apiKeyDoc.version).before(Version.V_8_2_0)) { throw new ValidationException().addValidationError( From 64990b58453237b573cd23a61d46ddc8a0fae481 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 15:12:01 +0200 Subject: [PATCH 075/171] WIP tests multiple role descriptors --- .../security/authc/ApiKeyIntegTests.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index f220386d3f640..84f2d51a4ec0b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -87,6 +87,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -1470,11 +1471,11 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it expectMetadataForApiKey(request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), updatedApiKeyDoc); - expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("limited_by_role_descriptors", Set.of(expectedLimitedByRoleDescriptor), updatedApiKeyDoc); if (nullRoleDescriptors) { // Default role descriptor assigned to api key in `createApiKey` final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - expectRoleDescriptorForApiKey("role_descriptors", expectedRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("role_descriptors", List.of(expectedRoleDescriptor), updatedApiKeyDoc); // Test authorized because we didn't update key role descriptor final var authorizationHeaders = Collections.singletonMap( @@ -1483,7 +1484,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, ); assertNotNull(client().filterWithHeader(authorizationHeaders).admin().cluster().health(new ClusterHealthRequest()).get()); } else { - expectRoleDescriptorForApiKey("role_descriptors", newRoleDescriptor, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("role_descriptors", List.of(newRoleDescriptor), updatedApiKeyDoc); // Test authorized because we updated key role descriptor to cluster priv none final var authorizationHeaders = Collections.singletonMap( @@ -1693,26 +1694,26 @@ private void expectMetadataForApiKey(Map expectedMetadata, Map expectedRoleDescriptors, Map actualRawApiKeyDoc ) throws IOException { assertNotNull(actualRawApiKeyDoc); assertThat(roleDescriptorType, in(new String[] { "role_descriptors", "limited_by_role_descriptors" })); final var rawRoleDescriptor = (Map) actualRawApiKeyDoc.get(roleDescriptorType); - assertThat(rawRoleDescriptor.size(), equalTo(1)); - assertThat(rawRoleDescriptor, hasKey(expectedRoleDescriptor.getName())); - - final var descriptor = (Map) rawRoleDescriptor.get(expectedRoleDescriptor.getName()); - final var roleDescriptor = RoleDescriptor.parse( - expectedRoleDescriptor.getName(), - XContentTestUtils.convertToXContent(descriptor, XContentType.JSON), - false, - XContentType.JSON - ); - assertThat(roleDescriptor, equalTo(expectedRoleDescriptor)); + assertEquals(expectedRoleDescriptors.size(), rawRoleDescriptor.size()); + for (RoleDescriptor expectedRoleDescriptor : expectedRoleDescriptors) { + assertThat(rawRoleDescriptor, hasKey(expectedRoleDescriptor.getName())); + final var descriptor = (Map) rawRoleDescriptor.get(expectedRoleDescriptor.getName()); + final var roleDescriptor = RoleDescriptor.parse( + expectedRoleDescriptor.getName(), + XContentTestUtils.convertToXContent(descriptor, XContentType.JSON), + false, + XContentType.JSON + ); + assertEquals(expectedRoleDescriptor, roleDescriptor); + } } private Map getApiKeyDocument(String apiKeyId) { From bba8f2b3e7a7c7c5df5bf6aba1586ae4bc099bf7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 15:24:20 +0200 Subject: [PATCH 076/171] More clean up --- .../security/authc/ApiKeyIntegTests.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 84f2d51a4ec0b..109b3ef00cc5f 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1431,11 +1431,13 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var apiKeyId = createdApiKey.v1().getId(); final boolean nullRoleDescriptors = randomBoolean(); - final var newRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "none" }, null, null); - final var expectedLimitedByRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var newRoleDescriptors = List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "none" }, null, null)); + final var expectedLimitedByRoleDescriptors = Set.of( + new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null) + ); final var request = new UpdateApiKeyRequest( apiKeyId, - nullRoleDescriptors ? null : List.of(newRoleDescriptor), + nullRoleDescriptors ? null : newRoleDescriptors, ApiKeyTests.randomMetadata() ); @@ -1445,7 +1447,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, .updateApiKey( fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), request, - Set.of(expectedLimitedByRoleDescriptor), + expectedLimitedByRoleDescriptors, listener ); final var response = listener.get(); @@ -1453,6 +1455,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertNotNull(response); assertTrue(response.isUpdated()); + // Correct data returned from GET API Client client = client().filterWithHeader( Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) ); @@ -1460,18 +1463,16 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener); GetApiKeyResponse getResponse = getListener.get(); assertEquals(1, getResponse.getApiKeyInfos().length); - assertEquals( - request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), - getResponse.getApiKeyInfos()[0].getMetadata() - ); + final var expectedMetadata = request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(); + assertEquals(expectedMetadata == null ? Map.of() : expectedMetadata, getResponse.getApiKeyInfos()[0].getMetadata()); assertEquals(ES_TEST_ROOT_USER, getResponse.getApiKeyInfos()[0].getUsername()); assertEquals("file", getResponse.getApiKeyInfos()[0].getRealm()); + // Document updated as expected final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it - expectMetadataForApiKey(request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(), updatedApiKeyDoc); - - expectRoleDescriptorForApiKey("limited_by_role_descriptors", Set.of(expectedLimitedByRoleDescriptor), updatedApiKeyDoc); + expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); + expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc); if (nullRoleDescriptors) { // Default role descriptor assigned to api key in `createApiKey` final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); @@ -1484,7 +1485,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, ); assertNotNull(client().filterWithHeader(authorizationHeaders).admin().cluster().health(new ClusterHealthRequest()).get()); } else { - expectRoleDescriptorForApiKey("role_descriptors", List.of(newRoleDescriptor), updatedApiKeyDoc); + expectRoleDescriptorForApiKey("role_descriptors", newRoleDescriptors, updatedApiKeyDoc); // Test authorized because we updated key role descriptor to cluster priv none final var authorizationHeaders = Collections.singletonMap( From dd69a006e910be20e519d56b56d884a051479cb3 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 16:08:48 +0200 Subject: [PATCH 077/171] Support multiple roles --- .../security/authc/ApiKeyIntegTests.java | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 109b3ef00cc5f..c27aa952ac122 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; @@ -76,6 +75,7 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.authz.RoleDescriptorTests; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.junit.After; import org.junit.Before; @@ -1430,16 +1430,12 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); - final boolean nullRoleDescriptors = randomBoolean(); - final var newRoleDescriptors = List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "none" }, null, null)); + final var newRoleDescriptors = randomRoleDescriptors(); + final boolean nullRoleDescriptors = newRoleDescriptors == null; final var expectedLimitedByRoleDescriptors = Set.of( new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null) ); - final var request = new UpdateApiKeyRequest( - apiKeyId, - nullRoleDescriptors ? null : newRoleDescriptors, - ApiKeyTests.randomMetadata() - ); + final var request = new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, ApiKeyTests.randomMetadata()); final var serviceWithNodeName = getServiceWithNodeName(); final PlainActionFuture listener = new PlainActionFuture<>(); @@ -1463,14 +1459,18 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener); GetApiKeyResponse getResponse = getListener.get(); assertEquals(1, getResponse.getApiKeyInfos().length); + // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it final var expectedMetadata = request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(); assertEquals(expectedMetadata == null ? Map.of() : expectedMetadata, getResponse.getApiKeyInfos()[0].getMetadata()); assertEquals(ES_TEST_ROOT_USER, getResponse.getApiKeyInfos()[0].getUsername()); assertEquals("file", getResponse.getApiKeyInfos()[0].getRealm()); + // Test authenticate works with updated API key + final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); + assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); + // Document updated as expected final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); - // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc); if (nullRoleDescriptors) { @@ -1478,31 +1478,39 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); expectRoleDescriptorForApiKey("role_descriptors", List.of(expectedRoleDescriptor), updatedApiKeyDoc); - // Test authorized because we didn't update key role descriptor - final var authorizationHeaders = Collections.singletonMap( + // Create user action unauthorized because we did not update key role; it only has `monitor` cluster priv + final Map authorizationHeaders = Collections.singletonMap( "Authorization", "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) ); - assertNotNull(client().filterWithHeader(authorizationHeaders).admin().cluster().health(new ClusterHealthRequest()).get()); + ExecutionException e = expectThrows( + ExecutionException.class, + () -> createUserWithRunAsRole(authorizationHeaders.get("Authorization")) + ); + assertThat(e.getMessage(), containsString("unauthorized")); + assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); } else { expectRoleDescriptorForApiKey("role_descriptors", newRoleDescriptors, updatedApiKeyDoc); - - // Test authorized because we updated key role descriptor to cluster priv none + // Create user action authorized because we updated key role to `all` cluster priv final var authorizationHeaders = Collections.singletonMap( "Authorization", "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) ); - ExecutionException e = expectThrows( - ExecutionException.class, - () -> client().filterWithHeader(authorizationHeaders).admin().cluster().health(new ClusterHealthRequest()).get() - ); - assertThat(e.getMessage(), containsString("unauthorized")); - assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); + createUserWithRunAsRole(authorizationHeaders.get("Authorization")); } + } - // Test authenticate works with updated API key - final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); - assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); + private List randomRoleDescriptors() { + int caseNo = randomIntBetween(0, 2); + return switch (caseNo) { + case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); + case 1 -> List.of( + new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null), + RoleDescriptorTests.randomRoleDescriptor() + ); + case 2 -> null; + default -> throw new IllegalStateException("unexpected case no"); + }; } public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { @@ -1938,14 +1946,16 @@ private Tuple, List>> createApiKe * This new helper method creates the user in the native realm. */ private void createUserWithRunAsRole() throws ExecutionException, InterruptedException { + createUserWithRunAsRole(basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)); + } + + private void createUserWithRunAsRole(String authHeaderValue) throws ExecutionException, InterruptedException { final PutUserRequest putUserRequest = new PutUserRequest(); putUserRequest.username("user_with_run_as_role"); putUserRequest.roles("run_as_role"); putUserRequest.passwordHash(SecuritySettingsSource.TEST_PASSWORD_HASHED.toCharArray()); PlainActionFuture listener = new PlainActionFuture<>(); - final Client client = client().filterWithHeader( - Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) - ); + final Client client = client().filterWithHeader(Map.of("Authorization", authHeaderValue)); client.execute(PutUserAction.INSTANCE, putUserRequest, listener); final PutUserResponse putUserResponse = listener.get(); assertTrue(putUserResponse.created()); From 3aefef847ef5b775ebbf9829f974ebc36e561f5c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 24 Jun 2022 16:53:59 +0200 Subject: [PATCH 078/171] Domain check and test clean up --- .../xpack/security/authc/ApiKeyIntegTests.java | 13 +++++-------- .../xpack/security/authc/ApiKeyServiceTests.java | 4 +++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index c27aa952ac122..4a87a65da3e1c 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1483,10 +1483,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, "Authorization", "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) ); - ExecutionException e = expectThrows( - ExecutionException.class, - () -> createUserWithRunAsRole(authorizationHeaders.get("Authorization")) - ); + ExecutionException e = expectThrows(ExecutionException.class, () -> createUserWithRunAsRole(authorizationHeaders)); assertThat(e.getMessage(), containsString("unauthorized")); assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); } else { @@ -1496,7 +1493,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, "Authorization", "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) ); - createUserWithRunAsRole(authorizationHeaders.get("Authorization")); + createUserWithRunAsRole(authorizationHeaders); } } @@ -1946,16 +1943,16 @@ private Tuple, List>> createApiKe * This new helper method creates the user in the native realm. */ private void createUserWithRunAsRole() throws ExecutionException, InterruptedException { - createUserWithRunAsRole(basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)); + createUserWithRunAsRole(Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))); } - private void createUserWithRunAsRole(String authHeaderValue) throws ExecutionException, InterruptedException { + private void createUserWithRunAsRole(Map authHeaders) throws ExecutionException, InterruptedException { final PutUserRequest putUserRequest = new PutUserRequest(); putUserRequest.username("user_with_run_as_role"); putUserRequest.roles("run_as_role"); putUserRequest.passwordHash(SecuritySettingsSource.TEST_PASSWORD_HASHED.toCharArray()); PlainActionFuture listener = new PlainActionFuture<>(); - final Client client = client().filterWithHeader(Map.of("Authorization", authHeaderValue)); + final Client client = client().filterWithHeader(authHeaders); client.execute(PutUserAction.INSTANCE, putUserRequest, listener); final PutUserResponse putUserResponse = listener.get(); assertTrue(putUserResponse.created()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 97ad0de314205..c9626c130d96d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1747,7 +1747,9 @@ public void testUpdatedDocument() throws IOException { assertEquals(realm.getName(), updatedApiKeyDoc.creator.getOrDefault("realm", null)); assertEquals(realm.getType(), updatedApiKeyDoc.creator.getOrDefault("realm_type", null)); if (realm.getDomain() != null) { - assertEquals(realm.getDomain(), updatedApiKeyDoc.creator.getOrDefault("realm_domain", null)); + @SuppressWarnings("unchecked") + final var actualDomain = (Map) updatedApiKeyDoc.creator.getOrDefault("realm_domain", null); + assertEquals(realm.getDomain().name(), actualDomain.get("name")); } else { assertFalse(updatedApiKeyDoc.creator.containsKey("realm_domain")); } From 49b4dd5a07e8739d312be25808c45113046151fa Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Sun, 26 Jun 2022 22:18:00 +0200 Subject: [PATCH 079/171] Better names --- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 4f763360cd443..12bb8fabf475e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -375,8 +375,8 @@ public void updateApiKey( validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); - doBulkUpdate( - buildBulkUpdateRequest(versionedDocs, authentication, request, userRoles), + executeBulkIndexRequest( + buildBulkIndexRequestForUpdate(versionedDocs, authentication, request, userRoles), ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) ); }, listener::onFailure)); @@ -435,7 +435,7 @@ private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { return new ResourceNotFoundException("api key [" + apiKeyId + "] not found"); } - private BulkRequest buildBulkUpdateRequest( + private BulkRequest buildBulkIndexRequestForUpdate( Collection currentVersionedDocs, Authentication authentication, UpdateApiKeyRequest request, @@ -475,7 +475,7 @@ private IndexRequest buildIndexRequestForUpdate( .request(); } - private void doBulkUpdate(BulkRequest bulkRequest, ActionListener listener) { + private void executeBulkIndexRequest(BulkRequest bulkRequest, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute( listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) From 4a0b2b2dc8adbc91ae2ee27e000f0982464d80db Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 12:44:51 +0200 Subject: [PATCH 080/171] WIP address feedback --- .../action/apikey/UpdateApiKeyResponse.java | 16 ++----- .../xpack/security/authc/ApiKeyService.java | 48 ++++++++----------- .../security/authc/ApiKeyServiceTests.java | 23 +++++---- 3 files changed, 34 insertions(+), 53 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java index 6f3070459e05d..a1ed1c6092df8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyResponse.java @@ -18,36 +18,28 @@ import java.util.Objects; public final class UpdateApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { - private final String id; private final boolean updated; - public UpdateApiKeyResponse(String id, boolean updated) { - this.id = id; + public UpdateApiKeyResponse(boolean updated) { this.updated = updated; } public UpdateApiKeyResponse(StreamInput in) throws IOException { super(in); - this.id = in.readString(); this.updated = in.readBoolean(); } - public String getId() { - return id; - } - public boolean isUpdated() { return updated; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject().field("id", id).field("updated", updated).endObject(); + return builder.startObject().field("updated", updated).endObject(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(id); out.writeBoolean(updated); } @@ -56,11 +48,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; UpdateApiKeyResponse that = (UpdateApiKeyResponse) o; - return updated == that.updated && id.equals(that.id); + return updated == that.updated; } @Override public int hashCode() { - return Objects.hash(id, updated); + return Objects.hash(updated); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 12bb8fabf475e..3e72c60d8815c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -37,7 +37,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; @@ -364,13 +363,16 @@ public void updateApiKey( if (authentication == null) { listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; + } else if (authentication.isApiKey()) { + listener.onFailure(new IllegalArgumentException("api key cannot update api keys")); + return; } findVersionedApiKeyDocsForSubject(authentication, new String[] { request.getId() }, ActionListener.wrap((versionedDocs) -> { final var apiKeyId = request.getId(); if (versionedDocs.isEmpty()) { - throw apiKeyNotFound(apiKeyId); + throw new ResourceNotFoundException("api key with id [" + apiKeyId + "] owned by requesting user not found"); } validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); @@ -385,26 +387,18 @@ public void updateApiKey( // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentication, ApiKeyDoc apiKeyDoc) { assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.getOrDefault("principal", null)); - assert authentication.getEffectiveSubject().getRealm().getName().equals(apiKeyDoc.creator.getOrDefault("realm", null)); - if (Version.fromId(apiKeyDoc.version).before(Version.V_8_2_0)) { - throw new ValidationException().addValidationError( - "cannot update legacy api key [" + apiKeyId + "] with version [" + Version.fromId(apiKeyDoc.version) + "]" - ); - } - if (isActive(apiKeyDoc) == false) { - throw new ValidationException().addValidationError("cannot update inactive api key [" + apiKeyId + "]"); + boolean isActive = apiKeyDoc.invalidated == false + && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); + if (isActive == false) { + throw new IllegalArgumentException("cannot update inactive api key [" + apiKeyId + "]"); } + if (Strings.isNullOrEmpty(apiKeyDoc.name)) { - throw new ValidationException().addValidationError("cannot update legacy api key [" + apiKeyId + "] without name"); + throw new IllegalArgumentException("cannot update legacy api key [" + apiKeyId + "] without name"); } } - private boolean isActive(ApiKeyDoc apiKeyDoc) { - return apiKeyDoc.invalidated == false - && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); - } - private void translateResponseAndClearCache(String apiKeyId, BulkResponse bulkResponse, ActionListener listener) { final var bulkItemResponse = single(bulkResponse.getItems()); if (bulkItemResponse.isFailed()) { @@ -413,7 +407,7 @@ private void translateResponseAndClearCache(String apiKeyId, BulkResponse bulkRe assert bulkItemResponse.getResponse().getId().equals(apiKeyId); // Since we made an index request against an existing document, we can't get a NOOP or CREATED here assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; - clearApiKeyDocCache(new UpdateApiKeyResponse(apiKeyId, true), listener); + clearApiKeyDocCache(apiKeyId, new UpdateApiKeyResponse(true), listener); } } @@ -431,10 +425,6 @@ private static T single(T[] elements) { return elements[0]; } - private ResourceNotFoundException apiKeyNotFound(String apiKeyId) { - return new ResourceNotFoundException("api key [" + apiKeyId + "] not found"); - } - private BulkRequest buildBulkIndexRequestForUpdate( Collection currentVersionedDocs, Authentication authentication, @@ -461,7 +451,7 @@ private IndexRequest buildIndexRequestForUpdate( return client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) .setSource( - updatedDocument( + buildUpdatedDocument( currentVersionedDoc.doc(), authentication, userRoles, @@ -513,7 +503,7 @@ static XContentBuilder newDocument( return builder.endObject(); } - static XContentBuilder updatedDocument( + static XContentBuilder buildUpdatedDocument( ApiKeyDoc currentApiKeyDoc, Authentication authentication, Set userRoles, @@ -1140,8 +1130,8 @@ private void findApiKeys( final BoolQueryBuilder boolQuery, boolean filterOutInvalidatedKeys, boolean filterOutExpiredKeys, - ActionListener> listener, - Function hitParser + final Function hitParser, + final ActionListener> listener ) { if (filterOutInvalidatedKeys) { boolQuery.filter(QueryBuilders.termQuery("api_key_invalidated", false)); @@ -1163,7 +1153,7 @@ private void findApiKeys( .request(); securityIndex.checkIndexVersionThenExecute( listener::onFailure, - () -> ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier, listener), hitParser) + () -> ScrollHelper.fetchAllByEntity(client, request, hitParser, new ContextPreservingActionListener<>(supplier, listener)) ); } } @@ -1257,7 +1247,7 @@ private void findApiKeysForUserRealmApiKeyIdAndNameCombination( boolQuery.filter(QueryBuilders.idsQuery().addIds(apiKeyIds)); } - findApiKeys(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, listener, hitParser); + findApiKeys(boolQuery, filterOutInvalidatedKeys, filterOutExpiredKeys, hitParser, listener); } } @@ -1342,8 +1332,8 @@ private void clearCache(InvalidateApiKeyResponse result, ActionListener listener) { - executeClearCacheRequest(result, listener, new ClearSecurityCacheRequest().cacheName("api_key_doc").keys(result.getId())); + private void clearApiKeyDocCache(String apiKeyId, UpdateApiKeyResponse result, ActionListener listener) { + executeClearCacheRequest(result, listener, new ClearSecurityCacheRequest().cacheName("api_key_doc").keys(apiKeyId)); } private void executeClearCacheRequest(T result, ActionListener listener, ClearSecurityCacheRequest clearApiKeyCacheRequest) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index c9626c130d96d..367f0b7cc2d4e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; @@ -1657,27 +1656,20 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { ); var ex = expectThrows( - ValidationException.class, + IllegalArgumentException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, apiKeyDocWithNullName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_2_0.id); ex = expectThrows( - ValidationException.class, + IllegalArgumentException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, apiKeyDocWithEmptyName) ); assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); - - final var legacyApiKeyDoc = buildApiKeyDoc(hash, -1, false, "", 0); - ex = expectThrows( - ValidationException.class, - () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, legacyApiKeyDoc) - ); - assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] with version")); } - public void testUpdatedDocument() throws IOException { + public void testBuildUpdatedDocument() throws IOException { final var apiKey = randomAlphaOfLength(16); final var hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); @@ -1698,7 +1690,14 @@ public void testUpdatedDocument() throws IOException { final var version = Version.CURRENT; final var authentication = AuthenticationTestHelper.builder().user(new User("user", "role")).build(false); - final var keyDocSource = ApiKeyService.updatedDocument(oldApiKeyDoc, authentication, newUserRoles, newKeyRoles, version, metadata); + final var keyDocSource = ApiKeyService.buildUpdatedDocument( + oldApiKeyDoc, + authentication, + newUserRoles, + newKeyRoles, + version, + metadata + ); final var updatedApiKeyDoc = ApiKeyDoc.fromXContent( XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(keyDocSource), XContentType.JSON) ); From 2bd3ea4fef3b31ac473d5602eb99830d9fd52635 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 12:45:10 +0200 Subject: [PATCH 081/171] Scroll helper --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3e72c60d8815c..9fd7b8de2d9bf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1153,7 +1153,7 @@ private void findApiKeys( .request(); securityIndex.checkIndexVersionThenExecute( listener::onFailure, - () -> ScrollHelper.fetchAllByEntity(client, request, hitParser, new ContextPreservingActionListener<>(supplier, listener)) + () -> ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier, listener), hitParser) ); } } From 0473c5dbedb571203b1d3546a12291fa987a5507 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 12:58:16 +0200 Subject: [PATCH 082/171] Add assertion --- .../security/authc/ApiKeyIntegTests.java | 6 +-- .../xpack/security/authc/ApiKeyService.java | 54 +++++++++---------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 4a87a65da3e1c..05a0872051fa5 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1481,7 +1481,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // Create user action unauthorized because we did not update key role; it only has `monitor` cluster priv final Map authorizationHeaders = Collections.singletonMap( "Authorization", - "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) + "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) ); ExecutionException e = expectThrows(ExecutionException.class, () -> createUserWithRunAsRole(authorizationHeaders)); assertThat(e.getMessage(), containsString("unauthorized")); @@ -1491,7 +1491,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // Create user action authorized because we updated key role to `all` cluster priv final var authorizationHeaders = Collections.singletonMap( "Authorization", - "ApiKey " + getBase64EncodedApiKeyValue(response.getId(), createdApiKey.v1().getKey()) + "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) ); createUserWithRunAsRole(authorizationHeaders); } @@ -1608,7 +1608,7 @@ private void testUpdateApiKeyNotFound( serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); final var ex = expectThrows(ExecutionException.class, listener::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("api key [" + request.getId() + "] not found")); + assertThat(ex.getMessage(), containsString("api key with id [" + request.getId() + "] owned by requesting user not found")); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 9fd7b8de2d9bf..a3da112aabf46 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -426,45 +426,37 @@ private static T single(T[] elements) { } private BulkRequest buildBulkIndexRequestForUpdate( - Collection currentVersionedDocs, - Authentication authentication, - UpdateApiKeyRequest request, - Set userRoles + final Collection currentVersionedDocs, + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoles ) throws IOException { assert currentVersionedDocs.isEmpty() == false; final var version = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); for (VersionedApiKeyDoc apiKeyDoc : currentVersionedDocs) { - bulkRequestBuilder.add(buildIndexRequestForUpdate(apiKeyDoc, authentication, request, userRoles, version)); + bulkRequestBuilder.add( + client.prepareIndex(SECURITY_MAIN_ALIAS) + .setId(request.getId()) + .setSource( + buildUpdatedDocument( + apiKeyDoc.doc(), + authentication, + userRoles, + request.getRoleDescriptors(), + version, + request.getMetadata() + ) + ) + .setIfSeqNo(apiKeyDoc.seqNo()) + .setIfPrimaryTerm(apiKeyDoc.primaryTerm()) + .request() + ); } bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder.request(); } - private IndexRequest buildIndexRequestForUpdate( - VersionedApiKeyDoc currentVersionedDoc, - Authentication authentication, - UpdateApiKeyRequest request, - Set userRoles, - Version version - ) throws IOException { - return client.prepareIndex(SECURITY_MAIN_ALIAS) - .setId(request.getId()) - .setSource( - buildUpdatedDocument( - currentVersionedDoc.doc(), - authentication, - userRoles, - request.getRoleDescriptors(), - version, - request.getMetadata() - ) - ) - .setIfSeqNo(currentVersionedDoc.seqNo()) - .setIfPrimaryTerm(currentVersionedDoc.primaryTerm()) - .request(); - } - private void executeBulkIndexRequest(BulkRequest bulkRequest, ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute( listener::onFailure, @@ -533,6 +525,10 @@ static XContentBuilder buildUpdatedDocument( if (metadata != null) { builder.field("metadata_flattened", metadata); } else { + assert currentApiKeyDoc.metadataFlattened == null + || MetadataUtils.containsReservedMetadata( + XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() + ) == false; builder.rawField( "metadata_flattened", currentApiKeyDoc.metadataFlattened == null From 6c3370319c1633d07f2490f908b33daf32c595b4 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 13:09:14 +0200 Subject: [PATCH 083/171] api key error message --- .../xpack/security/authc/ApiKeyService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index a3da112aabf46..6e19bd387f766 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -353,10 +353,10 @@ private void createApiKeyAndIndexIt( } public void updateApiKey( - Authentication authentication, - UpdateApiKeyRequest request, - Set userRoles, - ActionListener listener + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoles, + final ActionListener listener ) { ensureEnabled(); @@ -364,7 +364,7 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("api key cannot update api keys")); + listener.onFailure(new IllegalArgumentException("updating api keys is not supported with api key authentication")); return; } From d7bd7237e216afdbf9b6add5f309ea246b2c7ec1 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 13:20:03 +0200 Subject: [PATCH 084/171] More inline --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 6e19bd387f766..6b282e5dfd465 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -478,7 +478,7 @@ static XContentBuilder newDocument( Version version, @Nullable Map metadata ) throws IOException { - final var builder = XContentFactory.jsonBuilder(); + final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") .field("creation_time", created.toEpochMilli()) @@ -503,7 +503,7 @@ static XContentBuilder buildUpdatedDocument( Version version, Map metadata ) throws IOException { - final var builder = XContentFactory.jsonBuilder(); + final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") .field("creation_time", currentApiKeyDoc.creationTime) From 0b2457de7e54219f6e8991df056f9522419fcbae Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 14:19:56 +0200 Subject: [PATCH 085/171] Single and fix test --- .../xpack/security/authc/ApiKeyIntegTests.java | 3 +-- .../xpack/security/authc/ApiKeyService.java | 17 +++++++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 05a0872051fa5..b8f12ead7dff5 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -1595,7 +1594,7 @@ public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, Inter ); final var ex = expectThrows(ExecutionException.class, updateListener::get); - assertThat(ex.getCause(), instanceOf(ValidationException.class)); + assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); assertThat(ex.getMessage(), containsString("cannot update inactive api key [" + apiKeyId + "]")); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 6b282e5dfd465..0684e1e7d87a7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -400,7 +400,9 @@ void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentic } private void translateResponseAndClearCache(String apiKeyId, BulkResponse bulkResponse, ActionListener listener) { - final var bulkItemResponse = single(bulkResponse.getItems()); + final BulkItemResponse[] elements = bulkResponse.getItems(); + assert elements.length == 1 : "expected single item in bulk index response for api key update"; + final var bulkItemResponse = elements[0]; if (bulkItemResponse.isFailed()) { listener.onFailure(bulkItemResponse.getFailure().getCause()); } else { @@ -411,20 +413,15 @@ private void translateResponseAndClearCache(String apiKeyId, BulkResponse bulkRe } } - private static T single(Collection elements) { + private static VersionedApiKeyDoc single(Collection elements) { if (elements.size() != 1) { - throw new IllegalStateException("collection must have exactly one element but had [" + elements.size() + "]"); + final var message = "expected single api key doc to be found for update but found [" + elements.size() + "]"; + assert false : message; + throw new IllegalStateException(message); } return elements.iterator().next(); } - private static T single(T[] elements) { - if (elements.length != 1) { - throw new IllegalStateException("array must contain exactly one element but had [" + elements.length + "]"); - } - return elements[0]; - } - private BulkRequest buildBulkIndexRequestForUpdate( final Collection currentVersionedDocs, final Authentication authentication, From 3f283d952422c96bb2110e1a134d17a91c636b98 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 14:24:02 +0200 Subject: [PATCH 086/171] Better error message --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 2 +- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index b8f12ead7dff5..02d4e73e0981d 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1607,7 +1607,7 @@ private void testUpdateApiKeyNotFound( serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); final var ex = expectThrows(ExecutionException.class, listener::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("api key with id [" + request.getId() + "] owned by requesting user not found")); + assertThat(ex.getMessage(), containsString("no api key owned by requesting user has requested id [" + request.getId() + "]")); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 0684e1e7d87a7..f557e550e2ee1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -372,7 +372,7 @@ public void updateApiKey( final var apiKeyId = request.getId(); if (versionedDocs.isEmpty()) { - throw new ResourceNotFoundException("api key with id [" + apiKeyId + "] owned by requesting user not found"); + throw new ResourceNotFoundException("no api key owned by requesting user has requested id [" + apiKeyId + "]"); } validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); From 0d02ebad2b08e014f3d61bf1abde52cd99437c48 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 14:30:02 +0200 Subject: [PATCH 087/171] Nit --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 2 +- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 02d4e73e0981d..4f155946465c8 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1607,7 +1607,7 @@ private void testUpdateApiKeyNotFound( serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); final var ex = expectThrows(ExecutionException.class, listener::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("no api key owned by requesting user has requested id [" + request.getId() + "]")); + assertThat(ex.getMessage(), containsString("no api key owned by requesting user found for requested id [" + request.getId() + "]")); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index f557e550e2ee1..32ff0a6e2fa51 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -372,7 +372,7 @@ public void updateApiKey( final var apiKeyId = request.getId(); if (versionedDocs.isEmpty()) { - throw new ResourceNotFoundException("no api key owned by requesting user has requested id [" + apiKeyId + "]"); + throw new ResourceNotFoundException("no api key owned by requesting user found for requested id [" + apiKeyId + "]"); } validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); From 82fdb6b495de54b54e6b4d29a5d7eae493d282db Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 14:47:38 +0200 Subject: [PATCH 088/171] Api keys not allowed --- .../xpack/security/authc/ApiKeyIntegTests.java | 13 +++++++++++-- .../xpack/security/authc/ApiKeyService.java | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 4f155946465c8..8f556c7cd8781 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.RealmDomain; import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; @@ -1558,7 +1559,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter ); } - public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, InterruptedException { + public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, InterruptedException { final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); @@ -1584,7 +1585,7 @@ public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, Inter final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); final var serviceWithNodeName = getServiceWithNodeName(); - final PlainActionFuture updateListener = new PlainActionFuture<>(); + PlainActionFuture updateListener = new PlainActionFuture<>(); serviceWithNodeName.service() .updateApiKey( fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), @@ -1596,6 +1597,14 @@ public void testUpdateInactiveApiKeyScenarios() throws ExecutionException, Inter assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); assertThat(ex.getMessage(), containsString("cannot update inactive api key [" + apiKeyId + "]")); + + updateListener = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey(AuthenticationTestHelper.builder().apiKey().build(false), request, Set.of(roleDescriptor), updateListener); + final var apiKeysNotAllowedEx = expectThrows(ExecutionException.class, updateListener::get); + + assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); + assertThat(apiKeysNotAllowedEx.getMessage(), containsString("cannot use an api key to update api keys")); } private void testUpdateApiKeyNotFound( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 32ff0a6e2fa51..2f075433c1e60 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -364,7 +364,7 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("updating api keys is not supported with api key authentication")); + listener.onFailure(new IllegalArgumentException("cannot use an api key to update api keys")); return; } From 7a371a44014efa8b4b6569140cdb50be1c6875e1 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 14:55:00 +0200 Subject: [PATCH 089/171] Move assertion up --- .../xpack/security/authc/ApiKeyService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 2f075433c1e60..6728cc48bfe69 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -519,13 +519,14 @@ static XContentBuilder buildUpdatedDocument( addLimitedByRoleDescriptors(builder, userRoles); builder.field("name", currentApiKeyDoc.name).field("version", version.id); + + assert currentApiKeyDoc.metadataFlattened == null + || MetadataUtils.containsReservedMetadata( + XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() + ) == false : "api key doc to be updated contains reserved metadata"; if (metadata != null) { builder.field("metadata_flattened", metadata); } else { - assert currentApiKeyDoc.metadataFlattened == null - || MetadataUtils.containsReservedMetadata( - XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() - ) == false; builder.rawField( "metadata_flattened", currentApiKeyDoc.metadataFlattened == null From a9dd29f5157921d4c3b96e40b5b10151759cf82d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 16:19:44 +0200 Subject: [PATCH 090/171] WIP actions --- .../action/apikey/UpdateApiKeyAction.java | 20 ++++++ .../xpack/security/Security.java | 5 ++ .../apikey/TransportUpdateApiKeyAction.java | 71 ++++++++++++++++++ .../action/apikey/RestUpdateApiKeyAction.java | 72 +++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyAction.java new file mode 100644 index 0000000000000..9cacc909b14ea --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyAction.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.action.apikey; + +import org.elasticsearch.action.ActionType; + +public final class UpdateApiKeyAction extends ActionType { + + public static final String NAME = "cluster:admin/xpack/security/api_key/update"; + public static final UpdateApiKeyAction INSTANCE = new UpdateApiKeyAction(); + + private UpdateApiKeyAction() { + super(NAME, UpdateApiKeyResponse::new); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 27a590f133f3f..313b43a543f99 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -98,6 +98,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; import org.elasticsearch.xpack.core.security.action.enrollment.KibanaEnrollmentAction; import org.elasticsearch.xpack.core.security.action.enrollment.NodeEnrollmentAction; import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectAuthenticateAction; @@ -179,6 +180,7 @@ import org.elasticsearch.xpack.security.action.apikey.TransportGrantApiKeyAction; import org.elasticsearch.xpack.security.action.apikey.TransportInvalidateApiKeyAction; import org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction; +import org.elasticsearch.xpack.security.action.apikey.TransportUpdateApiKeyAction; import org.elasticsearch.xpack.security.action.enrollment.TransportKibanaEnrollmentAction; import org.elasticsearch.xpack.security.action.enrollment.TransportNodeEnrollmentAction; import org.elasticsearch.xpack.security.action.filter.SecurityActionFilter; @@ -276,6 +278,7 @@ import org.elasticsearch.xpack.security.rest.action.apikey.RestGrantApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestInvalidateApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestQueryApiKeyAction; +import org.elasticsearch.xpack.security.rest.action.apikey.RestUpdateApiKeyAction; import org.elasticsearch.xpack.security.rest.action.enrollment.RestKibanaEnrollAction; import org.elasticsearch.xpack.security.rest.action.enrollment.RestNodeEnrollmentAction; import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction; @@ -1223,6 +1226,7 @@ public void onIndexModule(IndexModule module) { new ActionHandler<>(InvalidateApiKeyAction.INSTANCE, TransportInvalidateApiKeyAction.class), new ActionHandler<>(GetApiKeyAction.INSTANCE, TransportGetApiKeyAction.class), new ActionHandler<>(QueryApiKeyAction.INSTANCE, TransportQueryApiKeyAction.class), + new ActionHandler<>(UpdateApiKeyAction.INSTANCE, TransportUpdateApiKeyAction.class), new ActionHandler<>(DelegatePkiAuthenticationAction.INSTANCE, TransportDelegatePkiAuthenticationAction.class), new ActionHandler<>(CreateServiceAccountTokenAction.INSTANCE, TransportCreateServiceAccountTokenAction.class), new ActionHandler<>(DeleteServiceAccountTokenAction.INSTANCE, TransportDeleteServiceAccountTokenAction.class), @@ -1300,6 +1304,7 @@ public List getRestHandlers( new RestPutPrivilegesAction(settings, getLicenseState()), new RestDeletePrivilegesAction(settings, getLicenseState()), new RestCreateApiKeyAction(settings, getLicenseState()), + new RestUpdateApiKeyAction(settings, getLicenseState()), new RestGrantApiKeyAction(settings, getLicenseState()), new RestInvalidateApiKeyAction(settings, getLicenseState()), new RestGetApiKeyAction(settings, getLicenseState()), diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java new file mode 100644 index 0000000000000..1bbefa97fd651 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.action.apikey; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.security.SecurityContext; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.Subject; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.security.authc.ApiKeyService; +import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; + +import java.util.Set; + +public final class TransportUpdateApiKeyAction extends HandledTransportAction { + + private final ApiKeyService apiKeyService; + private final SecurityContext securityContext; + + private final CompositeRolesStore rolesStore; + + @Inject + public TransportUpdateApiKeyAction( + TransportService transportService, + ActionFilters actionFilters, + ApiKeyService apiKeyService, + SecurityContext context, + CompositeRolesStore rolesStore + ) { + super(UpdateApiKeyAction.NAME, transportService, actionFilters, UpdateApiKeyRequest::new); + this.apiKeyService = apiKeyService; + this.securityContext = context; + this.rolesStore = rolesStore; + } + + @Override + protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener listener) { + final Authentication authentication = securityContext.getAuthentication(); + final Subject effectiveSubject = authentication.getEffectiveSubject(); + + // TODO none of this belongs here + if (effectiveSubject.getType() == Subject.Type.API_KEY) { + listener.onFailure(new IllegalStateException("api key cannot update api key")); + return; + } + + final ActionListener> roleDescriptorsListener = ActionListener.wrap( + roleDescriptors -> apiKeyService.updateApiKey(authentication, request, roleDescriptors, listener), + listener::onFailure + ); + + rolesStore.getRoleDescriptorsList(effectiveSubject, ActionListener.wrap(roleDescriptorsList -> { + assert roleDescriptorsList.size() == 1; + roleDescriptorsListener.onResponse(roleDescriptorsList.iterator().next()); + }, roleDescriptorsListener::onFailure)); + } + +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java new file mode 100644 index 0000000000000..119b2bee3626f --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.rest.action.apikey; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public final class RestUpdateApiKeyAction extends SecurityBaseRestHandler { + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "update_api_key_request_payload", + a -> new Payload((List) a[0], (Map) a[1]) + ); + + static { + PARSER.declareNamedObjects(optionalConstructorArg(), (p, c, n) -> { + p.nextToken(); + return RoleDescriptor.parse(n, p, false); + }, new ParseField("role_descriptors")); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(), new ParseField("metadata")); + } + + public RestUpdateApiKeyAction(Settings settings, XPackLicenseState licenseState) { + super(settings, licenseState); + } + + @Override + public List routes() { + return List.of(new Route(POST, "/_security/api_key/_update/{id}"), new Route(PUT, "/_security/api_key/_update/{id}")); + } + + @Override + public String getName() { + return "xpack_security_update_api_key"; + } + + @Override + protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final var apiKeyId = request.param("id"); + + final var payload = PARSER.parse(request.contentParser(), null); + + final var updateApiKeyRequest = new UpdateApiKeyRequest(apiKeyId, payload.roleDescriptors, payload.metadata); + + return channel -> client.execute(UpdateApiKeyAction.INSTANCE, updateApiKeyRequest, new RestToXContentListener<>(channel)); + } + + record Payload(List roleDescriptors, Map metadata) {} +} From 41d3f03719c38a03463d04b6c663db98cd4a97c3 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 16:21:49 +0200 Subject: [PATCH 091/171] Manage own api key priv --- .../authz/privilege/ManageOwnApiKeyClusterPrivilege.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java index cc428f1169567..201ec8ea12eaf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.RealmDomain; @@ -93,6 +94,8 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent } } else if (request instanceof final QueryApiKeyRequest queryApiKeyRequest) { return queryApiKeyRequest.isFilterForCurrentUser(); + } else if (request instanceof UpdateApiKeyRequest) { + return true; } else if (request instanceof GrantApiKeyRequest) { return false; } From d26f4593d1bb7faefa78fd920557bf0f258b6d78 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 16:56:07 +0200 Subject: [PATCH 092/171] WIP --- .../action/apikey/TransportUpdateApiKeyAction.java | 12 +++++++----- .../security/authc/support/ApiKeyGenerator.java | 1 - .../rest/action/apikey/RestUpdateApiKeyAction.java | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 1bbefa97fd651..8ba355575cc7d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -49,14 +49,16 @@ public TransportUpdateApiKeyAction( @Override protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener listener) { final Authentication authentication = securityContext.getAuthentication(); - final Subject effectiveSubject = authentication.getEffectiveSubject(); - - // TODO none of this belongs here - if (effectiveSubject.getType() == Subject.Type.API_KEY) { - listener.onFailure(new IllegalStateException("api key cannot update api key")); + if (authentication == null) { + listener.onFailure(new IllegalStateException("authentication is required")); + return; + } else if (authentication.isApiKey()) { + listener.onFailure(new IllegalArgumentException("cannot use an api key as a credential to update api keys")); return; } + final Subject effectiveSubject = authentication.getEffectiveSubject(); + final ActionListener> roleDescriptorsListener = ActionListener.wrap( roleDescriptors -> apiKeyService.updateApiKey(authentication, request, roleDescriptors, listener), listener::onFailure diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java index 619b6cbbc9c48..3db45cc75c10e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java @@ -64,7 +64,6 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re rolesStore.getRoleDescriptorsList(effectiveSubject, ActionListener.wrap(roleDescriptorsList -> { assert roleDescriptorsList.size() == 1; roleDescriptorsListener.onResponse(roleDescriptorsList.iterator().next()); - }, roleDescriptorsListener::onFailure)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java index 119b2bee3626f..55eaebeb7de21 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -61,6 +61,7 @@ public String getName() { protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { final var apiKeyId = request.param("id"); + // TODO check if fields are present or absent final var payload = PARSER.parse(request.contentParser(), null); final var updateApiKeyRequest = new UpdateApiKeyRequest(apiKeyId, payload.roleDescriptors, payload.metadata); From a9d005fe44b0676f117757a2fd21d6cc979967ad Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 17:00:14 +0200 Subject: [PATCH 093/171] Error message --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 5 ++++- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 8f556c7cd8781..3e8c2d0ba3be2 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1604,7 +1604,10 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr final var apiKeysNotAllowedEx = expectThrows(ExecutionException.class, updateListener::get); assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat(apiKeysNotAllowedEx.getMessage(), containsString("cannot use an api key to update api keys")); + assertThat( + apiKeysNotAllowedEx.getMessage(), + containsString("authentication through an api key is not supported for updating api keys") + ); } private void testUpdateApiKeyNotFound( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 6728cc48bfe69..d996b650d202b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -364,7 +364,7 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("cannot use an api key to update api keys")); + listener.onFailure(new IllegalArgumentException("authentication through an api key is not supported for updating api keys")); return; } From 49eba6215757daa80dba5f67bf686251ae833664 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 17:04:34 +0200 Subject: [PATCH 094/171] Nit --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index d996b650d202b..643293a516572 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -388,7 +388,7 @@ public void updateApiKey( void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentication, ApiKeyDoc apiKeyDoc) { assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.getOrDefault("principal", null)); - boolean isActive = apiKeyDoc.invalidated == false + final boolean isActive = apiKeyDoc.invalidated == false && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); if (isActive == false) { throw new IllegalArgumentException("cannot update inactive api key [" + apiKeyId + "]"); From 3fdb5ff7306c84c9f555b01b6f3ec9a31496a25f Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 17:36:21 +0200 Subject: [PATCH 095/171] Simplify --- .../apikey/TransportUpdateApiKeyAction.java | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 8ba355575cc7d..849e3cdda68b8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -7,24 +7,23 @@ package org.elasticsearch.xpack.security.action.apikey; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; -import org.elasticsearch.xpack.core.security.authc.Authentication; -import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator; import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; -import java.util.Set; - public final class TransportUpdateApiKeyAction extends HandledTransportAction { private final ApiKeyService apiKeyService; @@ -32,42 +31,50 @@ public final class TransportUpdateApiKeyAction extends HandledTransportAction listener) { - final Authentication authentication = securityContext.getAuthentication(); + final var authentication = securityContext.getAuthentication(); if (authentication == null) { listener.onFailure(new IllegalStateException("authentication is required")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("cannot use an api key as a credential to update api keys")); + listener.onFailure(new IllegalArgumentException("authentication through an api key is not supported for updating api keys")); return; } - final Subject effectiveSubject = authentication.getEffectiveSubject(); + apiKeyService.ensureEnabled(); - final ActionListener> roleDescriptorsListener = ActionListener.wrap( - roleDescriptors -> apiKeyService.updateApiKey(authentication, request, roleDescriptors, listener), - listener::onFailure - ); - - rolesStore.getRoleDescriptorsList(effectiveSubject, ActionListener.wrap(roleDescriptorsList -> { + rolesStore.getRoleDescriptorsList(authentication.getEffectiveSubject(), ActionListener.wrap(roleDescriptorsList -> { + // TODO duplicated code from ApiKeyGenerator assert roleDescriptorsList.size() == 1; - roleDescriptorsListener.onResponse(roleDescriptorsList.iterator().next()); - }, roleDescriptorsListener::onFailure)); + final var roleDescriptors = roleDescriptorsList.iterator().next(); + for (final RoleDescriptor rd : roleDescriptors) { + try { + DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry); + } catch (ElasticsearchException | IllegalArgumentException e) { + listener.onFailure(e); + return; + } + } + apiKeyService.updateApiKey(authentication, request, roleDescriptors, listener); + }, listener::onFailure)); } - } From df2b6a3292b19502c262b165fe1afab80a5e7883 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 17:38:07 +0200 Subject: [PATCH 096/171] Role descs --- .../action/apikey/TransportUpdateApiKeyAction.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 849e3cdda68b8..bfa2054a80f82 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -24,6 +24,8 @@ import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; +import java.util.Set; + public final class TransportUpdateApiKeyAction extends HandledTransportAction { private final ApiKeyService apiKeyService; @@ -65,10 +67,10 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< rolesStore.getRoleDescriptorsList(authentication.getEffectiveSubject(), ActionListener.wrap(roleDescriptorsList -> { // TODO duplicated code from ApiKeyGenerator assert roleDescriptorsList.size() == 1; - final var roleDescriptors = roleDescriptorsList.iterator().next(); - for (final RoleDescriptor rd : roleDescriptors) { + final Set roleDescriptors = roleDescriptorsList.iterator().next(); + for (final var roleDescriptor : roleDescriptors) { try { - DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry); + DLSRoleQueryValidator.validateQueryField(roleDescriptor.getIndicesPrivileges(), xContentRegistry); } catch (ElasticsearchException | IllegalArgumentException e) { listener.onFailure(e); return; From 58019b47b0e749c58ed72db3ca94e909082c599e Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 17:50:10 +0200 Subject: [PATCH 097/171] More tweaks --- .../security/action/apikey/TransportUpdateApiKeyAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index bfa2054a80f82..48ed3ac51600f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -62,10 +62,10 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< return; } + // TODO does this not belong here? apiKeyService.ensureEnabled(); rolesStore.getRoleDescriptorsList(authentication.getEffectiveSubject(), ActionListener.wrap(roleDescriptorsList -> { - // TODO duplicated code from ApiKeyGenerator assert roleDescriptorsList.size() == 1; final Set roleDescriptors = roleDescriptorsList.iterator().next(); for (final var roleDescriptor : roleDescriptors) { From 58a7e0243c09a62e17bb84147bfbef0c769ff609 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 27 Jun 2022 19:50:27 +0200 Subject: [PATCH 098/171] Assert --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 4 ++-- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 3e8c2d0ba3be2..a66a054314150 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1606,7 +1606,7 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); assertThat( apiKeysNotAllowedEx.getMessage(), - containsString("authentication through an api key is not supported for updating api keys") + containsString("authentication via an api key is not supported for updating api keys") ); } @@ -1619,7 +1619,7 @@ private void testUpdateApiKeyNotFound( serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); final var ex = expectThrows(ExecutionException.class, listener::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("no api key owned by requesting user found for requested id [" + request.getId() + "]")); + assertThat(ex.getMessage(), containsString("no api key owned by requesting user found for id [" + request.getId() + "]")); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 643293a516572..3013ea26d21c3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -364,7 +364,7 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("authentication through an api key is not supported for updating api keys")); + listener.onFailure(new IllegalArgumentException("authentication via an api key is not supported for updating api keys")); return; } @@ -372,7 +372,7 @@ public void updateApiKey( final var apiKeyId = request.getId(); if (versionedDocs.isEmpty()) { - throw new ResourceNotFoundException("no api key owned by requesting user found for requested id [" + apiKeyId + "]"); + throw new ResourceNotFoundException("no api key owned by requesting user found for id [" + apiKeyId + "]"); } validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); @@ -1173,6 +1173,7 @@ private void findVersionedApiKeyDocsForSubject( String[] apiKeyIds, ActionListener> listener ) { + assert authentication.isApiKey() == false; findApiKeysForUserRealmApiKeyIdAndNameCombination( getOwnersRealmNames(authentication), authentication.getEffectiveSubject().getUser().principal(), From a896ae0987dcc6da53d6f34011b0ebe6444711f2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 11:17:36 +0200 Subject: [PATCH 099/171] Address feedback --- .../action/apikey/UpdateApiKeyRequest.java | 5 +- .../apikey/UpdateApiKeyRequestTests.java | 6 +- .../security/authc/ApiKeyIntegTests.java | 42 +-- .../xpack/security/authc/ApiKeyService.java | 310 +++++++++--------- .../security/authc/ApiKeyServiceTests.java | 9 +- 5 files changed, 196 insertions(+), 176 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index e3645cfdc6e8f..ec09064a365cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.core.security.action.role.RoleDescriptorRequestValidator; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.support.MetadataUtils; @@ -25,10 +26,12 @@ public final class UpdateApiKeyRequest extends ActionRequest { private final String id; + @Nullable private final Map metadata; + @Nullable private final List roleDescriptors; - public UpdateApiKeyRequest(String id, List roleDescriptors, Map metadata) { + public UpdateApiKeyRequest(String id, @Nullable List roleDescriptors, @Nullable Map metadata) { this.id = Objects.requireNonNull(id, "api key id must not be null"); this.roleDescriptors = roleDescriptors; this.metadata = metadata; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java index d613014eee711..965268fb7f65a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java @@ -45,11 +45,7 @@ public void testSerialization() throws IOException { try (StreamInput in = out.bytes().streamInput()) { final var serialized = new UpdateApiKeyRequest(in); assertEquals(id, serialized.getId()); - if (roleDescriptorsPresent) { - assertEquals(descriptorList, serialized.getRoleDescriptors()); - } else { - assertNull(serialized.getRoleDescriptors()); - } + assertEquals(descriptorList, serialized.getRoleDescriptors()); assertEquals(metadata, request.getMetadata()); } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index a66a054314150..4688dd1d78bd4 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1533,7 +1533,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter // Test not found exception on non-existent API key final var otherApiKeyId = randomValueOtherThan(apiKeyId, () -> randomAlphaOfLength(20)); - testUpdateApiKeyNotFound( + doTestUpdateApiKeyNotFound( serviceWithNodeName, fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()) @@ -1541,14 +1541,14 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter // Test not found exception on other user's API key final var otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); - testUpdateApiKeyNotFound( + doTestUpdateApiKeyNotFound( serviceWithNodeName, fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) ); - // Test not found exception on API key of user from other realm - testUpdateApiKeyNotFound( + // Test not found exception on API key of user with the same username but from a different realm + doTestUpdateApiKeyNotFound( serviceWithNodeName, Authentication.newRealmAuthentication( new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), @@ -1560,7 +1560,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter } public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, InterruptedException { - final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final Tuple> createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); final boolean invalidated = randomBoolean(); @@ -1596,7 +1596,11 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr final var ex = expectThrows(ExecutionException.class, updateListener::get); assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat(ex.getMessage(), containsString("cannot update inactive api key [" + apiKeyId + "]")); + if (invalidated) { + assertThat(ex.getMessage(), containsString("cannot update invalidated API key [" + apiKeyId + "]")); + } else { + assertThat(ex.getMessage(), containsString("cannot update expired API key [" + apiKeyId + "]")); + } updateListener = new PlainActionFuture<>(); serviceWithNodeName.service() @@ -1606,22 +1610,10 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); assertThat( apiKeysNotAllowedEx.getMessage(), - containsString("authentication via an api key is not supported for updating api keys") + containsString("authentication via an API key is not supported for updating API keys") ); } - private void testUpdateApiKeyNotFound( - ServiceWithNodeName serviceWithNodeName, - Authentication authentication, - UpdateApiKeyRequest request - ) { - final PlainActionFuture listener = new PlainActionFuture<>(); - serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); - final var ex = expectThrows(ExecutionException.class, listener::get); - assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); - assertThat(ex.getMessage(), containsString("no api key owned by requesting user found for id [" + request.getId() + "]")); - } - public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { final List services = Arrays.stream(internalCluster().getNodeNames()) .map(n -> new ServiceWithNodeName(internalCluster().getInstance(ApiKeyService.class, n), n)) @@ -1683,6 +1675,18 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution assertEquals(serviceForDoc2AuthCacheCount, serviceForDoc2.getApiKeyAuthCache().count()); } + private void doTestUpdateApiKeyNotFound( + ServiceWithNodeName serviceWithNodeName, + Authentication authentication, + UpdateApiKeyRequest request + ) { + final PlainActionFuture listener = new PlainActionFuture<>(); + serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); + final var ex = expectThrows(ExecutionException.class, listener::get); + assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); + assertThat(ex.getMessage(), containsString("no API key owned by requesting user found for ID [" + request.getId() + "]")); + } + private static Authentication fileRealmAuth(String nodeName, String userName, String roleName) { boolean includeDomain = randomBoolean(); final var realmName = "file"; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3013ea26d21c3..4ed8464dc8746 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -16,6 +16,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -364,21 +365,23 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("authentication via an api key is not supported for updating api keys")); + listener.onFailure(new IllegalArgumentException("authentication via an API key is not supported for updating API keys")); return; } + logger.debug("Updating API key [{}]", request.getId()); + findVersionedApiKeyDocsForSubject(authentication, new String[] { request.getId() }, ActionListener.wrap((versionedDocs) -> { final var apiKeyId = request.getId(); if (versionedDocs.isEmpty()) { - throw new ResourceNotFoundException("no api key owned by requesting user found for id [" + apiKeyId + "]"); + throw new ResourceNotFoundException("no API key owned by requesting user found for ID [" + apiKeyId + "]"); } - validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(versionedDocs).doc()); + validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(apiKeyId, versionedDocs).doc()); - executeBulkIndexRequest( - buildBulkIndexRequestForUpdate(versionedDocs, authentication, request, userRoles), + executeBulkRequest( + buildBulkRequestForUpdate(versionedDocs, authentication, request, userRoles), ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) ); }, listener::onFailure)); @@ -386,79 +389,22 @@ public void updateApiKey( // package-private for testing void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentication, ApiKeyDoc apiKeyDoc) { - assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.getOrDefault("principal", null)); + assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.get("principal")); - final boolean isActive = apiKeyDoc.invalidated == false - && (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())); - if (isActive == false) { - throw new IllegalArgumentException("cannot update inactive api key [" + apiKeyId + "]"); + if (apiKeyDoc.invalidated) { + throw new IllegalArgumentException("cannot update invalidated API key [" + apiKeyId + "]"); } - if (Strings.isNullOrEmpty(apiKeyDoc.name)) { - throw new IllegalArgumentException("cannot update legacy api key [" + apiKeyId + "] without name"); + final var now = clock.instant(); + boolean expired = apiKeyDoc.expirationTime != -1 + && (Instant.ofEpochMilli(apiKeyDoc.expirationTime).equals(now) || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isBefore(now)); + if (expired) { + throw new IllegalArgumentException("cannot update expired API key [" + apiKeyId + "]"); } - } - - private void translateResponseAndClearCache(String apiKeyId, BulkResponse bulkResponse, ActionListener listener) { - final BulkItemResponse[] elements = bulkResponse.getItems(); - assert elements.length == 1 : "expected single item in bulk index response for api key update"; - final var bulkItemResponse = elements[0]; - if (bulkItemResponse.isFailed()) { - listener.onFailure(bulkItemResponse.getFailure().getCause()); - } else { - assert bulkItemResponse.getResponse().getId().equals(apiKeyId); - // Since we made an index request against an existing document, we can't get a NOOP or CREATED here - assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; - clearApiKeyDocCache(apiKeyId, new UpdateApiKeyResponse(true), listener); - } - } - private static VersionedApiKeyDoc single(Collection elements) { - if (elements.size() != 1) { - final var message = "expected single api key doc to be found for update but found [" + elements.size() + "]"; - assert false : message; - throw new IllegalStateException(message); - } - return elements.iterator().next(); - } - - private BulkRequest buildBulkIndexRequestForUpdate( - final Collection currentVersionedDocs, - final Authentication authentication, - final UpdateApiKeyRequest request, - final Set userRoles - ) throws IOException { - assert currentVersionedDocs.isEmpty() == false; - final var version = clusterService.state().nodes().getMinNodeVersion(); - final var bulkRequestBuilder = client.prepareBulk(); - for (VersionedApiKeyDoc apiKeyDoc : currentVersionedDocs) { - bulkRequestBuilder.add( - client.prepareIndex(SECURITY_MAIN_ALIAS) - .setId(request.getId()) - .setSource( - buildUpdatedDocument( - apiKeyDoc.doc(), - authentication, - userRoles, - request.getRoleDescriptors(), - version, - request.getMetadata() - ) - ) - .setIfSeqNo(apiKeyDoc.seqNo()) - .setIfPrimaryTerm(apiKeyDoc.primaryTerm()) - .request() - ); + if (Strings.isNullOrEmpty(apiKeyDoc.name)) { + throw new IllegalArgumentException("cannot update legacy API key [" + apiKeyId + "] without name"); } - bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - return bulkRequestBuilder.request(); - } - - private void executeBulkIndexRequest(BulkRequest bulkRequest, ActionListener listener) { - securityIndex.prepareIndexIfNeededThenExecute( - listener::onFailure, - () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) - ); } /** @@ -523,7 +469,7 @@ static XContentBuilder buildUpdatedDocument( assert currentApiKeyDoc.metadataFlattened == null || MetadataUtils.containsReservedMetadata( XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() - ) == false : "api key doc to be updated contains reserved metadata"; + ) == false : "API key doc to be updated contains reserved metadata"; if (metadata != null) { builder.field("metadata_flattened", metadata); } else { @@ -541,53 +487,6 @@ static XContentBuilder buildUpdatedDocument( return builder.endObject(); } - private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { - assert userRoles != null; - builder.startObject("limited_by_role_descriptors"); - for (RoleDescriptor descriptor : userRoles) { - builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); - } - builder.endObject(); - } - - private static void addApiKeyHash(XContentBuilder builder, char[] apiKeyHashChars) throws IOException { - byte[] utf8Bytes = null; - try { - utf8Bytes = CharArrays.toUtf8Bytes(apiKeyHashChars); - builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); - } finally { - if (utf8Bytes != null) { - Arrays.fill(utf8Bytes, (byte) 0); - } - } - } - - private static void addCreator(XContentBuilder builder, Authentication authentication) throws IOException { - final var user = authentication.getEffectiveSubject().getUser(); - final var sourceRealm = authentication.getEffectiveSubject().getRealm(); - builder.startObject("creator") - .field("principal", user.principal()) - .field("full_name", user.fullName()) - .field("email", user.email()) - .field("metadata", user.metadata()) - .field("realm", sourceRealm.getName()) - .field("realm_type", sourceRealm.getType()); - if (sourceRealm.getDomain() != null) { - builder.field("realm_domain", sourceRealm.getDomain()); - } - builder.endObject(); - } - - private static void addRoleDescriptors(XContentBuilder builder, List keyRoles) throws IOException { - builder.startObject("role_descriptors"); - if (keyRoles != null && keyRoles.isEmpty() == false) { - for (RoleDescriptor descriptor : keyRoles) { - builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); - } - } - builder.endObject(); - } - void tryAuthenticate(ThreadContext ctx, ApiKeyCredentials credentials, ActionListener> listener) { if (false == isEnabled()) { listener.onResponse(AuthenticationResult.notHandled()); @@ -1098,6 +997,7 @@ public void invalidateApiKeys( apiKeyIds, true, false, + ApiKeyService::convertSearchHitToApiKeyInfo, ActionListener.wrap(apiKeys -> { if (apiKeys.isEmpty()) { logger.debug( @@ -1169,9 +1069,9 @@ public static QueryBuilder filterForRealmNames(String[] realmNames) { } private void findVersionedApiKeyDocsForSubject( - Authentication authentication, - String[] apiKeyIds, - ActionListener> listener + final Authentication authentication, + final String[] apiKeyIds, + final ActionListener> listener ) { assert authentication.isApiKey() == false; findApiKeysForUserRealmApiKeyIdAndNameCombination( @@ -1181,29 +1081,8 @@ private void findVersionedApiKeyDocsForSubject( apiKeyIds, false, false, - listener, - ApiKeyService::convertSearchHitToVersionedApiKeyDoc - ); - } - - private void findApiKeysForUserRealmApiKeyIdAndNameCombination( - String[] realmNames, - String userName, - String apiKeyName, - String[] apiKeyIds, - boolean filterOutInvalidatedKeys, - boolean filterOutExpiredKeys, - ActionListener> listener - ) { - findApiKeysForUserRealmApiKeyIdAndNameCombination( - realmNames, - userName, - apiKeyName, - apiKeyIds, - filterOutInvalidatedKeys, - filterOutExpiredKeys, - listener, - ApiKeyService::convertSearchHitToApiKeyInfo + ApiKeyService::convertSearchHitToVersionedApiKeyDoc, + listener ); } @@ -1214,8 +1093,8 @@ private void findApiKeysForUserRealmApiKeyIdAndNameCombination( String[] apiKeyIds, boolean filterOutInvalidatedKeys, boolean filterOutExpiredKeys, - ActionListener> listener, - Function hitParser + Function hitParser, + ActionListener> listener ) { final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); if (frozenSecurityIndex.indexExists() == false) { @@ -1319,6 +1198,140 @@ private void indexInvalidation( } } + private void translateResponseAndClearCache( + final String apiKeyId, + final BulkResponse bulkResponse, + final ActionListener listener + ) { + final BulkItemResponse[] elements = bulkResponse.getItems(); + assert elements.length == 1 : "expected single item in bulk index response for API key update"; + final var bulkItemResponse = elements[0]; + if (bulkItemResponse.isFailed()) { + listener.onFailure(bulkItemResponse.getFailure().getCause()); + } else { + assert bulkItemResponse.getResponse().getId().equals(apiKeyId); + // Since we made an index request against an existing document, we can't get a NOOP or CREATED here + assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; + clearApiKeyDocCache(apiKeyId, new UpdateApiKeyResponse(true), listener); + } + } + + private static VersionedApiKeyDoc single(final String apiKeyId, final Collection elements) { + if (elements.size() != 1) { + final var message = "expected single API key doc with ID [" + + apiKeyId + + "] to be found for update but found [" + + elements.size() + + "]"; + assert false : message; + throw new IllegalStateException(message); + } + return elements.iterator().next(); + } + + private BulkRequest buildBulkRequestForUpdate( + final Collection currentVersionedDocs, + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoles + ) throws IOException { + assert currentVersionedDocs.isEmpty() == false; + final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); + final var bulkRequestBuilder = client.prepareBulk(); + for (final VersionedApiKeyDoc apiKeyDoc : currentVersionedDocs) { + logger.trace( + "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", + request.getId(), + apiKeyDoc.seqNo(), + apiKeyDoc.primaryTerm() + ); + final var currentDocVersion = Version.fromId(apiKeyDoc.doc().version); + assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; + if (currentDocVersion.before(targetDocVersion)) { + logger.debug( + "API key update for [{}] will update version from [{}] to [{}]", + request.getId(), + currentDocVersion, + targetDocVersion + ); + } + bulkRequestBuilder.add( + client.prepareIndex(SECURITY_MAIN_ALIAS) + .setId(request.getId()) + .setSource( + buildUpdatedDocument( + apiKeyDoc.doc(), + authentication, + userRoles, + request.getRoleDescriptors(), + targetDocVersion, + request.getMetadata() + ) + ) + .setIfSeqNo(apiKeyDoc.seqNo()) + .setIfPrimaryTerm(apiKeyDoc.primaryTerm()) + .setOpType(DocWriteRequest.OpType.INDEX) + .request() + ); + } + bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + return bulkRequestBuilder.request(); + } + + private void executeBulkRequest(BulkRequest bulkRequest, ActionListener listener) { + securityIndex.prepareIndexIfNeededThenExecute( + listener::onFailure, + () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) + ); + } + + private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { + assert userRoles != null; + builder.startObject("limited_by_role_descriptors"); + for (RoleDescriptor descriptor : userRoles) { + builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); + } + builder.endObject(); + } + + private static void addApiKeyHash(XContentBuilder builder, char[] apiKeyHashChars) throws IOException { + byte[] utf8Bytes = null; + try { + utf8Bytes = CharArrays.toUtf8Bytes(apiKeyHashChars); + builder.field("api_key_hash").utf8Value(utf8Bytes, 0, utf8Bytes.length); + } finally { + if (utf8Bytes != null) { + Arrays.fill(utf8Bytes, (byte) 0); + } + } + } + + private static void addCreator(XContentBuilder builder, Authentication authentication) throws IOException { + final var user = authentication.getEffectiveSubject().getUser(); + final var sourceRealm = authentication.getEffectiveSubject().getRealm(); + builder.startObject("creator") + .field("principal", user.principal()) + .field("full_name", user.fullName()) + .field("email", user.email()) + .field("metadata", user.metadata()) + .field("realm", sourceRealm.getName()) + .field("realm_type", sourceRealm.getType()); + if (sourceRealm.getDomain() != null) { + builder.field("realm_domain", sourceRealm.getDomain()); + } + builder.endObject(); + } + + private static void addRoleDescriptors(XContentBuilder builder, List keyRoles) throws IOException { + builder.startObject("role_descriptors"); + if (keyRoles != null && keyRoles.isEmpty() == false) { + for (RoleDescriptor descriptor : keyRoles) { + builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); + } + } + builder.endObject(); + } + private void clearCache(InvalidateApiKeyResponse result, ActionListener listener) { executeClearCacheRequest( result, @@ -1426,6 +1439,7 @@ public void getApiKeys( apiKeyIds, false, false, + ApiKeyService::convertSearchHitToApiKeyInfo, ActionListener.wrap(apiKeyInfos -> { if (apiKeyInfos.isEmpty()) { logger.debug( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 367f0b7cc2d4e..1d16d28d99aa3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -1659,14 +1659,14 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { IllegalArgumentException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, apiKeyDocWithNullName) ); - assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); + assertThat(ex.getMessage(), containsString("cannot update legacy API key [" + apiKeyId + "] without name")); final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_2_0.id); ex = expectThrows( IllegalArgumentException.class, () -> apiKeyService.validateCurrentApiKeyDocForUpdate(apiKeyId, auth, apiKeyDocWithEmptyName) ); - assertThat(ex.getMessage(), containsString("cannot update legacy api key [" + apiKeyId + "] without name")); + assertThat(ex.getMessage(), containsString("cannot update legacy API key [" + apiKeyId + "] without name")); } public void testBuildUpdatedDocument() throws IOException { @@ -1688,7 +1688,10 @@ public void testBuildUpdatedDocument() throws IOException { final var metadata = ApiKeyTests.randomMetadata(); final var version = Version.CURRENT; - final var authentication = AuthenticationTestHelper.builder().user(new User("user", "role")).build(false); + final var authentication = randomValueOtherThanMany( + Authentication::isApiKey, + () -> AuthenticationTestHelper.builder().user(new User("user", "role")).build(false) + ); final var keyDocSource = ApiKeyService.buildUpdatedDocument( oldApiKeyDoc, From 542b360ba4db471314b9991dc10ccc53d09aa596 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 11:27:30 +0200 Subject: [PATCH 100/171] More logs and finals --- .../xpack/security/authc/ApiKeyService.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 4ed8464dc8746..0b96aa9b4a85f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -439,12 +439,12 @@ static XContentBuilder newDocument( } static XContentBuilder buildUpdatedDocument( - ApiKeyDoc currentApiKeyDoc, - Authentication authentication, - Set userRoles, - List keyRoles, - Version version, - Map metadata + final ApiKeyDoc currentApiKeyDoc, + final Authentication authentication, + final Set userRoles, + final List keyRoles, + final Version version, + final Map metadata ) throws IOException { final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject() @@ -456,6 +456,7 @@ static XContentBuilder buildUpdatedDocument( addApiKeyHash(builder, currentApiKeyDoc.hash.toCharArray()); if (keyRoles != null) { + logger.trace("Building API key doc with updated role descriptors [{}]", keyRoles); addRoleDescriptors(builder, keyRoles); } else { assert currentApiKeyDoc.roleDescriptorsBytes != null; @@ -471,6 +472,7 @@ static XContentBuilder buildUpdatedDocument( XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() ) == false : "API key doc to be updated contains reserved metadata"; if (metadata != null) { + logger.trace("Building API key doc with updated metadata [{}]", metadata); builder.field("metadata_flattened", metadata); } else { builder.rawField( @@ -1278,14 +1280,14 @@ private BulkRequest buildBulkRequestForUpdate( return bulkRequestBuilder.request(); } - private void executeBulkRequest(BulkRequest bulkRequest, ActionListener listener) { + private void executeBulkRequest(final BulkRequest bulkRequest, final ActionListener listener) { securityIndex.prepareIndexIfNeededThenExecute( listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) ); } - private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set userRoles) throws IOException { + private static void addLimitedByRoleDescriptors(final XContentBuilder builder, final Set userRoles) throws IOException { assert userRoles != null; builder.startObject("limited_by_role_descriptors"); for (RoleDescriptor descriptor : userRoles) { @@ -1294,7 +1296,7 @@ private static void addLimitedByRoleDescriptors(XContentBuilder builder, Set keyRoles) throws IOException { + private static void addRoleDescriptors(final XContentBuilder builder, final List keyRoles) throws IOException { builder.startObject("role_descriptors"); if (keyRoles != null && keyRoles.isEmpty() == false) { for (RoleDescriptor descriptor : keyRoles) { From b9fcad14774bdbf8b32edc20cea49ab211a98629 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 11:57:18 +0200 Subject: [PATCH 101/171] More randomization in test --- .../security/authc/ApiKeyIntegTests.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 4688dd1d78bd4..0a3a4c4953031 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1427,7 +1427,7 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce } public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { - final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final Tuple> createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); final var newRoleDescriptors = randomRoleDescriptors(); @@ -1511,7 +1511,7 @@ private List randomRoleDescriptors() { } public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { - final var createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final Tuple> createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); @@ -1540,7 +1540,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter ); // Test not found exception on other user's API key - final var otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); + final Tuple> otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); doTestUpdateApiKeyNotFound( serviceWithNodeName, fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), @@ -1690,20 +1690,25 @@ private void doTestUpdateApiKeyNotFound( private static Authentication fileRealmAuth(String nodeName, String userName, String roleName) { boolean includeDomain = randomBoolean(); final var realmName = "file"; - final var realmType = FileRealmSettings.TYPE; - return Authentication.newRealmAuthentication( - new User(userName, roleName), - new Authentication.RealmRef( - realmName, - realmType, - nodeName, - includeDomain - ? new RealmDomain( - ESTestCase.randomAlphaOfLengthBetween(3, 8), - Set.of(new RealmConfig.RealmIdentifier(realmType, realmName)) + final String realmType = FileRealmSettings.TYPE; + return randomValueOtherThanMany( + Authentication::isApiKey, + () -> AuthenticationTestHelper.builder() + .user(new User(userName, roleName)) + .realmRef( + new Authentication.RealmRef( + realmName, + realmType, + nodeName, + includeDomain + ? new RealmDomain( + ESTestCase.randomAlphaOfLengthBetween(3, 8), + Set.of(new RealmConfig.RealmIdentifier(realmType, realmName)) + ) + : null ) - : null - ) + ) + .build() ); } @@ -1881,7 +1886,7 @@ private void verifyGetResponse( } private Tuple> createApiKey(String user, TimeValue expiration) { - final var res = createApiKeys(user, 1, expiration, "monitor"); + final Tuple, List>> res = createApiKeys(user, 1, expiration, "monitor"); return new Tuple<>(res.v1().get(0), res.v2().get(0)); } From 87b6fb03a19268843446e81f14d05bd2efd0b6c3 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 12:30:16 +0200 Subject: [PATCH 102/171] Caps API key --- .../xpack/core/security/action/apikey/UpdateApiKeyRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index ec09064a365cb..8612bc4fd95d3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -32,7 +32,7 @@ public final class UpdateApiKeyRequest extends ActionRequest { private final List roleDescriptors; public UpdateApiKeyRequest(String id, @Nullable List roleDescriptors, @Nullable Map metadata) { - this.id = Objects.requireNonNull(id, "api key id must not be null"); + this.id = Objects.requireNonNull(id, "API key ID must not be null"); this.roleDescriptors = roleDescriptors; this.metadata = metadata; } From 59b9093e1e858311014dca4659be9beb25a3cf42 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 14:41:00 +0200 Subject: [PATCH 103/171] WIP --- .../security/action/apikey/TransportUpdateApiKeyAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 48ed3ac51600f..e73e918293e69 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -58,7 +58,7 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< listener.onFailure(new IllegalStateException("authentication is required")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("authentication through an api key is not supported for updating api keys")); + listener.onFailure(new IllegalArgumentException("authentication via an API key is not supported for updating API keys")); return; } From ca65709a39885ec4ee953fe7dcae63e33ed9d5c5 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 14:46:49 +0200 Subject: [PATCH 104/171] Address review --- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 0b96aa9b4a85f..1826967ce0f20 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -395,9 +395,7 @@ void validateCurrentApiKeyDocForUpdate(String apiKeyId, Authentication authentic throw new IllegalArgumentException("cannot update invalidated API key [" + apiKeyId + "]"); } - final var now = clock.instant(); - boolean expired = apiKeyDoc.expirationTime != -1 - && (Instant.ofEpochMilli(apiKeyDoc.expirationTime).equals(now) || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isBefore(now)); + boolean expired = apiKeyDoc.expirationTime != -1 && clock.instant().isAfter(Instant.ofEpochMilli(apiKeyDoc.expirationTime)); if (expired) { throw new IllegalArgumentException("cannot update expired API key [" + apiKeyId + "]"); } @@ -456,7 +454,7 @@ static XContentBuilder buildUpdatedDocument( addApiKeyHash(builder, currentApiKeyDoc.hash.toCharArray()); if (keyRoles != null) { - logger.trace("Building API key doc with updated role descriptors [{}]", keyRoles); + logger.trace(() -> format("Building API key doc with updated role descriptors [{}]", keyRoles)); addRoleDescriptors(builder, keyRoles); } else { assert currentApiKeyDoc.roleDescriptorsBytes != null; @@ -472,7 +470,7 @@ static XContentBuilder buildUpdatedDocument( XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() ) == false : "API key doc to be updated contains reserved metadata"; if (metadata != null) { - logger.trace("Building API key doc with updated metadata [{}]", metadata); + logger.trace(() -> format("Building API key doc with updated metadata [{}]", metadata)); builder.field("metadata_flattened", metadata); } else { builder.rawField( From 8a1a88b74c97cd22f34133a2f50f5c98737b50e9 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 17:10:37 +0200 Subject: [PATCH 105/171] Hard-won rest test --- .../action/apikey/UpdateApiKeyRequest.java | 13 +++++ .../action/apikey/RestUpdateApiKeyAction.java | 14 ++--- .../apikey/RestUpdateApiKeyActionTests.java | 56 +++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 8612bc4fd95d3..59b8dcc4ab1e1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -93,4 +93,17 @@ public Map getMetadata() { public List getRoleDescriptors() { return roleDescriptors; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UpdateApiKeyRequest that = (UpdateApiKeyRequest) o; + return id.equals(that.id) && Objects.equals(metadata, that.metadata) && Objects.equals(roleDescriptors, that.roleDescriptors); + } + + @Override + public int hashCode() { + return Objects.hash(id, metadata, roleDescriptors); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java index 55eaebeb7de21..62574f9e24bc2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -49,7 +49,8 @@ public RestUpdateApiKeyAction(Settings settings, XPackLicenseState licenseState) @Override public List routes() { - return List.of(new Route(POST, "/_security/api_key/_update/{id}"), new Route(PUT, "/_security/api_key/_update/{id}")); + final var path = "/_security/api_key/_update/{id}"; + return List.of(new Route(POST, path), new Route(PUT, path)); } @Override @@ -60,13 +61,12 @@ public String getName() { @Override protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { final var apiKeyId = request.param("id"); - - // TODO check if fields are present or absent final var payload = PARSER.parse(request.contentParser(), null); - - final var updateApiKeyRequest = new UpdateApiKeyRequest(apiKeyId, payload.roleDescriptors, payload.metadata); - - return channel -> client.execute(UpdateApiKeyAction.INSTANCE, updateApiKeyRequest, new RestToXContentListener<>(channel)); + return channel -> client.execute( + UpdateApiKeyAction.INSTANCE, + new UpdateApiKeyRequest(apiKeyId, payload.roleDescriptors, payload.metadata), + new RestToXContentListener<>(channel) + ); } record Payload(List roleDescriptors, Map metadata) {} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java new file mode 100644 index 0000000000000..e588af927439c --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.rest.action.apikey; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; +import org.junit.Before; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; + +public class RestUpdateApiKeyActionTests extends RestActionTestCase { + + private RestUpdateApiKeyAction restAction; + private AtomicReference requestHolder; + + @Before + public void init() { + final Settings settings = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build(); + final XPackLicenseState licenseState = mock(XPackLicenseState.class); + requestHolder = new AtomicReference<>(); + restAction = new RestUpdateApiKeyAction(settings, licenseState); + controller().registerHandler(restAction); + verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { + assertThat(actionRequest, instanceOf(UpdateApiKeyRequest.class)); + requestHolder.set((UpdateApiKeyRequest) actionRequest); + return new UpdateApiKeyResponse(true); + })); + } + + public void testAbsentRoleDescriptorsAndMetadataSetToNull() { + final var apiKeyId = "api_key_id"; + final FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod( + randomFrom(RestRequest.Method.PUT, RestRequest.Method.POST) + ).withPath("/_security/api_key/_update/" + apiKeyId).withContent(new BytesArray("{}"), XContentType.JSON).build(); + + dispatchRequest(restRequest); + + assertEquals(new UpdateApiKeyRequest(apiKeyId, null, null), requestHolder.get()); + } +} From 782f6ec5e03e04b0d4aa889513b775df153c3233 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 17:22:07 +0200 Subject: [PATCH 106/171] Nits --- .../security/action/apikey/TransportUpdateApiKeyAction.java | 2 -- .../security/rest/action/apikey/RestUpdateApiKeyAction.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index e73e918293e69..71c3f42a6c095 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -30,9 +30,7 @@ public final class TransportUpdateApiKeyAction extends HandledTransportAction p.map(), new ParseField("metadata")); } - public RestUpdateApiKeyAction(Settings settings, XPackLicenseState licenseState) { + public RestUpdateApiKeyAction(final Settings settings, final XPackLicenseState licenseState) { super(settings, licenseState); } From 60a07555da53701fd06fba5a2b7b47e1de63630b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 28 Jun 2022 17:56:58 +0200 Subject: [PATCH 107/171] Plug in client --- .../security/authc/ApiKeyIntegTests.java | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 0a3a4c4953031..8d8c16caaf8c2 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; @@ -105,6 +106,8 @@ import java.util.stream.Stream; import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER; +import static org.elasticsearch.test.SecuritySettingsSource.TEST_ROLE; +import static org.elasticsearch.test.SecuritySettingsSource.TEST_USER_NAME; import static org.elasticsearch.test.SecuritySettingsSourceField.ES_TEST_ROOT_ROLE; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; @@ -1427,47 +1430,47 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce } public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { - final Tuple> createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); final var newRoleDescriptors = randomRoleDescriptors(); final boolean nullRoleDescriptors = newRoleDescriptors == null; + // TODO parse role? final var expectedLimitedByRoleDescriptors = Set.of( - new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null) + new RoleDescriptor( + TEST_ROLE, + new String[] { "ALL" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("*").allowRestrictedIndices(true).privileges("ALL").build() }, + null + ) ); final var request = new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, ApiKeyTests.randomMetadata()); - final var serviceWithNodeName = getServiceWithNodeName(); final PlainActionFuture listener = new PlainActionFuture<>(); - serviceWithNodeName.service() - .updateApiKey( - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - request, - expectedLimitedByRoleDescriptors, - listener - ); + final Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(UpdateApiKeyAction.INSTANCE, request, listener); final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); // Correct data returned from GET API - Client client = client().filterWithHeader( - Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) - ); final PlainActionFuture getListener = new PlainActionFuture<>(); client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener); - GetApiKeyResponse getResponse = getListener.get(); + final GetApiKeyResponse getResponse = getListener.get(); assertEquals(1, getResponse.getApiKeyInfos().length); // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it final var expectedMetadata = request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(); assertEquals(expectedMetadata == null ? Map.of() : expectedMetadata, getResponse.getApiKeyInfos()[0].getMetadata()); - assertEquals(ES_TEST_ROOT_USER, getResponse.getApiKeyInfos()[0].getUsername()); + assertEquals(TEST_USER_NAME, getResponse.getApiKeyInfos()[0].getUsername()); assertEquals("file", getResponse.getApiKeyInfos()[0].getRealm()); // Test authenticate works with updated API key final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); - assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(ES_TEST_ROOT_USER)); + assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(TEST_USER_NAME)); // Document updated as expected final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); @@ -1721,9 +1724,9 @@ private void expectMetadataForApiKey(Map expectedMetadata, Map expectedRoleDescriptors, - Map actualRawApiKeyDoc + final String roleDescriptorType, + final Collection expectedRoleDescriptors, + final Map actualRawApiKeyDoc ) throws IOException { assertNotNull(actualRawApiKeyDoc); assertThat(roleDescriptorType, in(new String[] { "role_descriptors", "limited_by_role_descriptors" })); From 7e2e786be4e02ddd19f19aff941814b1e9a9f7af Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 13:11:44 +0200 Subject: [PATCH 108/171] WIP --- .../security/authc/ApiKeyIntegTests.java | 89 +++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 79d9278ca4415..adb798668a0dd 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1435,6 +1435,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var newRoleDescriptors = randomRoleDescriptors(); final boolean nullRoleDescriptors = newRoleDescriptors == null; + // TODO parse role? final var expectedLimitedByRoleDescriptors = Set.of( new RoleDescriptor( @@ -1498,65 +1499,46 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, } } - private List randomRoleDescriptors() { - int caseNo = randomIntBetween(0, 2); - return switch (caseNo) { - case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); - case 1 -> List.of( - new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null), - RoleDescriptorTests.randomRoleDescriptor() - ); - case 2 -> null; - default -> throw new IllegalStateException("unexpected case no"); - }; - } - public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { - final Tuple> createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); + final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); final var expectedRoleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); // Validate can update own API key - final var serviceWithNodeName = getServiceWithNodeName(); final PlainActionFuture listener = new PlainActionFuture<>(); - serviceWithNodeName.service() - .updateApiKey( - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - request, - Set.of(expectedRoleDescriptor), - listener - ); + final Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(UpdateApiKeyAction.INSTANCE, request, listener); final var response = listener.get(); - assertNotNull(response); assertTrue(response.isUpdated()); // Test not found exception on non-existent API key final var otherApiKeyId = randomValueOtherThan(apiKeyId, () -> randomAlphaOfLength(20)); - doTestUpdateApiKeyNotFound( - serviceWithNodeName, - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata()) - ); + doTestUpdateApiKeyNotFound(new UpdateApiKeyRequest(otherApiKeyId, request.getRoleDescriptors(), request.getMetadata())); // Test not found exception on other user's API key final Tuple> otherUsersApiKey = createApiKey("user_with_manage_api_key_role", null); doTestUpdateApiKeyNotFound( - serviceWithNodeName, - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) ); // Test not found exception on API key of user with the same username but from a different realm + // Authentication authentication = Authentication.newRealmAuthentication( + // new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), + // // Use native realm; no need to actually create user since we are injecting the authentication object directly + // new Authentication.RealmRef(NativeRealmSettings.DEFAULT_NAME, NativeRealmSettings.TYPE, serviceWithNodeName.nodeName()) + // ); + createNativeRealmUser( + TEST_USER_NAME, + TEST_ROLE, + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ); + final Tuple> apiKeyForNativeRealmUser = createApiKey(TEST_USER_NAME, null); doTestUpdateApiKeyNotFound( - serviceWithNodeName, - Authentication.newRealmAuthentication( - new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - // Use native realm; no need to actually create user since we are injecting the authentication object directly - new Authentication.RealmRef(NativeRealmSettings.DEFAULT_NAME, NativeRealmSettings.TYPE, serviceWithNodeName.nodeName()) - ), - new UpdateApiKeyRequest(apiKeyId, request.getRoleDescriptors(), request.getMetadata()) + new UpdateApiKeyRequest(apiKeyForNativeRealmUser.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) ); } @@ -1676,13 +1658,25 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution assertEquals(serviceForDoc2AuthCacheCount, serviceForDoc2.getApiKeyAuthCache().count()); } - private void doTestUpdateApiKeyNotFound( - ServiceWithNodeName serviceWithNodeName, - Authentication authentication, - UpdateApiKeyRequest request - ) { + private List randomRoleDescriptors() { + int caseNo = randomIntBetween(0, 2); + return switch (caseNo) { + case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); + case 1 -> List.of( + new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null), + RoleDescriptorTests.randomRoleDescriptor() + ); + case 2 -> null; + default -> throw new IllegalStateException("unexpected case no"); + }; + } + + private void doTestUpdateApiKeyNotFound(UpdateApiKeyRequest request) { final PlainActionFuture listener = new PlainActionFuture<>(); - serviceWithNodeName.service().updateApiKey(authentication, request, Set.of(), listener); + final Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(UpdateApiKeyAction.INSTANCE, request, listener); final var ex = expectThrows(ExecutionException.class, listener::get); assertThat(ex.getCause(), instanceOf(ResourceNotFoundException.class)); assertThat(ex.getMessage(), containsString("no API key owned by requesting user found for ID [" + request.getId() + "]")); @@ -1968,9 +1962,14 @@ private void createUserWithRunAsRole() throws ExecutionException, InterruptedExc } private void createUserWithRunAsRole(Map authHeaders) throws ExecutionException, InterruptedException { + createNativeRealmUser("user_with_run_as_role", "run_as_role", authHeaders); + } + + private void createNativeRealmUser(final String username, final String role, final Map authHeaders) + throws ExecutionException, InterruptedException { final PutUserRequest putUserRequest = new PutUserRequest(); - putUserRequest.username("user_with_run_as_role"); - putUserRequest.roles("run_as_role"); + putUserRequest.username(username); + putUserRequest.roles(role); putUserRequest.passwordHash(SecuritySettingsSource.TEST_PASSWORD_HASHED.toCharArray()); PlainActionFuture listener = new PlainActionFuture<>(); final Client client = client().filterWithHeader(authHeaders); From 4ff33aa55daaef74c49563087ebd9a2ad96647df Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 13:26:13 +0200 Subject: [PATCH 109/171] Native realm user --- .../security/authc/ApiKeyIntegTests.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index adb798668a0dd..f9feaf21b5eee 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -72,7 +72,6 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.RealmDomain; -import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.User; @@ -106,9 +105,11 @@ import java.util.stream.Stream; import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER; +import static org.elasticsearch.test.SecuritySettingsSource.HASHER; import static org.elasticsearch.test.SecuritySettingsSource.TEST_ROLE; import static org.elasticsearch.test.SecuritySettingsSource.TEST_USER_NAME; import static org.elasticsearch.test.SecuritySettingsSourceField.ES_TEST_ROOT_ROLE; +import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -1525,20 +1526,23 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) ); - // Test not found exception on API key of user with the same username but from a different realm - // Authentication authentication = Authentication.newRealmAuthentication( - // new User(ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - // // Use native realm; no need to actually create user since we are injecting the authentication object directly - // new Authentication.RealmRef(NativeRealmSettings.DEFAULT_NAME, NativeRealmSettings.TYPE, serviceWithNodeName.nodeName()) - // ); + // Create native realm user with same username but different password to allow us to create an API key for _that_ user + // instead of file realm one + final var passwordSecureString = new SecureString("x-pack-test-other-password".toCharArray()); createNativeRealmUser( TEST_USER_NAME, TEST_ROLE, + new String(HASHER.hash(passwordSecureString)), Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) ); - final Tuple> apiKeyForNativeRealmUser = createApiKey(TEST_USER_NAME, null); + final CreateApiKeyResponse apiKeyForNativeRealmUser = createApiKeys( + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, passwordSecureString)), + 1, + null, + "ALL" + ).v1().get(0); doTestUpdateApiKeyNotFound( - new UpdateApiKeyRequest(apiKeyForNativeRealmUser.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) + new UpdateApiKeyRequest(apiKeyForNativeRealmUser.getId(), request.getRoleDescriptors(), request.getMetadata()) ); } @@ -1962,15 +1966,19 @@ private void createUserWithRunAsRole() throws ExecutionException, InterruptedExc } private void createUserWithRunAsRole(Map authHeaders) throws ExecutionException, InterruptedException { - createNativeRealmUser("user_with_run_as_role", "run_as_role", authHeaders); + createNativeRealmUser("user_with_run_as_role", "run_as_role", SecuritySettingsSource.TEST_PASSWORD_HASHED, authHeaders); } - private void createNativeRealmUser(final String username, final String role, final Map authHeaders) - throws ExecutionException, InterruptedException { + private void createNativeRealmUser( + final String username, + final String role, + final String passwordHashed, + final Map authHeaders + ) throws ExecutionException, InterruptedException { final PutUserRequest putUserRequest = new PutUserRequest(); putUserRequest.username(username); putUserRequest.roles(role); - putUserRequest.passwordHash(SecuritySettingsSource.TEST_PASSWORD_HASHED.toCharArray()); + putUserRequest.passwordHash(passwordHashed.toCharArray()); PlainActionFuture listener = new PlainActionFuture<>(); final Client client = client().filterWithHeader(authHeaders); client.execute(PutUserAction.INSTANCE, putUserRequest, listener); From 5489f5394a7e4f9c2b0cf4233730deedf9e23077 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 13:59:52 +0200 Subject: [PATCH 110/171] Api key test --- .../security/authc/ApiKeyIntegTests.java | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index f9feaf21b5eee..0c8d59f0a4a0c 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -109,7 +109,6 @@ import static org.elasticsearch.test.SecuritySettingsSource.TEST_ROLE; import static org.elasticsearch.test.SecuritySettingsSource.TEST_USER_NAME; import static org.elasticsearch.test.SecuritySettingsSourceField.ES_TEST_ROOT_ROLE; -import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -1455,6 +1454,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, ); client.execute(UpdateApiKeyAction.INSTANCE, request, listener); final var response = listener.get(); + assertNotNull(response); assertTrue(response.isUpdated()); @@ -1547,8 +1547,25 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter } public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, InterruptedException { - final Tuple> createdApiKey = createApiKey(ES_TEST_ROOT_USER, null); - final var apiKeyId = createdApiKey.v1().getId(); + final CreateApiKeyResponse createdApiKey = createApiKeys(TEST_USER_NAME, 1, null, "all").v1().get(0); + final var apiKeyId = createdApiKey.getId(); + + final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); + PlainActionFuture updateListener = new PlainActionFuture<>(); + client().filterWithHeader( + Collections.singletonMap( + "Authorization", + "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.getId(), createdApiKey.getKey()) + ) + ).execute(UpdateApiKeyAction.INSTANCE, request, updateListener); + + final var apiKeysNotAllowedEx = expectThrows(ExecutionException.class, updateListener::get); + assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); + assertThat( + apiKeysNotAllowedEx.getMessage(), + containsString("authentication via an API key is not supported for updating API keys") + ); final boolean invalidated = randomBoolean(); if (invalidated) { @@ -1568,18 +1585,11 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); } - final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); - final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); - - final var serviceWithNodeName = getServiceWithNodeName(); - PlainActionFuture updateListener = new PlainActionFuture<>(); - serviceWithNodeName.service() - .updateApiKey( - fileRealmAuth(serviceWithNodeName.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - request, - Set.of(roleDescriptor), - updateListener - ); + updateListener = new PlainActionFuture<>(); + final Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(UpdateApiKeyAction.INSTANCE, request, updateListener); final var ex = expectThrows(ExecutionException.class, updateListener::get); assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); @@ -1588,17 +1598,6 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr } else { assertThat(ex.getMessage(), containsString("cannot update expired API key [" + apiKeyId + "]")); } - - updateListener = new PlainActionFuture<>(); - serviceWithNodeName.service() - .updateApiKey(AuthenticationTestHelper.builder().apiKey().build(false), request, Set.of(roleDescriptor), updateListener); - final var apiKeysNotAllowedEx = expectThrows(ExecutionException.class, updateListener::get); - - assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat( - apiKeysNotAllowedEx.getMessage(), - containsString("authentication via an API key is not supported for updating API keys") - ); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { From 41b573f25140443fd2e9880b5a67add1205c1fa8 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 14:15:06 +0200 Subject: [PATCH 111/171] REST test --- .../xpack/security/apikey/ApiKeyRestIT.java | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 765c3dfa3aff8..3656385d5404c 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -85,18 +85,7 @@ public void testAuthenticateResponseApiKey() throws IOException { assertThat(actualApiKeyName, equalTo(expectedApiKeyName)); assertThat(actualApiKeyEncoded, not(emptyString())); - final Request authenticateRequest = new Request("GET", "_security/_authenticate"); - authenticateRequest.setOptions( - authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + actualApiKeyEncoded) - ); - - final Response authenticateResponse = client().performRequest(authenticateRequest); - assertOK(authenticateResponse); - final Map authenticate = responseAsMap(authenticateResponse); // keys: username, roles, full_name, etc - - // If authentication type is API_KEY, authentication.api_key={"id":"abc123","name":"my-api-key"}. No encoded, api_key, or metadata. - // If authentication type is other, authentication.api_key not present. - assertThat(authenticate, hasEntry("api_key", Map.of("id", actualApiKeyId, "name", expectedApiKeyName))); + doTestAuthenticationWithApiKey(expectedApiKeyName, actualApiKeyId, actualApiKeyEncoded); } public void testGrantApiKeyForOtherUserWithPassword() throws IOException { @@ -205,4 +194,50 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce deleteUser(manageOwnApiKeyUser); deleteRole(manageOwnApiKeyRole); } + + public void testUpdateApiKey() throws IOException { + final String expectedApiKeyName = "my-api-key-name"; + final Map expectedApiKeyMetadata = Map.of("not", "returned"); + final Map createApiKeyRequestBody = Map.of("name", expectedApiKeyName, "metadata", expectedApiKeyMetadata); + + final Request createApiKeyRequest = new Request("POST", "_security/api_key"); + createApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(createApiKeyRequestBody, XContentType.JSON).utf8ToString()); + + final Response createApiKeyResponse = adminClient().performRequest(createApiKeyRequest); + final Map createApiKeyResponseMap = responseAsMap(createApiKeyResponse); // keys: id, name, api_key, encoded + final String apiKeyId = (String) createApiKeyResponseMap.get("id"); + final String apiKeyEncoded = (String) createApiKeyResponseMap.get("encoded"); // Base64(id:api_key) + assertThat(apiKeyId, not(emptyString())); + assertThat(apiKeyEncoded, not(emptyString())); + + final Request updateApiKeyRequest = new Request(randomFrom("POST", "PUT"), "_security/api_key/_update/" + apiKeyId); + final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); + updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); + + final Response updateApiKeyResponse = adminClient().performRequest(updateApiKeyRequest); + final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); + assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); + + // validate authentication works after update + doTestAuthenticationWithApiKey(expectedApiKeyName, apiKeyId, apiKeyEncoded); + } + + private void doTestAuthenticationWithApiKey( + final String expectedApiKeyName, + final String actualApiKeyId, + final String actualApiKeyEncoded + ) throws IOException { + final Request authenticateRequest = new Request("GET", "_security/_authenticate"); + authenticateRequest.setOptions( + authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + actualApiKeyEncoded) + ); + + final Response authenticateResponse = client().performRequest(authenticateRequest); + assertOK(authenticateResponse); + final Map authenticate = responseAsMap(authenticateResponse); // keys: username, roles, full_name, etc + + // If authentication type is API_KEY, authentication.api_key={"id":"abc123","name":"my-api-key"}. No encoded, api_key, or metadata. + // If authentication type is other, authentication.api_key not present. + assertThat(authenticate, hasEntry("api_key", Map.of("id", actualApiKeyId, "name", expectedApiKeyName))); + } } From 665c426df6bdca6fdd3baf72e8f6de5083cbd994 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 14:47:09 +0200 Subject: [PATCH 112/171] Expect metadata rest test --- .../xpack/security/apikey/ApiKeyRestIT.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 3656385d5404c..d7fccb841f72d 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -14,8 +14,10 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; +import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase; @@ -25,6 +27,7 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -196,9 +199,9 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce } public void testUpdateApiKey() throws IOException { - final String expectedApiKeyName = "my-api-key-name"; - final Map expectedApiKeyMetadata = Map.of("not", "returned"); - final Map createApiKeyRequestBody = Map.of("name", expectedApiKeyName, "metadata", expectedApiKeyMetadata); + final String apiKeyName = "my-api-key-name"; + final Map apiKeyMetadata = Map.of("not", "returned"); + final Map createApiKeyRequestBody = Map.of("name", apiKeyName, "metadata", apiKeyMetadata); final Request createApiKeyRequest = new Request("POST", "_security/api_key"); createApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(createApiKeyRequestBody, XContentType.JSON).utf8ToString()); @@ -211,15 +214,18 @@ public void testUpdateApiKey() throws IOException { assertThat(apiKeyEncoded, not(emptyString())); final Request updateApiKeyRequest = new Request(randomFrom("POST", "PUT"), "_security/api_key/_update/" + apiKeyId); + final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); final Response updateApiKeyResponse = adminClient().performRequest(updateApiKeyRequest); + assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); - // validate authentication works after update - doTestAuthenticationWithApiKey(expectedApiKeyName, apiKeyId, apiKeyEncoded); + // validate authentication still works after update + doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + expectMetadata(apiKeyId, expectedApiKeyMetadata); } private void doTestAuthenticationWithApiKey( @@ -240,4 +246,17 @@ private void doTestAuthenticationWithApiKey( // If authentication type is other, authentication.api_key not present. assertThat(authenticate, hasEntry("api_key", Map.of("id", actualApiKeyId, "name", expectedApiKeyName))); } + + @SuppressWarnings({ "unchecked" }) + private void expectMetadata(final String apiKeyId, final Map expectedMetadata) throws IOException { + final var request = new Request("GET", "_security/api_key/"); + request.addParameter("id", apiKeyId); + final Response response = adminClient().performRequest(request); + assertOK(response); + try (XContentParser parser = responseAsParser(response)) { + final var apiKeyResponse = GetApiKeyResponse.fromXContent(parser); + assertThat(apiKeyResponse.getApiKeyInfos().length, equalTo(1)); + assertThat(apiKeyResponse.getApiKeyInfos()[0].getMetadata(), equalTo(expectedMetadata)); + } + } } From 84b22f63a7ccfb5f1ed2810bd63f194711eefaea Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 14:55:29 +0200 Subject: [PATCH 113/171] Nit --- .../org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index d7fccb841f72d..fd9d1fb2e5740 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; From a04792b65c4231fd61edfe252bf3ee6a99904785 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 15:02:57 +0200 Subject: [PATCH 114/171] Manage own api key test --- .../core/security/action/apikey/UpdateApiKeyRequest.java | 4 ++++ .../privilege/ManageOwnApiKeyClusterPrivilegeTests.java | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 59b8dcc4ab1e1..2b8c4bcdc6328 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -82,6 +82,10 @@ private void writeOptionalList(StreamOutput out) throws IOException { } } + public static UpdateApiKeyRequest usingId(String id) { + return new UpdateApiKeyRequest(id, null, null); + } + public String getId() { return id; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index cd87e115fefda..3f2f8ac361def 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.AuthenticationTests; @@ -69,8 +70,16 @@ public void testAuthenticationWithUserAllowsAccessToApiKeyActionsWhenItIsOwner() TransportRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName(realmRef.getName(), "joe"); TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName(realmRef.getName(), "joe"); + TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingId("id"); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/get", getApiKeyRequest, authentication)); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/invalidate", invalidateApiKeyRequest, authentication)); + assertTrue( + clusterPermission.check( + "cluster:admin/xpack/security/api_key/update", + updateApiKeyRequest, + authentication + ) + ); assertFalse(clusterPermission.check("cluster:admin/something", mock(TransportRequest.class), authentication)); From 7f5240a430f1cf60dd8471230cfa7fb19de4ef50 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 15:15:03 +0200 Subject: [PATCH 115/171] Tweak method --- .../security/action/apikey/UpdateApiKeyRequest.java | 2 +- .../ManageOwnApiKeyClusterPrivilegeTests.java | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 2b8c4bcdc6328..8a6e091e3ee22 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -82,7 +82,7 @@ private void writeOptionalList(StreamOutput out) throws IOException { } } - public static UpdateApiKeyRequest usingId(String id) { + public static UpdateApiKeyRequest usingApiKeyId(String id) { return new UpdateApiKeyRequest(id, null, null); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index 3f2f8ac361def..e6c19227ff559 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -70,16 +70,10 @@ public void testAuthenticationWithUserAllowsAccessToApiKeyActionsWhenItIsOwner() TransportRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName(realmRef.getName(), "joe"); TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName(realmRef.getName(), "joe"); - TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingId("id"); + TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(10)); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/get", getApiKeyRequest, authentication)); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/invalidate", invalidateApiKeyRequest, authentication)); - assertTrue( - clusterPermission.check( - "cluster:admin/xpack/security/api_key/update", - updateApiKeyRequest, - authentication - ) - ); + assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); assertFalse(clusterPermission.check("cluster:admin/something", mock(TransportRequest.class), authentication)); From 74b9e79024fcd9ae21437728a8f4c7f93e8d6f90 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 15:31:37 +0200 Subject: [PATCH 116/171] More tweaks --- .../authz/privilege/ManageOwnApiKeyClusterPrivilege.java | 7 +++++++ .../privilege/ManageOwnApiKeyClusterPrivilegeTests.java | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java index 201ec8ea12eaf..03ecc452d7b6d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java @@ -95,6 +95,13 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent } else if (request instanceof final QueryApiKeyRequest queryApiKeyRequest) { return queryApiKeyRequest.isFilterForCurrentUser(); } else if (request instanceof UpdateApiKeyRequest) { + // Note: this returns true even for requests authenticated via API keys. + // An API key currently *cannot* update itself. However, that's not because it's not authorized. Rather, it's because + // we have not resolved how to handle role descriptors for API keys. As such, an API key as an owner of itself is authorized + // to update itself but the behavior is currently not supported, and prevented as a bad request. + if (authentication.isApiKey()) { + return isCurrentAuthenticationUsingSameApiKeyIdFromRequest(authentication, ((UpdateApiKeyRequest) request).getId()); + } return true; } else if (request instanceof GrantApiKeyRequest) { return false; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index e6c19227ff559..f5440ae0294fb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -41,9 +41,10 @@ public void testAuthenticationWithApiKeyAllowsAccessToApiKeyActionsWhenItIsOwner final Authentication authentication = AuthenticationTests.randomApiKeyAuthentication(userJoe, apiKeyId); final TransportRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); - + final TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingApiKeyId(apiKeyId); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/get", getApiKeyRequest, authentication)); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/invalidate", invalidateApiKeyRequest, authentication)); + assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); assertFalse(clusterPermission.check("cluster:admin/something", mock(TransportRequest.class), authentication)); } @@ -56,9 +57,11 @@ public void testAuthenticationWithApiKeyDeniesAccessToApiKeyActionsWhenItIsNotOw final Authentication authentication = AuthenticationTests.randomApiKeyAuthentication(userJoe, randomAlphaOfLength(20)); final TransportRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); + final TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingApiKeyId(apiKeyId); assertFalse(clusterPermission.check("cluster:admin/xpack/security/api_key/get", getApiKeyRequest, authentication)); assertFalse(clusterPermission.check("cluster:admin/xpack/security/api_key/invalidate", invalidateApiKeyRequest, authentication)); + assertFalse(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); } public void testAuthenticationWithUserAllowsAccessToApiKeyActionsWhenItIsOwner() { From 6a89230a41955ea485b0b073aa18726af3e07256 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 15:33:41 +0200 Subject: [PATCH 117/171] Simplify --- .../authz/privilege/ManageOwnApiKeyClusterPrivilege.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java index 03ecc452d7b6d..a1cd8410168a3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java @@ -98,11 +98,9 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent // Note: this returns true even for requests authenticated via API keys. // An API key currently *cannot* update itself. However, that's not because it's not authorized. Rather, it's because // we have not resolved how to handle role descriptors for API keys. As such, an API key as an owner of itself is authorized - // to update itself but the behavior is currently not supported, and prevented as a bad request. - if (authentication.isApiKey()) { - return isCurrentAuthenticationUsingSameApiKeyIdFromRequest(authentication, ((UpdateApiKeyRequest) request).getId()); - } - return true; + // to update itself but the behavior is currently not supported, and prevented as a bad request at the transport level. + return authentication.isApiKey() == false + || isCurrentAuthenticationUsingSameApiKeyIdFromRequest(authentication, ((UpdateApiKeyRequest) request).getId()); } else if (request instanceof GrantApiKeyRequest) { return false; } From ab9b0d8530da1d58ecddde03fd1141e85f6d956a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 16:27:40 +0200 Subject: [PATCH 118/171] Only post --- .../elasticsearch/xpack/security/apikey/ApiKeyRestIT.java | 2 +- .../rest/action/apikey/RestUpdateApiKeyAction.java | 3 +-- .../rest/action/apikey/RestUpdateApiKeyActionTests.java | 7 ++++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index fd9d1fb2e5740..914b80f4307da 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -212,7 +212,7 @@ public void testUpdateApiKey() throws IOException { assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - final Request updateApiKeyRequest = new Request(randomFrom("POST", "PUT"), "_security/api_key/_update/" + apiKeyId); + final Request updateApiKeyRequest = new Request("POST", "_security/api_key/_update/" + apiKeyId); final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java index f3ceb815d18e9..d17b1bb4610b1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -49,8 +49,7 @@ public RestUpdateApiKeyAction(final Settings settings, final XPackLicenseState l @Override public List routes() { - final var path = "/_security/api_key/_update/{id}"; - return List.of(new Route(POST, path), new Route(PUT, path)); + return List.of(new Route(POST, "/_security/api_key/_update/{id}")); } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java index e588af927439c..2e366a9cc7cb7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java @@ -45,9 +45,10 @@ public void init() { public void testAbsentRoleDescriptorsAndMetadataSetToNull() { final var apiKeyId = "api_key_id"; - final FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod( - randomFrom(RestRequest.Method.PUT, RestRequest.Method.POST) - ).withPath("/_security/api_key/_update/" + apiKeyId).withContent(new BytesArray("{}"), XContentType.JSON).build(); + final FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) + .withPath("/_security/api_key/_update/" + apiKeyId) + .withContent(new BytesArray("{}"), XContentType.JSON) + .build(); dispatchRequest(restRequest); From 673de56388f3f460f40e1b7a6a9376f8740d25dd Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 16:58:56 +0200 Subject: [PATCH 119/171] Clean up --- .../security/action/apikey/TransportUpdateApiKeyAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 71c3f42a6c095..c94eb315d8e24 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -60,9 +60,9 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< return; } - // TODO does this not belong here? + // Don't resolve and validate owner roles if the service is not enabled; this avoids a costly operation and also provides + // a clearer error message (service disabled error is prioritized over role descriptor validation errors) apiKeyService.ensureEnabled(); - rolesStore.getRoleDescriptorsList(authentication.getEffectiveSubject(), ActionListener.wrap(roleDescriptorsList -> { assert roleDescriptorsList.size() == 1; final Set roleDescriptors = roleDescriptorsList.iterator().next(); From 9ada903ca005095e14e626aee348dcb0f91a00c0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 17:13:45 +0200 Subject: [PATCH 120/171] Use stream utils --- .../action/apikey/UpdateApiKeyRequest.java | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 8a6e091e3ee22..2c2b3499270aa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -40,7 +40,7 @@ public UpdateApiKeyRequest(String id, @Nullable List roleDescrip public UpdateApiKeyRequest(StreamInput in) throws IOException { super(in); this.id = in.readString(); - this.roleDescriptors = readOptionalList(in); + this.roleDescriptors = in.readOptionalList(RoleDescriptor::new); this.metadata = in.readMap(); } @@ -65,23 +65,10 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); - writeOptionalList(out); + out.writeOptionalCollection(roleDescriptors); out.writeGenericMap(metadata); } - private List readOptionalList(StreamInput in) throws IOException { - return in.readBoolean() ? in.readList(RoleDescriptor::new) : null; - } - - private void writeOptionalList(StreamOutput out) throws IOException { - if (roleDescriptors == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeList(roleDescriptors); - } - } - public static UpdateApiKeyRequest usingApiKeyId(String id) { return new UpdateApiKeyRequest(id, null, null); } From dc1db793500095617955d357cba945c8463b3927 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 17:23:13 +0200 Subject: [PATCH 121/171] Undo --- .../xpack/security/authc/support/ApiKeyGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java index 3db45cc75c10e..6139a2698308f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java @@ -62,6 +62,7 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re } rolesStore.getRoleDescriptorsList(effectiveSubject, ActionListener.wrap(roleDescriptorsList -> { + assert roleDescriptorsList.size() == 1; roleDescriptorsListener.onResponse(roleDescriptorsList.iterator().next()); }, roleDescriptorsListener::onFailure)); From e8d7bb846e258447016a5e9171eda3a7bae291d7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 17:24:05 +0200 Subject: [PATCH 122/171] Undo more --- .../xpack/security/authc/support/ApiKeyGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java index 6139a2698308f..619b6cbbc9c48 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java @@ -62,9 +62,9 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re } rolesStore.getRoleDescriptorsList(effectiveSubject, ActionListener.wrap(roleDescriptorsList -> { - assert roleDescriptorsList.size() == 1; roleDescriptorsListener.onResponse(roleDescriptorsList.iterator().next()); + }, roleDescriptorsListener::onFailure)); } } From a3546736d4c2a2a90d4751933bd16e080212d552 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 19:16:57 +0200 Subject: [PATCH 123/171] Nit --- .../security/action/apikey/TransportUpdateApiKeyAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index c94eb315d8e24..74a671ea5c838 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -61,7 +61,7 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< } // Don't resolve and validate owner roles if the service is not enabled; this avoids a costly operation and also provides - // a clearer error message (service disabled error is prioritized over role descriptor validation errors) + // a clearer error message (a service disabled error is prioritized over role descriptor validation errors) apiKeyService.ensureEnabled(); rolesStore.getRoleDescriptorsList(authentication.getEffectiveSubject(), ActionListener.wrap(roleDescriptorsList -> { assert roleDescriptorsList.size() == 1; From 96e4864e8e94d742a4f53213bd05d2d32871047f Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 19:23:26 +0200 Subject: [PATCH 124/171] Import --- .../security/rest/action/apikey/RestUpdateApiKeyAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java index d17b1bb4610b1..6cce4189b8293 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -24,7 +24,6 @@ import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public final class RestUpdateApiKeyAction extends SecurityBaseRestHandler { From 85ee9dea0f6a6362c10123053fcd9fd7735f99b7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 20:40:07 +0200 Subject: [PATCH 125/171] Add to non-operator constants --- .../org/elasticsearch/xpack/security/operator/Constants.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 0a64a32d6021a..d83fcaa9e84e3 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -178,6 +178,7 @@ public class Constants { "cluster:admin/xpack/security/api_key/grant", "cluster:admin/xpack/security/api_key/invalidate", "cluster:admin/xpack/security/api_key/query", + "cluster:admin/xpack/security/api_key/update", "cluster:admin/xpack/security/cache/clear", "cluster:admin/xpack/security/delegate_pki", "cluster:admin/xpack/security/enroll/node", From d5381188b0689f8716e4480800ece9a9eee9212c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 21:23:26 +0200 Subject: [PATCH 126/171] Clean up --- .../xpack/security/authc/ApiKeyService.java | 72 +++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 1826967ce0f20..3a3dae96d719c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -378,10 +378,11 @@ public void updateApiKey( throw new ResourceNotFoundException("no API key owned by requesting user found for ID [" + apiKeyId + "]"); } - validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, single(apiKeyId, versionedDocs).doc()); + final VersionedApiKeyDoc currentApiKeyDoc = single(apiKeyId, versionedDocs); + validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, currentApiKeyDoc.doc()); executeBulkRequest( - buildBulkRequestForUpdate(versionedDocs, authentication, request, userRoles), + buildBulkRequestForUpdate(currentApiKeyDoc, authentication, request, userRoles), ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) ); }, listener::onFailure)); @@ -1230,50 +1231,47 @@ private static VersionedApiKeyDoc single(final String apiKeyId, final Collection } private BulkRequest buildBulkRequestForUpdate( - final Collection currentVersionedDocs, + final VersionedApiKeyDoc apiKeyDoc, final Authentication authentication, final UpdateApiKeyRequest request, final Set userRoles ) throws IOException { - assert currentVersionedDocs.isEmpty() == false; final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); final var bulkRequestBuilder = client.prepareBulk(); - for (final VersionedApiKeyDoc apiKeyDoc : currentVersionedDocs) { - logger.trace( - "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", + logger.trace( + "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", + request.getId(), + apiKeyDoc.seqNo(), + apiKeyDoc.primaryTerm() + ); + final var currentDocVersion = Version.fromId(apiKeyDoc.doc().version); + assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; + if (currentDocVersion.before(targetDocVersion)) { + logger.debug( + "API key update for [{}] will update version from [{}] to [{}]", request.getId(), - apiKeyDoc.seqNo(), - apiKeyDoc.primaryTerm() - ); - final var currentDocVersion = Version.fromId(apiKeyDoc.doc().version); - assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; - if (currentDocVersion.before(targetDocVersion)) { - logger.debug( - "API key update for [{}] will update version from [{}] to [{}]", - request.getId(), - currentDocVersion, - targetDocVersion - ); - } - bulkRequestBuilder.add( - client.prepareIndex(SECURITY_MAIN_ALIAS) - .setId(request.getId()) - .setSource( - buildUpdatedDocument( - apiKeyDoc.doc(), - authentication, - userRoles, - request.getRoleDescriptors(), - targetDocVersion, - request.getMetadata() - ) - ) - .setIfSeqNo(apiKeyDoc.seqNo()) - .setIfPrimaryTerm(apiKeyDoc.primaryTerm()) - .setOpType(DocWriteRequest.OpType.INDEX) - .request() + currentDocVersion, + targetDocVersion ); } + bulkRequestBuilder.add( + client.prepareIndex(SECURITY_MAIN_ALIAS) + .setId(request.getId()) + .setSource( + buildUpdatedDocument( + apiKeyDoc.doc(), + authentication, + userRoles, + request.getRoleDescriptors(), + targetDocVersion, + request.getMetadata() + ) + ) + .setIfSeqNo(apiKeyDoc.seqNo()) + .setIfPrimaryTerm(apiKeyDoc.primaryTerm()) + .setOpType(DocWriteRequest.OpType.INDEX) + .request() + ); bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); return bulkRequestBuilder.request(); } From 77021e4be37925495fb25fb3d6742455a2f94ff2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 21:24:53 +0200 Subject: [PATCH 127/171] Few more tweaks --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3a3dae96d719c..96d47b39f60ca 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1236,14 +1236,13 @@ private BulkRequest buildBulkRequestForUpdate( final UpdateApiKeyRequest request, final Set userRoles ) throws IOException { - final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); - final var bulkRequestBuilder = client.prepareBulk(); logger.trace( "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", request.getId(), apiKeyDoc.seqNo(), apiKeyDoc.primaryTerm() ); + final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); final var currentDocVersion = Version.fromId(apiKeyDoc.doc().version); assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; if (currentDocVersion.before(targetDocVersion)) { @@ -1254,6 +1253,7 @@ private BulkRequest buildBulkRequestForUpdate( targetDocVersion ); } + final var bulkRequestBuilder = client.prepareBulk(); bulkRequestBuilder.add( client.prepareIndex(SECURITY_MAIN_ALIAS) .setId(request.getId()) From 222d475eab4a97f12ad40ed4ecf3743e6c6a0f51 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 29 Jun 2022 21:26:15 +0200 Subject: [PATCH 128/171] Nit --- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 96d47b39f60ca..a7e5d13f7bf01 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1242,8 +1242,8 @@ private BulkRequest buildBulkRequestForUpdate( apiKeyDoc.seqNo(), apiKeyDoc.primaryTerm() ); - final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); final var currentDocVersion = Version.fromId(apiKeyDoc.doc().version); + final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; if (currentDocVersion.before(targetDocVersion)) { logger.debug( From 7654f75f4ac32ea59cf72e41187c3e7a33701854 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 11:29:38 +0200 Subject: [PATCH 129/171] Address early feedback --- .../ManageOwnApiKeyClusterPrivilege.java | 14 ++++---- .../ManageOwnApiKeyClusterPrivilegeTests.java | 2 -- .../apikey/TransportUpdateApiKeyAction.java | 35 ++++++------------- .../authc/support/ApiKeyGenerator.java | 13 +++++-- .../apikey/RestClearApiKeyCacheAction.java | 4 +-- .../action/apikey/RestUpdateApiKeyAction.java | 4 +-- 6 files changed, 32 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java index a1cd8410168a3..4f7a1b030321c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java @@ -62,6 +62,11 @@ private ManageOwnClusterPermissionCheck() { protected boolean extendedCheck(String action, TransportRequest request, Authentication authentication) { if (request instanceof CreateApiKeyRequest) { return true; + } else if (request instanceof UpdateApiKeyRequest) { + // Note: we return `true` here even if the authenticated entity is an API key. API keys *cannot* update themselves + // however this is a "business-logic" restriction, rather than one related to privileges. We therefore enforce this + // limitation at the transport layer, in `TransportUpdateApiKeyAction` + return true; } else if (request instanceof final GetApiKeyRequest getApiKeyRequest) { return checkIfUserIsOwnerOfApiKeys( authentication, @@ -94,17 +99,10 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent } } else if (request instanceof final QueryApiKeyRequest queryApiKeyRequest) { return queryApiKeyRequest.isFilterForCurrentUser(); - } else if (request instanceof UpdateApiKeyRequest) { - // Note: this returns true even for requests authenticated via API keys. - // An API key currently *cannot* update itself. However, that's not because it's not authorized. Rather, it's because - // we have not resolved how to handle role descriptors for API keys. As such, an API key as an owner of itself is authorized - // to update itself but the behavior is currently not supported, and prevented as a bad request at the transport level. - return authentication.isApiKey() == false - || isCurrentAuthenticationUsingSameApiKeyIdFromRequest(authentication, ((UpdateApiKeyRequest) request).getId()); } else if (request instanceof GrantApiKeyRequest) { return false; } - String message = "manage own api key privilege only supports API key requests (not " + request.getClass().getName() + ")"; + final var message = "manage own api key privilege only supports API key requests (not " + request.getClass().getName() + ")"; assert false : message; throw new IllegalArgumentException(message); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index f5440ae0294fb..a7dfb79f97e22 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -57,11 +57,9 @@ public void testAuthenticationWithApiKeyDeniesAccessToApiKeyActionsWhenItIsNotOw final Authentication authentication = AuthenticationTests.randomApiKeyAuthentication(userJoe, randomAlphaOfLength(20)); final TransportRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); - final TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingApiKeyId(apiKeyId); assertFalse(clusterPermission.check("cluster:admin/xpack/security/api_key/get", getApiKeyRequest, authentication)); assertFalse(clusterPermission.check("cluster:admin/xpack/security/api_key/invalidate", invalidateApiKeyRequest, authentication)); - assertFalse(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); } public void testAuthenticationWithUserAllowsAccessToApiKeyActionsWhenItIsOwner() { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 74a671ea5c838..177a682509ecb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.security.action.apikey; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -19,19 +18,15 @@ import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator; import org.elasticsearch.xpack.security.authc.ApiKeyService; +import org.elasticsearch.xpack.security.authc.support.ApiKeyGenerator; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; -import java.util.Set; - public final class TransportUpdateApiKeyAction extends HandledTransportAction { private final ApiKeyService apiKeyService; private final SecurityContext securityContext; - private final CompositeRolesStore rolesStore; - private final NamedXContentRegistry xContentRegistry; + private final ApiKeyGenerator apiKeyGenerator; @Inject public TransportUpdateApiKeyAction( @@ -45,8 +40,7 @@ public TransportUpdateApiKeyAction( super(UpdateApiKeyAction.NAME, transportService, actionFilters, UpdateApiKeyRequest::new); this.apiKeyService = apiKeyService; this.securityContext = context; - this.rolesStore = rolesStore; - this.xContentRegistry = xContentRegistry; + this.apiKeyGenerator = new ApiKeyGenerator(apiKeyService, rolesStore, xContentRegistry); } @Override @@ -60,21 +54,14 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< return; } - // Don't resolve and validate owner roles if the service is not enabled; this avoids a costly operation and also provides - // a clearer error message (a service disabled error is prioritized over role descriptor validation errors) + // TODO generalize `ApiKeyGenerator` to handle updates apiKeyService.ensureEnabled(); - rolesStore.getRoleDescriptorsList(authentication.getEffectiveSubject(), ActionListener.wrap(roleDescriptorsList -> { - assert roleDescriptorsList.size() == 1; - final Set roleDescriptors = roleDescriptorsList.iterator().next(); - for (final var roleDescriptor : roleDescriptors) { - try { - DLSRoleQueryValidator.validateQueryField(roleDescriptor.getIndicesPrivileges(), xContentRegistry); - } catch (ElasticsearchException | IllegalArgumentException e) { - listener.onFailure(e); - return; - } - } - apiKeyService.updateApiKey(authentication, request, roleDescriptors, listener); - }, listener::onFailure)); + apiKeyGenerator.getUserRoleDescriptors( + authentication, + ActionListener.wrap( + roleDescriptors -> apiKeyService.updateApiKey(authentication, request, roleDescriptors, listener), + listener::onFailure + ) + ); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java index 619b6cbbc9c48..5b1be869b0570 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java @@ -41,6 +41,16 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re } apiKeyService.ensureEnabled(); + getUserRoleDescriptors( + authentication, + ActionListener.wrap( + roleDescriptors -> apiKeyService.createApiKey(authentication, request, roleDescriptors, listener), + listener::onFailure + ) + ); + } + + public void getUserRoleDescriptors(Authentication authentication, ActionListener> listener) { final ActionListener> roleDescriptorsListener = ActionListener.wrap(roleDescriptors -> { for (RoleDescriptor rd : roleDescriptors) { try { @@ -50,7 +60,7 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re return; } } - apiKeyService.createApiKey(authentication, request, roleDescriptors, listener); + listener.onResponse(roleDescriptors); }, listener::onFailure); final Subject effectiveSubject = authentication.getEffectiveSubject(); @@ -64,7 +74,6 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re rolesStore.getRoleDescriptorsList(effectiveSubject, ActionListener.wrap(roleDescriptorsList -> { assert roleDescriptorsList.size() == 1; roleDescriptorsListener.onResponse(roleDescriptorsList.iterator().next()); - }, roleDescriptorsListener::onFailure)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java index ef55f1609801f..bbeb8d0c3758f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java @@ -34,12 +34,12 @@ public String getName() { @Override public List routes() { - return List.of(new Route(POST, "/_security/api_key/{ids}/_clear_cache")); + return List.of(new Route(POST, "/_security/api_key/{id}/_clear_cache")); } @Override protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { - String[] ids = request.paramAsStringArrayOrEmptyIfAll("ids"); + String[] ids = request.paramAsStringArrayOrEmptyIfAll("id"); final ClearSecurityCacheRequest req = new ClearSecurityCacheRequest().cacheName("api_key").keys(ids); return channel -> client.execute(ClearSecurityCacheAction.INSTANCE, req, new NodesResponseRestListener<>(channel)); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java index 6cce4189b8293..de6e3b4ad7351 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public final class RestUpdateApiKeyAction extends SecurityBaseRestHandler { @@ -48,7 +48,7 @@ public RestUpdateApiKeyAction(final Settings settings, final XPackLicenseState l @Override public List routes() { - return List.of(new Route(POST, "/_security/api_key/_update/{id}")); + return List.of(new Route(PUT, "/_security/api_key/{id}")); } @Override From 5aab2964435d2c40e0da1ba7a5e9da520b9ee7c1 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 11:41:38 +0200 Subject: [PATCH 130/171] Fix tests --- .../elasticsearch/xpack/security/apikey/ApiKeyRestIT.java | 6 +++--- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 5 +++-- .../rest/action/apikey/RestUpdateApiKeyActionTests.java | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 914b80f4307da..ea0f26f05a60b 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -212,7 +212,7 @@ public void testUpdateApiKey() throws IOException { assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - final Request updateApiKeyRequest = new Request("POST", "_security/api_key/_update/" + apiKeyId); + final Request updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); @@ -222,9 +222,9 @@ public void testUpdateApiKey() throws IOException { final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); + expectMetadata(apiKeyId, expectedApiKeyMetadata); // validate authentication still works after update doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); - expectMetadata(apiKeyId, expectedApiKeyMetadata); } private void doTestAuthenticationWithApiKey( @@ -232,7 +232,7 @@ private void doTestAuthenticationWithApiKey( final String actualApiKeyId, final String actualApiKeyEncoded ) throws IOException { - final Request authenticateRequest = new Request("GET", "_security/_authenticate"); + final var authenticateRequest = new Request("GET", "_security/_authenticate"); authenticateRequest.setOptions( authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + actualApiKeyEncoded) ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index a7e5d13f7bf01..80bb0821d37e8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -378,7 +378,8 @@ public void updateApiKey( throw new ResourceNotFoundException("no API key owned by requesting user found for ID [" + apiKeyId + "]"); } - final VersionedApiKeyDoc currentApiKeyDoc = single(apiKeyId, versionedDocs); + final VersionedApiKeyDoc currentApiKeyDoc = singleDoc(apiKeyId, versionedDocs); + validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, currentApiKeyDoc.doc()); executeBulkRequest( @@ -1217,7 +1218,7 @@ private void translateResponseAndClearCache( } } - private static VersionedApiKeyDoc single(final String apiKeyId, final Collection elements) { + private static VersionedApiKeyDoc singleDoc(final String apiKeyId, final Collection elements) { if (elements.size() != 1) { final var message = "expected single API key doc with ID [" + apiKeyId diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java index 2e366a9cc7cb7..5e64ed8a0b620 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyActionTests.java @@ -45,8 +45,8 @@ public void init() { public void testAbsentRoleDescriptorsAndMetadataSetToNull() { final var apiKeyId = "api_key_id"; - final FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) - .withPath("/_security/api_key/_update/" + apiKeyId) + final FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath("/_security/api_key/" + apiKeyId) .withContent(new BytesArray("{}"), XContentType.JSON) .build(); From 0f17546f200c5f4a64e9a01dfd64d6aeaf4ce733 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 11:53:01 +0200 Subject: [PATCH 131/171] Better error message --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 2 +- .../security/action/apikey/TransportUpdateApiKeyAction.java | 4 +++- .../org/elasticsearch/xpack/security/authc/ApiKeyService.java | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 0c8d59f0a4a0c..06c5e1a0d0cb5 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1564,7 +1564,7 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr assertThat(apiKeysNotAllowedEx.getCause(), instanceOf(IllegalArgumentException.class)); assertThat( apiKeysNotAllowedEx.getMessage(), - containsString("authentication via an API key is not supported for updating API keys") + containsString("authentication via API key not supported: only the owner user can update an API key") ); final boolean invalidated = randomBoolean(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java index 177a682509ecb..d90abdea65284 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java @@ -50,7 +50,9 @@ protected void doExecute(Task task, UpdateApiKeyRequest request, ActionListener< listener.onFailure(new IllegalStateException("authentication is required")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("authentication via an API key is not supported for updating API keys")); + listener.onFailure( + new IllegalArgumentException("authentication via API key not supported: only the owner user can update an API key") + ); return; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 80bb0821d37e8..ad5cd7995ad0b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -365,7 +365,9 @@ public void updateApiKey( listener.onFailure(new IllegalArgumentException("authentication must be provided")); return; } else if (authentication.isApiKey()) { - listener.onFailure(new IllegalArgumentException("authentication via an API key is not supported for updating API keys")); + listener.onFailure( + new IllegalArgumentException("authentication via API key not supported: only the owner user can update an API key") + ); return; } From b3aaf2e863eb94d134e9f3525a30e0994cb952d9 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 12:12:58 +0200 Subject: [PATCH 132/171] WIP rest tests --- .../xpack/security/apikey/ApiKeyRestIT.java | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index ea0f26f05a60b..ff51a268f5b6f 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -51,6 +51,7 @@ public class ApiKeyRestIT extends SecurityOnTrialLicenseRestTestCase { private static final SecureString SYSTEM_USER_PASSWORD = new SecureString("system-user-password".toCharArray()); private static final String END_USER = "end_user"; private static final SecureString END_USER_PASSWORD = new SecureString("end-user-password".toCharArray()); + private static final String MANAGE_OWN_API_KEY_USER = "manage_own_api_key_user"; @Before public void createUsers() throws IOException { @@ -58,15 +59,20 @@ public void createUsers() throws IOException { createRole("system_role", Set.of("grant_api_key")); createUser(END_USER, END_USER_PASSWORD, List.of("user_role")); createRole("user_role", Set.of("monitor")); + createUser(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD, List.of("manage_own_api_key_role")); + createRole("manage_own_api_key_role", Set.of("manage_own_api_key")); } @After public void cleanUp() throws IOException { - deleteUser("system_user"); - deleteUser("end_user"); + deleteUser(SYSTEM_USER); + deleteUser(END_USER); + deleteUser(MANAGE_OWN_API_KEY_USER); deleteRole("system_role"); deleteRole("user_role"); + deleteRole("manage_own_api_key_role"); invalidateApiKeysForUser(END_USER); + invalidateApiKeysForUser(MANAGE_OWN_API_KEY_USER); } @SuppressWarnings({ "unchecked" }) @@ -170,21 +176,15 @@ public void testGrantApiKeyWithoutApiKeyNameWillFail() throws IOException { } public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOException { - final String manageOwnApiKeyUser = "manage-own-api-key-user"; - final SecureString manageOwnApiKeyUserPassword = new SecureString("manage-own-api-key-password".toCharArray()); - final String manageOwnApiKeyRole = "manage_own_api_key_role"; - createUser(manageOwnApiKeyUser, manageOwnApiKeyUserPassword, List.of(manageOwnApiKeyRole)); - createRole(manageOwnApiKeyRole, Set.of("manage_own_api_key")); - final Request request = new Request("POST", "_security/api_key/grant"); request.setOptions( RequestOptions.DEFAULT.toBuilder() - .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(manageOwnApiKeyUser, manageOwnApiKeyUserPassword)) + .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) ); final Map requestBody = Map.ofEntries( Map.entry("grant_type", "password"), - Map.entry("username", manageOwnApiKeyUser), - Map.entry("password", manageOwnApiKeyUserPassword.toString()), + Map.entry("username", MANAGE_OWN_API_KEY_USER), + Map.entry("password", END_USER_PASSWORD.toString()), Map.entry("api_key", Map.of("name", "test_api_key_password")) ); request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString()); @@ -193,8 +193,6 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("action [" + GrantApiKeyAction.NAME + "] is unauthorized for user")); - deleteUser(manageOwnApiKeyUser); - deleteRole(manageOwnApiKeyRole); } public void testUpdateApiKey() throws IOException { @@ -204,8 +202,12 @@ public void testUpdateApiKey() throws IOException { final Request createApiKeyRequest = new Request("POST", "_security/api_key"); createApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(createApiKeyRequestBody, XContentType.JSON).utf8ToString()); + createApiKeyRequest.setOptions( + RequestOptions.DEFAULT.toBuilder() + .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + ); - final Response createApiKeyResponse = adminClient().performRequest(createApiKeyRequest); + final Response createApiKeyResponse = client().performRequest(createApiKeyRequest); final Map createApiKeyResponseMap = responseAsMap(createApiKeyResponse); // keys: id, name, api_key, encoded final String apiKeyId = (String) createApiKeyResponseMap.get("id"); final String apiKeyEncoded = (String) createApiKeyResponseMap.get("encoded"); // Base64(id:api_key) @@ -216,8 +218,12 @@ public void testUpdateApiKey() throws IOException { final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); + updateApiKeyRequest.setOptions( + RequestOptions.DEFAULT.toBuilder() + .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + ); - final Response updateApiKeyResponse = adminClient().performRequest(updateApiKeyRequest); + final Response updateApiKeyResponse = client().performRequest(updateApiKeyRequest); assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); From 9e7ce5e3d37d54d3146f51f48cd09ecbc06bac96 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 12:28:16 +0200 Subject: [PATCH 133/171] Also test bearer token auth --- .../xpack/security/apikey/ApiKeyRestIT.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index ff51a268f5b6f..7206ce7b64289 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -203,8 +203,7 @@ public void testUpdateApiKey() throws IOException { final Request createApiKeyRequest = new Request("POST", "_security/api_key"); createApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(createApiKeyRequestBody, XContentType.JSON).utf8ToString()); createApiKeyRequest.setOptions( - RequestOptions.DEFAULT.toBuilder() - .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authorizationHeader(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) ); final Response createApiKeyResponse = client().performRequest(createApiKeyRequest); @@ -219,8 +218,7 @@ public void testUpdateApiKey() throws IOException { final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); updateApiKeyRequest.setOptions( - RequestOptions.DEFAULT.toBuilder() - .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authorizationHeader(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) ); final Response updateApiKeyResponse = client().performRequest(updateApiKeyRequest); @@ -233,6 +231,16 @@ public void testUpdateApiKey() throws IOException { doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } + private String authorizationHeader(final String username, final SecureString password) throws IOException { + final boolean useBearerTokenAuth = randomBoolean(); + if (useBearerTokenAuth) { + final Tuple token = super.createOAuthToken(username, password); + return "Bearer " + token.v1(); + } else { + return UsernamePasswordToken.basicAuthHeaderValue(username, password); + } + } + private void doTestAuthenticationWithApiKey( final String expectedApiKeyName, final String actualApiKeyId, From 085cc4f4edb336c914845aa858dd3625fc788369 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 13:10:38 +0200 Subject: [PATCH 134/171] Grant target can update api key test --- .../xpack/security/apikey/ApiKeyRestIT.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 7206ce7b64289..f3e6f3c4914ed 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -231,6 +231,49 @@ public void testUpdateApiKey() throws IOException { doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } + public void testGrantTargetCanUpdateApiKey() throws IOException { + final var request = new Request("POST", "_security/api_key/grant"); + request.setOptions( + RequestOptions.DEFAULT.toBuilder() + .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SYSTEM_USER, SYSTEM_USER_PASSWORD)) + ); + final var apiKeyName = "test_api_key_password"; + final Map requestBody = Map.ofEntries( + Map.entry("grant_type", "password"), + Map.entry("username", MANAGE_OWN_API_KEY_USER), + Map.entry("password", END_USER_PASSWORD.toString()), + Map.entry("api_key", Map.of("name", apiKeyName)) + ); + request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString()); + + final Response response = client().performRequest(request); + final Map responseBody = entityAsMap(response); + + assertThat(responseBody.get("name"), equalTo(apiKeyName)); + assertThat(responseBody.get("id"), notNullValue()); + assertThat(responseBody.get("id"), instanceOf(String.class)); + + final var apiKeyId = (String) responseBody.get("id"); + final var apiKeyEncoded = (String) responseBody.get("encoded"); + + final Request updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); + final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); + final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); + updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); + updateApiKeyRequest.setOptions( + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authorizationHeader(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + ); + + final Response updateApiKeyResponse = client().performRequest(updateApiKeyRequest); + assertOK(updateApiKeyResponse); + final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); + assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); + + expectMetadata(apiKeyId, expectedApiKeyMetadata); + // validate authentication still works after update + doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + } + private String authorizationHeader(final String username, final SecureString password) throws IOException { final boolean useBearerTokenAuth = randomBoolean(); if (useBearerTokenAuth) { From dde0b7287e00e7f8f18917aa4d7ecead84c71b48 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 13:18:26 +0200 Subject: [PATCH 135/171] Refactor --- .../xpack/security/apikey/ApiKeyRestIT.java | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index f3e6f3c4914ed..060d8eb3443aa 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -208,27 +208,12 @@ public void testUpdateApiKey() throws IOException { final Response createApiKeyResponse = client().performRequest(createApiKeyRequest); final Map createApiKeyResponseMap = responseAsMap(createApiKeyResponse); // keys: id, name, api_key, encoded - final String apiKeyId = (String) createApiKeyResponseMap.get("id"); - final String apiKeyEncoded = (String) createApiKeyResponseMap.get("encoded"); // Base64(id:api_key) + final var apiKeyId = (String) createApiKeyResponseMap.get("id"); + final var apiKeyEncoded = (String) createApiKeyResponseMap.get("encoded"); // Base64(id:api_key) assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - final Request updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); - final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); - final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); - updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); - updateApiKeyRequest.setOptions( - RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authorizationHeader(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) - ); - - final Response updateApiKeyResponse = client().performRequest(updateApiKeyRequest); - assertOK(updateApiKeyResponse); - final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); - assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); - - expectMetadata(apiKeyId, expectedApiKeyMetadata); - // validate authentication still works after update - doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } public void testGrantTargetCanUpdateApiKey() throws IOException { @@ -247,15 +232,16 @@ public void testGrantTargetCanUpdateApiKey() throws IOException { request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString()); final Response response = client().performRequest(request); - final Map responseBody = entityAsMap(response); - - assertThat(responseBody.get("name"), equalTo(apiKeyName)); - assertThat(responseBody.get("id"), notNullValue()); - assertThat(responseBody.get("id"), instanceOf(String.class)); + final Map createApiKeyResponseMap = responseAsMap(response); // keys: id, name, api_key, encoded + final var apiKeyId = (String) createApiKeyResponseMap.get("id"); + final var apiKeyEncoded = (String) createApiKeyResponseMap.get("encoded"); // Base64(id:api_key) + assertThat(apiKeyId, not(emptyString())); + assertThat(apiKeyEncoded, not(emptyString())); - final var apiKeyId = (String) responseBody.get("id"); - final var apiKeyEncoded = (String) responseBody.get("encoded"); + doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + } + private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKeyEncoded) throws IOException { final Request updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); From f90b93ebead9aba142ff963edba275d18452de1d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 13:46:31 +0200 Subject: [PATCH 136/171] Randomize auth method --- .../xpack/security/apikey/ApiKeyRestIT.java | 65 +++++++++++-------- .../security/authc/ApiKeyIntegTests.java | 3 - 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 060d8eb3443aa..f9e87ee643533 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -196,14 +196,15 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce } public void testUpdateApiKey() throws IOException { - final String apiKeyName = "my-api-key-name"; + final var apiKeyName = "my-api-key-name"; final Map apiKeyMetadata = Map.of("not", "returned"); final Map createApiKeyRequestBody = Map.of("name", apiKeyName, "metadata", apiKeyMetadata); final Request createApiKeyRequest = new Request("POST", "_security/api_key"); createApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(createApiKeyRequestBody, XContentType.JSON).utf8ToString()); createApiKeyRequest.setOptions( - RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authorizationHeader(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + RequestOptions.DEFAULT.toBuilder() + .addHeader("Authorization", headerFromRandomAuthMethod(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) ); final Response createApiKeyResponse = client().performRequest(createApiKeyRequest); @@ -241,16 +242,33 @@ public void testGrantTargetCanUpdateApiKey() throws IOException { doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } + private void doTestAuthenticationWithApiKey( + final String expectedApiKeyName, + final String actualApiKeyId, + final String actualApiKeyEncoded + ) throws IOException { + final var authenticateRequest = new Request("GET", "_security/_authenticate"); + authenticateRequest.setOptions( + authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + actualApiKeyEncoded) + ); + + final Response authenticateResponse = client().performRequest(authenticateRequest); + assertOK(authenticateResponse); + final Map authenticate = responseAsMap(authenticateResponse); // keys: username, roles, full_name, etc + + // If authentication type is API_KEY, authentication.api_key={"id":"abc123","name":"my-api-key"}. No encoded, api_key, or metadata. + // If authentication type is other, authentication.api_key not present. + assertThat(authenticate, hasEntry("api_key", Map.of("id", actualApiKeyId, "name", expectedApiKeyName))); + } + private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKeyEncoded) throws IOException { final Request updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); - updateApiKeyRequest.setOptions( - RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authorizationHeader(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) - ); - final Response updateApiKeyResponse = client().performRequest(updateApiKeyRequest); + final Response updateApiKeyResponse = doUpdateWithRandomAuthMethod(updateApiKeyRequest); + assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); @@ -260,7 +278,21 @@ private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKe doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } - private String authorizationHeader(final String username, final SecureString password) throws IOException { + private Response doUpdateWithRandomAuthMethod(Request updateApiKeyRequest) throws IOException { + final boolean useRunAs = randomBoolean(); + if (useRunAs) { + updateApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", MANAGE_OWN_API_KEY_USER)); + return adminClient().performRequest(updateApiKeyRequest); + } else { + updateApiKeyRequest.setOptions( + RequestOptions.DEFAULT.toBuilder() + .addHeader("Authorization", headerFromRandomAuthMethod(MANAGE_OWN_API_KEY_USER, END_USER_PASSWORD)) + ); + return client().performRequest(updateApiKeyRequest); + } + } + + private String headerFromRandomAuthMethod(final String username, final SecureString password) throws IOException { final boolean useBearerTokenAuth = randomBoolean(); if (useBearerTokenAuth) { final Tuple token = super.createOAuthToken(username, password); @@ -270,25 +302,6 @@ private String authorizationHeader(final String username, final SecureString pas } } - private void doTestAuthenticationWithApiKey( - final String expectedApiKeyName, - final String actualApiKeyId, - final String actualApiKeyEncoded - ) throws IOException { - final var authenticateRequest = new Request("GET", "_security/_authenticate"); - authenticateRequest.setOptions( - authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + actualApiKeyEncoded) - ); - - final Response authenticateResponse = client().performRequest(authenticateRequest); - assertOK(authenticateResponse); - final Map authenticate = responseAsMap(authenticateResponse); // keys: username, roles, full_name, etc - - // If authentication type is API_KEY, authentication.api_key={"id":"abc123","name":"my-api-key"}. No encoded, api_key, or metadata. - // If authentication type is other, authentication.api_key not present. - assertThat(authenticate, hasEntry("api_key", Map.of("id", actualApiKeyId, "name", expectedApiKeyName))); - } - @SuppressWarnings({ "unchecked" }) private void expectMetadata(final String apiKeyId, final Map expectedMetadata) throws IOException { final var request = new Request("GET", "_security/api_key/"); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 06c5e1a0d0cb5..73d57159f090a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1432,11 +1432,8 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); - final var newRoleDescriptors = randomRoleDescriptors(); final boolean nullRoleDescriptors = newRoleDescriptors == null; - - // TODO parse role? final var expectedLimitedByRoleDescriptors = Set.of( new RoleDescriptor( TEST_ROLE, From 5e7a5bdf436d4cdac0c18c1e5fa149a91da420b1 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 13:48:08 +0200 Subject: [PATCH 137/171] Use header constant --- .../elasticsearch/xpack/security/apikey/ApiKeyRestIT.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index f9e87ee643533..5007734e26f7d 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; @@ -267,7 +268,7 @@ private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKe final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); - final Response updateApiKeyResponse = doUpdateWithRandomAuthMethod(updateApiKeyRequest); + final Response updateApiKeyResponse = doUpdateUsingRandomAuthMethod(updateApiKeyRequest); assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); @@ -278,10 +279,10 @@ private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKe doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } - private Response doUpdateWithRandomAuthMethod(Request updateApiKeyRequest) throws IOException { + private Response doUpdateUsingRandomAuthMethod(Request updateApiKeyRequest) throws IOException { final boolean useRunAs = randomBoolean(); if (useRunAs) { - updateApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", MANAGE_OWN_API_KEY_USER)); + updateApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader(RUN_AS_USER_HEADER, MANAGE_OWN_API_KEY_USER)); return adminClient().performRequest(updateApiKeyRequest); } else { updateApiKeyRequest.setOptions( From 4be5deee47417fb943ebc7a970951730d8f75ae2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 14:10:22 +0200 Subject: [PATCH 138/171] Randomize cluster privs --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 73d57159f090a..1c060418c2ace 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1544,10 +1544,11 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter } public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, InterruptedException { - final CreateApiKeyResponse createdApiKey = createApiKeys(TEST_USER_NAME, 1, null, "all").v1().get(0); + final var apiKeyClusterPrivilege = randomFrom("all", "manage_security", "manage_api_key", "manage_own_api_key"); + final CreateApiKeyResponse createdApiKey = createApiKeys(TEST_USER_NAME, 1, null, apiKeyClusterPrivilege).v1().get(0); final var apiKeyId = createdApiKey.getId(); - final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null); + final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { apiKeyClusterPrivilege }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); PlainActionFuture updateListener = new PlainActionFuture<>(); client().filterWithHeader( From 7b137d2e993a5a4dbedfae2a6771392f6c8dd0ae Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 14:38:35 +0200 Subject: [PATCH 139/171] Plug in more client --- .../security/authc/ApiKeyIntegTests.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 1c060418c2ace..21f3974eeb10f 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1472,11 +1472,11 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, // Document updated as expected final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); - expectRoleDescriptorForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc); + expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc); if (nullRoleDescriptors) { // Default role descriptor assigned to api key in `createApiKey` final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - expectRoleDescriptorForApiKey("role_descriptors", List.of(expectedRoleDescriptor), updatedApiKeyDoc); + expectRoleDescriptorsForApiKey("role_descriptors", List.of(expectedRoleDescriptor), updatedApiKeyDoc); // Create user action unauthorized because we did not update key role; it only has `monitor` cluster priv final Map authorizationHeaders = Collections.singletonMap( @@ -1487,7 +1487,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, assertThat(e.getMessage(), containsString("unauthorized")); assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); } else { - expectRoleDescriptorForApiKey("role_descriptors", newRoleDescriptors, updatedApiKeyDoc); + expectRoleDescriptorsForApiKey("role_descriptors", newRoleDescriptors, updatedApiKeyDoc); // Create user action authorized because we updated key role to `all` cluster priv final var authorizationHeaders = Collections.singletonMap( "Authorization", @@ -1634,17 +1634,15 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution // Update the first key final PlainActionFuture listener = new PlainActionFuture<>(); - serviceForDoc1.updateApiKey( - fileRealmAuth(serviceWithNameForDoc1.nodeName(), ES_TEST_ROOT_USER, ES_TEST_ROOT_ROLE), - new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), - Set.of(), - listener + final Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) ); + client.execute(UpdateApiKeyAction.INSTANCE, new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), listener); final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); - // The cache entry should be gone for the first key + // The doc cache entry should be gone for the first key if (sameServiceNode) { assertEquals(1, serviceForDoc1.getDocCache().count()); assertNull(serviceForDoc1.getDocCache().get(apiKey1.v1())); @@ -1708,7 +1706,7 @@ private static Authentication fileRealmAuth(String nodeName, String userName, St ); } - private void expectMetadataForApiKey(Map expectedMetadata, Map actualRawApiKeyDoc) { + private void expectMetadataForApiKey(final Map expectedMetadata, final Map actualRawApiKeyDoc) { assertNotNull(actualRawApiKeyDoc); @SuppressWarnings("unchecked") final var actualMetadata = (Map) actualRawApiKeyDoc.get("metadata_flattened"); @@ -1716,7 +1714,7 @@ private void expectMetadataForApiKey(Map expectedMetadata, Map expectedRoleDescriptors, final Map actualRawApiKeyDoc From 5070e4813be880f6ffb4e5c33503ff78e305e01d Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 15:02:42 +0200 Subject: [PATCH 140/171] Import --- .../org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 21f3974eeb10f..764ef85c63152 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -108,7 +108,6 @@ import static org.elasticsearch.test.SecuritySettingsSource.HASHER; import static org.elasticsearch.test.SecuritySettingsSource.TEST_ROLE; import static org.elasticsearch.test.SecuritySettingsSource.TEST_USER_NAME; -import static org.elasticsearch.test.SecuritySettingsSourceField.ES_TEST_ROOT_ROLE; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; From bad21a0f3c6e59fa65b5f21c0f65d67e95d62ba4 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 15:47:11 +0200 Subject: [PATCH 141/171] Random realm id --- .../authc/AuthenticationTestHelper.java | 4 ++ .../security/authc/ApiKeyIntegTests.java | 60 +++++++++++-------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java index bca5e4f916e54..8cb8b684a64ab 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java @@ -136,6 +136,10 @@ public static Authentication.RealmRef randomRealmRef(boolean underDomain, boolea } } + public static RealmConfig.RealmIdentifier randomRealmIdentifier(boolean includeInternal) { + return new RealmConfig.RealmIdentifier(randomRealmTypeSupplier(includeInternal).get(), ESTestCase.randomAlphaOfLengthBetween(3, 8)); + } + private static Supplier randomRealmTypeSupplier(boolean includeInternal) { final Supplier randomAllRealmTypeSupplier = () -> ESTestCase.randomFrom( "reserved", diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 764ef85c63152..948a1d2e90474 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1597,6 +1597,41 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr } } + public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionException, InterruptedException { + final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); + final var apiKeyId = createdApiKey.v1().getId(); + + final ServiceWithNodeName serviceWithNodeName = getServiceWithNodeName(); + final PlainActionFuture listener = new PlainActionFuture<>(); + final RealmConfig.RealmIdentifier creatorRealmOnCreatedApiKey = new RealmConfig.RealmIdentifier(FileRealmSettings.TYPE, "file"); + final RealmConfig.RealmIdentifier otherRealmInDomain = AuthenticationTestHelper.randomRealmIdentifier(true); + final var realmDomain = new RealmDomain( + ESTestCase.randomAlphaOfLengthBetween(3, 8), + Set.of(creatorRealmOnCreatedApiKey, otherRealmInDomain) + ); + // Update should work for any of the realms within the domain + final var authenticatingRealm = randomFrom(creatorRealmOnCreatedApiKey, otherRealmInDomain); + final var authentication = randomValueOtherThanMany( + Authentication::isApiKey, + () -> AuthenticationTestHelper.builder() + .user(new User(TEST_USER_NAME, TEST_ROLE)) + .realmRef( + new Authentication.RealmRef( + authenticatingRealm.getName(), + authenticatingRealm.getType(), + serviceWithNodeName.nodeName(), + realmDomain + ) + ) + .build() + ); + serviceWithNodeName.service().updateApiKey(authentication, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), Set.of(), listener); + final UpdateApiKeyResponse response = listener.get(); + + assertNotNull(response); + assertTrue(response.isUpdated()); + } + public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { final List services = Arrays.stream(internalCluster().getNodeNames()) .map(n -> new ServiceWithNodeName(internalCluster().getInstance(ApiKeyService.class, n), n)) @@ -1680,31 +1715,6 @@ private void doTestUpdateApiKeyNotFound(UpdateApiKeyRequest request) { assertThat(ex.getMessage(), containsString("no API key owned by requesting user found for ID [" + request.getId() + "]")); } - private static Authentication fileRealmAuth(String nodeName, String userName, String roleName) { - boolean includeDomain = randomBoolean(); - final var realmName = "file"; - final String realmType = FileRealmSettings.TYPE; - return randomValueOtherThanMany( - Authentication::isApiKey, - () -> AuthenticationTestHelper.builder() - .user(new User(userName, roleName)) - .realmRef( - new Authentication.RealmRef( - realmName, - realmType, - nodeName, - includeDomain - ? new RealmDomain( - ESTestCase.randomAlphaOfLengthBetween(3, 8), - Set.of(new RealmConfig.RealmIdentifier(realmType, realmName)) - ) - : null - ) - ) - .build() - ); - } - private void expectMetadataForApiKey(final Map expectedMetadata, final Map actualRawApiKeyDoc) { assertNotNull(actualRawApiKeyDoc); @SuppressWarnings("unchecked") From cb67b3ed84c736f977999eb98eaf5006846f74f5 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 15:48:04 +0200 Subject: [PATCH 142/171] Get random node --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 948a1d2e90474..ddc5f03751ddb 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1751,7 +1751,7 @@ private Map getApiKeyDocument(String apiKeyId) { } private ServiceWithNodeName getServiceWithNodeName() { - final var nodeName = internalCluster().getNodeNames()[0]; + final var nodeName = randomFrom(internalCluster().getNodeNames()); final var service = internalCluster().getInstance(ApiKeyService.class, nodeName); return new ServiceWithNodeName(service, nodeName); } From 046a22d3e2c2072f0abf29c10dde5afad1fc11ad Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 15:50:54 +0200 Subject: [PATCH 143/171] Nit --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index ddc5f03751ddb..8a4ecd2e1201a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -1746,8 +1745,7 @@ private void expectRoleDescriptorsForApiKey( } private Map getApiKeyDocument(String apiKeyId) { - final GetResponse getResponse = client().execute(GetAction.INSTANCE, new GetRequest(SECURITY_MAIN_ALIAS, apiKeyId)).actionGet(); - return getResponse.getSource(); + return client().execute(GetAction.INSTANCE, new GetRequest(SECURITY_MAIN_ALIAS, apiKeyId)).actionGet().getSource(); } private ServiceWithNodeName getServiceWithNodeName() { From a2f5f38fd3c2f30911ccbe7dddf0c00e2d091a8c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 16:08:26 +0200 Subject: [PATCH 144/171] Add back comment --- .../xpack/security/authc/ApiKeyIntegTests.java | 3 ++- .../elasticsearch/xpack/security/authc/ApiKeyService.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 8a4ecd2e1201a..32a74795715e9 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1521,6 +1521,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter new UpdateApiKeyRequest(otherUsersApiKey.v1().getId(), request.getRoleDescriptors(), request.getMetadata()) ); + // Test not found exception on API key of user with the same username but from a different realm // Create native realm user with same username but different password to allow us to create an API key for _that_ user // instead of file realm one final var passwordSecureString = new SecureString("x-pack-test-other-password".toCharArray()); @@ -1534,7 +1535,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, passwordSecureString)), 1, null, - "ALL" + "all" ).v1().get(0); doTestUpdateApiKeyNotFound( new UpdateApiKeyRequest(apiKeyForNativeRealmUser.getId(), request.getRoleDescriptors(), request.getMetadata()) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index ad5cd7995ad0b..3b2b6d95febcf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -380,12 +380,12 @@ public void updateApiKey( throw new ResourceNotFoundException("no API key owned by requesting user found for ID [" + apiKeyId + "]"); } - final VersionedApiKeyDoc currentApiKeyDoc = singleDoc(apiKeyId, versionedDocs); + final VersionedApiKeyDoc versionedDoc = singleDoc(apiKeyId, versionedDocs); - validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, currentApiKeyDoc.doc()); + validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, versionedDoc.doc()); executeBulkRequest( - buildBulkRequestForUpdate(currentApiKeyDoc, authentication, request, userRoles), + buildBulkRequestForUpdate(versionedDoc, authentication, request, userRoles), ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) ); }, listener::onFailure)); From 4f66060035a7a43586ac067806b520ee36e01971 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 30 Jun 2022 16:57:24 +0200 Subject: [PATCH 145/171] Auto sync user roles test --- .../security/authc/ApiKeyIntegTests.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 32a74795715e9..0476734cfaccd 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -61,6 +61,9 @@ import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.role.PutRoleAction; +import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest; +import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequestBuilder; import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse; @@ -1432,6 +1435,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var apiKeyId = createdApiKey.v1().getId(); final var newRoleDescriptors = randomRoleDescriptors(); final boolean nullRoleDescriptors = newRoleDescriptors == null; + // Role descriptor corresponding to SecuritySettingsSource.TEST_ROLE_YML final var expectedLimitedByRoleDescriptors = Set.of( new RoleDescriptor( TEST_ROLE, @@ -1495,6 +1499,51 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, } } + public void testUpdateApiKeyAutoSyncsUserRoles() throws IOException, ExecutionException, InterruptedException { + // Create native realm user and role + final var nativeRealmUser = "native_user"; + final var nativeRealmRole = "native_role"; + createNativeRealmUser( + nativeRealmUser, + nativeRealmRole, + new String(HASHER.hash(TEST_PASSWORD_SECURE_STRING)), + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ); + putRoleWithClusterPrivileges(nativeRealmRole, "all"); + + // Create api key + final CreateApiKeyResponse createdApiKey = createApiKeys( + Collections.singletonMap("Authorization", basicAuthHeaderValue(nativeRealmUser, TEST_PASSWORD_SECURE_STRING)), + 1, + null, + "all" + ).v1().get(0); + final String apiKeyId = createdApiKey.getId(); + expectRoleDescriptorsForApiKey( + "limited_by_role_descriptors", + Set.of(new RoleDescriptor(nativeRealmRole, new String[] { "all" }, null, null)), + getApiKeyDocument(apiKeyId) + ); + + // Update user role + putRoleWithClusterPrivileges(nativeRealmRole, "manage_own_api_key"); + + // Update API key + final PlainActionFuture listener = new PlainActionFuture<>(); + client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(nativeRealmUser, TEST_PASSWORD_SECURE_STRING)) + ).execute(UpdateApiKeyAction.INSTANCE, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), listener); + final var response = listener.get(); + assertNotNull(response); + assertTrue(response.isUpdated()); + + expectRoleDescriptorsForApiKey( + "limited_by_role_descriptors", + Set.of(new RoleDescriptor(nativeRealmRole, new String[] { "manage_own_api_key" }, null, null)), + getApiKeyDocument(apiKeyId) + ); + } + public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); @@ -1989,6 +2038,17 @@ private void createNativeRealmUser( assertTrue(putUserResponse.created()); } + private void putRoleWithClusterPrivileges(final String nativeRealmRoleName, final String clusterPrivilege) throws InterruptedException, + ExecutionException { + final PutRoleRequest putRoleRequest = new PutRoleRequest(); + putRoleRequest.name(nativeRealmRoleName); + putRoleRequest.cluster(clusterPrivilege); + final PlainActionFuture roleListener = new PlainActionFuture<>(); + client().filterWithHeader(Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))) + .execute(PutRoleAction.INSTANCE, putRoleRequest, roleListener); + assertNotNull(roleListener.get()); + } + private Client getClientForRunAsUser() { return client().filterWithHeader( Map.of( From 57938c53388b0a961f2f3b6325a565d988eb6c51 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 10:06:14 +0200 Subject: [PATCH 146/171] Clean up test --- .../ManageOwnApiKeyClusterPrivilege.java | 2 +- .../security/authc/ApiKeyIntegTests.java | 23 +++++++------------ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java index 4f7a1b030321c..75a3dadcb5e5b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java @@ -102,7 +102,7 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent } else if (request instanceof GrantApiKeyRequest) { return false; } - final var message = "manage own api key privilege only supports API key requests (not " + request.getClass().getName() + ")"; + String message = "manage own api key privilege only supports API key requests (not " + request.getClass().getName() + ")"; assert false : message; throw new IllegalArgumentException(message); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 0476734cfaccd..f365b2785e23c 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1500,7 +1500,7 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, } public void testUpdateApiKeyAutoSyncsUserRoles() throws IOException, ExecutionException, InterruptedException { - // Create native realm user and role + // Create separate native realm user and role for user role change test final var nativeRealmUser = "native_user"; final var nativeRealmRole = "native_role"; createNativeRealmUser( @@ -1509,7 +1509,7 @@ public void testUpdateApiKeyAutoSyncsUserRoles() throws IOException, ExecutionEx new String(HASHER.hash(TEST_PASSWORD_SECURE_STRING)), Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) ); - putRoleWithClusterPrivileges(nativeRealmRole, "all"); + final RoleDescriptor roleDescriptorBeforeUpdate = putRoleWithClusterPrivileges(nativeRealmRole, "all"); // Create api key final CreateApiKeyResponse createdApiKey = createApiKeys( @@ -1519,14 +1519,10 @@ public void testUpdateApiKeyAutoSyncsUserRoles() throws IOException, ExecutionEx "all" ).v1().get(0); final String apiKeyId = createdApiKey.getId(); - expectRoleDescriptorsForApiKey( - "limited_by_role_descriptors", - Set.of(new RoleDescriptor(nativeRealmRole, new String[] { "all" }, null, null)), - getApiKeyDocument(apiKeyId) - ); + expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorBeforeUpdate), getApiKeyDocument(apiKeyId)); // Update user role - putRoleWithClusterPrivileges(nativeRealmRole, "manage_own_api_key"); + final RoleDescriptor roleDescriptorAfterUpdate = putRoleWithClusterPrivileges(nativeRealmRole, "manage_own_api_key"); // Update API key final PlainActionFuture listener = new PlainActionFuture<>(); @@ -1537,11 +1533,7 @@ public void testUpdateApiKeyAutoSyncsUserRoles() throws IOException, ExecutionEx assertNotNull(response); assertTrue(response.isUpdated()); - expectRoleDescriptorsForApiKey( - "limited_by_role_descriptors", - Set.of(new RoleDescriptor(nativeRealmRole, new String[] { "manage_own_api_key" }, null, null)), - getApiKeyDocument(apiKeyId) - ); + expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorAfterUpdate), getApiKeyDocument(apiKeyId)); } public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { @@ -2038,8 +2030,8 @@ private void createNativeRealmUser( assertTrue(putUserResponse.created()); } - private void putRoleWithClusterPrivileges(final String nativeRealmRoleName, final String clusterPrivilege) throws InterruptedException, - ExecutionException { + private RoleDescriptor putRoleWithClusterPrivileges(final String nativeRealmRoleName, final String clusterPrivilege) + throws InterruptedException, ExecutionException { final PutRoleRequest putRoleRequest = new PutRoleRequest(); putRoleRequest.name(nativeRealmRoleName); putRoleRequest.cluster(clusterPrivilege); @@ -2047,6 +2039,7 @@ private void putRoleWithClusterPrivileges(final String nativeRealmRoleName, fina client().filterWithHeader(Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))) .execute(PutRoleAction.INSTANCE, putRoleRequest, roleListener); assertNotNull(roleListener.get()); + return putRoleRequest.roleDescriptor(); } private Client getClientForRunAsUser() { From 6a2deb1a6448ea0714575267b4523539676c00e0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 10:28:58 +0200 Subject: [PATCH 147/171] Test grantor cannot update --- .../xpack/security/apikey/ApiKeyRestIT.java | 26 +++++++++++++++++++ .../security/authc/ApiKeyIntegTests.java | 6 ++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 5007734e26f7d..e0dbb4c44c118 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -243,6 +243,32 @@ public void testGrantTargetCanUpdateApiKey() throws IOException { doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } + public void testGrantorCannotUpdateApiKeyOfGrantTarget() throws IOException { + final var request = new Request("POST", "_security/api_key/grant"); + final var apiKeyName = "test_api_key_password"; + final Map requestBody = Map.ofEntries( + Map.entry("grant_type", "password"), + Map.entry("username", MANAGE_OWN_API_KEY_USER), + Map.entry("password", END_USER_PASSWORD.toString()), + Map.entry("api_key", Map.of("name", apiKeyName)) + ); + request.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString()); + final Response response = adminClient().performRequest(request); + + final Map createApiKeyResponseMap = responseAsMap(response); // keys: id, name, api_key, encoded + final var apiKeyId = (String) createApiKeyResponseMap.get("id"); + final var apiKeyEncoded = (String) createApiKeyResponseMap.get("encoded"); // Base64(id:api_key) + assertThat(apiKeyId, not(emptyString())); + assertThat(apiKeyEncoded, not(emptyString())); + + final var updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); + updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(Map.of(), XContentType.JSON).utf8ToString()); + final ResponseException e = expectThrows(ResponseException.class, () -> adminClient().performRequest(updateApiKeyRequest)); + + assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); + assertThat(e.getMessage(), containsString("no API key owned by requesting user found for ID")); + } + private void doTestAuthenticationWithApiKey( final String expectedApiKeyName, final String actualApiKeyId, diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index f365b2785e23c..b10bded6a2d1d 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1499,10 +1499,10 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, } } - public void testUpdateApiKeyAutoSyncsUserRoles() throws IOException, ExecutionException, InterruptedException { + public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, ExecutionException, InterruptedException { // Create separate native realm user and role for user role change test - final var nativeRealmUser = "native_user"; - final var nativeRealmRole = "native_role"; + final var nativeRealmUser = randomAlphaOfLengthBetween(5, 10); + final var nativeRealmRole = randomAlphaOfLengthBetween(5, 10); createNativeRealmUser( nativeRealmUser, nativeRealmRole, From a83864834f156662f326ac549cf84161bb1ee6cd Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 10:47:43 +0200 Subject: [PATCH 148/171] Include ID in assertion --- .../org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index e0dbb4c44c118..9669779dafa92 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -266,7 +266,7 @@ public void testGrantorCannotUpdateApiKeyOfGrantTarget() throws IOException { final ResponseException e = expectThrows(ResponseException.class, () -> adminClient().performRequest(updateApiKeyRequest)); assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); - assertThat(e.getMessage(), containsString("no API key owned by requesting user found for ID")); + assertThat(e.getMessage(), containsString("no API key owned by requesting user found for ID [" + apiKeyId + "]")); } private void doTestAuthenticationWithApiKey( From 2d6d9d0c544e03a260f83078b1b09478f91d8023 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 11:18:51 +0200 Subject: [PATCH 149/171] Update docs/changelog/88186.yaml --- docs/changelog/88186.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/88186.yaml diff --git a/docs/changelog/88186.yaml b/docs/changelog/88186.yaml new file mode 100644 index 0000000000000..59851c4a113c1 --- /dev/null +++ b/docs/changelog/88186.yaml @@ -0,0 +1,5 @@ +pr: 88186 +summary: "Support updates of API key attributes [REST and transport layers]" +area: Authentication +type: feature +issues: [] From 3e3e5e0b0b8b1f4ae6329858c7a185c106fb3c1b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 11:22:35 +0200 Subject: [PATCH 150/171] Changelog --- docs/changelog/88186.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/88186.yaml b/docs/changelog/88186.yaml index 59851c4a113c1..210b45cc883bc 100644 --- a/docs/changelog/88186.yaml +++ b/docs/changelog/88186.yaml @@ -1,5 +1,5 @@ pr: 88186 -summary: "Support updates of API key attributes [REST and transport layers]" +summary: Support updates of API key attributes (single operation route) area: Authentication type: feature -issues: [] +issues: [87870] From 57b7e67815241e8d0fdc255b39196cef47525187 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 11:50:55 +0200 Subject: [PATCH 151/171] Validator tests --- .../action/apikey/UpdateApiKeyRequest.java | 6 ++- .../apikey/UpdateApiKeyRequestTests.java | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java index 2c2b3499270aa..e1d5b5325d6ae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequest.java @@ -31,7 +31,11 @@ public final class UpdateApiKeyRequest extends ActionRequest { @Nullable private final List roleDescriptors; - public UpdateApiKeyRequest(String id, @Nullable List roleDescriptors, @Nullable Map metadata) { + public UpdateApiKeyRequest( + final String id, + @Nullable final List roleDescriptors, + @Nullable final Map metadata + ) { this.id = Objects.requireNonNull(id, "API key ID must not be null"); this.roleDescriptors = roleDescriptors; this.metadata = metadata; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java index 965268fb7f65a..0da6fe51ae743 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/UpdateApiKeyRequestTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.security.action.apikey; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; @@ -15,6 +16,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.equalTo; public class UpdateApiKeyRequestTests extends ESTestCase { @@ -50,4 +56,46 @@ public void testSerialization() throws IOException { } } } + + public void testMetadataKeyValidation() { + final var reservedKey = "_" + randomAlphaOfLengthBetween(1, 10); + final var metadataValue = randomAlphaOfLengthBetween(1, 10); + UpdateApiKeyRequest request = new UpdateApiKeyRequest(randomAlphaOfLength(10), null, Map.of(reservedKey, metadataValue)); + final ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), equalTo(1)); + assertThat(ve.validationErrors().get(0), containsString("API key metadata keys may not start with [_]")); + } + + public void testRoleDescriptorValidation() { + final var request1 = new UpdateApiKeyRequest( + randomAlphaOfLength(10), + List.of( + new RoleDescriptor( + randomAlphaOfLength(5), + new String[] { "manage_index_template" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("rad").build() }, + new RoleDescriptor.ApplicationResourcePrivileges[] { + RoleDescriptor.ApplicationResourcePrivileges.builder() + .application(randomFrom("app*tab", "app 1")) + .privileges(randomFrom(" ", "\n")) + .resources("resource") + .build() }, + null, + null, + Map.of("_key", "value"), + null + ) + ), + null + ); + final ActionRequestValidationException ve1 = request1.validate(); + assertNotNull(ve1); + assertThat(ve1.validationErrors().get(0), containsString("unknown cluster privilege")); + assertThat(ve1.validationErrors().get(1), containsString("unknown index privilege")); + assertThat(ve1.validationErrors().get(2), containsStringIgnoringCase("application name")); + assertThat(ve1.validationErrors().get(3), containsStringIgnoringCase("Application privilege names")); + assertThat(ve1.validationErrors().get(4), containsStringIgnoringCase("role descriptor metadata keys may not start with ")); + } } From 0fc25b926b16f247f3f40d7aeabb5d1ffa550551 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 11:58:55 +0200 Subject: [PATCH 152/171] Priv test --- .../ManageOwnApiKeyClusterPrivilegeTests.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index a7dfb79f97e22..c0e6bee94ff5e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -48,6 +48,20 @@ public void testAuthenticationWithApiKeyAllowsAccessToApiKeyActionsWhenItIsOwner assertFalse(clusterPermission.check("cluster:admin/something", mock(TransportRequest.class), authentication)); } + public void testAuthenticationWithApiKeyAllowsAllowsAccessForUpdateApiKey() { + final ClusterPermission clusterPermission = ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()) + .build(); + + final String apiKeyId = randomAlphaOfLengthBetween(4, 7); + final User userJoe = new User("joe"); + final Authentication authentication = AuthenticationTests.randomApiKeyAuthentication(userJoe, apiKeyId); + final TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingApiKeyId( + randomValueOtherThan(apiKeyId, () -> randomAlphaOfLengthBetween(4, 7)) + ); + + assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); + } + public void testAuthenticationWithApiKeyDeniesAccessToApiKeyActionsWhenItIsNotOwner() { final ClusterPermission clusterPermission = ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()) .build(); From 75e2e0e79517751acf035066b2dd4f7f2879a81a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 12:10:27 +0200 Subject: [PATCH 153/171] Clean up tests --- .../ManageOwnApiKeyClusterPrivilege.java | 2 +- .../ManageOwnApiKeyClusterPrivilegeTests.java | 10 ++++++---- .../xpack/security/apikey/ApiKeyRestIT.java | 16 +++++----------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java index 75a3dadcb5e5b..032beddf59d03 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java @@ -64,7 +64,7 @@ protected boolean extendedCheck(String action, TransportRequest request, Authent return true; } else if (request instanceof UpdateApiKeyRequest) { // Note: we return `true` here even if the authenticated entity is an API key. API keys *cannot* update themselves - // however this is a "business-logic" restriction, rather than one related to privileges. We therefore enforce this + // however this is a business logic restriction, rather than one driven solely by privileges. We therefore enforce this // limitation at the transport layer, in `TransportUpdateApiKeyAction` return true; } else if (request instanceof final GetApiKeyRequest getApiKeyRequest) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index c0e6bee94ff5e..56b7712a1e176 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -48,18 +48,20 @@ public void testAuthenticationWithApiKeyAllowsAccessToApiKeyActionsWhenItIsOwner assertFalse(clusterPermission.check("cluster:admin/something", mock(TransportRequest.class), authentication)); } - public void testAuthenticationWithApiKeyAllowsAllowsAccessForUpdateApiKey() { + public void testAuthenticationForUpdateApiKeyAllowsAll() { final ClusterPermission clusterPermission = ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()) .build(); - final String apiKeyId = randomAlphaOfLengthBetween(4, 7); final User userJoe = new User("joe"); - final Authentication authentication = AuthenticationTests.randomApiKeyAuthentication(userJoe, apiKeyId); + final Authentication.RealmRef realmRef = AuthenticationTests.randomRealmRef(randomBoolean()); + final Authentication authenticationWithUser = AuthenticationTests.randomAuthentication(new User("joe"), realmRef); + final Authentication apiKeyAuthentication = AuthenticationTests.randomApiKeyAuthentication(userJoe, apiKeyId); final TransportRequest updateApiKeyRequest = UpdateApiKeyRequest.usingApiKeyId( randomValueOtherThan(apiKeyId, () -> randomAlphaOfLengthBetween(4, 7)) ); - assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); + assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, apiKeyAuthentication)); + assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authenticationWithUser)); } public void testAuthenticationWithApiKeyDeniesAccessToApiKeyActionsWhenItIsNotOwner() { diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 9669779dafa92..c7f89106338cd 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -269,15 +269,10 @@ public void testGrantorCannotUpdateApiKeyOfGrantTarget() throws IOException { assertThat(e.getMessage(), containsString("no API key owned by requesting user found for ID [" + apiKeyId + "]")); } - private void doTestAuthenticationWithApiKey( - final String expectedApiKeyName, - final String actualApiKeyId, - final String actualApiKeyEncoded - ) throws IOException { + private void doTestAuthenticationWithApiKey(final String apiKeyName, final String apiKeyId, final String apiKeyEncoded) + throws IOException { final var authenticateRequest = new Request("GET", "_security/_authenticate"); - authenticateRequest.setOptions( - authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + actualApiKeyEncoded) - ); + authenticateRequest.setOptions(authenticateRequest.getOptions().toBuilder().addHeader("Authorization", "ApiKey " + apiKeyEncoded)); final Response authenticateResponse = client().performRequest(authenticateRequest); assertOK(authenticateResponse); @@ -285,11 +280,11 @@ private void doTestAuthenticationWithApiKey( // If authentication type is API_KEY, authentication.api_key={"id":"abc123","name":"my-api-key"}. No encoded, api_key, or metadata. // If authentication type is other, authentication.api_key not present. - assertThat(authenticate, hasEntry("api_key", Map.of("id", actualApiKeyId, "name", expectedApiKeyName))); + assertThat(authenticate, hasEntry("api_key", Map.of("id", apiKeyId, "name", apiKeyName))); } private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKeyEncoded) throws IOException { - final Request updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); + final var updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); @@ -299,7 +294,6 @@ private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKe assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); - expectMetadata(apiKeyId, expectedApiKeyMetadata); // validate authentication still works after update doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); From ea8d79e5bfe92403c82e10a1c6fa36d85a324bef Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 12:54:58 +0200 Subject: [PATCH 154/171] Randomize more but not too much --- .../security/authc/ApiKeyIntegTests.java | 51 +++++++++++-------- .../security/authz/RoleDescriptorTests.java | 6 ++- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index b10bded6a2d1d..f73c6501bbcf8 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -76,6 +76,8 @@ import org.elasticsearch.xpack.core.security.authc.RealmDomain; import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver; +import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authz.RoleDescriptorTests; import org.elasticsearch.xpack.security.transport.filter.IPFilter; @@ -92,6 +94,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -1448,18 +1451,16 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var request = new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, ApiKeyTests.randomMetadata()); final PlainActionFuture listener = new PlainActionFuture<>(); - final Client client = client().filterWithHeader( - Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) - ); - client.execute(UpdateApiKeyAction.INSTANCE, request, listener); - final var response = listener.get(); + final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request, listener); assertNotNull(response); assertTrue(response.isUpdated()); final PlainActionFuture getListener = new PlainActionFuture<>(); - client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener); - GetApiKeyResponse getResponse = getListener.get(); + client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) + ).execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener); + final GetApiKeyResponse getResponse = getListener.get(); assertEquals(1, getResponse.getApiKeyInfos().length); // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it final var expectedMetadata = request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2(); @@ -1526,13 +1527,10 @@ public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, Execution // Update API key final PlainActionFuture listener = new PlainActionFuture<>(); - client().filterWithHeader( - Collections.singletonMap("Authorization", basicAuthHeaderValue(nativeRealmUser, TEST_PASSWORD_SECURE_STRING)) - ).execute(UpdateApiKeyAction.INSTANCE, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), listener); - final var response = listener.get(); + final UpdateApiKeyResponse response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), listener); + assertNotNull(response); assertTrue(response.isUpdated()); - expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorAfterUpdate), getApiKeyDocument(apiKeyId)); } @@ -1544,11 +1542,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter // Validate can update own API key final PlainActionFuture listener = new PlainActionFuture<>(); - final Client client = client().filterWithHeader( - Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) - ); - client.execute(UpdateApiKeyAction.INSTANCE, request, listener); - final var response = listener.get(); + final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request, listener); assertNotNull(response); assertTrue(response.isUpdated()); @@ -1584,11 +1578,14 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter } public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, InterruptedException { - final var apiKeyClusterPrivilege = randomFrom("all", "manage_security", "manage_api_key", "manage_own_api_key"); - final CreateApiKeyResponse createdApiKey = createApiKeys(TEST_USER_NAME, 1, null, apiKeyClusterPrivilege).v1().get(0); + final List apiKeyPrivileges = new ArrayList<>(randomSubsetOf(ClusterPrivilegeResolver.names())); + // At a minimum include `manage_own_api_key` to ensure no 403 + apiKeyPrivileges.add("manage_own_api_key"); + final CreateApiKeyResponse createdApiKey = createApiKeys(TEST_USER_NAME, 1, null, apiKeyPrivileges.toArray(new String[0])).v1() + .get(0); final var apiKeyId = createdApiKey.getId(); - final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { apiKeyClusterPrivilege }, null, null); + final var roleDescriptor = new RoleDescriptor(randomAlphaOfLength(10), new String[] { "manage_own_api_key" }, null, null); final var request = new UpdateApiKeyRequest(apiKeyId, List.of(roleDescriptor), ApiKeyTests.randomMetadata()); PlainActionFuture updateListener = new PlainActionFuture<>(); client().filterWithHeader( @@ -1738,7 +1735,7 @@ private List randomRoleDescriptors() { case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); case 1 -> List.of( new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null), - RoleDescriptorTests.randomRoleDescriptor() + RoleDescriptorTests.randomRoleDescriptor(false) ); case 2 -> null; default -> throw new IllegalStateException("unexpected case no"); @@ -2053,6 +2050,18 @@ private Client getClientForRunAsUser() { ); } + private UpdateApiKeyResponse executeUpdateApiKey( + final String username, + final UpdateApiKeyRequest request, + final PlainActionFuture listener + ) throws InterruptedException, ExecutionException { + final Client client = client().filterWithHeader( + Collections.singletonMap("Authorization", basicAuthHeaderValue(username, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(UpdateApiKeyAction.INSTANCE, request, listener); + return listener.get(); + } + private void assertErrorMessage(final ElasticsearchSecurityException ese, String action, String userName, String apiKeyId) { assertThat( ese, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java index 3c7d936fa114a..1135afbe1020d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java @@ -627,6 +627,10 @@ public void testIsEmpty() { } public static RoleDescriptor randomRoleDescriptor() { + return randomRoleDescriptor(true); + } + + public static RoleDescriptor randomRoleDescriptor(boolean allowReservedMetadata) { final RoleDescriptor.IndicesPrivileges[] indexPrivileges = new RoleDescriptor.IndicesPrivileges[randomIntBetween(0, 3)]; for (int i = 0; i < indexPrivileges.length; i++) { final RoleDescriptor.IndicesPrivileges.Builder builder = RoleDescriptor.IndicesPrivileges.builder() @@ -695,7 +699,7 @@ public static RoleDescriptor randomRoleDescriptor() { final Map metadata = new HashMap<>(); while (randomBoolean()) { String key = randomAlphaOfLengthBetween(4, 12); - if (randomBoolean()) { + if (allowReservedMetadata && randomBoolean()) { key = MetadataUtils.RESERVED_PREFIX + key; } final Object value = randomBoolean() ? randomInt() : randomAlphaOfLengthBetween(3, 50); From bedd5b044d40a1bf9f0558c7ac1e0d8f1d283e60 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 12:56:10 +0200 Subject: [PATCH 155/171] Imports --- .../elasticsearch/xpack/security/authc/ApiKeyIntegTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index f73c6501bbcf8..8a9a94a6056e2 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -77,7 +77,6 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver; -import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authz.RoleDescriptorTests; import org.elasticsearch.xpack.security.transport.filter.IPFilter; @@ -94,7 +93,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; From 96e681a712690c6ad863beb96a6a2867bb9c0606 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 13:05:45 +0200 Subject: [PATCH 156/171] Randomize cluster privs in auto update test --- .../security/authc/ApiKeyIntegTests.java | 22 +++++++++++++++---- .../xpack/security/authc/ApiKeyService.java | 14 ++++++------ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 8a9a94a6056e2..8e1d8503e73d9 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -1508,7 +1508,13 @@ public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, Execution new String(HASHER.hash(TEST_PASSWORD_SECURE_STRING)), Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING)) ); - final RoleDescriptor roleDescriptorBeforeUpdate = putRoleWithClusterPrivileges(nativeRealmRole, "all"); + final List clusterPrivileges = new ArrayList<>(randomSubsetOf(ClusterPrivilegeResolver.names())); + // At a minimum include `manage_own_api_key` to ensure no 403 + clusterPrivileges.add("manage_own_api_key"); + final RoleDescriptor roleDescriptorBeforeUpdate = putRoleWithClusterPrivileges( + nativeRealmRole, + clusterPrivileges.toArray(new String[0]) + ); // Create api key final CreateApiKeyResponse createdApiKey = createApiKeys( @@ -1520,8 +1526,14 @@ public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, Execution final String apiKeyId = createdApiKey.getId(); expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorBeforeUpdate), getApiKeyDocument(apiKeyId)); + final List newClusterPrivileges = new ArrayList<>(randomSubsetOf(ClusterPrivilegeResolver.names())); + // At a minimum include `manage_own_api_key` to ensure no 403 + newClusterPrivileges.add("manage_own_api_key"); // Update user role - final RoleDescriptor roleDescriptorAfterUpdate = putRoleWithClusterPrivileges(nativeRealmRole, "manage_own_api_key"); + final RoleDescriptor roleDescriptorAfterUpdate = putRoleWithClusterPrivileges( + nativeRealmRole, + newClusterPrivileges.toArray(new String[0]) + ); // Update API key final PlainActionFuture listener = new PlainActionFuture<>(); @@ -2025,11 +2037,13 @@ private void createNativeRealmUser( assertTrue(putUserResponse.created()); } - private RoleDescriptor putRoleWithClusterPrivileges(final String nativeRealmRoleName, final String clusterPrivilege) + private RoleDescriptor putRoleWithClusterPrivileges(final String nativeRealmRoleName, String... clusterPrivileges) throws InterruptedException, ExecutionException { final PutRoleRequest putRoleRequest = new PutRoleRequest(); putRoleRequest.name(nativeRealmRoleName); - putRoleRequest.cluster(clusterPrivilege); + for (final String clusterPrivilege : clusterPrivileges) { + putRoleRequest.cluster(clusterPrivilege); + } final PlainActionFuture roleListener = new PlainActionFuture<>(); client().filterWithHeader(Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))) .execute(PutRoleAction.INSTANCE, putRoleRequest, roleListener); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3b2b6d95febcf..ce771df7dfc25 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1234,7 +1234,7 @@ private static VersionedApiKeyDoc singleDoc(final String apiKeyId, final Collect } private BulkRequest buildBulkRequestForUpdate( - final VersionedApiKeyDoc apiKeyDoc, + final VersionedApiKeyDoc versionedDoc, final Authentication authentication, final UpdateApiKeyRequest request, final Set userRoles @@ -1242,10 +1242,10 @@ private BulkRequest buildBulkRequestForUpdate( logger.trace( "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", request.getId(), - apiKeyDoc.seqNo(), - apiKeyDoc.primaryTerm() + versionedDoc.seqNo(), + versionedDoc.primaryTerm() ); - final var currentDocVersion = Version.fromId(apiKeyDoc.doc().version); + final var currentDocVersion = Version.fromId(versionedDoc.doc().version); final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; if (currentDocVersion.before(targetDocVersion)) { @@ -1262,7 +1262,7 @@ private BulkRequest buildBulkRequestForUpdate( .setId(request.getId()) .setSource( buildUpdatedDocument( - apiKeyDoc.doc(), + versionedDoc.doc(), authentication, userRoles, request.getRoleDescriptors(), @@ -1270,8 +1270,8 @@ private BulkRequest buildBulkRequestForUpdate( request.getMetadata() ) ) - .setIfSeqNo(apiKeyDoc.seqNo()) - .setIfPrimaryTerm(apiKeyDoc.primaryTerm()) + .setIfSeqNo(versionedDoc.seqNo()) + .setIfPrimaryTerm(versionedDoc.primaryTerm()) .setOpType(DocWriteRequest.OpType.INDEX) .request() ); From 8ce56515cc772a214ebcc26cf4dc9c83daacb604 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 14:09:37 +0200 Subject: [PATCH 157/171] Update docs/changelog/88186.yaml --- docs/changelog/88186.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/88186.yaml b/docs/changelog/88186.yaml index 210b45cc883bc..f13b944126f69 100644 --- a/docs/changelog/88186.yaml +++ b/docs/changelog/88186.yaml @@ -2,4 +2,4 @@ pr: 88186 summary: Support updates of API key attributes (single operation route) area: Authentication type: feature -issues: [87870] +issues: [] From cfe84c8740712336285b85b567b6ad6edb128c92 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 14:15:29 +0200 Subject: [PATCH 158/171] Clear cache param --- .../rest-api-spec/api/security.clear_api_key_cache.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json index 2f3ce2f27e071..72156cbe776a0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json @@ -12,7 +12,7 @@ "url":{ "paths":[ { - "path":"/_security/api_key/{ids}/_clear_cache", + "path":"/_security/api_key/{id}/_clear_cache", "methods":[ "POST" ], From 17e449f8927a5f0ba3a0eb5c936a07355ee64852 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 14:17:15 +0200 Subject: [PATCH 159/171] Changelog --- docs/changelog/88186.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/88186.yaml b/docs/changelog/88186.yaml index f13b944126f69..210b45cc883bc 100644 --- a/docs/changelog/88186.yaml +++ b/docs/changelog/88186.yaml @@ -2,4 +2,4 @@ pr: 88186 summary: Support updates of API key attributes (single operation route) area: Authentication type: feature -issues: [] +issues: [87870] From 23f857262c05cdd92e20945486c4179babcca11a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 14:30:49 +0200 Subject: [PATCH 160/171] Use ids to avoid cascading failure --- .../rest-api-spec/api/security.clear_api_key_cache.json | 2 +- .../rest/action/apikey/RestClearApiKeyCacheAction.java | 4 ++-- .../security/rest/action/apikey/RestUpdateApiKeyAction.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json index 72156cbe776a0..2f3ce2f27e071 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.clear_api_key_cache.json @@ -12,7 +12,7 @@ "url":{ "paths":[ { - "path":"/_security/api_key/{id}/_clear_cache", + "path":"/_security/api_key/{ids}/_clear_cache", "methods":[ "POST" ], diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java index bbeb8d0c3758f..ef55f1609801f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestClearApiKeyCacheAction.java @@ -34,12 +34,12 @@ public String getName() { @Override public List routes() { - return List.of(new Route(POST, "/_security/api_key/{id}/_clear_cache")); + return List.of(new Route(POST, "/_security/api_key/{ids}/_clear_cache")); } @Override protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { - String[] ids = request.paramAsStringArrayOrEmptyIfAll("id"); + String[] ids = request.paramAsStringArrayOrEmptyIfAll("ids"); final ClearSecurityCacheRequest req = new ClearSecurityCacheRequest().cacheName("api_key").keys(ids); return channel -> client.execute(ClearSecurityCacheAction.INSTANCE, req, new NodesResponseRestListener<>(channel)); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java index de6e3b4ad7351..8e144913d36a0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestUpdateApiKeyAction.java @@ -48,7 +48,7 @@ public RestUpdateApiKeyAction(final Settings settings, final XPackLicenseState l @Override public List routes() { - return List.of(new Route(PUT, "/_security/api_key/{id}")); + return List.of(new Route(PUT, "/_security/api_key/{ids}")); } @Override @@ -58,7 +58,7 @@ public String getName() { @Override protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final var apiKeyId = request.param("id"); + final var apiKeyId = request.param("ids"); final var payload = PARSER.parse(request.contentParser(), null); return channel -> client.execute( UpdateApiKeyAction.INSTANCE, From 3673f0ff10f2fc7a44ddd48ba3143dcfe3e46a8f Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 4 Jul 2022 15:34:24 +0200 Subject: [PATCH 161/171] Exclude invalid role descriptors --- .../xpack/security/authc/ApiKeyIntegTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 8e1d8503e73d9..9e0d04fa203f1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -64,6 +64,7 @@ import org.elasticsearch.xpack.core.security.action.role.PutRoleAction; import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest; import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse; +import org.elasticsearch.xpack.core.security.action.role.RoleDescriptorRequestValidator; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequestBuilder; import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse; @@ -1745,7 +1746,10 @@ private List randomRoleDescriptors() { case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); case 1 -> List.of( new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null), - RoleDescriptorTests.randomRoleDescriptor(false) + randomValueOtherThanMany( + rd -> RoleDescriptorRequestValidator.validate(rd) != null, + () -> RoleDescriptorTests.randomRoleDescriptor(false) + ) ); case 2 -> null; default -> throw new IllegalStateException("unexpected case no"); From 8f500b4259f0b00e3bf5571f969f6fa34cd890b6 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 5 Jul 2022 15:25:57 +0200 Subject: [PATCH 162/171] Audit trail for update api key --- .../audit/logfile/LoggingAuditTrail.java | 30 ++++++++++++++++++- .../audit/logfile/LoggingAuditTrailTests.java | 29 ++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index a4ebb5dfdd37b..cef94b03a9864 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -49,6 +49,8 @@ import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest; import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction; @@ -287,7 +289,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { DeleteServiceAccountTokenAction.NAME, ActivateProfileAction.NAME, UpdateProfileDataAction.NAME, - SetProfileEnabledAction.NAME + SetProfileEnabledAction.NAME, + UpdateApiKeyAction.NAME ); private static final String FILTER_POLICY_PREFIX = setting("audit.logfile.events.ignore_filters."); // because of the default wildcard value (*) for the field filter, a policy with @@ -747,6 +750,9 @@ public void accessGranted( } else if (msg instanceof final SetProfileEnabledRequest setProfileEnabledRequest) { assert SetProfileEnabledAction.NAME.equals(action); securityChangeLogEntryBuilder(requestId).withRequestBody(setProfileEnabledRequest).build(); + } else if (msg instanceof UpdateApiKeyRequest) { + assert UpdateApiKeyAction.NAME.equals(action); + securityChangeLogEntryBuilder(requestId).withRequestBody((UpdateApiKeyRequest) msg).build(); } else { throw new IllegalStateException( "Unknown message class type [" @@ -1215,6 +1221,16 @@ LogEntryBuilder withRequestBody(GrantApiKeyRequest grantApiKeyRequest) throws IO return this; } + LogEntryBuilder withRequestBody(UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + logEntry.with(EVENT_ACTION_FIELD_NAME, "update_apikey"); + XContentBuilder builder = JsonXContent.contentBuilder().humanReadable(true); + builder.startObject(); + withRequestBody(builder, updateApiKeyRequest); + builder.endObject(); + logEntry.with(CHANGE_CONFIG_FIELD_NAME, Strings.toString(builder)); + return this; + } + private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest createApiKeyRequest) throws IOException { TimeValue expiration = createApiKeyRequest.getExpiration(); builder.startObject("apikey") @@ -1228,6 +1244,18 @@ private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest create .endObject(); // apikey } + private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + builder.startObject("apikey").field("id", updateApiKeyRequest.getId()); + if (updateApiKeyRequest.getRoleDescriptors() != null) { + builder.startArray("role_descriptors"); + for (RoleDescriptor roleDescriptor : updateApiKeyRequest.getRoleDescriptors()) { + withRoleDescriptor(builder, roleDescriptor); + } + builder.endArray(); + } + builder.endObject(); + } + private void withRoleDescriptor(XContentBuilder builder, RoleDescriptor roleDescriptor) throws IOException { builder.startObject().array(RoleDescriptor.Fields.CLUSTER.getPreferredName(), roleDescriptor.getClusterPrivileges()); if (roleDescriptor.getConditionalClusterPrivileges() != null && roleDescriptor.getConditionalClusterPrivileges().length > 0) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 5fc92aa3f416a..2481a0e57df4c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -46,12 +46,15 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest; import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction; @@ -605,6 +608,32 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException // clear log CapturingLogger.output(logger.getName(), Level.INFO).clear(); + final String keyId = randomAlphaOfLength(10); + final var updateApiKeyRequest = new UpdateApiKeyRequest( + keyId, + randomBoolean() ? null : keyRoleDescriptors, + ApiKeyTests.randomMetadata() + ); + auditTrail.accessGranted(requestId, authentication, UpdateApiKeyAction.NAME, updateApiKeyRequest, authorizationInfo); + final var expectedUpdateKeyAuditEventString = """ + "change":{"apikey":{"id":"%s"%s}}\ + """.formatted(keyId, updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder); + output = CapturingLogger.output(logger.getName(), Level.INFO); + assertThat(output.size(), is(2)); + String generatedUpdateKeyAuditEventString = output.get(1); + assertThat(generatedUpdateKeyAuditEventString, containsString(expectedUpdateKeyAuditEventString)); + generatedUpdateKeyAuditEventString = generatedUpdateKeyAuditEventString.replace(", " + expectedUpdateKeyAuditEventString, ""); + checkedFields = new MapBuilder<>(commonFields); + checkedFields.remove(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME); + checkedFields.remove(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME); + checkedFields.put("type", "audit") + .put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, "security_config_change") + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "update_apikey") + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + assertMsg(generatedUpdateKeyAuditEventString, checkedFields.map()); + // clear log + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(); grantApiKeyRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); grantApiKeyRequest.getGrant().setType(randomFrom(randomAlphaOfLength(8), null)); From ba17743bf5fbccb173250b58a6062ef5dffabaa6 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 5 Jul 2022 15:30:32 +0200 Subject: [PATCH 163/171] Test --- .../xpack/security/audit/logfile/LoggingAuditTrailTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 2481a0e57df4c..166c39d924e5f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -1825,7 +1825,8 @@ public void testSecurityConfigChangedEventSelection() { new Tuple<>( SetProfileEnabledAction.NAME, new SetProfileEnabledRequest(randomAlphaOfLength(20), randomBoolean(), WriteRequest.RefreshPolicy.WAIT_UNTIL) - ) + ), + new Tuple<>(UpdateApiKeyAction.NAME, UpdateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(10))) ); auditTrail.accessGranted(requestId, authentication, actionAndRequest.v1(), actionAndRequest.v2(), authorizationInfo); List output = CapturingLogger.output(logger.getName(), Level.INFO); From e90a9c42a5027cbf3284b80f73f704a98a906798 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 5 Jul 2022 15:53:19 +0200 Subject: [PATCH 164/171] final --- .../xpack/security/audit/logfile/LoggingAuditTrail.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index cef94b03a9864..f871ce1c9f149 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -1221,7 +1221,7 @@ LogEntryBuilder withRequestBody(GrantApiKeyRequest grantApiKeyRequest) throws IO return this; } - LogEntryBuilder withRequestBody(UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + LogEntryBuilder withRequestBody(final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { logEntry.with(EVENT_ACTION_FIELD_NAME, "update_apikey"); XContentBuilder builder = JsonXContent.contentBuilder().humanReadable(true); builder.startObject(); From e694d905d4010dfdf56633e6795143dcd8970d89 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 7 Jul 2022 12:48:31 +0200 Subject: [PATCH 165/171] Merge --- .../authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java index 8e0073768d8dc..af3b81dbec024 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java @@ -43,7 +43,6 @@ public void testAuthenticationWithApiKeyAllowsAccessToApiKeyActionsWhenItIsOwner final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean()); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/get", getApiKeyRequest, authentication)); assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/invalidate", invalidateApiKeyRequest, authentication)); - assertTrue(clusterPermission.check("cluster:admin/xpack/security/api_key/update", updateApiKeyRequest, authentication)); assertFalse(clusterPermission.check("cluster:admin/something", mock(TransportRequest.class), authentication)); } From 7f646da3c2fb54531fd62f03c6613797bc18bcb7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 7 Jul 2022 12:51:45 +0200 Subject: [PATCH 166/171] Update docs/changelog/88276.yaml --- docs/changelog/88276.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/88276.yaml diff --git a/docs/changelog/88276.yaml b/docs/changelog/88276.yaml new file mode 100644 index 0000000000000..9833c80f57650 --- /dev/null +++ b/docs/changelog/88276.yaml @@ -0,0 +1,5 @@ +pr: 88276 +summary: Update API keys - logging audit trail event +area: Audit +type: enhancement +issues: [] From b3eeaa195c047c85e32d0370b740bf08718a1aeb Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 7 Jul 2022 12:53:13 +0200 Subject: [PATCH 167/171] Changelog --- docs/changelog/88276.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/88276.yaml b/docs/changelog/88276.yaml index 9833c80f57650..942d83375b361 100644 --- a/docs/changelog/88276.yaml +++ b/docs/changelog/88276.yaml @@ -1,5 +1,5 @@ pr: 88276 -summary: Update API keys - logging audit trail event +summary: Updatable API keys - logging audit trail event area: Audit type: enhancement issues: [] From b06a593aa8d20a805d98b17ddd6abf7a1813ad9b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 8 Jul 2022 11:14:57 +0200 Subject: [PATCH 168/171] Review comments and docs --- .../en/security/auditing/event-types.asciidoc | 40 +++++++++++++++++-- .../audit/logfile/LoggingAuditTrail.java | 6 +-- .../audit/logfile/LoggingAuditTrailTests.java | 2 +- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 7cc041ff4ccf8..999888c44b5a4 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -231,6 +231,29 @@ event action. ["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} ==== +[[event-update-apikey]] +`update_apikey`:: +Logged when the <> API is +invoked to update the attributes of an existing API key. ++ +You must include the `security_config_change` event type to audit the related +event action. ++ +.Example +[%collapsible%open] +==== +[source,js] +{"type":"audit", "timestamp":"2020-12-31T00:33:52,521+0200", "node.id": +"9clhpgjJRR-iKzOw20xBNQ", "event.type":"security_config_change", "event.action": +"update_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey_update": +{"id":"zcwN3YEBBmnjw-K-hW5_","role_descriptors":[{"cluster": +["monitor","manage_ilm"],"indices":[{"names":["index-a*"],"privileges": +["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], +"field_security":{"grant":["field1*","@timestamp"],"except":["field11"]}}], +"applications":[],"run_as":[]},{"cluster":["all"],"indices":[{"names": +["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} +==== + [[event-delete-privileges]] `delete_privileges`:: Logged when the @@ -535,8 +558,8 @@ In addition, if `event.type` equals <, "role_descriptors" []}` +---- +// NOTCONSOLE + `service_token` :: An object like: + [source,js] diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index f91cb883d9961..1b95bee344785 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -750,9 +750,9 @@ public void accessGranted( } else if (msg instanceof final SetProfileEnabledRequest setProfileEnabledRequest) { assert SetProfileEnabledAction.NAME.equals(action); securityChangeLogEntryBuilder(requestId).withRequestBody(setProfileEnabledRequest).build(); - } else if (msg instanceof UpdateApiKeyRequest) { + } else if (msg instanceof final UpdateApiKeyRequest updateApiKeyRequest) { assert UpdateApiKeyAction.NAME.equals(action); - securityChangeLogEntryBuilder(requestId).withRequestBody((UpdateApiKeyRequest) msg).build(); + securityChangeLogEntryBuilder(requestId).withRequestBody(updateApiKeyRequest).build(); } else { throw new IllegalStateException( "Unknown message class type [" @@ -1245,7 +1245,7 @@ private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest create } private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { - builder.startObject("apikey").field("id", updateApiKeyRequest.getId()); + builder.startObject("apikey_update").field("id", updateApiKeyRequest.getId()); if (updateApiKeyRequest.getRoleDescriptors() != null) { builder.startArray("role_descriptors"); for (RoleDescriptor roleDescriptor : updateApiKeyRequest.getRoleDescriptors()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 751cb45532551..fe512922d7d86 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -616,7 +616,7 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException ); auditTrail.accessGranted(requestId, authentication, UpdateApiKeyAction.NAME, updateApiKeyRequest, authorizationInfo); final var expectedUpdateKeyAuditEventString = """ - "change":{"apikey":{"id":"%s"%s}}\ + "change":{"apikey_update":{"id":"%s"%s}}\ """.formatted(keyId, updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder); output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); From 55ea3f9fdeb0c96c942afbe42a66c48c6d9450b4 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 8 Jul 2022 11:33:30 +0200 Subject: [PATCH 169/171] Use apikey --- .../en/security/auditing/event-types.asciidoc | 17 ++++++----------- .../audit/logfile/LoggingAuditTrail.java | 2 +- .../audit/logfile/LoggingAuditTrailTests.java | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 999888c44b5a4..f41c7502753a7 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -245,7 +245,7 @@ event action. [source,js] {"type":"audit", "timestamp":"2020-12-31T00:33:52,521+0200", "node.id": "9clhpgjJRR-iKzOw20xBNQ", "event.type":"security_config_change", "event.action": -"update_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey_update": +"update_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey": {"id":"zcwN3YEBBmnjw-K-hW5_","role_descriptors":[{"cluster": ["monitor","manage_ilm"],"indices":[{"names":["index-a*"],"privileges": ["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], @@ -663,8 +663,8 @@ ones), which is specific to the `event.type` attribute. The attribute's value is `role_mapping` or for application `privileges`. `change` :: The object representation of the security config that is being changed. It can be the `password`, `enable` or `disable`, - config object for native or built-in users, or `apikey_update` if - an API key is updated. + config object for native or built-in users. If an API key is updated, + the config object will be an `apikey`. `create` :: The object representation of the new security config that is being created. This is currently only used for API keys auditing. If the API key is created using the @@ -764,6 +764,9 @@ the <>. The `role_descriptors` objects have the same schema as the `role_descriptor` object that is part of the above `role` config object. +The object for an API key update will differ in that it will include an `id` +field instead of `name` and will not include `expiration`. + `grant` :: An object like: + [source,js] @@ -782,14 +785,6 @@ object that is part of the above `role` config object. ---- // NOTCONSOLE -`apikey_update` :: An object like: -+ -[source,js] ----- -`{"id": , "role_descriptors" []}` ----- -// NOTCONSOLE - `service_token` :: An object like: + [source,js] diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 1b95bee344785..c3ae1d284168e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -1245,7 +1245,7 @@ private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest create } private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { - builder.startObject("apikey_update").field("id", updateApiKeyRequest.getId()); + builder.startObject("apikey").field("id", updateApiKeyRequest.getId()); if (updateApiKeyRequest.getRoleDescriptors() != null) { builder.startArray("role_descriptors"); for (RoleDescriptor roleDescriptor : updateApiKeyRequest.getRoleDescriptors()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index fe512922d7d86..751cb45532551 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -616,7 +616,7 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException ); auditTrail.accessGranted(requestId, authentication, UpdateApiKeyAction.NAME, updateApiKeyRequest, authorizationInfo); final var expectedUpdateKeyAuditEventString = """ - "change":{"apikey_update":{"id":"%s"%s}}\ + "change":{"apikey":{"id":"%s"%s}}\ """.formatted(keyId, updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder); output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); From aeb25583b139d14fa4ffd37c95da40fb6bbdbfd7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 11 Jul 2022 13:36:51 +0200 Subject: [PATCH 170/171] change-apikeys --- x-pack/docs/en/security/auditing/event-types.asciidoc | 10 +++++----- .../security/audit/logfile/LoggingAuditTrail.java | 2 +- .../security/audit/logfile/LoggingAuditTrailTests.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index f41c7502753a7..d42601b231685 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -231,8 +231,8 @@ event action. ["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} ==== -[[event-update-apikey]] -`update_apikey`:: +[[event-change-apikey]] +`change_apikey`:: Logged when the <> API is invoked to update the attributes of an existing API key. + @@ -245,7 +245,7 @@ event action. [source,js] {"type":"audit", "timestamp":"2020-12-31T00:33:52,521+0200", "node.id": "9clhpgjJRR-iKzOw20xBNQ", "event.type":"security_config_change", "event.action": -"update_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey": +"change_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey": {"id":"zcwN3YEBBmnjw-K-hW5_","role_descriptors":[{"cluster": ["monitor","manage_ilm"],"indices":[{"names":["index-a*"],"privileges": ["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], @@ -559,7 +559,7 @@ the `event.action` attribute takes one of the following values: `put_user`, `change_password`, `put_role`, `put_role_mapping`, `change_enable_user`, `change_disable_user`, `put_privileges`, `create_apikey`, `delete_user`, `delete_role`, `delete_role_mapping`, `invalidate_apikeys`, -`delete_privileges`, or `update_apikey`. +`delete_privileges`, or `change_apikey`. `request.id` :: A synthetic identifier that can be used to correlate the events associated with a particular REST request. @@ -649,7 +649,7 @@ ones): The events with the `event.type` attribute equal to `security_config_change` have one of the following `event.action` attribute values: `put_user`, `change_password`, `put_role`, `put_role_mapping`, `change_enable_user`, `change_disable_user`, `put_privileges`, `create_apikey`, `delete_user`, -`delete_role`, `delete_role_mapping`, `invalidate_apikeys`, `delete_privileges`, or `update_apikey`. +`delete_role`, `delete_role_mapping`, `invalidate_apikeys`, `delete_privileges`, or `change_apikey`. These events also have *one* of the following extra attributes (in addition to the common ones), which is specific to the `event.type` attribute. The attribute's value is a nested JSON object: diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index c3ae1d284168e..6ea823564db02 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -1222,7 +1222,7 @@ LogEntryBuilder withRequestBody(GrantApiKeyRequest grantApiKeyRequest) throws IO } LogEntryBuilder withRequestBody(final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { - logEntry.with(EVENT_ACTION_FIELD_NAME, "update_apikey"); + logEntry.with(EVENT_ACTION_FIELD_NAME, "change_apikey"); XContentBuilder builder = JsonXContent.contentBuilder().humanReadable(true); builder.startObject(); withRequestBody(builder, updateApiKeyRequest); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 751cb45532551..540b5fa5c1507 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -628,7 +628,7 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException checkedFields.remove(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME); checkedFields.put("type", "audit") .put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, "security_config_change") - .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "update_apikey") + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "change_apikey") .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); assertMsg(generatedUpdateKeyAuditEventString, checkedFields.map()); // clear log From 6db216b3466aac87050242f31d41074efc3d0fe0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 12 Jul 2022 11:53:33 +0200 Subject: [PATCH 171/171] Event type tweak --- x-pack/docs/en/security/auditing/event-types.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index d42601b231685..f3ca8a303725b 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -764,8 +764,8 @@ the <>. The `role_descriptors` objects have the same schema as the `role_descriptor` object that is part of the above `role` config object. -The object for an API key update will differ in that it will include an `id` -field instead of `name` and will not include `expiration`. +The object for an API key update will differ in that it will not include +a `name` or `expiration`. `grant` :: An object like: +