Skip to content

Commit

Permalink
[ML] Change Datafeed actions to read config from the config index (#3…
Browse files Browse the repository at this point in the history 
…3273)
  • Loading branch information
@davidkyle
davidkyle committed Sep 18, 2018
1 parent 2ec7a61 commit cfafe3c
Show file tree
Hide file tree
Showing 14 changed files with 578 additions and 187 deletions.
3 changes: 1 addition & 2 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.core.ml;

import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.AbstractDiffable;
Expand Down Expand Up @@ -295,7 +294,7 @@ public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) {

public Builder putDatafeed(DatafeedConfig datafeedConfig, Map<String, String> headers) {
if (datafeeds.containsKey(datafeedConfig.getId())) {
throw new ResourceAlreadyExistsException("A datafeed with id [" + datafeedConfig.getId() + "] already exists");
throw ExceptionsHelper.datafeedAlreadyExists(datafeedConfig.getId());
}
String jobId = datafeedConfig.getJobId();
checkJobIsAvailableForDatafeed(jobId);
Expand Down
2 changes: 1 addition & 1 deletion ...ck/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ private void addOptionalField(XContentBuilder builder, ParseField field, Object
}
}

String getJobId() {
public String getJobId() {
return jobId;
}

Expand Down
1 change: 1 addition & 0 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public final class Messages {
public static final String DATAFEED_MISSING_MAX_AGGREGATION_FOR_TIME_FIELD = "Missing max aggregation for time_field [{0}]";
public static final String DATAFEED_FREQUENCY_MUST_BE_MULTIPLE_OF_AGGREGATIONS_INTERVAL =
"Datafeed frequency [{0}] must be a multiple of the aggregation interval [{1}]";
public static final String DATAFEED_ID_ALREADY_TAKEN = "A datafeed with id [{0}] already exists";

public static final String FILTER_NOT_FOUND = "No filter with id [{0}] exists";
public static final String FILTER_CONTAINS_TOO_MANY_ITEMS = "Filter [{0}] contains too many items; up to [{1}] items are allowed";
Expand Down
4 changes: 4 additions & 0 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ public static ResourceNotFoundException missingDatafeedException(String datafeed
return new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId));
}

public static ResourceAlreadyExistsException datafeedAlreadyExists(String datafeedId) {
return new ResourceAlreadyExistsException(Messages.getMessage(Messages.DATAFEED_ID_ALREADY_TAKEN, datafeedId));
}

public static ElasticsearchException serverError(String msg) {
return new ElasticsearchException(msg);
}
Expand Down
59 changes: 28 additions & 31 deletions ...gin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,42 +11,48 @@
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.persistent.PersistentTasksService;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.XPackPlugin;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;

import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

public class TransportDeleteDatafeedAction extends TransportMasterNodeAction<DeleteDatafeedAction.Request, AcknowledgedResponse> {

private Client client;
private PersistentTasksService persistentTasksService;
private final Client client;
private final DatafeedConfigProvider datafeedConfigProvider;
private final ClusterService clusterService;
private final PersistentTasksService persistentTasksService;

@Inject
public TransportDeleteDatafeedAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
Client client, PersistentTasksService persistentTasksService) {
Client client, PersistentTasksService persistentTasksService,
NamedXContentRegistry xContentRegistry) {
super(settings, DeleteDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, DeleteDatafeedAction.Request::new);
this.client = client;
this.datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
this.persistentTasksService = persistentTasksService;
this.clusterService = clusterService;
}

@Override
Expand All @@ -65,14 +71,14 @@ protected void masterOperation(DeleteDatafeedAction.Request request, ClusterStat
if (request.isForce()) {
forceDeleteDatafeed(request, state, listener);
} else {
deleteDatafeedFromMetadata(request, listener);
deleteDatafeedConfig(request, listener);
}
}

private void forceDeleteDatafeed(DeleteDatafeedAction.Request request, ClusterState state,
ActionListener<AcknowledgedResponse> listener) {
ActionListener<Boolean> finalListener = ActionListener.wrap(
response -> deleteDatafeedFromMetadata(request, listener),
response -> deleteDatafeedConfig(request, listener),
listener::onFailure
);

Expand Down Expand Up @@ -111,28 +117,19 @@ public void onFailure(Exception e) {
}
}

private void deleteDatafeedFromMetadata(DeleteDatafeedAction.Request request, ActionListener<AcknowledgedResponse> listener) {
clusterService.submitStateUpdateTask("delete-datafeed-" + request.getDatafeedId(),
new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {

@Override
protected AcknowledgedResponse newResponse(boolean acknowledged) {
return new AcknowledgedResponse(acknowledged);
}

@Override
public ClusterState execute(ClusterState currentState) {
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
MlMetadata currentMetadata = MlMetadata.getMlMetadata(currentState);
PersistentTasksCustomMetaData persistentTasks =
currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
.removeDatafeed(request.getDatafeedId(), persistentTasks).build();
return ClusterState.builder(currentState).metaData(
MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build())
.build();
}
});
private void deleteDatafeedConfig(DeleteDatafeedAction.Request request, ActionListener<AcknowledgedResponse> listener) {
// Check datafeed is stopped
PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
if (MlTasks.getDatafeedTask(request.getDatafeedId(), tasks) != null) {
listener.onFailure(ExceptionsHelper.conflictStatusException(
Messages.getMessage(Messages.DATAFEED_CANNOT_DELETE_IN_CURRENT_STATE, request.getDatafeedId(), DatafeedState.STARTED)));
return;
}

datafeedConfigProvider.deleteDatafeedConfig(request.getDatafeedId(), ActionListener.wrap(
deleteResponse -> listener.onResponse(new AcknowledgedResponse(true)),
listener::onFailure
));
}

@Override
Expand Down
61 changes: 53 additions & 8 deletions ...lugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,46 @@
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class TransportGetDatafeedsAction extends TransportMasterNodeReadAction<GetDatafeedsAction.Request,
GetDatafeedsAction.Response> {

private final DatafeedConfigProvider datafeedConfigProvider;

@Inject
public TransportGetDatafeedsAction(Settings settings, TransportService transportService,
ClusterService clusterService, ThreadPool threadPool,
ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver) {
IndexNameExpressionResolver indexNameExpressionResolver,
Client client, NamedXContentRegistry xContentRegistry) {
super(settings, GetDatafeedsAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, GetDatafeedsAction.Request::new);

datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
}

@Override
Expand All @@ -50,18 +62,51 @@ protected GetDatafeedsAction.Response newResponse() {

@Override
protected void masterOperation(GetDatafeedsAction.Request request, ClusterState state,
ActionListener<GetDatafeedsAction.Response> listener) throws Exception {
ActionListener<GetDatafeedsAction.Response> listener) {
logger.debug("Get datafeed '{}'", request.getDatafeedId());

MlMetadata mlMetadata = MlMetadata.getMlMetadata(state);
Set<String> expandedDatafeedIds = mlMetadata.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds());
List<DatafeedConfig> datafeedConfigs = new ArrayList<>();
Map<String, DatafeedConfig> clusterStateConfigs =
expandClusterStateDatafeeds(request.getDatafeedId(), request.allowNoDatafeeds(), state);

datafeedConfigProvider.expandDatafeedConfigs(request.getDatafeedId(), request.allowNoDatafeeds(), ActionListener.wrap(
datafeedBuilders -> {
// Check for duplicate datafeeds
for (DatafeedConfig.Builder datafeed : datafeedBuilders) {
if (clusterStateConfigs.containsKey(datafeed.getId())) {
listener.onFailure(new IllegalStateException("Datafeed [" + datafeed.getId() + "] configuration " +
"exists in both clusterstate and index"));
return;
}
}

// Merge cluster state and index configs
List<DatafeedConfig> datafeeds = new ArrayList<>(datafeedBuilders.size() + clusterStateConfigs.values().size());
for (DatafeedConfig.Builder builder: datafeedBuilders) {
datafeeds.add(builder.build());
}

datafeeds.addAll(clusterStateConfigs.values());
Collections.sort(datafeeds, Comparator.comparing(DatafeedConfig::getId));
listener.onResponse(new GetDatafeedsAction.Response(new QueryPage<>(datafeeds, datafeeds.size(),
DatafeedConfig.RESULTS_FIELD)));
},
listener::onFailure
));
}

Map<String, DatafeedConfig> expandClusterStateDatafeeds(String datafeedExpression, boolean allowNoDatafeeds,
ClusterState clusterState) {

Map<String, DatafeedConfig> configById = new HashMap<>();

MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState);
Set<String> expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds);

for (String expandedDatafeedId : expandedDatafeedIds) {
datafeedConfigs.add(mlMetadata.getDatafeed(expandedDatafeedId));
configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId));
}

listener.onResponse(new GetDatafeedsAction.Response(new QueryPage<>(datafeedConfigs, datafeedConfigs.size(),
DatafeedConfig.RESULTS_FIELD)));
return configById;
}

@Override
Expand Down
Loading

0 comments on commit cfafe3c

Please sign in to comment.