Skip to content

Commit

Permalink
[ML] Job in Index: Convert get calendar events to index docs (elastic…
Browse files Browse the repository at this point in the history 
…#34710)
  • Loading branch information
@davidkyle
davidkyle committed Oct 29, 2018
1 parent e8c3951 commit 7fd4b9d
Show file tree
Hide file tree
Showing 5 changed files with 77 additions and 29 deletions.
9 changes: 7 additions & 2 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.elasticsearch.xpack.core.ml.job.config.JobState;
import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;

import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -83,10 +84,14 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis
* All anomaly detector jobs are returned regardless of the status of the
* task (OPEN, CLOSED, FAILED etc).
*
* @param tasks Persistent tasks
* @param tasks Persistent tasks. If null an empty set is returned.
* @return The job Ids of anomaly detector job tasks
*/
public static Set<String> openJobIds(PersistentTasksCustomMetaData tasks) {
public static Set<String> openJobIds(@Nullable PersistentTasksCustomMetaData tasks) {
if (tasks == null) {
return Collections.emptySet();
}

return tasks.findTasks(JOB_TASK_NAME, task -> true)
.stream()
.map(t -> t.getId().substring(JOB_TASK_ID_PREFIX.length()))
Expand Down
8 changes: 8 additions & 0 deletions x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ public void testGetJobState() {
assertEquals(JobState.OPENED, MlTasks.getJobState("foo", tasksBuilder.build()));
}

public void testGetJobState_GivenNull() {
assertEquals(JobState.CLOSED, MlTasks.getJobState("foo", null));
}

public void testGetDatefeedState() {
PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
// A missing task is a stopped datafeed
Expand Down Expand Up @@ -83,6 +87,10 @@ public void testOpenJobIds() {
assertThat(MlTasks.openJobIds(tasksBuilder.build()), containsInAnyOrder("foo-1", "bar"));
}

public void testOpenJobIds_GivenNull() {
assertThat(MlTasks.openJobIds(null), empty());
}

public void testTaskExistsForJob() {
PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
assertFalse(MlTasks.taskExistsForJob("job-1", tasksBuilder.build()));
Expand Down
51 changes: 25 additions & 26 deletions .../ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,40 +8,37 @@
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction;
import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction;
import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder;

import java.util.Collections;
import java.util.List;
import java.util.function.Supplier;

public class TransportGetCalendarEventsAction extends HandledTransportAction<GetCalendarEventsAction.Request,
GetCalendarEventsAction.Response> {

private final JobResultsProvider jobResultsProvider;
private final ClusterService clusterService;
private final JobConfigProvider jobConfigProvider;

@Inject
public TransportGetCalendarEventsAction(Settings settings, TransportService transportService,
ActionFilters actionFilters, ClusterService clusterService,
JobResultsProvider jobResultsProvider) {
ActionFilters actionFilters, JobResultsProvider jobResultsProvider,
JobConfigProvider jobConfigProvider) {
super(settings, GetCalendarEventsAction.NAME, transportService, actionFilters,
(Supplier<GetCalendarEventsAction.Request>) GetCalendarEventsAction.Request::new);
this.jobResultsProvider = jobResultsProvider;
this.clusterService = clusterService;
this.jobConfigProvider = jobConfigProvider;
}

@Override
Expand All @@ -67,26 +64,28 @@ protected void doExecute(Task task, GetCalendarEventsAction.Request request,
);

if (request.getJobId() != null) {
ClusterState state = clusterService.state();
MlMetadata currentMlMetadata = MlMetadata.getMlMetadata(state);

List<String> jobGroups;
String requestId = request.getJobId();
jobConfigProvider.getJob(request.getJobId(), ActionListener.wrap(
jobBuiler -> {
Job job = jobBuiler.build();
jobResultsProvider.scheduledEventsForJob(request.getJobId(), job.getGroups(), query, eventsListener);

Job job = currentMlMetadata.getJobs().get(request.getJobId());
if (job == null) {
// Check if the requested id is a job group
if (currentMlMetadata.isGroupOrJob(request.getJobId()) == false) {
listener.onFailure(ExceptionsHelper.missingJobException(request.getJobId()));
return;
}
jobGroups = Collections.singletonList(request.getJobId());
requestId = null;
} else {
jobGroups = job.getGroups();
}

jobResultsProvider.scheduledEventsForJob(requestId, jobGroups, query, eventsListener);
},
jobNotFound -> {
// is the request Id a group?
jobConfigProvider.groupExists(request.getJobId(), ActionListener.wrap(
groupExists -> {
if (groupExists) {
jobResultsProvider.scheduledEventsForJob(
null, Collections.singletonList(request.getJobId()), query, eventsListener);
} else {
listener.onFailure(ExceptionsHelper.missingJobException(request.getJobId()));
}
},
listener::onFailure
));
}
));
} else {
jobResultsProvider.scheduledEvents(query, eventsListener);
}
Expand Down
35 changes: 35 additions & 0 deletions ...plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -630,6 +630,41 @@ public void expandGroupIds(List<String> groupIds, ActionListener<Set<String>> l
, client::search);
}

/**
* Check if a group exists, that is there exists a job that is a member of
* the group. If there are one or more jobs that define the group then
* the listener responds with true else false.
*
* @param groupId The group Id
* @param listener Returns true, false or a failure
*/
public void groupExists(String groupId, ActionListener<Boolean> listener) {
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.filter(new TermQueryBuilder(Job.JOB_TYPE.getPreferredName(), Job.ANOMALY_DETECTOR_JOB_TYPE));
boolQueryBuilder.filter(new TermQueryBuilder(Job.GROUPS.getPreferredName(), groupId));

SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
.query(boolQueryBuilder);
sourceBuilder.fetchSource(false);

SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
.setSize(0)
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.setSource(sourceBuilder).request();

executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest,
ActionListener.<SearchResponse>wrap(
response -> {
listener.onResponse(response.getHits().totalHits > 0);
},
listener::onFailure)
, client::search);
}

/**
* Find jobs with custom rules defined.
* @param listener Jobs listener
*/
public void findJobsWithCustomRules(ActionListener<List<Job>> listener) {
String customRulesPath = Strings.collectionToDelimitedString(Arrays.asList(Job.ANALYSIS_CONFIG.getPreferredName(),
AnalysisConfig.DETECTORS.getPreferredName(), Detector.CUSTOM_RULES_FIELD.getPreferredName()), ".");
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ private void waitForTemplates() throws Exception {
List<String> templates = new ArrayList<>();
templates.addAll(Arrays.asList(AuditorField.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME,
AnomalyDetectorsIndex.jobStateIndexName(),
AnomalyDetectorsIndex.jobResultsIndexPrefix()));
AnomalyDetectorsIndex.jobResultsIndexPrefix(),
AnomalyDetectorsIndex.configIndexName()));

for (String template : templates) {
awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
Expand Down

0 comments on commit 7fd4b9d

Please sign in to comment.