-
Notifications
You must be signed in to change notification settings - Fork 74
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
separate doc-level monitor query indices created by detectors #1324
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -156,6 +156,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction<IndexDe | |
|
||
private volatile Boolean enabledWorkflowUsage; | ||
|
||
private volatile Boolean enableDetectorWithDedicatedQueryIndices; | ||
|
||
private final Settings settings; | ||
|
||
private final NamedWriteableRegistry namedWriteableRegistry; | ||
|
@@ -202,11 +204,13 @@ public TransportIndexDetectorAction(TransportService transportService, | |
this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); | ||
this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); | ||
this.enabledWorkflowUsage = SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE.get(this.settings); | ||
this.enableDetectorWithDedicatedQueryIndices = SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES.get(this.settings); | ||
this.monitorService = new MonitorService(client); | ||
this.workflowService = new WorkflowService(client, monitorService); | ||
|
||
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); | ||
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, this::setEnabledWorkflowUsage); | ||
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES, this::setEnabledDetectorsWithDedicatedQueryIndices); | ||
this.exceptionChecker = exceptionChecker; | ||
} | ||
|
||
|
@@ -793,7 +797,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List<Pair<String, Rule> | |
detector.getAlertsHistoryIndex(), | ||
detector.getAlertsHistoryIndexPattern(), | ||
DetectorMonitorConfig.getRuleIndexMappingsByType(), | ||
true), PLUGIN_OWNER_FIELD); | ||
true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); | ||
|
||
return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); | ||
} | ||
|
@@ -887,14 +891,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( | |
|
||
Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, | ||
Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), | ||
new DataSources(detector.getRuleIndex(), | ||
new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(), | ||
detector.getFindingsIndex(), | ||
detector.getFindingsIndexPattern(), | ||
detector.getAlertsIndex(), | ||
detector.getAlertsHistoryIndex(), | ||
detector.getAlertsHistoryIndexPattern(), | ||
DetectorMonitorConfig.getRuleIndexMappingsByType(), | ||
true), PLUGIN_OWNER_FIELD); | ||
true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); | ||
|
||
return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); | ||
} | ||
|
@@ -1068,7 +1072,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { | |
detector.getAlertsHistoryIndex(), | ||
detector.getAlertsHistoryIndexPattern(), | ||
DetectorMonitorConfig.getRuleIndexMappingsByType(), | ||
true), PLUGIN_OWNER_FIELD); | ||
true), false, PLUGIN_OWNER_FIELD); | ||
|
||
listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); | ||
} | ||
|
@@ -1252,7 +1256,13 @@ void createDetector() { | |
request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); | ||
request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); | ||
request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); | ||
|
||
if (enableDetectorWithDedicatedQueryIndices) { | ||
// disabling the setting after enabling it will mean delete & re-create the detector | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); | ||
} else { | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); | ||
} | ||
|
||
User originalContextUser = this.user; | ||
log.debug("user from original context is {}", originalContextUser); | ||
|
@@ -1369,7 +1379,16 @@ void onGetResponse(Detector currentDetector, User user) { | |
request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); | ||
request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); | ||
request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); | ||
if (currentDetector.getRuleIndex().contains("optimized")) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if name contains is this if else condition flipped? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the |
||
request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. plz add comment that if we turn OFF setting after turning on, updating detector will not change from optimized to normal and we need to re-create detector There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added a comment. added a test too. |
||
} else { | ||
if (enableDetectorWithDedicatedQueryIndices) { | ||
// disabling the setting after enabling it will mean delete & re-create the detector | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); | ||
} else { | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); | ||
} | ||
} | ||
request.getDetector().setUser(user); | ||
|
||
if (!detector.getInputs().isEmpty()) { | ||
|
@@ -1805,4 +1824,8 @@ private void setFilterByEnabled(boolean filterByEnabled) { | |
private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) { | ||
this.enabledWorkflowUsage = enabledWorkflowUsage; | ||
} | ||
|
||
private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) { | ||
this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException { | |
public void initRuleTopicIndexTemplate(ActionListener<AcknowledgedResponse> actionListener) throws IOException { | ||
getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { | ||
// Compose list of all patterns to cover all query indices | ||
List<String> indexPatterns = new ArrayList<>(); | ||
for(String ruleIndex : allRuleIndices) { | ||
indexPatterns.add(ruleIndex + "*"); | ||
} | ||
|
||
ComposableIndexTemplate template = new ComposableIndexTemplate( | ||
indexPatterns, | ||
allRuleIndices, | ||
new Template( | ||
Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), | ||
null, | ||
|
@@ -87,7 +83,8 @@ private void getAllRuleIndices(ActionListener<List<String>> listener) { | |
listener.onResponse( | ||
logTypes | ||
.stream() | ||
.map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) | ||
// use index pattern here to define rule topic index template for all query indices which match the pattern | ||
.map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shouldn't this be detector specific? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. a single index template containing specific settings for sigma rule executions are applied to all rule indices. |
||
.collect(Collectors.toList()) | ||
); | ||
}, listener::onFailure)); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -75,7 +75,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why has this been removed ? Plz dont reduce test coverage.. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. query indices are created & deleted now. So, it is difficult to assert number of queries in query index now. |
||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(2, response.getHits().getTotalHits().value); | ||
|
@@ -275,7 +275,7 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
@@ -372,7 +372,7 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
@@ -466,7 +466,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
@@ -561,7 +561,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
@@ -653,7 +653,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
@@ -745,7 +745,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT | |
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
@@ -802,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT | |
/** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/ | ||
Assert.assertEquals(3, getAlertsBody.get("total_alerts")); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -266,6 +266,7 @@ void setDebugLogLevel() throws IOException, InterruptedException { | |
|
||
|
||
makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json")); | ||
updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. are there tests with setting turned off There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added a test for setting turned off. |
||
} | ||
|
||
protected final List<String> clusterPermissions = List.of( | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
we are moving from index patttern to index?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no. this method is utilized by few integ tests & hence the change.