separate doc-level monitor query indices created by detectors (#1324)

Signed-off-by: Subhobrata Dey <[email protected]>
opensearch-project · Sep 26, 2024 · 1ca4090 · 1ca4090
1 parent 39c29d4
commit 1ca4090
Show file tree

Hide file tree

Showing 19 changed files with 343 additions and 145 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
@@ -508,7 +508,8 @@ public List<Setting<?>> getSettings() {
                 SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
                 SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
                 SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN,
-                SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT
+                SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT,
+                SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES
         );
     }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
@@ -5,6 +5,8 @@
 package org.opensearch.securityanalytics.config.monitors;
 
 import java.util.List;
+import java.util.Random;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.securityanalytics.logtype.LogTypeService;
@@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) {
         return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType);
     }
 
+    public static String getRuleIndexOptimized(String logType) {
+        return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID());
+    }
+
     public static String getAlertsIndex(String logType) {
         return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType);
     }

diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java
@@ -249,4 +249,10 @@ public static final List<Setting<?>> settings() {
             Setting.Property.NodeScope, Setting.Property.Dynamic
     );
 
+    public static final Setting<Boolean> ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES = Setting.boolSetting(
+            "plugins.security_analytics.enable_detectors_with_dedicated_query_indices",
+            false,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+    );
+
 }
diff --git a/...curityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/...curityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java
@@ -239,6 +239,7 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request)
                     triggers,
                     Collections.emptyMap(),
                     new DataSources(),
+                    false,
                     PLUGIN_OWNER_FIELD
             );
         } catch (Exception e) {

diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
@@ -156,6 +156,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction<IndexDe
 
     private volatile Boolean enabledWorkflowUsage;
 
+    private volatile Boolean enableDetectorWithDedicatedQueryIndices;
+
     private final Settings settings;
 
     private final NamedWriteableRegistry namedWriteableRegistry;
@@ -202,11 +204,13 @@ public TransportIndexDetectorAction(TransportService transportService,
         this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings);
         this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings);
         this.enabledWorkflowUsage = SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE.get(this.settings);
+        this.enableDetectorWithDedicatedQueryIndices = SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES.get(this.settings);
         this.monitorService = new MonitorService(client);
         this.workflowService = new WorkflowService(client, monitorService);
 
         this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled);
         this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, this::setEnabledWorkflowUsage);
+        this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES, this::setEnabledDetectorsWithDedicatedQueryIndices);
         this.exceptionChecker = exceptionChecker;
     }
 
@@ -793,7 +797,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List<Pair<String, Rule>
                         detector.getAlertsHistoryIndex(),
                         detector.getAlertsHistoryIndexPattern(),
                         DetectorMonitorConfig.getRuleIndexMappingsByType(),
-                        true), PLUGIN_OWNER_FIELD);
+                        true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD);
 
         return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null);
     }
@@ -887,14 +891,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest(
 
         Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null,
                 Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(),
-                new DataSources(detector.getRuleIndex(),
+                new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(),
                         detector.getFindingsIndex(),
                         detector.getFindingsIndexPattern(),
                         detector.getAlertsIndex(),
                         detector.getAlertsHistoryIndex(),
                         detector.getAlertsHistoryIndexPattern(),
                         DetectorMonitorConfig.getRuleIndexMappingsByType(),
-                        true), PLUGIN_OWNER_FIELD);
+                        true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD);
 
         return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null);
     }
@@ -1068,7 +1072,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) {
                                             detector.getAlertsHistoryIndex(),
                                             detector.getAlertsHistoryIndexPattern(),
                                             DetectorMonitorConfig.getRuleIndexMappingsByType(),
-                                            true), PLUGIN_OWNER_FIELD);
+                                            true), false, PLUGIN_OWNER_FIELD);
 
                             listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null));
                         }
@@ -1252,7 +1256,13 @@ void createDetector() {
             request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic));
             request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic));
             request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic));
-            request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));
+
+            if (enableDetectorWithDedicatedQueryIndices) {
+                // disabling the setting after enabling it will mean delete & re-create the detector
+                request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic));
+            } else {
+                request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));
+            }
 
             User originalContextUser = this.user;
             log.debug("user from original context is {}", originalContextUser);
@@ -1369,7 +1379,16 @@ void onGetResponse(Detector currentDetector, User user) {
             request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic));
             request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic));
             request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic));
-            request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));
+            if (currentDetector.getRuleIndex().contains("optimized")) {
+                request.getDetector().setRuleIndex(currentDetector.getRuleIndex());
+            } else {
+                if (enableDetectorWithDedicatedQueryIndices) {
+                    // disabling the setting after enabling it will mean delete & re-create the detector
+                    request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic));
+                } else {
+                    request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));
+                }
+            }
             request.getDetector().setUser(user);
 
             if (!detector.getInputs().isEmpty()) {
@@ -1805,4 +1824,8 @@ private void setFilterByEnabled(boolean filterByEnabled) {
     private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) {
         this.enabledWorkflowUsage = enabledWorkflowUsage;
     }
+
+    private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) {
+        this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices;
+    }
 }
diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java
@@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException {
     public void initRuleTopicIndexTemplate(ActionListener<AcknowledgedResponse> actionListener) throws IOException {
         getAllRuleIndices(ActionListener.wrap(allRuleIndices -> {
             // Compose list of all patterns to cover all query indices
-            List<String> indexPatterns = new ArrayList<>();
-            for(String ruleIndex : allRuleIndices) {
-                indexPatterns.add(ruleIndex + "*");
-            }
 
             ComposableIndexTemplate template = new ComposableIndexTemplate(
-                    indexPatterns,
+                    allRuleIndices,
                     new Template(
                             Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(),
                             null,
@@ -87,7 +83,8 @@ private void getAllRuleIndices(ActionListener<List<String>> listener) {
             listener.onResponse(
                     logTypes
                         .stream()
-                        .map(logType -> DetectorMonitorConfig.getRuleIndex(logType))
+                            // use index pattern here to define rule topic index template for all query indices which match the pattern
+                        .map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*")
                         .collect(Collectors.toList())
             );
         }, listener::onFailure));

diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java
@@ -75,7 +75,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(2, response.getHits().getTotalHits().value);
@@ -275,7 +275,7 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(1, response.getHits().getTotalHits().value);
@@ -372,7 +372,7 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(1, response.getHits().getTotalHits().value);
@@ -466,7 +466,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(1, response.getHits().getTotalHits().value);
@@ -561,7 +561,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(1, response.getHits().getTotalHits().value);
@@ -653,7 +653,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(1, response.getHits().getTotalHits().value);
@@ -745,7 +745,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
 
         assertEquals(1, response.getHits().getTotalHits().value);
@@ -802,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT
         /** findings are present but alerts are NOT generated as  detection type mentioned in trigger is threat_intel only but finding is from rules*/
         Assert.assertEquals(3, getAlertsBody.get("total_alerts"));
     }
-}
+}
diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
@@ -266,6 +266,7 @@ void setDebugLogLevel() throws IOException, InterruptedException {
 
 
         makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json"));
+        updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true");
     }
 
     protected final List<String> clusterPermissions = List.of(

diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java
@@ -95,6 +95,7 @@ public void testGetAlerts_success() {
                         List.of(),
                         Map.of(),
                         new DataSources(),
+                        true,
                         TransportIndexDetectorAction.PLUGIN_OWNER_FIELD
                 ),
                 new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")),
@@ -129,6 +130,7 @@ public void testGetAlerts_success() {
                         List.of(),
                         Map.of(),
                         new DataSources(),
+                        true,
                         TransportIndexDetectorAction.PLUGIN_OWNER_FIELD
                 ),
                 new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")),

diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java
@@ -296,7 +296,7 @@ public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, Inte
     }
 
     public void testGetAlerts_noDetector_failure() throws IOException {
-         // Call GetAlerts API
+        // Call GetAlerts API
         Map<String, String> params = new HashMap<>();
         params.put("detector_id", "nonexistent_detector_id");
         try {
@@ -821,7 +821,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
                 Collections.emptyList());
         Detector detector = randomDetectorWithInputsAndTriggers(List.of(input),
                 List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))
-                );
+        );
 
         Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
 
@@ -832,7 +832,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
         assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor
 
@@ -870,8 +870,8 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
             }
         }
 
-         assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue());
-         assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue());
+        assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue());
+        assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue());
 
         Map<String, String> params = new HashMap<>();
         params.put("detector_id", detectorId);
@@ -895,13 +895,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
             List<Map<String, Object>> queries = (List<Map<String, Object>>) finding.get("queries");
             Set<String> findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet());
 
-                // In the case of bucket level monitors, queries will always contain one value
-                String aggRuleId = findingRuleIds.iterator().next();
-                List<String> findingDocs = (List<String>) finding.get("related_doc_ids");
+            // In the case of bucket level monitors, queries will always contain one value
+            String aggRuleId = findingRuleIds.iterator().next();
+            List<String> findingDocs = (List<String>) finding.get("related_doc_ids");
 
-                if (aggRuleId.equals(sumRuleId)) {
-                    assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs));
-                }
+            if (aggRuleId.equals(sumRuleId)) {
+                assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs));
+            }
         }
 
         assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding));

diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java
@@ -1215,7 +1215,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe
                 "     }\n" +
                 "   }\n" +
                 "}";
-        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
+        SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);
 
         assertEquals(1, response.getHits().getTotalHits().value);