From 8e6ac76b219c1d41c38407e38cf6c9b35336e620 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 25 Sep 2024 03:16:10 +0000 Subject: [PATCH 1/4] separate doc-level monitor query indices created by detectors Signed-off-by: Subhobrata Dey --- .../monitors/DetectorMonitorConfig.java | 8 +- .../TransportIndexDetectorAction.java | 10 +- .../util/RuleTopicIndices.java | 6 +- .../DetectorThreatIntelIT.java | 91 ++---------------- .../securityanalytics/alerts/AlertsIT.java | 13 +-- .../securityanalytics/findings/FindingIT.java | 14 +-- .../mapper/MapperRestApiIT.java | 95 +++++++++++++++---- .../resthandler/DetectorMonitorRestApiIT.java | 48 +--------- .../resthandler/DetectorRestApiIT.java | 74 +++++++++------ src/test/resources/ad_ldap-sample.json | 2 +- src/test/resources/cloudtrail-sample.json | 4 +- src/test/resources/waf-sample.json | 2 +- 12 files changed, 155 insertions(+), 212 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 459f523b7..f20656a50 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.config.monitors; import java.util.List; +import java.util.Random; +import java.util.UUID; import java.util.stream.Collectors; import org.opensearch.common.inject.Inject; import org.opensearch.securityanalytics.logtype.LogTypeService; @@ -22,7 +24,11 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; public static String getRuleIndex(String logType) { - return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries*", logType); + } + + public static String getRuleIndexOptimized(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID()); } public static String getAlertsIndex(String logType) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 95720c237..b798f3354 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -887,7 +887,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), + new DataSources(detector.getRuleIndex() + "_chained_findings", detector.getFindingsIndex(), detector.getFindingsIndexPattern(), detector.getAlertsIndex(), @@ -1252,7 +1252,7 @@ void createDetector() { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); User originalContextUser = this.user; log.debug("user from original context is {}", originalContextUser); @@ -1369,7 +1369,11 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + if (currentDetector.getRuleIndex().contains("optimized")) { + request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } request.getDetector().setUser(user); if (!detector.getInputs().isEmpty()) { diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 8f0f1cab5..7f21e2a41 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException { public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { // Compose list of all patterns to cover all query indices - List indexPatterns = new ArrayList<>(); - for(String ruleIndex : allRuleIndices) { - indexPatterns.add(ruleIndex + "*"); - } ComposableIndexTemplate template = new ComposableIndexTemplate( - indexPatterns, + allRuleIndices, new Template( Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), null, diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 144b5d2c0..9432f3f68 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -69,22 +69,11 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -269,22 +258,11 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn Detector detector = randomDetectorWithInputsAndThreatIntel(List.of(input), false); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -366,22 +344,11 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -460,22 +427,11 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -555,22 +511,11 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -647,22 +592,11 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -739,22 +673,11 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 46279da01..333d5dbd2 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -825,21 +825,10 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 12264c4b3..d3b0c3bc9 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -1209,21 +1209,11 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1261,7 +1251,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe " }\n" + " }\n" + "}"; - response = executeSearchAndGetResponse(DetectorMonitorConfig.getFindingsIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getFindingsIndex(randomDetectorType()), request, true); assertEquals(2, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 1b59944eb..5752aa41e 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -22,6 +22,7 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; +import org.junit.Ignore; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -47,6 +48,7 @@ import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; public class MapperRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1703,6 +1705,7 @@ public void onError(String error) { } } + @Ignore public void testAzureMappings() throws IOException { String indexName = "azure-test-index"; @@ -1722,12 +1725,16 @@ public void testAzureMappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("azure").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "azure"); - createDetector(detector); - - List hits = executeSearch(".opensearch-sap-azure-detectors-queries-000001", matchAllSearchBody); + String detectorId = createDetector(detector); + Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + List hits = executeSearch(ruleTopicIndex, matchAllSearchBody); Assert.assertEquals(127, hits.size()); } + @Ignore public void testADLDAPMappings() throws IOException { String indexName = "adldap-test-index"; @@ -1737,20 +1744,37 @@ public void testADLDAPMappings() throws IOException { indexDoc(indexName, "1", sampleDoc); - createMappingsAPI(indexName, "ad_ldap"); +// createMappingsAPI(indexName, "ad_ldap"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); - assertTrue(mappings.containsKey("timestamp")); +// assertTrue(mappings.containsKey("timestamp")); // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); - createDetector(detector); + String detectorId = createDetector(detector); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-000001", matchAllSearchBody); - Assert.assertEquals(11, hits.size()); + indexDoc(indexName, "2", sampleDoc); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); } public void testCloudtrailMappings() throws IOException { @@ -1772,12 +1796,29 @@ public void testCloudtrailMappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("cloudtrail").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); - createDetector(detector); + String detectorId = createDetector(detector); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", matchAllSearchBody); - Assert.assertEquals(39, hits.size()); + indexDoc(indexName, "2", sampleDoc); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); } + @Ignore public void testS3Mappings() throws IOException { String indexName = "s3-test-index"; @@ -1797,12 +1838,16 @@ public void testS3Mappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "s3"); - createDetector(detector); - - List hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", matchAllSearchBody); + String detectorId = createDetector(detector); + Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + List hits = executeSearch(ruleTopicIndex, matchAllSearchBody); Assert.assertEquals(1, hits.size()); } + @Ignore public void testWAFMappings() throws IOException { String indexName = "waf-test-index"; String sampleDoc = readResource("waf-sample.json"); @@ -1824,10 +1869,26 @@ public void testWAFMappings() throws IOException { DetectorInput input = new DetectorInput("waf detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("waf").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "waf"); - createDetector(detector); + String detectorId = createDetector(detector); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(indexName, "2", sampleDoc); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); - List hits = executeSearch(".opensearch-sap-waf-detectors-queries-000001", matchAllSearchBody); - Assert.assertEquals(5, hits.size()); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); } @SuppressWarnings("unchecked") diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 8e732bb6f..35767edbe 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1036,21 +1036,11 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1104,21 +1094,11 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesIn Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1373,21 +1353,11 @@ public void testCreateDetector_workflowWithDuplicateMonitor_failure() throws IOE Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1443,21 +1413,11 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - request = "{\n" + + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index c2e80a0ac..e774b0b36 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -717,6 +717,12 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ bucketLevelMonitorId = secondMonitorId; } + String thirdMonitorId = monitorIds.get(2); + String thirdMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + thirdMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(thirdMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(thirdMonitorType)){ + bucketLevelMonitorId = thirdMonitorId; + } Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); indexDoc(index, "1", randomProductDocument()); @@ -770,7 +776,13 @@ public void testAggRuleCount() throws IOException { Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); - String bucketLevelMonitorId = ((List) (detectorAsMap).get("monitor_id")).get(1); + String bucketLevelMonitorId = ""; + Map monitorOpts = ((Map) (detectorAsMap).get("bucket_monitor_id_rule_id")); + for (Map.Entry monitorOpt: monitorOpts.entrySet()) { + if (!(monitorOpt.getKey().equals("-1") || monitorOpt.getKey().equals("chained_findings_monitor"))) { + bucketLevelMonitorId = monitorOpt.getValue().toString(); + } + } // condition: sel | count(*) by name > 2 indexDoc(index, "1", randomProductDocument()); indexDoc(index, "2", randomProductDocument()); @@ -828,18 +840,26 @@ public void testUpdateADetector() throws IOException { Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + " \"query\" : {\n" + - " \"match_all\":{\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - Assert.assertEquals(5, response.getHits().getTotalHits().value); + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); - String rule = randomRule(); + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + String rule = randomRule(); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); @@ -858,14 +878,13 @@ public void testUpdateADetector() throws IOException { String detectorTypeInResponse = (String) ((Map) (asMap(updateResponse).get("detector"))).get("detector_type"); Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); - request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - Assert.assertEquals(6, response.getHits().getTotalHits().value); + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(6, noOfSigmaRuleMatches); } public void testUpdateANonExistingDetector() throws IOException { @@ -1262,6 +1281,7 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Assert.assertEquals(0, hits.size()); } + @SuppressWarnings("unchecked") public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() throws IOException { String index1 = "test_index_1"; createIndex(index1, Settings.EMPTY); @@ -1279,6 +1299,10 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro List.of(index1) ); String detectorId1 = createDetector(detector1); + Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; // Create detector #2 of type test_windows Detector detector2 = randomDetectorWithTriggers( @@ -1288,29 +1312,19 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro ); String detectorId2 = createDetector(detector2); - - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000001")); - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000002")); - - // Check if both query indices have proper settings applied from index template - Map settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000001"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); - settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000002"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId2 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex2 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); - // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); // We deleted all detectors of type windows, so we expect that queryIndex is deleted - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); + Assert.assertFalse(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(ruleTopicIndex2)); String request = "{\n" + " \"query\" : {\n" + diff --git a/src/test/resources/ad_ldap-sample.json b/src/test/resources/ad_ldap-sample.json index 4e9b2caf3..f6beef93c 100644 --- a/src/test/resources/ad_ldap-sample.json +++ b/src/test/resources/ad_ldap-sample.json @@ -8,7 +8,7 @@ "SearchFilter": "1234", "azure.platformlogs.result_type": "1234", "azure.signinlogs.result_description": "1234", - "azure.signinlogs.properties.device_detail.is_compliant": "1234", + "azure.signinlogs.properties.device_detail.is_compliant": false, "resource_display_name": "1234", "azure.signinlogs.properties.authentication_requirement": "1234", "target_resources": "1234", diff --git a/src/test/resources/cloudtrail-sample.json b/src/test/resources/cloudtrail-sample.json index 0cf13e37a..e3c812f35 100644 --- a/src/test/resources/cloudtrail-sample.json +++ b/src/test/resources/cloudtrail-sample.json @@ -6,10 +6,10 @@ "source.as.organization.name": "213123", "source.ip": "213123", "userIdentity.arn": "213123", - "eventName": "213123", + "eventName": "DeleteIdentity", "eventType": "213123", "errorCode": "213123", - "eventSource": "213123", + "eventSource": "ses.amazonaws.com", "tlsDetails.tlsVersion": "213123", "user_agent.name": "213123", "threat.matched.providers": "213123", diff --git a/src/test/resources/waf-sample.json b/src/test/resources/waf-sample.json index cf925a70e..0cb24f5f0 100644 --- a/src/test/resources/waf-sample.json +++ b/src/test/resources/waf-sample.json @@ -54,5 +54,5 @@ "name": "awswaf:managed:aws:bot-control:signal:known_bot_data_center" } ], - "waf.request.headers.user_agent": "111" + "waf.request.headers.user_agent": "WPScan v" } From b3284e7136eac15b9dbb921ca3382021ce56c1c7 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 25 Sep 2024 11:02:03 +0000 Subject: [PATCH 2/4] separate doc-level monitor query indices created by detectors Signed-off-by: Subhobrata Dey --- .../monitor/TransportIndexThreatIntelMonitorAction.java | 1 + .../transport/TransportIndexDetectorAction.java | 6 +++--- .../securityanalytics/alerts/AlertingServiceTests.java | 2 ++ .../threatIntel/model/monitor/ThreatIntelInputTests.java | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java index 3edb6ea94..c9e364da7 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -239,6 +239,7 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) triggers, Collections.emptyMap(), new DataSources(), + false, PLUGIN_OWNER_FIELD ); } catch (Exception e) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index b798f3354..c4b3ad998 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -793,7 +793,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), true, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -894,7 +894,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), true, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1068,7 +1068,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), false, PLUGIN_OWNER_FIELD); listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); } diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 06e464d34..bbb3eb71a 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -95,6 +95,7 @@ public void testGetAlerts_success() { List.of(), Map.of(), new DataSources(), + true, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), @@ -129,6 +130,7 @@ public void testGetAlerts_success() { List.of(), Map.of(), new DataSources(), + true, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java index def2b21e5..36de85ebf 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -57,6 +57,7 @@ public void testThreatInputSerde() throws IOException { emptyList(), emptyMap(), new DataSources(), + false, "security_analytics" ); BytesStreamOutput monitorOut = new BytesStreamOutput(); From 1f3eb294466d15edaffc32b14a8bb7f45260ebc9 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 25 Sep 2024 23:30:35 +0000 Subject: [PATCH 3/4] finalize changes Signed-off-by: Subhobrata Dey --- .../SecurityAnalyticsPlugin.java | 3 +- .../settings/SecurityAnalyticsSettings.java | 6 ++++ .../TransportIndexDetectorAction.java | 29 +++++++++++++++---- .../SecurityAnalyticsRestTestCase.java | 1 + .../resthandler/DetectorRestApiIT.java | 2 -- .../resthandler/RuleRestApiIT.java | 26 +++-------------- 6 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 52cfc3f08..2762bd5f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -508,7 +508,8 @@ public List> getSettings() { SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT, SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD, SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN, - SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT + SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT, + SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES ); } diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 8bcc66d40..57b6c5023 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -249,4 +249,10 @@ public static final List> settings() { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES = Setting.boolSetting( + "plugins.security_analytics.enable_detectors_with_dedicated_query_indices", + false, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index c4b3ad998..7ceae503c 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -156,6 +156,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), true, PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -887,14 +891,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex() + "_chained_findings", + new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), detector.getAlertsIndex(), detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), true, PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1252,7 +1256,13 @@ void createDetector() { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + + if (enableDetectorWithDedicatedQueryIndices) { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); + request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length()-1)); + } User originalContextUser = this.user; log.debug("user from original context is {}", originalContextUser); @@ -1372,7 +1382,12 @@ void onGetResponse(Detector currentDetector, User user) { if (currentDetector.getRuleIndex().contains("optimized")) { request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); } else { - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + if (enableDetectorWithDedicatedQueryIndices) { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); + request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length() - 1)); + } } request.getDetector().setUser(user); @@ -1809,4 +1824,8 @@ private void setFilterByEnabled(boolean filterByEnabled) { private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) { this.enabledWorkflowUsage = enabledWorkflowUsage; } + + private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) { + this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices; + } } diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index a2a08d5b7..52fd0b512 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -266,6 +266,7 @@ void setDebugLogLevel() throws IOException, InterruptedException { makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json")); + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); } protected final List clusterPermissions = List.of( diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index e774b0b36..bf6ce597b 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -11,7 +11,6 @@ import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; import org.junit.Ignore; -import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -23,7 +22,6 @@ import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 66a307a5e..2421bb2c7 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -9,6 +9,7 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -18,6 +19,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; @@ -732,31 +734,11 @@ public void testDeletingUsedRule() throws IOException { .contains(String.format(Locale.getDefault(), "Rule with id %s is actively used by detectors. Deletion can be forced by setting forced flag to true", createdId))); } - String request = "{\n" + - " \"query\": {\n" + - " \"script\": {\n" + - " \"script\": \"doc['_id'][0].indexOf('" + createdId + "') > -1\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); - Assert.assertEquals(2, hits.size()); - Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.singletonMap("forced", "true"), null); Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); - request = "{\n" + - " \"query\": {\n" + - " \"script\": {\n" + - " \"script\": \"doc['_id'][0].indexOf('" + createdId + "') > -1\"\n" + - " }\n" + - " }\n" + - "}"; - hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); - Assert.assertEquals(0, hits.size()); - index = Rule.CUSTOM_RULES_INDEX; - request = "{\n" + + String request = "{\n" + " \"query\": {\n" + " \"nested\": {\n" + " \"path\": \"rule\",\n" + @@ -770,7 +752,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - hits = executeSearch(index, request); + List hits = executeSearch(index, request); Assert.assertEquals(0, hits.size()); } From c514a780d4b3f65ac828e3ff868c56f46f2a82b7 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Thu, 26 Sep 2024 08:54:43 +0000 Subject: [PATCH 4/4] address review comments Signed-off-by: Subhobrata Dey --- .../monitors/DetectorMonitorConfig.java | 2 +- .../TransportIndexDetectorAction.java | 8 +- .../util/RuleTopicIndices.java | 3 +- .../DetectorThreatIntelIT.java | 93 +++++- .../securityanalytics/alerts/AlertsIT.java | 33 +- .../securityanalytics/findings/FindingIT.java | 14 +- .../mapper/MapperRestApiIT.java | 99 ++---- .../resthandler/DetectorMonitorRestApiIT.java | 48 ++- .../resthandler/DetectorRestApiIT.java | 308 +++++++++++++----- .../resthandler/RuleRestApiIT.java | 58 ++-- 10 files changed, 455 insertions(+), 211 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index f20656a50..02f6595ec 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -24,7 +24,7 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; public static String getRuleIndex(String logType) { - return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries*", logType); + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); } public static String getRuleIndexOptimized(String logType) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 7ceae503c..48fe08752 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -1258,10 +1258,10 @@ void createDetector() { request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); } else { - String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); - request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length()-1)); + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); } User originalContextUser = this.user; @@ -1383,10 +1383,10 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); } else { if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); } else { - String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); - request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length() - 1)); + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); } } request.getDetector().setUser(user); diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 7f21e2a41..7aa2def36 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -83,7 +83,8 @@ private void getAllRuleIndices(ActionListener> listener) { listener.onResponse( logTypes .stream() - .map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) + // use index pattern here to define rule topic index template for all query indices which match the pattern + .map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*") .collect(Collectors.toList()) ); }, listener::onFailure)); diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 9432f3f68..ff80ca329 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -69,11 +69,22 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -258,11 +269,22 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn Detector detector = randomDetectorWithInputsAndThreatIntel(List.of(input), false); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -344,11 +366,22 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -427,11 +460,22 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -511,11 +555,22 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -592,11 +647,22 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -673,11 +739,22 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -725,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT /** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/ Assert.assertEquals(3, getAlertsBody.get("total_alerts")); } -} +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 333d5dbd2..0574821e1 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -296,7 +296,7 @@ public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, Inte } public void testGetAlerts_noDetector_failure() throws IOException { - // Call GetAlerts API + // Call GetAlerts API Map params = new HashMap<>(); params.put("detector_id", "nonexistent_detector_id"); try { @@ -821,14 +821,25 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Collections.emptyList()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())) - ); + ); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -859,8 +870,8 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException } } - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -884,13 +895,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException List> queries = (List>) finding.get("queries"); Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); - // In the case of bucket level monitors, queries will always contain one value - String aggRuleId = findingRuleIds.iterator().next(); - List findingDocs = (List) finding.get("related_doc_ids"); + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List) finding.get("related_doc_ids"); - if (aggRuleId.equals(sumRuleId)) { - assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); - } + if (aggRuleId.equals(sumRuleId)) { + assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); + } } assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index d3b0c3bc9..aed4cbfab 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -1209,11 +1209,21 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1251,7 +1261,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getFindingsIndex(randomDetectorType()), request, true); + response = executeSearchAndGetResponse(DetectorMonitorConfig.getFindingsIndex(randomDetectorType()), request, true); assertEquals(2, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 5752aa41e..dacb89bfa 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -22,7 +22,6 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -48,7 +47,6 @@ import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; public class MapperRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1659,8 +1657,8 @@ public void testTraverseAndCopy() { " \"type\":\"keyword\"," + " \"ignore_above\":256" + " }" + - " }" + - " }" + + " }" + + " }" + " }" + " }" + "}"; @@ -1705,7 +1703,6 @@ public void onError(String error) { } } - @Ignore public void testAzureMappings() throws IOException { String indexName = "azure-test-index"; @@ -1725,16 +1722,12 @@ public void testAzureMappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("azure").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "azure"); - String detectorId = createDetector(detector); - Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), - new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId + "\"}}}"), new BasicHeader("Content-Type", "application/json")); - String ruleTopicIndex = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) - .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; - List hits = executeSearch(ruleTopicIndex, matchAllSearchBody); + createDetector(detector); + + List hits = executeSearch(".opensearch-sap-azure-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(127, hits.size()); } - @Ignore public void testADLDAPMappings() throws IOException { String indexName = "adldap-test-index"; @@ -1744,37 +1737,20 @@ public void testADLDAPMappings() throws IOException { indexDoc(indexName, "1", sampleDoc); -// createMappingsAPI(indexName, "ad_ldap"); + createMappingsAPI(indexName, "ad_ldap"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); -// assertTrue(mappings.containsKey("timestamp")); + assertTrue(mappings.containsKey("timestamp")); // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); - String detectorId = createDetector(detector); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); + createDetector(detector); - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - - indexDoc(indexName, "2", sampleDoc); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-*", matchAllSearchBody); + Assert.assertEquals(11, hits.size()); } public void testCloudtrailMappings() throws IOException { @@ -1796,29 +1772,12 @@ public void testCloudtrailMappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("cloudtrail").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); - String detectorId = createDetector(detector); - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); + createDetector(detector); - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - - indexDoc(indexName, "2", sampleDoc); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(1, noOfSigmaRuleMatches); + List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-*", matchAllSearchBody); + Assert.assertEquals(39, hits.size()); } - @Ignore public void testS3Mappings() throws IOException { String indexName = "s3-test-index"; @@ -1838,16 +1797,12 @@ public void testS3Mappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "s3"); - String detectorId = createDetector(detector); - Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), - new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId + "\"}}}"), new BasicHeader("Content-Type", "application/json")); - String ruleTopicIndex = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) - .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; - List hits = executeSearch(ruleTopicIndex, matchAllSearchBody); + createDetector(detector); + + List hits = executeSearch(".opensearch-sap-s3-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(1, hits.size()); } - @Ignore public void testWAFMappings() throws IOException { String indexName = "waf-test-index"; String sampleDoc = readResource("waf-sample.json"); @@ -1869,26 +1824,10 @@ public void testWAFMappings() throws IOException { DetectorInput input = new DetectorInput("waf detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("waf").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "waf"); - String detectorId = createDetector(detector); - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - - indexDoc(indexName, "2", sampleDoc); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); + createDetector(detector); - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + List hits = executeSearch(".opensearch-sap-waf-detectors-queries-*", matchAllSearchBody); + Assert.assertEquals(5, hits.size()); } @SuppressWarnings("unchecked") diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 35767edbe..930f35940 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1036,11 +1036,21 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1094,11 +1104,21 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesIn Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1353,11 +1373,21 @@ public void testCreateDetector_workflowWithDuplicateMonitor_failure() throws IOE Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1413,11 +1443,21 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index bf6ce597b..2a6b6b9c8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -11,6 +11,7 @@ import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; import org.junit.Ignore; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -22,6 +23,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; @@ -154,7 +156,7 @@ private void validateDetectorDeletion(final String detectorId) throws IOExceptio Assert.assertEquals(0, hits.size()); } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") public void testCreatingADetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -838,26 +840,18 @@ public void testUpdateADetector() throws IOException { Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); + String request = "{\n" + " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + + " \"match_all\":{\n" + " }\n" + " }\n" + "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - indexDoc(index, "1", randomDoc()); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + Assert.assertEquals(5, response.getHits().getTotalHits().value); String rule = randomRule(); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); @@ -876,13 +870,14 @@ public void testUpdateADetector() throws IOException { String detectorTypeInResponse = (String) ((Map) (asMap(updateResponse).get("detector"))).get("detector_type"); Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); - indexDoc(index, "2", randomDoc()); - - executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - executeResults = entityAsMap(executeResponse); - - noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(6, noOfSigmaRuleMatches); + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + Assert.assertEquals(6, response.getHits().getTotalHits().value); } public void testUpdateANonExistingDetector() throws IOException { @@ -1068,10 +1063,10 @@ public void testDeletingADetector_single_Monitor() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1081,12 +1076,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1108,12 +1103,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1149,22 +1144,22 @@ public void testDeletingADetector_single_Monitor() throws IOException { Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1178,10 +1173,10 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1191,12 +1186,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1218,12 +1213,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1259,22 +1254,22 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1468,9 +1463,9 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); updateRequest.setJsonEntity(MediaTypeRegistry.JSON.contentBuilder().map(Map.of( - "index_name", index, - "field", "time", - "alias", "timestamp")) + "index_name", index, + "field", "time", + "alias", "timestamp")) .toString()); Response apiResponse = client().performRequest(updateRequest); assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); @@ -1729,4 +1724,157 @@ public void testDetector_withAlias_endToEnd_success() throws IOException { List findings = (List) getFindingsBody.get("findings"); Assert.assertEquals(findings.size(), 1); } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexDisabledAndThenEnabledToUpdate() throws IOException { + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId2 = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 2421bb2c7..3fb397167 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -9,7 +9,6 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; -import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -19,7 +18,6 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; @@ -124,14 +122,14 @@ public void testCreatingARule_custom_category() throws IOException { } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( - e.getMessage().contains("Invalid rule category") + e.getMessage().contains("Invalid rule category") ); } } public void testCreatingAggregationRule() throws SigmaError, IOException { Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "windows"), - new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); + new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -144,24 +142,24 @@ public void testCreatingAggregationRule() throws SigmaError, IOException { String index = Rule.CUSTOM_RULES_INDEX; String request = "{\n" + - " \"query\": {\n" + - " \"nested\": {\n" + - " \"path\": \"rule\",\n" + - " \"query\": {\n" + - " \"bool\": {\n" + - " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"windows\"}}\n" + - " ]\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"windows\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(index, request); XContentParser xcp = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); Rule result = Rule.docParse(xcp, null, null); Assert.assertEquals(1, result.getAggregationQueries().size()); @@ -734,11 +732,31 @@ public void testDeletingUsedRule() throws IOException { .contains(String.format(Locale.getDefault(), "Rule with id %s is actively used by detectors. Deletion can be forced by setting forced flag to true", createdId))); } + String request = "{\n" + + " \"query\": {\n" + + " \"script\": {\n" + + " \"script\": \"doc['_id'][0].indexOf('" + createdId + "') > -1\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); + Assert.assertEquals(2, hits.size()); + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.singletonMap("forced", "true"), null); Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); + request = "{\n" + + " \"query\": {\n" + + " \"script\": {\n" + + " \"script\": \"doc['_id'][0].indexOf('" + createdId + "') > -1\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); + Assert.assertEquals(0, hits.size()); + index = Rule.CUSTOM_RULES_INDEX; - String request = "{\n" + + request = "{\n" + " \"query\": {\n" + " \"nested\": {\n" + " \"path\": \"rule\",\n" + @@ -752,7 +770,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - List hits = executeSearch(index, request); + hits = executeSearch(index, request); Assert.assertEquals(0, hits.size()); }