diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 4d5194eec..a199a2155 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -45,6 +45,7 @@ import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.resthandler.*; @@ -90,6 +91,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private IndexTemplateManager indexTemplateManager; + private LogTypeService logTypeService; + @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -102,12 +105,13 @@ public Collection createComponents(Client client, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { + logTypeService = new LogTypeService(); detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool); ruleTopicIndices = new RuleTopicIndices(client, clusterService); correlationIndices = new CorrelationIndices(client, clusterService); indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry); mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager); - ruleIndices = new RuleIndices(client, clusterService, threadPool); + ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool); correlationRuleIndices = new CorrelationRuleIndices(client, clusterService); return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java new file mode 100644 index 000000000..3783143cc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.logtype; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.SettingsException; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.securityanalytics.model.LogType; +import org.opensearch.securityanalytics.util.FileUtils; + +public class BuiltinLogTypeLoader { + + private static final Logger logger = LogManager.getLogger(BuiltinLogTypeLoader.class); + + private static final String BASE_PATH = "OSMapping/"; + + private static final String LOG_TYPE_FILE_SUFFIX = "_logtype.json"; + + private static List logTypes; + private static Map logTypeMap; + + + static { + ensureLogTypesLoaded(); + } + + public static List getAllLogTypes() { + ensureLogTypesLoaded(); + return logTypes; + } + + public static LogType getLogTypeByName(String logTypeName) { + ensureLogTypesLoaded(); + return logTypeMap.get(logTypeName); + } + + public static boolean logTypeExists(String logTypeName) { + ensureLogTypesLoaded(); + return logTypeMap.containsKey(logTypeName); + } + + private static void ensureLogTypesLoaded() { + try { + if (logTypes != null) { + return; + } + logTypes = loadBuiltinLogTypes(); + logTypeMap = logTypes.stream() + .collect(Collectors.toMap(LogType::getName, Function.identity())); + } catch (Exception e) { + logger.error("Failed loading builtin log types from disk!", e); + } + } + + private static List loadBuiltinLogTypes() throws URISyntaxException, IOException { + List logTypes = new ArrayList<>(); + + final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); + + Path dirPath = null; + if (url.contains("!")) { + final String[] paths = url.split("!"); + dirPath = FileUtils.getFs().getPath(paths[1]); + } else { + dirPath = Path.of(url); + } + + Stream folder = Files.list(dirPath); + List logTypePaths = folder.filter(e -> e.toString().endsWith(LOG_TYPE_FILE_SUFFIX)).collect(Collectors.toList()); + + for (Path logTypePath : logTypePaths) { + try ( + InputStream is = BuiltinLogTypeLoader.class.getResourceAsStream(logTypePath.toString()) + ) { + String logTypeFilePayload = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); + + if (logTypeFilePayload != null) { + Map logTypeFileAsMap = + XContentHelper.convertToMap(JsonXContent.jsonXContent, logTypeFilePayload, false); + + logTypes.add(new LogType(logTypeFileAsMap)); + + logger.info("Loaded [{}] log type", logTypePath.getFileName()); + } + } catch (Exception e) { + throw new SettingsException("Failed to load builtin log types", e); + } + } + + return logTypes; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java new file mode 100644 index 000000000..6523ba92b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.logtype; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.securityanalytics.model.LogType; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + + +/** + * + * */ +public class LogTypeService { + + private static final Logger logger = LogManager.getLogger(LogTypeService.class); + + private BuiltinLogTypeLoader builtinLogTypeLoader; + + public LogTypeService() { + this.builtinLogTypeLoader = new BuiltinLogTypeLoader(); + } + + + public List getAllLogTypes() { + return BuiltinLogTypeLoader.getAllLogTypes(); + } + + public LogType getLogTypeByName(String logType) { + return BuiltinLogTypeLoader.getLogTypeByName(logType); + } + + /** + * Returns sigmaRule rawField to ECS field mapping + * + * @param logType Log type + * @return Map of rawField to ecs field + */ + public Map getRuleFieldMappings(String logType) { + LogType lt = getLogTypeByName(logType); + + if (lt == null) { + throw SecurityAnalyticsException.wrap(new IllegalArgumentException("Can't get rule field mappings for invalid logType: [" + logType + "]")); + } + if (lt.getMappings() == null) { + return Map.of(); + } else { + return lt.getMappings() + .stream() + .collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs)); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java new file mode 100644 index 000000000..80fa003d4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/LogType.java @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.model; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class LogType implements Writeable { + + private static final String ID = "id"; + private static final String NAME = "name"; + private static final String DESCRIPTION = "description"; + private static final String IS_BUILTIN = "is_builtin"; + private static final String MAPPINGS = "mappings"; + private static final String RAW_FIELD = "raw_field"; + private static final String ECS = "ecs"; + private static final String OCSF = "ocsf"; + + private String id; + private String name; + private String description; + private Boolean isBuiltIn; + private List mappings; + + public LogType(StreamInput sin) throws IOException { + this.id = sin.readString(); + this.isBuiltIn = sin.readOptionalBoolean(); + this.name = sin.readString(); + this.description = sin.readString(); + this.mappings = sin.readList(Mapping::readFrom); + } + + public LogType(String id, String name, String description, boolean isBuiltIn, List mappings) { + this.id = id; + this.name = name; + this.description = description; + this.isBuiltIn = isBuiltIn; + this.mappings = mappings == null ? List.of() : mappings; + } + + public LogType(Map logTypeAsMap) { + this.id = (String) logTypeAsMap.get(ID); + this.name = (String) logTypeAsMap.get(NAME); + this.description = (String) logTypeAsMap.get(DESCRIPTION); + if (logTypeAsMap.containsKey(IS_BUILTIN)) { + this.isBuiltIn = (Boolean) logTypeAsMap.get(IS_BUILTIN); + } + List> mappings = (List>)logTypeAsMap.get(MAPPINGS); + if (mappings.size() > 0) { + this.mappings = new ArrayList<>(mappings.size()); + this.mappings = mappings.stream().map(e -> + new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) + ).collect(Collectors.toList()); + } + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public boolean getIsBuiltIn() { return isBuiltIn; } + + public List getMappings() { + return mappings; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeOptionalBoolean(isBuiltIn); + out.writeString(name); + out.writeString(description); + out.writeCollection(mappings); + } + + @Override + public String toString() { + return name; + } + + public static class Mapping implements Writeable { + + private String rawField; + private String ecs; + private String ocsf; + + public Mapping(StreamInput sin) throws IOException { + this.rawField = sin.readString(); + this.ecs = sin.readOptionalString(); + this.ocsf = sin.readOptionalString(); + } + + public Mapping(String rawField, String ecs, String ocsf) { + this.rawField = rawField; + this.ecs = ecs; + this.ocsf = ocsf; + } + + public String getRawField() { + return rawField; + } + + public String getEcs() { + return ecs; + } + + public String getOcsf() { + return ocsf; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(rawField); + out.writeOptionalString(ecs); + out.writeOptionalString(ocsf); + } + + public static Mapping readFrom(StreamInput sin) throws IOException { + return new Mapping(sin); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index f8bdd8406..958a2c132 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -113,8 +113,8 @@ public class OSQueryBackend extends QueryBackend { private static final List> precedence = Arrays.asList(ConditionNOT.class, ConditionAND.class, ConditionOR.class); - public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enableFieldMappings) throws IOException { - super(ruleCategory, true, enableFieldMappings, true, collectErrors); + public OSQueryBackend(Map fieldMappings, boolean collectErrors, boolean enableFieldMappings) throws IOException { + super(fieldMappings, true, enableFieldMappings, true, collectErrors); this.tokenSeparator = " "; this.orToken = "OR"; this.andToken = "AND"; @@ -445,11 +445,7 @@ private String getMappedField(String field) { } private String getFinalField(String field) { - field = this.getMappedField(field); - if (field.contains(".")) { - field = field.replace(".", "_"); - } - return field; + return this.getMappedField(field); } private String getFinalValueField() { diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index 1b868054e..8a3e4d17c 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -59,7 +59,7 @@ public abstract class QueryBackend { protected Map ruleQueryFields; @SuppressWarnings("unchecked") - public QueryBackend(String ruleCategory, boolean convertAndAsIn, boolean enableFieldMappings, boolean convertOrAsIn, boolean collectErrors) throws IOException { + public QueryBackend(Map fieldMappings, boolean convertAndAsIn, boolean enableFieldMappings, boolean convertOrAsIn, boolean collectErrors) { this.convertAndAsIn = convertAndAsIn; this.convertOrAsIn = convertOrAsIn; this.collectErrors = collectErrors; @@ -68,15 +68,7 @@ public QueryBackend(String ruleCategory, boolean convertAndAsIn, boolean enableF this.queryFields = new HashMap<>(); if (this.enableFieldMappings) { - InputStream is = this.getClass().getClassLoader().getResourceAsStream(String.format(Locale.getDefault(), "OSMapping/%s/fieldmappings.yml", ruleCategory)); - assert is != null; - String content = new String(is.readAllBytes(), Charset.defaultCharset()); - - Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions())); - Map fieldMappingsObj = yaml.load(content); - this.fieldMappings = (Map) fieldMappingsObj.get("fieldmappings"); - - is.close(); + this.fieldMappings = fieldMappings; } else { this.fieldMappings = new HashMap<>(); } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 0f1f620c8..b0f2afc4e 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -79,6 +79,7 @@ import org.opensearch.securityanalytics.action.IndexDetectorRequest; import org.opensearch.securityanalytics.action.IndexDetectorResponse; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.mapper.MapperUtils; import org.opensearch.securityanalytics.model.Detector; @@ -136,6 +137,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction> rul List ruleCategories = bucketLevelRules.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( Collectors.toList()); Map queryBackendMap = new HashMap<>(); - for(String category: ruleCategories){ - queryBackendMap.put(category, new OSQueryBackend(category, true, true)); + for(String category: ruleCategories) { + Map fieldMappings = logTypeService.getRuleFieldMappings(category); + queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); } // Pair of RuleId - MonitorId for existing monitors of the detector @@ -448,8 +454,9 @@ private List buildBucketLevelMonitorRequests(List queryBackendMap = new HashMap<>(); - for(String category: ruleCategories){ - queryBackendMap.put(category, new OSQueryBackend(category, true, true)); + for(String category: ruleCategories) { + Map fieldMappings = logTypeService.getRuleFieldMappings(category); + queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); } List monitorRequests = new ArrayList<>(); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java index 818f3f6a4..d846ce4fc 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java @@ -42,6 +42,7 @@ import org.opensearch.securityanalytics.action.IndexRuleAction; import org.opensearch.securityanalytics.action.IndexRuleRequest; import org.opensearch.securityanalytics.action.IndexRuleResponse; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend; @@ -87,12 +88,17 @@ public class TransportIndexRuleAction extends HandledTransportAction fieldMappings = logTypeService.getRuleFieldMappings(category); + final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); List queries = backend.convertRule(parsedRule); Set queryFieldNames = backend.getQueryFields().keySet(); Rule ruleDoc = new Rule( diff --git a/src/main/java/org/opensearch/securityanalytics/util/FileUtils.java b/src/main/java/org/opensearch/securityanalytics/util/FileUtils.java new file mode 100644 index 000000000..67f3c9915 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/util/FileUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.util; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class FileUtils { + + private static FileSystem fs; + + static { + if (fs == null || !fs.isOpen()) { + final Map env = new HashMap<>(); + try { + final String url = Objects.requireNonNull(FileUtils.class.getResource("/rules")).toURI().toString(); + if (url.contains("!")) { + fs = FileSystems.newFileSystem(URI.create(url.split("!")[0]), env); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + } + } + + public static FileSystem getFs() { + return fs; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index aaa3c0d82..590a14488 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -38,6 +38,7 @@ import org.opensearch.index.reindex.DeleteByQueryRequestBuilder; import org.opensearch.rest.RestStatus; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.MapperUtils; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; @@ -78,12 +79,13 @@ public class RuleIndices { private final ThreadPool threadPool; - private static FileSystem fs; + private final LogTypeService logTypeService; - public RuleIndices(Client client, ClusterService clusterService, ThreadPool threadPool) { + public RuleIndices(LogTypeService logTypeService, Client client, ClusterService clusterService, ThreadPool threadPool) { this.client = client; this.clusterService = clusterService; this.threadPool = threadPool; + this.logTypeService = logTypeService; } public static String ruleMappings() throws IOException { @@ -260,26 +262,18 @@ private void ingestQueries(Map> logIndexToRules, WriteReque List queries = new ArrayList<>(); for (Map.Entry> logIndexToRule: logIndexToRules.entrySet()) { - final QueryBackend backend = new OSQueryBackend(logIndexToRule.getKey(), true, true); + Map fieldMappings = logTypeService.getRuleFieldMappings(logIndexToRule.getKey()); + final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); queries.addAll(getQueries(backend, logIndexToRule.getKey(), logIndexToRule.getValue())); } loadRules(queries, refreshPolicy, indexTimeout, listener, true); } private void loadQueries(String[] paths, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener listener) throws IOException, SigmaError { - getFS(paths[0]); - Path path = fs.getPath(paths[1]); + Path path = FileUtils.getFs().getPath(paths[1]); loadQueries(path, refreshPolicy, indexTimeout, listener); } - private static FileSystem getFS(String path) throws IOException { - if (fs == null || !fs.isOpen()) { - final Map env = new HashMap<>(); - fs = FileSystems.newFileSystem(URI.create(path), env); - } - return fs; - } - private List getQueries(QueryBackend backend, String category, List rules) throws SigmaError { List queries = new ArrayList<>(); for (String ruleStr: rules) { diff --git a/src/main/resources/OSMapping/ad_ldap_logtype.json b/src/main/resources/OSMapping/ad_ldap_logtype.json new file mode 100644 index 000000000..70bfc9a02 --- /dev/null +++ b/src/main/resources/OSMapping/ad_ldap_logtype.json @@ -0,0 +1,99 @@ +{ + "name": "ad_ldap", + "description": "AD/LDAP", + "is_builtin": true, + "mappings": [ + { + "raw_field":"TargetUserName", + "ecs":"azure.signinlogs.properties.user_id" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + }, + { + "raw_field":"Category", + "ecs":"azure.activitylogs.category" + }, + { + "raw_field":"OperationName", + "ecs":"azure.platformlogs.operation_name" + }, + { + "raw_field":"ModifiedProperties_NewValue", + "ecs":"modified_properties.new_value" + }, + { + "raw_field":"ResourceProviderValue", + "ecs":"azure.resource.provider" + }, + { + "raw_field":"conditionalAccessStatus", + "ecs":"azure.signinlogs.properties.conditional_access_status" + }, + { + "raw_field":"SearchFilter", + "ecs":"SearchFilter" + }, + { + "raw_field":"Operation", + "ecs":"azure.platformlogs.operation_name" + }, + { + "raw_field":"ResultType", + "ecs":"azure.platformlogs.result_type" + }, + { + "raw_field":"DeviceDetail_isCompliant", + "ecs":"azure.signinlogs.properties.device_detail.is_compliant" + }, + { + "raw_field":"ResourceDisplayName", + "ecs":"resource_display_name" + }, + { + "raw_field":"AuthenticationRequirement", + "ecs":"azure.signinlogs.properties.authentication_requirement" + }, + { + "raw_field":"TargetResources", + "ecs":"target_resources" + }, + { + "raw_field":"Workload", + "ecs":"workload" + }, + { + "raw_field":"DeviceDetail.deviceId", + "ecs":"azure.signinlogs.properties.device_detail.device_id" + }, + { + "raw_field":"OperationNameValue", + "ecs":"azure.platformlogs.operation_name" + }, + { + "raw_field":"ResourceId", + "ecs":"azure.signinlogs.properties.resource_id" + }, + { + "raw_field":"ResultDescription", + "ecs":"azure.signinlogs.result_description" + }, + { + "raw_field":"EventID", + "ecs":"EventID" + }, + { + "raw_field":"NetworkLocationDetails", + "ecs":"azure.signinlogs.properties.network_location_details" + }, + { + "raw_field":"CategoryValue", + "ecs":"azure.activitylogs.category" + }, + { + "raw_field":"ActivityDisplayName", + "ecs":"azure.auditlogs.properties.activity_display_name" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/apache_access_logtype.json b/src/main/resources/OSMapping/apache_access_logtype.json new file mode 100644 index 000000000..0cd1ecab4 --- /dev/null +++ b/src/main/resources/OSMapping/apache_access_logtype.json @@ -0,0 +1,6 @@ +{ + "name": "apache_access", + "description": "Apache Access Log type", + "is_builtin": true, + "mappings": [] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/azure_logtype.json b/src/main/resources/OSMapping/azure_logtype.json new file mode 100644 index 000000000..649539383 --- /dev/null +++ b/src/main/resources/OSMapping/azure_logtype.json @@ -0,0 +1,215 @@ +{ + "name": "azure", + "description": "Azure Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"Resultdescription", + "ecs":"azure.signinlogs.result_description" + }, + { + "raw_field":"eventSource", + "ecs":"eventSource" + }, + { + "raw_field":"eventName", + "ecs":"eventName" + }, + { + "raw_field":"Status", + "ecs":"azure.platformlogs.status" + }, + { + "raw_field":"LoggedByService", + "ecs":"azure.auditlogs.properties.logged_by_service" + }, + { + "raw_field":"properties_message", + "ecs":"properties_message" + }, + { + "raw_field":"status", + "ecs":"azure.platformlogs.status" + }, + { + "raw_field":"TargetUserName", + "ecs":"azure.signinlogs.properties.user_id" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + }, + { + "raw_field":"Category", + "ecs":"azure.activitylogs.category" + }, + { + "raw_field":"OperationName", + "ecs":"azure.platformlogs.operation_name" + }, + { + "raw_field":"ModifiedProperties_NewValue", + "ecs":"modified_properties.new_value" + }, + { + "raw_field":"ResourceProviderValue", + "ecs":"azure.resource.provider" + }, + { + "raw_field":"conditionalAccessStatus", + "ecs":"azure.signinlogs.properties.conditional_access_status" + }, + { + "raw_field":"SearchFilter", + "ecs":"search_filter" + }, + { + "raw_field":"Operation", + "ecs":"azure.platformlogs.operation_name" + }, + { + "raw_field":"ResultType", + "ecs":"azure.platformlogs.result_type" + }, + { + "raw_field":"DeviceDetail_isCompliant", + "ecs":"azure.signinlogs.properties.device_detail.is_compliant" + }, + { + "raw_field":"ResourceDisplayName", + "ecs":"resource_display_name" + }, + { + "raw_field":"AuthenticationRequirement", + "ecs":"azure.signinlogs.properties.authentication_requirement" + }, + { + "raw_field":"TargetResources", + "ecs":"target_resources" + }, + { + "raw_field":"Workload", + "ecs":"Workload" + }, + { + "raw_field":"DeviceDetail_deviceId", + "ecs":"azure.signinlogs.properties.device_detail.device_id" + }, + { + "raw_field":"OperationNameValue", + "ecs":"azure.platformlogs.operation_name" + }, + { + "raw_field":"ResourceId", + "ecs":"azure.signinlogs.properties.resource_id" + }, + { + "raw_field":"ResultDescription", + "ecs":"azure.signinlogs.result_description" + }, + { + "raw_field":"EventID", + "ecs":"EventID" + }, + { + "raw_field":"NetworkLocationDetails", + "ecs":"azure.signinlogs.properties.network_location_details" + }, + { + "raw_field":"CategoryValue", + "ecs":"azure.activitylogs.category" + }, + { + "raw_field":"ActivityDisplayName", + "ecs":"azure.auditlogs.properties.activity_display_name" + }, + { + "raw_field":"Initiatedby", + "ecs":"azure.activitylogs.identity.claims_initiated_by_user.name" + }, + { + "raw_field":"Count", + "ecs":"Count" + }, + { + "raw_field":"ResourceTenantId", + "ecs":"azure.signinlogs.properties.resource_tenant_id" + }, + { + "raw_field":"failure_status_reason", + "ecs":"failure_status_reason" + }, + { + "raw_field":"AppId", + "ecs":"azure.signinlogs.properties.app_id" + }, + { + "raw_field":"properties.message", + "ecs":"properties.message" + }, + { + "raw_field":"ClientApp", + "ecs":"azure.signinlogs.properties.client_app_used" + }, + { + "raw_field":"ActivityDetails", + "ecs":"ActivityDetails" + }, + { + "raw_field":"Target", + "ecs":"Target" + }, + { + "raw_field":"DeviceDetail.trusttype", + "ecs":"azure.signinlogs.properties.device_detail.trust_type" + }, + { + "raw_field":"HomeTenantId", + "ecs":"azure.signinlogs.properties.home_tenant_id" + }, + { + "raw_field":"ConsentContext.IsAdminConsent", + "ecs":"ConsentContext.IsAdminConsent" + }, + { + "raw_field":"InitiatedBy", + "ecs":"InitiatedBy" + }, + { + "raw_field":"ActivityType", + "ecs":"azure.auditlogs.properties.activity_display_name" + }, + { + "raw_field":"operationName", + "ecs":"azure.activitylogs.operation_name" + }, + { + "raw_field":"ModifiedProperties{}.NewValue", + "ecs":"modified_properties.new_value" + }, + { + "raw_field":"userAgent", + "ecs":"user_agent.name" + }, + { + "raw_field":"RiskState", + "ecs":"azure.signinlogs.properties.risk_state" + }, + { + "raw_field":"Username", + "ecs":"azure.activitylogs.identity.claims_initiated_by_user.name" + }, + { + "raw_field":"DeviceDetail.deviceId", + "ecs":"azure.signinlogs.properties.device_detail.device_id" + }, + { + "raw_field":"DeviceDetail.isCompliant", + "ecs":"azure.signinlogs.properties.device_detail.is_compliant" + }, + { + "raw_field":"Location", + "ecs":"azure.signinlogs.properties.network_location_details" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail_logtype.json b/src/main/resources/OSMapping/cloudtrail_logtype.json new file mode 100644 index 000000000..a6be253a6 --- /dev/null +++ b/src/main/resources/OSMapping/cloudtrail_logtype.json @@ -0,0 +1,71 @@ +{ + "name": "cloudtrail", + "description": "Cloudtrail Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"eventName", + "ecs":"aws.cloudtrail.event_name" + }, + { + "raw_field":"eventSource", + "ecs":"aws.cloudtrail.event_source" + }, + { + "raw_field":"eventType", + "ecs":"aws.cloudtrail.event_type" + }, + { + "raw_field":"errorMessage", + "ecs":"aws.cloudtrail.error_message" + }, + { + "raw_field":"errorCode", + "ecs":"aws.cloudtrail.error_code" + }, + { + "raw_field":"responseElements", + "ecs":"aws.cloudtrail.response_elements.text" + }, + { + "raw_field":"responseElements.pendingModifiedValues.masterUserPassword", + "ecs":"aws.cloudtrail.response_elements.pending_modified_values.master_user_password" + }, + { + "raw_field":"responseElements.publiclyAccessible", + "ecs":"aws.cloudtrail.response_elements.publicly_accessible" + }, + { + "raw_field":"requestParameters.arn", + "ecs":"aws.cloudtrail.request_parameters.arn" + }, + { + "raw_field":"requestParameters.attribute", + "ecs":"aws.cloudtrail.request_parameters.attribute" + }, + { + "raw_field":"requestParameters.userName", + "ecs":"aws.cloudtrail.request_parameters.username" + }, + { + "raw_field":"requestParameters.containerDefinitions.command", + "ecs":"aws.cloudtrail.request_parameters.container_definitions.command" + }, + { + "raw_field":"userIdentity.type", + "ecs":"aws.cloudtrail.user_identity.type" + }, + { + "raw_field":"userIdentity.arn", + "ecs":"aws.cloudtrail.user_identity.arn" + }, + { + "raw_field":"userIdentity.sessionContext.sessionIssuer.type", + "ecs":"aws.cloudtrail.user_identity.session_context.session_issuer.type" + }, + { + "raw_field":"eventTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns_logtype.json b/src/main/resources/OSMapping/dns_logtype.json new file mode 100644 index 000000000..fa2294507 --- /dev/null +++ b/src/main/resources/OSMapping/dns_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "dns", + "description": "DNS Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/github_logtype.json b/src/main/resources/OSMapping/github_logtype.json new file mode 100644 index 000000000..415d442e9 --- /dev/null +++ b/src/main/resources/OSMapping/github_logtype.json @@ -0,0 +1,11 @@ +{ + "name": "github", + "description": "Github Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"action", + "ecs":"github.action" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/gworkspace_logtype.json b/src/main/resources/OSMapping/gworkspace_logtype.json new file mode 100644 index 000000000..4c3507ca5 --- /dev/null +++ b/src/main/resources/OSMapping/gworkspace_logtype.json @@ -0,0 +1,19 @@ +{ + "name": "gworkspace", + "description": "GWorkspace Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"eventSource", + "ecs":"google_workspace.admin.service.name" + }, + { + "raw_field":"eventName", + "ecs":"google_workspace.event.name" + }, + { + "raw_field":"new_value", + "ecs":"google_workspace.admin.new_value" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/linux_logtype.json b/src/main/resources/OSMapping/linux_logtype.json new file mode 100644 index 000000000..0f8a54c4d --- /dev/null +++ b/src/main/resources/OSMapping/linux_logtype.json @@ -0,0 +1,63 @@ +{ + "name": "linux", + "description": "Linux Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"name", + "ecs":"user.filesystem.name" + }, + { + "raw_field":"a0", + "ecs":"auditd.log.a0" + }, + { + "raw_field":"comm", + "ecs":"auditd.log.comm" + }, + { + "raw_field":"exe", + "ecs":"auditd.log.exe" + }, + { + "raw_field":"uid", + "ecs":"auditd.log.uid" + }, + { + "raw_field":"USER", + "ecs":"system.auth.user" + }, + { + "raw_field":"User", + "ecs":"system.auth.user" + }, + { + "raw_field":"Image", + "ecs":"process.exe" + }, + { + "raw_field":"DestinationHostname", + "ecs":"rsa.web.remote_domain" + }, + { + "raw_field":"CommandLine", + "ecs":"process.command_line" + }, + { + "raw_field":"ParentImage", + "ecs":"process.parent.executable" + }, + { + "raw_field":"CurrentDirectory", + "ecs":"process.working_directory" + }, + { + "raw_field":"LogonId", + "ecs":"process.real_user.id" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/m365_logtype.json b/src/main/resources/OSMapping/m365_logtype.json new file mode 100644 index 000000000..6c9b3eb8b --- /dev/null +++ b/src/main/resources/OSMapping/m365_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "m365", + "description": "Microsoft 365 Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"eventSource", + "ecs":"rsa.misc.event_source" + }, + { + "raw_field":"eventName", + "ecs":"rsa.misc.event_desc" + }, + { + "raw_field":"status", + "ecs":"rsa.misc.status" + }, + { + "raw_field":"Payload", + "ecs":"rsa.misc.payload_dst" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/netflow_logtype.json b/src/main/resources/OSMapping/netflow_logtype.json new file mode 100644 index 000000000..f63bbc7c9 --- /dev/null +++ b/src/main/resources/OSMapping/netflow_logtype.json @@ -0,0 +1,35 @@ +{ + "name": "netflow", + "description": "Netflow Log Type used only in Integration Tests", + "is_builtin": true, + "mappings": [ + { + "raw_field":"source.ip", + "ecs":"source.ip" + }, + { + "raw_field":"source.port", + "ecs":"source.port" + }, + { + "raw_field":"destination.ip", + "ecs":"destination.ip" + }, + { + "raw_field":"destination.port", + "ecs":"destination.port" + }, + { + "raw_field":"http.request.method", + "ecs":"http.request.method" + }, + { + "raw_field":"http.response.status_code", + "ecs":"http.response.status_code" + }, + { + "raw_field":"timestamp", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/network_logtype.json b/src/main/resources/OSMapping/network_logtype.json new file mode 100644 index 000000000..0dddf260e --- /dev/null +++ b/src/main/resources/OSMapping/network_logtype.json @@ -0,0 +1,135 @@ +{ + "name": "network", + "description": "Network Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"action", + "ecs":"netflow.firewall_event" + }, + { + "raw_field":"certificate.serial", + "ecs":"zeek.x509.certificate.serial" + }, + { + "raw_field":"name", + "ecs":"zeek.smb_files.name" + }, + { + "raw_field":"path", + "ecs":"zeek.smb_files.path" + }, + { + "raw_field":"dst_port", + "ecs":"destination.port" + }, + { + "raw_field":"qtype_name", + "ecs":"zeek.dns.qtype_name" + }, + { + "raw_field":"operation", + "ecs":"zeek.dce_rpc.operation" + }, + { + "raw_field":"endpoint", + "ecs":"zeek.dce_rpc.endpoint" + }, + { + "raw_field":"zeek.dce_rpc.endpoint", + "ecs":"zeek.dce_rpc.endpoint" + }, + { + "raw_field":"answers", + "ecs":"zeek.dns.answers" + }, + { + "raw_field":"query", + "ecs":"zeek.dns.query" + }, + { + "raw_field":"client_header_names", + "ecs":"zeek.http.client_header_names" + }, + { + "raw_field":"resp_mime_types", + "ecs":"zeek.http.resp_mime_types" + }, + { + "raw_field":"cipher", + "ecs":"zeek.kerberos.cipher" + }, + { + "raw_field":"request_type", + "ecs":"zeek.kerberos.request_type" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + }, + { + "raw_field":"method", + "ecs":"http.request.method" + }, + { + "raw_field":"id.resp_p", + "ecs":"id.resp_p" + }, + { + "raw_field":"blocked", + "ecs":"blocked-flag" + }, + { + "raw_field":"id.orig_h", + "ecs":"id.orig_h" + }, + { + "raw_field":"Z", + "ecs":"Z-flag" + }, + { + "raw_field":"id.resp_h", + "ecs":"id.resp_h" + }, + { + "raw_field":"uri", + "ecs":"url.path" + }, + { + "raw_field":"c-uri", + "ecs":"url.path" + }, + { + "raw_field":"c-useragent", + "ecs":"user_agent.name" + }, + { + "raw_field":"status_code", + "ecs":"http.response.status_code" + }, + { + "raw_field":"rejected", + "ecs":"rejected" + }, + { + "raw_field":"dst_ip", + "ecs":"destination.ip" + }, + { + "raw_field":"src_ip", + "ecs":"source.ip" + }, + { + "raw_field":"user_agent", + "ecs":"user_agent.name" + }, + { + "raw_field":"request_body_len", + "ecs":"http.request.body.bytes" + }, + { + "raw_field":"service", + "ecs":"service" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/okta_logtype.json b/src/main/resources/OSMapping/okta_logtype.json new file mode 100644 index 000000000..96bda55bb --- /dev/null +++ b/src/main/resources/OSMapping/okta_logtype.json @@ -0,0 +1,15 @@ +{ + "name": "okta", + "description": "Okta Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"eventtype", + "ecs":"okta.event_type" + }, + { + "raw_field":"displaymessage", + "ecs":"okta.display_message" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_application_logtype.json b/src/main/resources/OSMapping/others_application_logtype.json new file mode 100644 index 000000000..1c37fc5f6 --- /dev/null +++ b/src/main/resources/OSMapping/others_application_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_application", + "description": "others_application", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_apt_logtype.json b/src/main/resources/OSMapping/others_apt_logtype.json new file mode 100644 index 000000000..82b6bb873 --- /dev/null +++ b/src/main/resources/OSMapping/others_apt_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_apt", + "description": "others_apt", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_cloud_logtype.json b/src/main/resources/OSMapping/others_cloud_logtype.json new file mode 100644 index 000000000..65fe1bc7d --- /dev/null +++ b/src/main/resources/OSMapping/others_cloud_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_cloud", + "description": "others_cloud", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_compliance_logtype.json b/src/main/resources/OSMapping/others_compliance_logtype.json new file mode 100644 index 000000000..71a29c40c --- /dev/null +++ b/src/main/resources/OSMapping/others_compliance_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_compliance", + "description": "others_compliance", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_macos_logtype.json b/src/main/resources/OSMapping/others_macos_logtype.json new file mode 100644 index 000000000..4b29c6f77 --- /dev/null +++ b/src/main/resources/OSMapping/others_macos_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_macos", + "description": "others_macos", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_proxy_logtype.json b/src/main/resources/OSMapping/others_proxy_logtype.json new file mode 100644 index 000000000..e43a543a9 --- /dev/null +++ b/src/main/resources/OSMapping/others_proxy_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_proxy", + "description": "others_proxy", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_web_logtype.json b/src/main/resources/OSMapping/others_web_logtype.json new file mode 100644 index 000000000..62ce44168 --- /dev/null +++ b/src/main/resources/OSMapping/others_web_logtype.json @@ -0,0 +1,23 @@ +{ + "name": "others_web", + "description": "others_web", + "is_builtin": true, + "mappings": [ + { + "raw_field":"record_type", + "ecs":"dns.answers.type" + }, + { + "raw_field":"query", + "ecs":"dns.question.name" + }, + { + "raw_field":"parent_domain", + "ecs":"dns.question.registered_domain" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/s3_logtype.json b/src/main/resources/OSMapping/s3_logtype.json new file mode 100644 index 000000000..d91101e47 --- /dev/null +++ b/src/main/resources/OSMapping/s3_logtype.json @@ -0,0 +1,19 @@ +{ + "name": "s3", + "description": "S3 Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"eventName", + "ecs":"aws.cloudtrail.event_name" + }, + { + "raw_field":"eventSource", + "ecs":"aws.cloudtrail.event_source" + }, + { + "raw_field":"eventTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/test_windows_logtype.json b/src/main/resources/OSMapping/test_windows_logtype.json new file mode 100644 index 000000000..096c2b923 --- /dev/null +++ b/src/main/resources/OSMapping/test_windows_logtype.json @@ -0,0 +1,47 @@ +{ + "name": "test_windows", + "description": "Test Log Type used by tests. It is created as a lightweight log type for integration tests", + "is_builtin": true, + "mappings": [ + { + "raw_field":"EventID", + "ecs":"event_uid" + }, + { + "raw_field":"HiveName", + "ecs":"unmapped.HiveName" + }, + { + "raw_field":"fieldB", + "ecs":"mappedB" + }, + { + "raw_field":"fieldA1", + "ecs":"mappedA" + }, + { + "raw_field":"CommandLine", + "ecs":"windows-event_data-CommandLine" + }, + { + "raw_field":"HostName", + "ecs":"windows-hostname" + }, + { + "raw_field":"Message", + "ecs":"windows-message" + }, + { + "raw_field":"Provider_Name", + "ecs":"windows-provider-name" + }, + { + "raw_field":"ServiceName", + "ecs":"windows-servicename" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/windows_logtype.json b/src/main/resources/OSMapping/windows_logtype.json new file mode 100644 index 000000000..4806478c8 --- /dev/null +++ b/src/main/resources/OSMapping/windows_logtype.json @@ -0,0 +1,835 @@ +{ + "name": "windows", + "description": "Windows Log Type", + "is_builtin": true, + "mappings":[ + { + "raw_field":"AccountName", + "ecs":"winlog.computerObject.name" + }, + { + "raw_field":"AuthenticationPackageName", + "ecs":"winlog.event_data.AuthenticationPackageName" + }, + { + "raw_field":"Channel", + "ecs":"winlog.channel" + }, + { + "raw_field":"Company", + "ecs":"winlog.event_data.Company" + }, + { + "raw_field":"ComputerName", + "ecs":"winlog.computer_name" + }, + { + "raw_field":"Description", + "ecs":"winlog.event_data.Description" + }, + { + "raw_field":"Details", + "ecs":"winlog.event_data.Detail" + }, + { + "raw_field":"Device", + "ecs":"winlog.event_data.Device" + }, + { + "raw_field":"FileName", + "ecs":"winlog.event_data.FileName" + }, + { + "raw_field":"FileVersion", + "ecs":"winlog.event_data.FileVersion" + }, + { + "raw_field":"IntegrityLevel", + "ecs":"winlog.event_data.IntegrityLevel" + }, + { + "raw_field":"IpAddress", + "ecs":"winlog.event_data.IpAddress" + }, + { + "raw_field":"KeyLength", + "ecs":"winlog.event_data.KeyLength" + }, + { + "raw_field":"Keywords", + "ecs":"winlog.keywords" + }, + { + "raw_field":"LogonId", + "ecs":"winlog.event_data.LogonId" + }, + { + "raw_field":"LogonProcessName", + "ecs":"winlog.event_data.LogonProcessName" + }, + { + "raw_field":"LogonType", + "ecs":"winlog.event_data.LogonType" + }, + { + "raw_field":"OriginalFilename", + "ecs":"winlog.event_data.OriginalFileName" + }, + { + "raw_field":"Path", + "ecs":"winlog.event_data.Path" + }, + { + "raw_field":"PrivilegeList", + "ecs":"winlog.event_data.PrivilegeList" + }, + { + "raw_field":"ProcessId", + "ecs":"winlog.event_data.ProcessId" + }, + { + "raw_field":"Product", + "ecs":"winlog.event_data.Product" + }, + { + "raw_field":"Provider", + "ecs":"winlog.provider_name" + }, + { + "raw_field":"ProviderName", + "ecs":"winlog.provider_name" + }, + { + "raw_field":"ScriptBlockText", + "ecs":"winlog.event_data.ScriptBlockText" + }, + { + "raw_field":"ServerName", + "ecs":"winlog.event_data.TargetServerName" + }, + { + "raw_field":"Service", + "ecs":"winlog.event_data.ServiceName" + }, + { + "raw_field":"Signed", + "ecs":"winlog.event_data.Signed" + }, + { + "raw_field":"State", + "ecs":"winlog.event_data.State" + }, + { + "raw_field":"Status", + "ecs":"winlog.event_data.Status" + }, + { + "raw_field":"SubjectDomainName", + "ecs":"winlog.event_data.SubjectDomainName" + }, + { + "raw_field":"SubjectLogonId", + "ecs":"winlog.event_data.SubjectLogonId" + }, + { + "raw_field":"SubjectUserName", + "ecs":"winlog.event_data.SubjectUserName" + }, + { + "raw_field":"SubjectUserSid", + "ecs":"winlog.event_data.SubjectUserSid" + }, + { + "raw_field":"TargetLogonId", + "ecs":"winlog.event_data.TargetLogonId" + }, + { + "raw_field":"TargetName", + "ecs":"winlog.event_data.TargetUserName" + }, + { + "raw_field":"TargetServerName", + "ecs":"winlog.event_data.TargetServerName" + }, + { + "raw_field":"TargetUserName", + "ecs":"winlog.event_data.TargetUserName" + }, + { + "raw_field":"TargetUserSid", + "ecs":"winlog.event_data.TargetUserSid" + }, + { + "raw_field":"TaskName", + "ecs":"winlog.task" + }, + { + "raw_field":"Type", + "ecs":"winlog.user.type" + }, + { + "raw_field":"User", + "ecs":"winlog.user.name" + }, + { + "raw_field":"UserName", + "ecs":"winlog.user.name" + }, + { + "raw_field":"Workstation", + "ecs":"winlog.event_data.Workstation" + }, + { + "raw_field":"WorkstationName", + "ecs":"winlog.event_data.Workstation" + }, + { + "raw_field":"event_uid", + "ecs":"winlog.event_id" + }, + { + "raw_field":"CommandLine", + "ecs":"process.command_line" + }, + { + "raw_field":"hostname", + "ecs":"host.hostname" + }, + { + "raw_field":"message", + "ecs":"windows.message" + }, + { + "raw_field":"Provider_Name", + "ecs":"winlog.provider_name" + }, + { + "raw_field":"EventId", + "ecs":"winlog.event_id" + }, + { + "raw_field":"processPath", + "ecs":"winlog.event_data.ProcessPath" + }, + { + "raw_field":"ProcessName", + "ecs":"winlog.event_data.ProcessName" + }, + { + "raw_field":"ObjectName", + "ecs":"winlog.computerObject.name" + }, + { + "raw_field":"param1", + "ecs":"winlog.event_data.param1" + }, + { + "raw_field":"param2", + "ecs":"winlog.event_data.param2" + }, + { + "raw_field":"creationTime", + "ecs":"timestamp" + }, + { + "raw_field":"Origin", + "ecs":"winlog.event_data.Origin" + }, + { + "raw_field":"ParentImage", + "ecs":"winlog.event_data.ParentImage" + }, + { + "raw_field":"TargetPort", + "ecs":"winlog.event_data.TargetPort" + }, + { + "raw_field":"Query", + "ecs":"winlog.event_data.Query" + }, + { + "raw_field":"DestinationPort", + "ecs":"destination.port" + }, + { + "raw_field":"StartAddress", + "ecs":"winlog.event_data.StartAddress" + }, + { + "raw_field":"TicketOptions", + "ecs":"winlog.event_data.TicketOptions" + }, + { + "raw_field":"ParentCommandLine", + "ecs":"winlog.event_data.ParentCommandLine" + }, + { + "raw_field":"AllowedToDelegateTo", + "ecs":"winlog.event_data.AllowedToDelegateTo" + }, + { + "raw_field":"HostApplication", + "ecs":"winlog.event_data.HostApplication" + }, + { + "raw_field":"AccessMask", + "ecs":"winlog.event_data.AccessMask" + }, + { + "raw_field":"Hashes", + "ecs":"winlog.event_data.Hashes" + }, + { + "raw_field":"SidHistory", + "ecs":"winlog.event_data.SidHistory" + }, + { + "raw_field":"Initiated", + "ecs":"winlog.event_data.Initiated" + }, + { + "raw_field":"DestinationIp", + "ecs":"destination.ip" + }, + { + "raw_field":"RelativeTargetName", + "ecs":"winlog.event_data.RelativeTargetName" + }, + { + "raw_field":"Source_Name", + "ecs":"winlog.event_data.Source_Name" + }, + { + "raw_field":"AttributeLDAPDisplayName", + "ecs":"winlog.event_data.AttributeLDAPDisplayName" + }, + { + "raw_field":"DeviceDescription", + "ecs":"winlog.event_data.DeviceDescription" + }, + { + "raw_field":"AttributeValue", + "ecs":"winlog.event_data.AttributeValue" + }, + { + "raw_field":"ObjectValueName", + "ecs":"winlog.event_data.ObjectValueName" + }, + { + "raw_field":"QueryStatus", + "ecs":"winlog.event_data.QueryStatus" + }, + { + "raw_field":"TargetParentProcessId", + "ecs":"winlog.event_data.TargetParentProcessId" + }, + { + "raw_field":"OldUacValue", + "ecs":"winlog.event_data.OldUacValue" + }, + { + "raw_field":"FailureCode", + "ecs":"winlog.event_data.FailureCode" + }, + { + "raw_field":"OldTargetUserName", + "ecs":"winlog.event_data.OldTargetUserName" + }, + { + "raw_field":"NewUacValue", + "ecs":"winlog.event_data.NewUacValue" + }, + { + "raw_field":"ServiceName", + "ecs":"winlog.event_data.ServiceName" + }, + { + "raw_field":"Imphash", + "ecs":"winlog.event_data.Imphash" + }, + { + "raw_field":"NewValue", + "ecs":"winlog.event_data.NewValue" + }, + { + "raw_field":"Action", + "ecs":"winlog.event_data.Action" + }, + { + "raw_field":"SourceImage", + "ecs":"winlog.event_data.SourceImage" + }, + { + "raw_field":"QNAME", + "ecs":"winlog.event_data.QNAME" + }, + { + "raw_field":"Properties", + "ecs":"winlog.event_data.Properties" + }, + { + "raw_field":"AuditPolicyChanges", + "ecs":"winlog.event_data.AuditPolicyChanges" + }, + { + "raw_field":"Accesses", + "ecs":"winlog.event_data.Accesses" + }, + { + "raw_field":"ClassName", + "ecs":"winlog.event_data.ClassName" + }, + { + "raw_field":"ObjectClass", + "ecs":"winlog.event_data.ObjectClass" + }, + { + "raw_field":"PipeName", + "ecs":"winlog.event_data.PipeName" + }, + { + "raw_field":"HiveName", + "ecs":"winlog.event_data.HiveName" + }, + { + "raw_field":"StartModule", + "ecs":"winlog.event_data.StartModule" + }, + { + "raw_field":"HostVersion", + "ecs":"winlog.event_data.HostVersion" + }, + { + "raw_field":"DestinationHostname", + "ecs":"winlog.event_data.DestinationHostname" + }, + { + "raw_field":"QueryName", + "ecs":"winlog.event_data.QueryName" + }, + { + "raw_field":"RemoteName", + "ecs":"winlog.event_data.RemoteName" + }, + { + "raw_field":"PasswordLastSet", + "ecs":"winlog.event_data.PasswordLastSet" + }, + { + "raw_field":"ErrorCode", + "ecs":"winlog.event_data.ErrorCode" + }, + { + "raw_field":"AccessList", + "ecs":"winlog.event_data.AccessList" + }, + { + "raw_field":"Address", + "ecs":"winlog.event_data.Address" + }, + { + "raw_field":"PossibleCause", + "ecs":"winlog.event_data.PossibleCause" + }, + { + "raw_field":"DestPort", + "ecs":"destination.port" + }, + { + "raw_field":"Image", + "ecs":"winlog.event_data.Image" + }, + { + "raw_field":"CertThumbprint", + "ecs":"winlog.event_data.CertThumbprint" + }, + { + "raw_field":"TicketEncryptionType", + "ecs":"winlog.event_data.TicketEncryptionType" + }, + { + "raw_field":"ServiceType", + "ecs":"winlog.event_data.ServiceType" + }, + { + "raw_field":"ObjectServer", + "ecs":"winlog.event_data.ObjectServer" + }, + { + "raw_field":"ImagePath", + "ecs":"winlog.event_data.ImagePath" + }, + { + "raw_field":"NewName", + "ecs":"winlog.event_data.NewName" + }, + { + "raw_field":"CallTrace", + "ecs":"winlog.event_data.CallTrace" + }, + { + "raw_field":"SamAccountName", + "ecs":"winlog.event_data.SamAccountName" + }, + { + "raw_field":"GrantedAccess", + "ecs":"winlog.event_data.GrantedAccess" + }, + { + "raw_field":"EngineVersion", + "ecs":"winlog.event_data.EngineVersion" + }, + { + "raw_field":"OriginalName", + "ecs":"winlog.event_data.OriginalName" + }, + { + "raw_field":"AuditSourceName", + "ecs":"winlog.event_data.AuditSourceName" + }, + { + "raw_field":"sha1", + "ecs":"hash.sha1" + }, + { + "raw_field":"SourceIp", + "ecs":"source.ip" + }, + { + "raw_field":"Payload", + "ecs":"winlog.event_data.Payload" + }, + { + "raw_field":"Level", + "ecs":"winlog.event_data.Level" + }, + { + "raw_field":"Application", + "ecs":"winlog.event_data.Application" + }, + { + "raw_field":"RemoteAddress", + "ecs":"winlog.event_data.RemoteAddress" + }, + { + "raw_field":"SearchFilter", + "ecs":"winlog.event_data.SearchFilter" + }, + { + "raw_field":"ApplicationPath", + "ecs":"winlog.event_data.ApplicationPath" + }, + { + "raw_field":"TargetFilename", + "ecs":"winlog.event_data.TargetFilename" + }, + { + "raw_field":"CurrentDirectory", + "ecs":"winlog.event_data.CurrentDirectory" + }, + { + "raw_field":"ObjectType", + "ecs":"winlog.event_data.ObjectType" + }, + { + "raw_field":"ServicePrincipalNames", + "ecs":"winlog.event_data.ServicePrincipalNames" + }, + { + "raw_field":"TemplateContent", + "ecs":"winlog.event_data.TemplateContent" + }, + { + "raw_field":"QueryResults", + "ecs":"winlog.event_data.QueryResults" + }, + { + "raw_field":"ServiceStartType", + "ecs":"winlog.event_data.ServiceStartType" + }, + { + "raw_field":"EventType", + "ecs":"winlog.event_data.EventType" + }, + { + "raw_field":"TargetSid", + "ecs":"winlog.event_data.TargetSid" + }, + { + "raw_field":"ParentUser", + "ecs":"winlog.event_data.ParentUser" + }, + { + "raw_field":"NewTargetUserName", + "ecs":"winlog.event_data.NewTargetUserName" + }, + { + "raw_field":"DestAddress", + "ecs":"winlog.event_data.DestAddress" + }, + { + "raw_field":"ContextInfo", + "ecs":"winlog.event_data.ContextInfo" + }, + { + "raw_field":"HostName", + "ecs":"host.name" + }, + { + "raw_field":"NewTemplateContent", + "ecs":"winlog.event_data.NewTemplateContent" + }, + { + "raw_field":"LayerRTID", + "ecs":"winlog.event_data.LayerRTID" + }, + { + "raw_field":"ImageFileName", + "ecs":"winlog.event_data.ImageFileName" + }, + { + "raw_field":"StartFunction", + "ecs":"winlog.event_data.StartFunction" + }, + { + "raw_field":"Value", + "ecs":"winlog.event_data.Value" + }, + { + "raw_field":"ModifyingApplication", + "ecs":"winlog.event_data.ModifyingApplication" + }, + { + "raw_field":"Destination", + "ecs":"winlog.event_data.Destination" + }, + { + "raw_field":"Commandline", + "ecs":"winlog.event_data.Commandline" + }, + { + "raw_field":"Message", + "ecs":"winlog.event_data.Message" + }, + { + "raw_field":"ShareName", + "ecs":"winlog.event_data.ShareName" + }, + { + "raw_field":"SourcePort", + "ecs":"source.port" + }, + { + "raw_field":"CallerProcessName", + "ecs":"winlog.event_data.CallerProcessName" + }, + { + "raw_field":"ServiceFileName", + "ecs":"winlog.event_data.ServiceFileName" + }, + { + "raw_field":"DestinationIsIpv6", + "ecs":"winlog.event_data.DestinationIsIpv6" + }, + { + "raw_field":"TargetImage", + "ecs":"winlog.event_data.TargetImage" + }, + { + "raw_field":"SourceAddress", + "ecs":"source.ip" + }, + { + "raw_field":"TargetObject", + "ecs":"winlog.event_data.TargetObject" + }, + { + "raw_field":"Caption", + "ecs":"winlog.event_data.Caption" + }, + { + "raw_field":"LocalName", + "ecs":"winlog.event_data.LocalName" + }, + { + "raw_field":"ImageLoaded", + "ecs":"winlog.event_data.ImageLoaded" + }, + { + "raw_field":"EventID", + "ecs":"winlog.event_id" + }, + { + "raw_field":"sha256", + "ecs":"hash.sha256" + }, + { + "raw_field":"ScriptBlockLogging", + "ecs":"winlog.event_data.ScriptBlockLogging" + }, + { + "raw_field":"SourceParentImage", + "ecs":"winlog.event_data.SourceParentImage" + }, + { + "raw_field":"SourceFilename", + "ecs":"winlog.event_data.SourceFilename" + }, + { + "raw_field":"Protocol", + "ecs":"winlog.event_data.Protocol" + }, + { + "raw_field":"ValidatedPolicy", + "ecs":"winlog.event_data.ValidatedPolicy" + }, + { + "raw_field":"ProcessPath", + "ecs":"winlog.event_data.ProcessPath" + }, + { + "raw_field":"OldValue", + "ecs":"winlog.event_data.OldValue" + }, + { + "raw_field":"ParentProcessId", + "ecs":"winlog.event_data.ParentProcessId" + }, + { + "raw_field":"TaskContentNew", + "ecs":"winlog.event_data.TaskContentNew" + }, + { + "raw_field":"Name", + "ecs":"winlog.event_data.Name" + }, + { + "raw_field":"payload", + "ecs":"winlog.event_data.payload" + }, + { + "raw_field":"SourceHostname", + "ecs":"winlog.event_data.SourceHostname" + }, + { + "raw_field":"ClientProcessId", + "ecs":"winlog.event_data.ClientProcessId" + }, + { + "raw_field":"TargetParentImage", + "ecs":"winlog.event_data.TargetParentImage" + }, + { + "raw_field":"ImpersonationLevel", + "ecs":"winlog.event_data.ImpersonationLevel" + }, + { + "raw_field":"ExceptionCode", + "ecs":"winlog.event_data.ExceptionCode" + }, + { + "raw_field":"FilterOrigin", + "ecs":"winlog.event_data.FilterOrigin" + }, + { + "raw_field":"PackagePath", + "ecs":"winlog.event_data.PackagePath" + }, + { + "raw_field":"SignatureStatus", + "ecs":"winlog.event_data.SignatureStatus" + }, + { + "raw_field":"Hash", + "ecs":"winlog.event_data.Hash" + }, + { + "raw_field":"AppID", + "ecs":"winlog.event_data.AppID" + }, + { + "raw_field":"SidList", + "ecs":"winlog.event_data.SidList" + }, + { + "raw_field":"ProcessNameBuffer", + "ecs":"winlog.event_data.ProcessNameBuffer" + }, + { + "raw_field":"PreviousCreationUtcTime", + "ecs":"winlog.event_data.PreviousCreationUtcTime" + }, + { + "raw_field":"Contents", + "ecs":"winlog.event_data.Contents" + }, + { + "raw_field":"TargetOutboundUserName", + "ecs":"winlog.event_data.TargetOutboundUserName" + }, + { + "raw_field":"ImageName", + "ecs":"winlog.event_data.ImageName" + }, + { + "raw_field":"md5", + "ecs":"hash.md5" + }, + { + "raw_field":"DeviceName", + "ecs":"winlog.event_data.DeviceName" + }, + { + "raw_field":"RequestedPolicy", + "ecs":"winlog.event_data.RequestedPolicy" + }, + { + "raw_field":"FileNameBuffer", + "ecs":"winlog.event_data.FileNameBuffer" + }, + { + "raw_field":"TaskContent", + "ecs":"winlog.event_data.TaskContent" + }, + { + "raw_field":"SourceCommandLine", + "ecs":"winlog.event_data.SourceCommandLine" + }, + { + "raw_field":"CreationUtcTime", + "ecs":"winlog.event_data.CreationUtcTime" + }, + { + "raw_field":"AppName", + "ecs":"winlog.event_data.AppName" + }, + { + "raw_field":"subjectName", + "ecs":"winlog.event_data.subjectName" + }, + { + "raw_field":"process", + "ecs":"winlog.event_data.process" + }, + { + "raw_field":"PackageFullName", + "ecs":"winlog.event_data.PackageFullName" + }, + { + "raw_field":"SourceName", + "ecs":"winlog.event_data.SourceName" + }, + { + "raw_field":"Data", + "ecs":"winlog.event_data.Data" + }, + { + "raw_field":"param3", + "ecs":"winlog.event_data.param3" + }, + { + "raw_field":"Signature", + "ecs":"winlog.event_data.Signature" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json index 421dc202c..3ffc39478 100644 --- a/src/main/resources/mappings/finding_mapping.json +++ b/src/main/resources/mappings/finding_mapping.json @@ -1,7 +1,7 @@ { "dynamic": "strict", "_meta" : { - "schema_version": 2 + "schema_version": 3 }, "properties": { "schema_version": { @@ -60,6 +60,9 @@ "type" : "keyword" } } + }, + "execution_id": { + "type": "keyword" } } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/Writable/LogTypeTests.java b/src/test/java/org/opensearch/securityanalytics/Writable/LogTypeTests.java new file mode 100644 index 000000000..e8a709ef3 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/Writable/LogTypeTests.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.Writable; + +import java.io.IOException; +import java.util.List; +import org.junit.Test; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.securityanalytics.model.LogType; + +import static org.opensearch.test.OpenSearchTestCase.assertEquals; + +public class LogTypeTests { + + + @Test + public void testLogTypeAsStreamRawFieldOnly() throws IOException { + LogType logType = new LogType( + "1", "my_log_type", "description", false, + List.of(new LogType.Mapping("rawField", null, null)) + ); + BytesStreamOutput out = new BytesStreamOutput(); + logType.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + LogType newLogType = new LogType(sin); + assertEquals(logType.getName(), newLogType.getName()); + assertEquals(logType.getDescription(), newLogType.getDescription()); + assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); + assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); + assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + } + + @Test + public void testLogTypeAsStreamFull() throws IOException { + LogType logType = new LogType( + "1", "my_log_type", "description", false, + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")) + ); + BytesStreamOutput out = new BytesStreamOutput(); + logType.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + LogType newLogType = new LogType(sin); + assertEquals(logType.getName(), newLogType.getName()); + assertEquals(logType.getDescription(), newLogType.getDescription()); + assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); + assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); + assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + } + + @Test + public void testLogTypeAsStreamNoMappings() throws IOException { + LogType logType = new LogType("1", "my_log_type", "description", false, null); + BytesStreamOutput out = new BytesStreamOutput(); + logType.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + LogType newLogType = new LogType(sin); + assertEquals(logType.getName(), newLogType.getName()); + assertEquals(logType.getDescription(), newLogType.getDescription()); + assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); + assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index cb231c5b8..0f79b5c15 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -30,7 +30,7 @@ public class CorrelationEngineRestApiIT extends SecurityAnalyticsRestTestCase { @SuppressWarnings("unchecked") - public void testBasicCorrelationEngineWorkflow() throws IOException { + public void testBasicCorrelationEngineWorkflow() throws IOException, InterruptedException { LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); @@ -46,7 +46,7 @@ public void testBasicCorrelationEngineWorkflow() throws IOException { Response executeResponse = executeAlertingMonitor(adLdapMonitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(0, noOfSigmaRuleMatches); + Assert.assertEquals(1, noOfSigmaRuleMatches); indexDoc(indices.windowsIndex, "2", randomDoc()); executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); @@ -71,6 +71,7 @@ public void testBasicCorrelationEngineWorkflow() throws IOException { executeResults = entityAsMap(executeResponse); noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); + Thread.sleep(5000); // Call GetFindings API Map params = new HashMap<>(); @@ -80,10 +81,15 @@ public void testBasicCorrelationEngineWorkflow() throws IOException { String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - Assert.assertEquals(1, correlatedFindings.size()); + Assert.assertEquals(2, correlatedFindings.size()); Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - Assert.assertEquals(1, ((List) correlatedFindings.get(0).get("rules")).size()); - Assert.assertEquals(ruleId, ((List) correlatedFindings.get(0).get("rules")).get(0)); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } } @SuppressWarnings("unchecked") @@ -95,7 +101,7 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept String testWindowsMonitorId = createTestWindowsDetector(indices.windowsIndex); createNetworkToAdLdapToWindowsRule(indices); - Thread.sleep(30000); + Thread.sleep(5000); indexDoc(indices.windowsIndex, "2", randomDoc()); Response executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); @@ -103,7 +109,7 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(5, noOfSigmaRuleMatches); - Thread.sleep(30000); + Thread.sleep(5000); indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); @@ -111,7 +117,7 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - Thread.sleep(30000); + Thread.sleep(5000); Long endTime = System.currentTimeMillis(); Request request = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); @@ -148,8 +154,8 @@ private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOE } private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ-EPO7HKA", "test_windows"); - CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "ad_ldap"); + CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows"); + CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application"); CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3"); CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4)); diff --git a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java index 2326b541d..995814953 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java @@ -46,4 +46,48 @@ public void testEmptyUserAsStream() throws IOException { User newUser = new User(sin); Assert.assertEquals("Round tripping User doesn't work", user, newUser); } + + public void testLogTypeAsStreamRawFieldOnly() throws IOException { + LogType logType = new LogType( + "1", "my_log_type", "description", false, + List.of(new LogType.Mapping("rawField", null, null)) + ); + BytesStreamOutput out = new BytesStreamOutput(); + logType.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + LogType newLogType = new LogType(sin); + assertEquals(logType.getName(), newLogType.getName()); + assertEquals(logType.getDescription(), newLogType.getDescription()); + assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); + assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); + assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + } + + public void testLogTypeAsStreamFull() throws IOException { + LogType logType = new LogType( + "1", "my_log_type", "description", false, + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")) + ); + BytesStreamOutput out = new BytesStreamOutput(); + logType.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + LogType newLogType = new LogType(sin); + assertEquals(logType.getName(), newLogType.getName()); + assertEquals(logType.getDescription(), newLogType.getDescription()); + assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); + assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); + assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + } + + public void testLogTypeAsStreamNoMappings() throws IOException { + LogType logType = new LogType("1", "my_log_type", "description", false, null); + BytesStreamOutput out = new BytesStreamOutput(); + logType.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + LogType newLogType = new LogType(sin); + assertEquals(logType.getName(), newLogType.getName()); + assertEquals(logType.getDescription(), newLogType.getDescription()); + assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); + assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 4d394ed36..43db549c8 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -4,36 +4,43 @@ */ package org.opensearch.securityanalytics.rules.aggregation; +import java.io.IOException; +import java.util.List; +import java.util.Map; import org.junit.Assert; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend; import org.opensearch.securityanalytics.rules.exceptions.SigmaError; import org.opensearch.securityanalytics.rules.objects.SigmaRule; import org.opensearch.test.OpenSearchTestCase; -import java.io.IOException; -import java.util.List; - public class AggregationBackendTests extends OpenSearchTestCase { + private static final Map windowsFieldMappings = Map.of( + "EventID", "event_uid", + "HiveName", "unmapped.HiveName", + "fieldB", "mappedB", + "fieldA1", "mappedA" + ); + public void testCountAggregation() throws SigmaError, IOException { - OSQueryBackend queryBackend = new OSQueryBackend("windows", true, true); + OSQueryBackend queryBackend = new OSQueryBackend(windowsFieldMappings, true, true); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: valueA\n" + - " fieldB: valueB\n" + - " fieldC: valueC\n" + - " condition: sel | count(*) > 1", true)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: valueA\n" + + " fieldB: valueB\n" + + " fieldC: valueC\n" + + " condition: sel | count(*) > 1", true)); String query = queries.get(0).toString(); Assert.assertEquals("(fieldA: \"valueA\") AND (mappedB: \"valueB\") AND (fieldC: \"valueC\")", query); @@ -47,24 +54,24 @@ public void testCountAggregation() throws SigmaError, IOException { } public void testCountAggregationWithGroupBy() throws IOException, SigmaError { - OSQueryBackend queryBackend = new OSQueryBackend("windows", true, true); + OSQueryBackend queryBackend = new OSQueryBackend(windowsFieldMappings, true, true); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: valueA\n" + - " fieldB: valueB\n" + - " fieldC: valueC\n" + - " condition: sel | count(*) by fieldB > 1", true)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: valueA\n" + + " fieldB: valueB\n" + + " fieldC: valueC\n" + + " condition: sel | count(*) by fieldB > 1", true)); String query = queries.get(0).toString(); Assert.assertEquals("(fieldA: \"valueA\") AND (mappedB: \"valueB\") AND (fieldC: \"valueC\")", query); @@ -78,24 +85,24 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError { } public void testSumAggregationWithGroupBy() throws IOException, SigmaError { - OSQueryBackend queryBackend = new OSQueryBackend("windows", true, true); + OSQueryBackend queryBackend = new OSQueryBackend(windowsFieldMappings, true, true); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: valueA\n" + - " fieldB: valueB\n" + - " fieldC: valueC\n" + - " condition: sel | sum(fieldA) by fieldB > 110", true)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: valueA\n" + + " fieldB: valueB\n" + + " fieldC: valueC\n" + + " condition: sel | sum(fieldA) by fieldB > 110", true)); String query = queries.get(0).toString(); Assert.assertEquals("(fieldA: \"valueA\") AND (mappedB: \"valueB\") AND (fieldC: \"valueC\")", query); @@ -112,7 +119,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError { } public void testMinAggregationWithGroupBy() throws IOException, SigmaError { - OSQueryBackend queryBackend = new OSQueryBackend("windows", true, true); + OSQueryBackend queryBackend = new OSQueryBackend(windowsFieldMappings, true, true); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -143,7 +150,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError { } public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { - OSQueryBackend queryBackend = new OSQueryBackend("windows", true, true); + OSQueryBackend queryBackend = new OSQueryBackend(windowsFieldMappings, true, true); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -174,7 +181,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { } public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { - OSQueryBackend queryBackend = new OSQueryBackend("windows", true, true); + OSQueryBackend queryBackend = new OSQueryBackend(windowsFieldMappings, true, true); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -203,4 +210,4 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } -} +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index d22cb2896..5dc4c7a9b 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -4,39 +4,45 @@ */ package org.opensearch.securityanalytics.rules.backend; +import java.io.IOException; +import java.util.List; +import java.util.Map; import org.junit.Assert; import org.opensearch.securityanalytics.rules.exceptions.SigmaError; -import org.opensearch.securityanalytics.rules.exceptions.SigmaIdentifierError; import org.opensearch.securityanalytics.rules.exceptions.SigmaTypeError; import org.opensearch.securityanalytics.rules.exceptions.SigmaValueError; import org.opensearch.securityanalytics.rules.objects.SigmaRule; import org.opensearch.test.OpenSearchTestCase; -import java.io.IOException; -import java.util.Collections; -import java.util.List; - public class QueryBackendTests extends OpenSearchTestCase { + private static Map testFieldMapping = Map.of( + "EventID", "event_uid", + "HiveName", "unmapped.HiveName", + "fieldB", "mappedB", + "fieldA1", "mappedA", + "creationTime", "timestamp" + ); + public void testBackendPipeline() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: valueA\n" + - " fieldB: valueB\n" + - " fieldC: valueC\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: valueA\n" + + " fieldB: valueB\n" + + " fieldC: valueC\n" + + " condition: sel", false)); Assert.assertEquals("(fieldA: \"valueA\") AND (mappedB: \"valueB\") AND (fieldC: \"valueC\")", queries.get(0).toString()); } @@ -44,21 +50,21 @@ public void testBackendAndCustomPipeline() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: valueA\n" + - " fieldB1: valueB\n" + - " fieldC1: valueC\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: valueA\n" + + " fieldB1: valueB\n" + + " fieldC1: valueC\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: \"valueA\") AND (fieldB1: \"valueB\") AND (fieldC1: \"valueC\")", queries.get(0).toString()); } @@ -66,19 +72,19 @@ public void testConvertValueStr() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: value\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: value\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: \"value\"", queries.get(0).toString()); } @@ -86,19 +92,19 @@ public void testConvertValueStrStartsWith() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|startswith: \"value\"\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|startswith: \"value\"\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: value*", queries.get(0).toString()); } @@ -106,19 +112,19 @@ public void testConvertValueStrStartsWithFurtherWildcard() throws IOException, S OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|startswith: \"va*lue\"\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|startswith: \"va*lue\"\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: va*lue*", queries.get(0).toString()); } @@ -126,19 +132,19 @@ public void testConvertValueStrEndsWith() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|endswith: \"value\"\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|endswith: \"value\"\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: *value", queries.get(0).toString()); } @@ -146,19 +152,19 @@ public void testConvertValueStrEndsWithFurtherWildcard() throws IOException, Sig OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|endswith: \"va*lue\"\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|endswith: \"va*lue\"\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: *va*lue", queries.get(0).toString()); } @@ -206,21 +212,21 @@ public void testConvertValueExpansionWithAll() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " CommandLine|windash|contains|all:\n" + - " - -foo\n" + - " - -bar\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " CommandLine|windash|contains|all:\n" + + " - -foo\n" + + " - -bar\n" + + " condition: sel", false)); Assert.assertEquals("((CommandLine: *\\-foo*) OR (CommandLine: *\\/foo*)) AND ((CommandLine: *\\-bar*) OR (CommandLine: *\\/bar*))", queries.get(0).toString()); } @@ -228,19 +234,19 @@ public void testConvertValueNum() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: 123\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: 123\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: 123", queries.get(0).toString()); } @@ -248,20 +254,20 @@ public void testConvertValueBool() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: true\n" + - " fieldB1: false\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: true\n" + + " fieldB1: false\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: true) AND (fieldB1: false)", queries.get(0).toString()); } @@ -269,19 +275,19 @@ public void testConvertValueNull() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: null\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: null\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: null", queries.get(0).toString()); } @@ -289,19 +295,19 @@ public void testConvertValueRegex() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|re: pat.*tern\"foo\"bar\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|re: pat.*tern\"foo\"bar\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: /pat.*tern\\\"foo\\\"bar/", queries.get(0).toString()); } @@ -309,19 +315,19 @@ public void testConvertValueRegexUnbound() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " \"|re\": pat.*tern\"foo\"bar\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " \"|re\": pat.*tern\"foo\"bar\n" + + " condition: sel", false)); Assert.assertEquals("_0: /pat.*tern\\\"foo\\\"bar/", queries.get(0).toString()); } @@ -329,19 +335,19 @@ public void testConvertValueCidrWildcardNone() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|cidr: 192.168.0.0/14\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|cidr: 192.168.0.0/14\n" + + " condition: sel", false)); Assert.assertEquals("mappedA: \"192.168.0.0/14\"", queries.get(0).toString()); } @@ -349,22 +355,22 @@ public void testConvertCompare() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA|lt: 123\n" + - " fieldB|lte: 123\n" + - " fieldC|gt: 123\n" + - " fieldD|gte: 123\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA|lt: 123\n" + + " fieldB|lte: 123\n" + + " fieldC|gt: 123\n" + + " fieldD|gte: 123\n" + + " condition: sel", false)); Assert.assertEquals("(\"fieldA\" \"lt\" 123) AND (\"mappedB\" \"lte\" 123) AND (\"fieldC\" \"gt\" 123) AND (\"fieldD\" \"gte\" 123)", queries.get(0).toString()); } @@ -373,41 +379,41 @@ public void testConvertCompareStr() throws IOException { assertThrows(SigmaTypeError.class, () -> { queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA|lt: test\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA|lt: test\n" + + " condition: sel", false)); });} public void testConvertOrInList() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: \n" + - " - value1\n" + - " - value2\n" + - " - value4\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2\n" + + " - value4\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value4\")", queries.get(0).toString()); } @@ -415,22 +421,22 @@ public void testConvertOrInListWithWildcards() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: \n" + - " - value1\n" + - " - value2*\n" + - " - val*ue3\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2*\n" + + " - val*ue3\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: value2*) OR (mappedA: val*ue3)", queries.get(0).toString()); } @@ -438,23 +444,23 @@ public void testConvertOrInSeparate() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA1: value1\n" + - " sel2:\n" + - " fieldA1: value2\n" + - " sel3:\n" + - " fieldA1: value4\n" + - " condition: sel1 or sel2 or sel3", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA1: value1\n" + + " sel2:\n" + + " fieldA1: value2\n" + + " sel3:\n" + + " fieldA1: value4\n" + + " condition: sel1 or sel2 or sel3", false)); Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\")) OR (mappedA: \"value4\")", queries.get(0).toString()); } @@ -462,22 +468,22 @@ public void testConvertOrInMixedKeywordField() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA: value1\n" + - " sel2:\n" + - " fieldB: value2\n" + - " sel3: value3\n" + - " condition: sel1 or sel2 or sel3", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: value1\n" + + " sel2:\n" + + " fieldB: value2\n" + + " sel3: value3\n" + + " condition: sel1 or sel2 or sel3", false)); Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (_0: \"value3\")", queries.get(0).toString()); } @@ -485,23 +491,23 @@ public void testConvertOrInMixedFields() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA1: value1\n" + - " sel2:\n" + - " fieldB1: value2\n" + - " sel3:\n" + - " fieldA1: value4\n" + - " condition: sel1 or sel2 or sel3", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA1: value1\n" + + " sel2:\n" + + " fieldB1: value2\n" + + " sel3:\n" + + " fieldA1: value4\n" + + " condition: sel1 or sel2 or sel3", false)); Assert.assertEquals("((mappedA: \"value1\") OR (fieldB1: \"value2\")) OR (mappedA: \"value4\")", queries.get(0).toString()); } @@ -509,22 +515,22 @@ public void testConvertOrInUnallowedValueType() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: \n" + - " - value1\n" + - " - value2\n" + - " - null\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2\n" + + " - null\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: null)", queries.get(0).toString()); } @@ -532,22 +538,22 @@ public void testConvertOrInListNumbers() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: \n" + - " - 1\n" + - " - 2\n" + - " - 4\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - 1\n" + + " - 2\n" + + " - 4\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: 1) OR (mappedA: 2) OR (mappedA: 4)", queries.get(0).toString()); } @@ -555,22 +561,22 @@ public void testConvertAndInList() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1|all:\n" + - " - value1\n" + - " - value2\n" + - " - value4\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1|all:\n" + + " - value1\n" + + " - value2\n" + + " - value4\n" + + " condition: sel", false)); Assert.assertEquals("(mappedA: \"value1\") AND (mappedA: \"value2\") AND (mappedA: \"value4\")", queries.get(0).toString()); } @@ -578,21 +584,21 @@ public void testConvertUnboundValues() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " - value1\n" + - " - value2\n" + - " - 4\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " - value1\n" + + " - value2\n" + + " - 4\n" + + " condition: sel", false)); Assert.assertEquals("(_0: \"value1\") OR (_1: \"value2\") OR (_2: 4)", queries.get(0).toString()); } @@ -601,18 +607,18 @@ public void testConvertInvalidUnboundBool() throws IOException { Exception exception = assertThrows(SigmaValueError.class, () -> { queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel: true\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel: true\n" + + " condition: sel", false)); }); String expectedMessage = "Unexpected Values"; @@ -626,19 +632,19 @@ public void testConvertInvalidCidr() throws IOException { Exception exception = assertThrows(SigmaValueError.class, () -> { queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel: \n" + - " \"|cidr\": 192.168.0/16\n" + - " condition: sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel: \n" + + " \"|cidr\": 192.168.0/16\n" + + " condition: sel", false)); }); String expectedMessage = "Unexpected Values"; @@ -651,21 +657,21 @@ public void testConvertAnd() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA: value1\n" + - " sel2:\n" + - " fieldC: value2\n" + - " condition: sel1 and sel2", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: value1\n" + + " sel2:\n" + + " fieldC: value2\n" + + " condition: sel1 and sel2", false)); Assert.assertEquals("(fieldA: \"value1\") AND (fieldC: \"value2\")", queries.get(0).toString()); } @@ -673,21 +679,21 @@ public void testConvertOr() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA: value1\n" + - " sel2:\n" + - " fieldC: value2\n" + - " condition: sel1 or sel2", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: value1\n" + + " sel2:\n" + + " fieldC: value2\n" + + " condition: sel1 or sel2", false)); Assert.assertEquals("(fieldA: \"value1\") OR (fieldC: \"value2\")", queries.get(0).toString()); } @@ -695,19 +701,19 @@ public void testConvertNot() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: value1\n" + - " condition: not sel", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: value1\n" + + " condition: not sel", false)); Assert.assertEquals("(NOT fieldA: \"value1\")", queries.get(0).toString()); } @@ -715,25 +721,25 @@ public void testConvertPrecedence() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA: value1\n" + - " sel2:\n" + - " fieldB: value2\n" + - " sel3:\n" + - " fieldC: value4\n" + - " sel4:\n" + - " fieldD: value5\n" + - " condition: (sel1 or sel2) and not (sel3 and sel4)", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: value1\n" + + " sel2:\n" + + " fieldB: value2\n" + + " sel3:\n" + + " fieldC: value4\n" + + " sel4:\n" + + " fieldD: value5\n" + + " condition: (sel1 or sel2) and not (sel3 and sel4)", false)); Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((NOT ((fieldC: \"value4\") AND (fieldD: \"value5\"))))", queries.get(0).toString()); } @@ -741,29 +747,29 @@ public void testConvertMultiConditions() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel1:\n" + - " fieldA: value1\n" + - " sel2:\n" + - " fieldC: value2\n" + - " condition:\n" + - " - sel1\n" + - " - sel2", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: value1\n" + + " sel2:\n" + + " fieldC: value2\n" + + " condition:\n" + + " - sel1\n" + + " - sel2", false)); Assert.assertEquals("fieldA: \"value1\"", queries.get(0).toString()); Assert.assertEquals("fieldC: \"value2\"", queries.get(1).toString()); } public void testConvertListCidrWildcardNone() throws IOException, SigmaError { - OSQueryBackend queryBackend = testBackend(); + OSQueryBackend queryBackend = new OSQueryBackend(null, false, false); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -788,23 +794,23 @@ public void testConvertNetworkRule() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " selection_webdav:\n" + - " - c-useragent|contains: 'WebDAV'\n" + - " - c-uri|contains: 'webdav'\n" + - " selection_executable:\n" + - " - resp_mime_types|contains: 'dosexec'\n" + - " - c-uri|endswith: '.exe'\n" + - " condition: selection_webdav and selection_executable", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " selection_webdav:\n" + + " - c-useragent|contains: 'WebDAV'\n" + + " - c-uri|contains: 'webdav'\n" + + " selection_executable:\n" + + " - resp_mime_types|contains: 'dosexec'\n" + + " - c-uri|endswith: '.exe'\n" + + " condition: selection_webdav and selection_executable", false)); Assert.assertEquals("((c-useragent: *WebDAV*) OR (c-uri: *webdav*)) AND ((resp_mime_types: *dosexec*) OR (c-uri: *.exe))", queries.get(0).toString()); } @@ -812,29 +818,29 @@ public void testConvertRegexpRule() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " select_file_with_asterisk:\n" + - " Image: '/usr/bin/file'\n" + - " CommandLine|re: '(.){200,}' # execution of the 'file */* *>> /tmp/output.txt' will produce huge commandline\n" + - " select_recursive_ls:\n" + - " Image: '/bin/ls'\n" + - " CommandLine|contains: '-R'\n" + - " select_find_execution:\n" + - " Image: '/usr/bin/find'\n" + - " select_mdfind_execution:\n" + - " Image: '/usr/bin/mdfind'\n" + - " select_tree_execution|endswith:\n" + - " Image: '/tree'\n" + - " condition: 1 of select*", false)); + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " select_file_with_asterisk:\n" + + " Image: '/usr/bin/file'\n" + + " CommandLine|re: '(.){200,}' # execution of the 'file */* *>> /tmp/output.txt' will produce huge commandline\n" + + " select_recursive_ls:\n" + + " Image: '/bin/ls'\n" + + " CommandLine|contains: '-R'\n" + + " select_find_execution:\n" + + " Image: '/usr/bin/find'\n" + + " select_mdfind_execution:\n" + + " Image: '/usr/bin/mdfind'\n" + + " select_tree_execution|endswith:\n" + + " Image: '/tree'\n" + + " condition: 1 of select*", false)); Assert.assertEquals("(Image: \"\\/usr\\/bin\\/find\") OR (Image: \"\\/tree\") OR (Image: \"\\/usr\\/bin\\/mdfind\") OR ((Image: \"\\/usr\\/bin\\/file\") AND (CommandLine: /(.){200,}/)) OR ((Image: \"\\/bin\\/ls\") AND (CommandLine: *\\-R*))", queries.get(0).toString()); } @@ -877,6 +883,6 @@ public void testConvertProxyRule() throws IOException, SigmaError { } private OSQueryBackend testBackend() throws IOException { - return new OSQueryBackend("others_proxy", true, true); + return new OSQueryBackend(testFieldMapping, false, true); } } \ No newline at end of file diff --git a/src/test/resources/ad_ldap-sample.json b/src/test/resources/ad_ldap-sample.json index 3101b1af4..10c05dc02 100644 --- a/src/test/resources/ad_ldap-sample.json +++ b/src/test/resources/ad_ldap-sample.json @@ -1,21 +1,22 @@ { - "azure-signinlogs-properties-user_id": "1234", - "azure-activitylogs-category": "1234", - "azure-platformlogs-operation_name": "1234", - "modified_properties-new_value": "1234", - "azure-resource-provider": "1234", - "azure-signinlogs-properties-conditional_access_status": "1234", + "azure.signinlogs.properties.user_id": "1234", + "azure.activitylogs.category": "1234", + "azure.platformlogs.operation_name": "1234", + "modified_properties.new_value": "1234", + "azure.resource.provider": "1234", + "azure.signinlogs.properties.conditional_access_status": "1234", "SearchFilter": "1234", - "azure-platformlogs-result_type": "1234", - "azure-signinlogs-properties-device_detail-is_compliant": "1234", - "ResourceDisplayName": "1234", - "azure-signinlogs-properties-authentication_requirement": "1234", - "TargetResources": "1234", - "Workload": "1234", - "azure-signinlogs-properties-device_detail-device_id": "1234", - "azure-signinlogs-properties-resource_id": "1234", + "azure.platformlogs.result_type": "1234", + "azure.signinlogs.result_description": "1234", + "azure.signinlogs.properties.device_detail.is_compliant": "1234", + "resource_display_name": "1234", + "azure.signinlogs.properties.authentication_requirement": "1234", + "target_resources": "1234", + "workload": "1234", + "azure.signinlogs.properties.device_detail.device_id": "1234", + "azure.signinlogs.properties.resource_id": "1234", "EventID": 12345, - "azure-signinlogs-properties-network_location_details": "1234", - "azure-auditlogs-properties-activity_display_name": "1234", + "azure.signinlogs.properties.network_location_details": "1234", + "azure.auditlogs.properties.activity_display_name": "1234", "@timestamp": "2022-12-27T20:29:31.734Z" } \ No newline at end of file