New Log Type JSON format (#465)

* New Log Type JSON format
Signed-off-by: Petar Dzepina <[email protected]>
Co-authored-by: Subhobrata Dey <[email protected]>
(cherry picked from commit faccf6c)

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -45,6 +45,7 @@
 import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
 import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder;
 import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
+import org.opensearch.securityanalytics.logtype.LogTypeService;
 import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
 import org.opensearch.securityanalytics.mapper.MapperService;
 import org.opensearch.securityanalytics.resthandler.*;
@@ -90,6 +91,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
 
     private IndexTemplateManager indexTemplateManager;
 
+    private LogTypeService logTypeService;
+
     @Override
     public Collection<Object> createComponents(Client client,
                                                ClusterService clusterService,
@@ -102,12 +105,13 @@ public Collection<Object> createComponents(Client client,
                                                NamedWriteableRegistry namedWriteableRegistry,
                                                IndexNameExpressionResolver indexNameExpressionResolver,
                                                Supplier<RepositoriesService> repositoriesServiceSupplier) {
+        logTypeService = new LogTypeService();
         detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
         ruleTopicIndices = new RuleTopicIndices(client, clusterService);
         correlationIndices = new CorrelationIndices(client, clusterService);
         indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry);
         mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager);
-        ruleIndices = new RuleIndices(client, clusterService, threadPool);
+        ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool);
         correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
 
         return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);

src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.logtype;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.SettingsException;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.securityanalytics.model.LogType;
+import org.opensearch.securityanalytics.util.FileUtils;
+
+public class BuiltinLogTypeLoader {
+
+    private static final Logger logger = LogManager.getLogger(BuiltinLogTypeLoader.class);
+
+    private static final String BASE_PATH = "OSMapping/";
+
+    private static final String LOG_TYPE_FILE_SUFFIX = "_logtype.json";
+
+    private static List<LogType> logTypes;
+    private static Map<String, LogType> logTypeMap;
+
+
+    static {
+        ensureLogTypesLoaded();
+    }
+
+    public static List<LogType> getAllLogTypes() {
+        ensureLogTypesLoaded();
+        return logTypes;
+    }
+
+    public static LogType getLogTypeByName(String logTypeName) {
+        ensureLogTypesLoaded();
+        return logTypeMap.get(logTypeName);
+    }
+
+    public static boolean logTypeExists(String logTypeName) {
+        ensureLogTypesLoaded();
+        return logTypeMap.containsKey(logTypeName);
+    }
+
+    private static void ensureLogTypesLoaded() {
+        try {
+            if (logTypes != null) {
+                return;
+            }
+            logTypes = loadBuiltinLogTypes();
+            logTypeMap = logTypes.stream()
+                    .collect(Collectors.toMap(LogType::getName, Function.identity()));
+        } catch (Exception e) {
+            logger.error("Failed loading builtin log types from disk!", e);
+        }
+    }
+
+    private static List<LogType> loadBuiltinLogTypes() throws URISyntaxException, IOException {
+        List<LogType> logTypes = new ArrayList<>();
+
+        final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString();
+
+        Path dirPath = null;
+        if (url.contains("!")) {
+            final String[] paths = url.split("!");
+            dirPath = FileUtils.getFs().getPath(paths[1]);
+        } else {
+            dirPath = Path.of(url);
+        }
+
+        Stream<Path> folder = Files.list(dirPath);
+        List<Path> logTypePaths = folder.filter(e -> e.toString().endsWith(LOG_TYPE_FILE_SUFFIX)).collect(Collectors.toList());
+
+        for (Path logTypePath : logTypePaths) {
+            try (
+                    InputStream is = BuiltinLogTypeLoader.class.getResourceAsStream(logTypePath.toString())
+            ) {
+                String logTypeFilePayload = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8);
+
+                if (logTypeFilePayload != null) {
+                    Map<String, Object> logTypeFileAsMap =
+                            XContentHelper.convertToMap(JsonXContent.jsonXContent, logTypeFilePayload, false);
+
+                    logTypes.add(new LogType(logTypeFileAsMap));
+
+                    logger.info("Loaded [{}] log type", logTypePath.getFileName());
+                }
+            } catch (Exception e) {
+                throw new SettingsException("Failed to load builtin log types", e);
+            }
+        }
+
+        return logTypes;
+    }
+}

src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.logtype;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.securityanalytics.model.LogType;
+import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
+
+
+/**
+ *
+ * */
+public class LogTypeService {
+
+    private static final Logger logger = LogManager.getLogger(LogTypeService.class);
+
+    private BuiltinLogTypeLoader builtinLogTypeLoader;
+
+    public LogTypeService() {
+        this.builtinLogTypeLoader = new BuiltinLogTypeLoader();
+    }
+
+
+    public List<LogType> getAllLogTypes() {
+        return BuiltinLogTypeLoader.getAllLogTypes();
+    }
+
+    public LogType getLogTypeByName(String logType) {
+        return BuiltinLogTypeLoader.getLogTypeByName(logType);
+    }
+
+    /**
+     * Returns sigmaRule rawField to ECS field mapping
+     *
+     * @param logType Log type
+     * @return Map of rawField to ecs field
+     */
+    public Map<String, String> getRuleFieldMappings(String logType) {
+        LogType lt = getLogTypeByName(logType);
+
+        if (lt == null) {
+            throw SecurityAnalyticsException.wrap(new IllegalArgumentException("Can't get rule field mappings for invalid logType: [" + logType + "]"));
+        }
+        if (lt.getMappings() == null) {
+            return Map.of();
+        } else {
+            return lt.getMappings()
+                    .stream()
+                    .collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs));
+        }
+    }
+}

src/main/java/org/opensearch/securityanalytics/model/LogType.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.model;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+public class LogType implements Writeable {
+
+    private static final String ID = "id";
+    private static final String NAME = "name";
+    private static final String DESCRIPTION = "description";
+    private static final String IS_BUILTIN = "is_builtin";
+    private static final String MAPPINGS = "mappings";
+    private static final String RAW_FIELD = "raw_field";
+    private static final String ECS = "ecs";
+    private static final String OCSF = "ocsf";
+
+    private String id;
+    private String name;
+    private String description;
+    private Boolean isBuiltIn;
+    private List<Mapping> mappings;
+
+    public LogType(StreamInput sin) throws IOException {
+        this.id = sin.readString();
+        this.isBuiltIn = sin.readOptionalBoolean();
+        this.name = sin.readString();
+        this.description = sin.readString();
+        this.mappings = sin.readList(Mapping::readFrom);
+    }
+
+    public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings) {
+        this.id = id;
+        this.name = name;
+        this.description = description;
+        this.isBuiltIn = isBuiltIn;
+        this.mappings = mappings == null ? List.of() : mappings;
+    }
+
+    public LogType(Map<String, Object> logTypeAsMap) {
+        this.id = (String) logTypeAsMap.get(ID);
+        this.name = (String) logTypeAsMap.get(NAME);
+        this.description = (String) logTypeAsMap.get(DESCRIPTION);
+        if (logTypeAsMap.containsKey(IS_BUILTIN)) {
+            this.isBuiltIn = (Boolean) logTypeAsMap.get(IS_BUILTIN);
+        }
+        List<Map<String, String>> mappings = (List<Map<String, String>>)logTypeAsMap.get(MAPPINGS);
+        if (mappings.size() > 0) {
+            this.mappings = new ArrayList<>(mappings.size());
+            this.mappings = mappings.stream().map(e ->
+                    new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF))
+            ).collect(Collectors.toList());
+        }
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public boolean getIsBuiltIn() { return isBuiltIn; }
+
+    public List<Mapping> getMappings() {
+        return mappings;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(id);
+        out.writeOptionalBoolean(isBuiltIn);
+        out.writeString(name);
+        out.writeString(description);
+        out.writeCollection(mappings);
+    }
+
+    @Override
+    public String toString() {
+        return name;
+    }
+
+    public static class Mapping implements Writeable {
+
+        private String rawField;
+        private String ecs;
+        private String ocsf;
+
+        public Mapping(StreamInput sin) throws IOException {
+            this.rawField = sin.readString();
+            this.ecs = sin.readOptionalString();
+            this.ocsf = sin.readOptionalString();
+        }
+
+        public Mapping(String rawField, String ecs, String ocsf) {
+            this.rawField = rawField;
+            this.ecs = ecs;
+            this.ocsf = ocsf;
+        }
+
+        public String getRawField() {
+            return rawField;
+        }
+
+        public String getEcs() {
+            return ecs;
+        }
+
+        public String getOcsf() {
+            return ocsf;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(rawField);
+            out.writeOptionalString(ecs);
+            out.writeOptionalString(ocsf);
+        }
+
+        public static Mapping readFrom(StreamInput sin) throws IOException {
+            return new Mapping(sin);
+        }
+    }
+
+}

src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java

@@ -113,8 +113,8 @@ public class OSQueryBackend extends QueryBackend {
 
     private static final List<Class<?>> precedence = Arrays.asList(ConditionNOT.class, ConditionAND.class, ConditionOR.class);
 
-    public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enableFieldMappings) throws IOException {
-        super(ruleCategory, true, enableFieldMappings, true, collectErrors);
+    public OSQueryBackend(Map<String, String> fieldMappings, boolean collectErrors, boolean enableFieldMappings) throws IOException {
+        super(fieldMappings, true, enableFieldMappings, true, collectErrors);
         this.tokenSeparator = " ";
         this.orToken = "OR";
         this.andToken = "AND";
@@ -445,11 +445,7 @@ private String getMappedField(String field) {
     }
 
     private String getFinalField(String field) {
-        field = this.getMappedField(field);
-        if (field.contains(".")) {
-            field = field.replace(".", "_");
-        }
-        return field;
+        return this.getMappedField(field);
     }
 
     private String getFinalValueField() {

src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java

@@ -59,7 +59,7 @@ public abstract class QueryBackend {
     protected Map<String, Object> ruleQueryFields;
 
     @SuppressWarnings("unchecked")
-    public QueryBackend(String ruleCategory, boolean convertAndAsIn, boolean enableFieldMappings, boolean convertOrAsIn, boolean collectErrors) throws IOException {
+    public QueryBackend(Map<String, String> fieldMappings, boolean convertAndAsIn, boolean enableFieldMappings, boolean convertOrAsIn, boolean collectErrors) {
         this.convertAndAsIn = convertAndAsIn;
         this.convertOrAsIn = convertOrAsIn;
         this.collectErrors = collectErrors;
@@ -68,15 +68,7 @@ public QueryBackend(String ruleCategory, boolean convertAndAsIn, boolean enableF
         this.queryFields = new HashMap<>();
 
         if (this.enableFieldMappings) {
-            InputStream is = this.getClass().getClassLoader().getResourceAsStream(String.format(Locale.getDefault(), "OSMapping/%s/fieldmappings.yml", ruleCategory));
-            assert is != null;
-            String content = new String(is.readAllBytes(), Charset.defaultCharset());
-
-            Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions()));
-            Map<String, Object> fieldMappingsObj = yaml.load(content);
-            this.fieldMappings = (Map<String, String>) fieldMappingsObj.get("fieldmappings");
-
-            is.close();
+            this.fieldMappings = fieldMappings;
         } else {
             this.fieldMappings = new HashMap<>();
         }