[SIEM] Office 365 module (elastic#16386)

This includes a new fileset, o365.audit, that uses the o365audit input
to ingest logs using the Office 365 Management API.

filebeat/docs/filebeat-options.asciidoc

@@ -57,6 +57,7 @@ You can configure {beatname_uc} to use the following inputs:
 * <<{beatname_lc}-input-google-pubsub>>
 * <<{beatname_lc}-input-azure-eventhub>>
 * <<{beatname_lc}-input-cloudfoundry>>
+* <<{beatname_lc}-input-o365audit>>
 
 
 include::multiline.asciidoc[]
@@ -90,3 +91,5 @@ include::../../x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc[]
 include::../../x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc[]
+
+include::../../x-pack/filebeat/docs/inputs/input-o365audit.asciidoc[]

filebeat/docs/modules/o365.asciidoc

@@ -0,0 +1,226 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-o365]]
+[role="xpack"]
+
+:modulename: o365
+:has-dashboards: true
+
+== Office 365 module
+
+This is a module for Office 365 logs received via one of the Office 365 API
+endpoints. It currently supports user, admin, system, and policy actions and
+events from Office 365 and Azure AD activity logs exposed by the Office 365
+Management Activity API.
+
+The {plugins}/ingest-geoip.html[ingest-geoip] and
+{plugins}/ingest-user-agent.html[ingest-user_agent] Elasticsearch plugins are
+required to run this module.
+
+include::../include/gs-link.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+:fileset_ex: audit
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `audit` fileset settings
+
+The `audit` fileset uses the Office 365 Management Activity API to retrieve
+audit messages from Office 365 and Azure AD activity logs. These are the same
+logs that are available under _Audit_ _Log_ _Search_ in the _Security_ _and_
+_Compliance_ _Center._
+
+[float]
+===== Setup
+
+To use this fileset you need to https://docs.microsoft.com/en-us/microsoft-365/compliance/turn-audit-log-search-on-or-off?view=o365-worldwide#turn-on-audit-log-search[enable Audit Log Search]
+ and https://docs.microsoft.com/en-us/office/office-365-management-api/get-started-with-office-365-management-apis#register-your-application-in-azure-ad[register an application in Azure AD.]
+
+Once this application is registered note the _Application (client) ID_ and the
+_Directory (tenant) ID._ Then configure the authentication in the _Certificates & Secrets_
+section.
+
+
+Example configuration `o365.yml` using client-secret authentication:
+
+[source,yaml]
+----
+  audit:
+    enabled: true
+    var.application_id: "<My Azure AD Application ID>"
+    var.tenants:
+      - id: "<My Tenant ID>"
+        name: "mytenant.onmicrosoft.com"
+    var.client_secret: "<My client secret>"
+----
+
+Certificate-based authentication is specially useful when monitoring multiple
+tenants. Example configuration:
+
+[source,yaml]
+----
+  audit:
+    enabled: true
+    var.application_id: "<My Azure AD Application ID>"
+    var.tenants:
+      - id: "<Tenant A ID>"
+        name: "tenantA.onmicrosoft.com"
+      - id: "<Tenant B ID>"
+        name: "tenantB.onmicrosoft.com"
+    var.certificate: "/path/to/certificate.pem"
+    var.key: "/path/to/private_key.pem"
+    var.key_passphrase: "my_passphrase" # (optional) for encrypted keys
+----
+
+Finally you need to add permissions in the _API permissions_ section and grant
+it admin consent. Click on _Add permission_ and select
+_Office 365 Management APIs._ The needed permissions are:
+
+- User.Read
+- ActivityFeed.Read
+- ActivityFeed.ReadDlp
+- ServiceHealth.Read
+
+[role="screenshot"]
+image::./images/filebeat-o365-azure-permissions.png[]
+
+Once the required permissions are added, click the _Grant admin consent_ button.
+Note that it can take a while for the required permissions to be in effect, so
+it's possible that you observe some permission errors when running {beatname_uc}
+right away.
+
+[float]
+===== Alternative endpoints
+
+This module supports custom endpoints for on-prem deployments as well as
+alternative endpoints (GCC High endponts, U.S. DoD, European Union, etc). In
+order to point the module to an alternative endpoint, you need to adjust the
+`authentication_endpoint` and `resource` variables accordingly. For example:
+
+[source,yaml]
+----
+    var.api:
+      # default is https://login.microsoftonline.com/
+      authentication_endpoint: https://login.microsoftonline.us/
+      # default is https://manage.office.com
+      resource: https://manage.office365.us
+----
+
+[float]
+===== Configuration options
+
+*`var.application_id`*::
+
+The Application ID (also known as client ID) of the Azure application.
+
+*`var.tenants`*::
+
+A list of one or more tenant IDs and name pairs. Set the `id` field to the
+tenant ID (also known as Directory ID). Set the name to the host name for the
+tenant, that is, the Office 365 domain for your organization.
+
+*`var.client_secret`*::
+
+The client-secret (api_key) used to authenticate your Azure AD application. This
+option cannot be specified at the same time as the `var.certificate` option.
+
+*`var.certificate`*::
+
+Path to the certificate file used for client authentication. This option cannot
+be specified at the same time as the `var.client_secret` option.
+
+*`var.key`*::
+
+Path to the private key file used for client authentication.
+
+*`var.key_passphrase`*::
+
+The passphrase used to decrypt an encrypted key stored in the configured
+`var.key` file. Only set this option when the key is encrypted.
+
+*`var.content_type`*::
+
+The list of content-types to subscribe to. By default, it subscribes to all
+known content-types:
+- Audit.AzureActiveDirectory
+- Audit.Exchange
+- Audit.SharePoint
+- Audit.General
+- DLP.All
+
+
+[float]
+===== Advanced configuration options
+
+The following configuration options are only recomended in case of problems.
+They must be nested under a single `var.api` key, like this:
+
+[source,yaml]
+----
+    var.api:
+      authentication_endpoint: https://login.microsoftonline.com/
+      resource: https://manage.office.com
+      max_retention: 168h
+      poll_interval: 3m
+      max_requests_per_minute: 2000
+      max_query_size: 24h
+----
+
+*`var.api.authentication_endpoint`*::
+
+The authentication endpoint used to authorize the Azure app. This is
+`https://login.microsoftonline.com/` by default, and can be changed to access
+alternative endpoints.
+
+*`var.api.resource`*::
+
+The API resource to retrieve information from. This is
+`https://manage.office.com` by default, and can be changed to access alternative
+endpoints.
+
+*`var.api.max_retention`*::
+
+The maximum data retention period to support. `168h` by default. {beatname_uc}
+will fetch all retained data for a tenant when run for the first time. The
+default is 7 days. Adjust it if your tenant has a different retention period.
+
+*`var.api.poll_interval`*::
+
+The interval to wait before polling the API server for new events. Default `3m`.
+
+*`var.api.max_requests_per_minute`*::
+
+The maximum number of requests to perform per minute, for each tenant. The
+default is `2000`, as this is the server-side limit per tenant.
+
+*`var.api.max_query_size`*::
+
+The maximum time window that API allows in a single query. Defaults to `24h`
+to match Microsoft's documented limit.
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard:
+
+[role="screenshot"]
+image::./images/filebeat-o365-audit.png[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-o365,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -29,6 +29,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-nats>>
   * <<filebeat-module-netflow>>
   * <<filebeat-module-nginx>>
+  * <<filebeat-module-o365>>
   * <<filebeat-module-okta>>
   * <<filebeat-module-osquery>>
   * <<filebeat-module-panw>>
@@ -71,6 +72,7 @@ include::modules/mysql.asciidoc[]
 include::modules/nats.asciidoc[]
 include::modules/netflow.asciidoc[]
 include::modules/nginx.asciidoc[]
+include::modules/o365.asciidoc[]
 include::modules/okta.asciidoc[]
 include::modules/osquery.asciidoc[]
 include::modules/panw.asciidoc[]

x-pack/filebeat/docs/inputs/input-o365audit.asciidoc

@@ -23,8 +23,7 @@ This input doesn't perform any transformation on the incoming messages, notably
 no {ecs-ref}/ecs-reference.html[Elastic Common Schema fields] are populated, and
 some data is encoded as arrays of objects, which are difficult to query in
 Elasticsearch. You probably want to use the
-{filebeat-ref}/filebeat-module-o365.html[o365 module] instead.
-// TODO: link to O365 module docs.
+{filebeat-ref}/filebeat-module-o365.html[Office 365 module] instead.
 
 Example configuration:
 
@@ -116,7 +115,7 @@ endpoints.
 
 ===== `api.max_retention`
 
-The maximum data retention period to support. `178h` by default. {beatname_uc}
+The maximum data retention period to support. `168h` by default. {beatname_uc}
 will fetch all retained data for a tenant when run for the first time.
 
 ===== `api.poll_interval`
@@ -132,3 +131,8 @@ default is `2000`, as this is the server-side limit per tenant.
 
 The maximum time window that API allows in a single query. Defaults to `24h`
 to match Microsoft's documented limit.
+
+[id="{beatname_lc}-input-{type}-common-options"]
+include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[]
+
+:type!:

x-pack/filebeat/filebeat.reference.yml

@@ -695,6 +695,53 @@ filebeat.modules:
   #  # Filebeat will choose the paths depending on your OS.
   #  #var.paths:
 
+#------------------------------ Office 365 Module ------------------------------
+- module: o365
+  audit:
+    enabled: true
+
+    # Set the application_id (also known as client ID):
+    var.application_id: "<MyApplicationID>"
+
+    # Configure the tenants to monitor:
+    # Use the tenant ID (also known as directory ID) and the domain name.
+    # var.tenants:
+    #  - id: "tenant_id_1"
+    #    name: "mydomain.onmicrosoft.com"
+    #  - id: "tenant_id_2"
+    #    name: "mycompany.com"
+    var.tenants:
+     - id: "<MyTenantID>"
+       name: "mytenant.onmicrosoft.com"
+
+    # List of content-types to fetch. By default all known content-types
+    # are retrieved:
+    # var.content_type:
+    #  - "Audit.AzureActiveDirectory"
+    #  - "Audit.Exchange"
+    #  - "Audit.SharePoint"
+    #  - "Audit.General"
+    #  - "DLP.All"
+
+    # Use the following settings to enable certificate-based authentication:
+    # var.certificate: "/path/to/certificate.pem"
+    # var.key: "/path/to/private_key.pem"
+    # var.key_passphrase: "myPrivateKeyPassword"
+
+    # Client-secret based authentication:
+    # Comment the following line if using certificate authentication.
+    var.client_secret: "<YourClientSecretHere>"
+
+    # Advanced settings, use with care:
+    # var.api:
+    #   # Settings for custom endpoints:
+    #   authentication_endpoint: "https://login.microsoftonline.us/"
+    #   resource: "https://manage.office365.us"
+    #
+    #   max_retention: 7d
+    #   max_requests_per_minute: 2000
+    #   poll_interval: 3m
+
 #--------------------------------- Okta Module ---------------------------------
 - module: okta
   system:

x-pack/filebeat/input/o365audit/config.go

@@ -6,6 +6,7 @@ package o365audit
 
 import (
 	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/pkg/errors"
@@ -146,6 +147,14 @@ func (c *Config) Validate() (err error) {
 			return errors.Wrap(err, "invalid certificate config")
 		}
 	}
+	c.API.Resource, err = forceURLScheme(c.API.Resource, "https")
+	if err != nil {
+		return errors.Wrapf(err, "resource '%s' is not a valid URL", c.API.Resource)
+	}
+	c.API.AuthenticationEndpoint, err = forceURLScheme(c.API.AuthenticationEndpoint, "https")
+	if err != nil {
+		return errors.Wrapf(err, "authentication_endpoint '%s' is not a valid URL", c.API.AuthenticationEndpoint)
+	}
 	return nil
 }
 
@@ -193,3 +202,20 @@ func (c *Config) NewTokenProvider(tenantID string) (auth.TokenProvider, error) {
 		c.CertificateConfig,
 	)
 }
+
+// Ensures that the passed URL has a scheme, using the provided one if needed.
+// Returns an error is the URL can't be parsed.
+func forceURLScheme(baseURL, scheme string) (urlWithScheme string, err error) {
+	parsed, err := url.Parse(baseURL)
+	if err != nil {
+		return "", err
+	}
+	// Scheme is mandatory
+	if parsed.Scheme == "" {
+		withResource := "https://" + baseURL
+		if parsed, err = url.Parse(withResource); err != nil {
+			return "", err
+		}
+	}
+	return parsed.String(), nil
+}