Add initial synthetic source fallback logic (elastic#112994)

Add initial code required to fallback synthetic source mode to stored
source mode using an index settings provider.

Note that the final version relies on a new index setting  that
determines source mode, which is currently controlled by  `mode` mapping
attribute in `_source` meta field mapper. Additionally index modes
should not enforce synthetic source mode.

x-pack/plugin/logsdb/build.gradle

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-java-rest-test'
+
+esplugin {
+  name 'logsdb'
+  description 'A plugin for logsdb related functionality'
+  classname 'org.elasticsearch.xpack.logsdb.LogsDBPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+base {
+  archivesName = 'x-pack-logsdb'
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  testImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+tasks.named("javaRestTest").configure {
+  usesDefaultDistribution()
+}

x-pack/plugin/logsdb/qa/build.gradle

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+

x-pack/plugin/logsdb/qa/with-basic/build.gradle

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+
+dependencies {
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+tasks.named("javaRestTest").configure {
+  // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
+  BuildParams.withFipsEnabledOnly(it)
+
+  usesDefaultDistribution()
+}

x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class LogsdbRestIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.license.self_generated.type", "basic")
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testFeatureUsageWithLogsdbIndex() throws IOException {
+        {
+            var response = getAsMap("/_license/feature_usage");
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
+            assertThat(features, Matchers.empty());
+        }
+        {
+            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            var response = getAsMap("/_license/feature_usage");
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
+            assertThat(features, Matchers.empty());
+        }
+    }
+
+}

x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LogsdbRestIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testFeatureUsageWithLogsdbIndex() throws IOException {
+        {
+            var response = getAsMap("/_license/feature_usage");
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
+            assertThat(features, Matchers.empty());
+        }
+        {
+            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            var response = getAsMap("/_license/feature_usage");
+            @SuppressWarnings("unchecked")
+            List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
+            logger.info("response's features: {}", features);
+            assertThat(features, Matchers.not(Matchers.empty()));
+            Map<?, ?> feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get();
+            assertThat(feature.get("name"), equalTo("synthetic-source"));
+            assertThat(feature.get("license_level"), equalTo("enterprise"));
+        }
+    }
+
+}

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackPlugin;
+
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
+
+public class LogsDBPlugin extends Plugin {
+
+    private final Settings settings;
+    private final SyntheticSourceLicenseService licenseService;
+
+    public LogsDBPlugin(Settings settings) {
+        this.settings = settings;
+        this.licenseService = new SyntheticSourceLicenseService(settings);
+    }
+
+    @Override
+    public Collection<?> createComponents(PluginServices services) {
+        licenseService.setLicenseState(XPackPlugin.getSharedLicenseState());
+        var clusterSettings = services.clusterService().getClusterSettings();
+        clusterSettings.addSettingsUpdateConsumer(FALLBACK_SETTING, licenseService::setSyntheticSourceFallback);
+        // Nothing to share here:
+        return super.createComponents(services);
+    }
+
+    @Override
+    public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
+        if (DiscoveryNode.isStateless(settings)) {
+            return List.of();
+        }
+        return List.of(new SyntheticSourceIndexSettingsProvider(licenseService));
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(FALLBACK_SETTING);
+    }
+}

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.index.IndexSettings;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
+ */
+public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
+
+    private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
+
+    private final SyntheticSourceLicenseService syntheticSourceLicenseService;
+
+    public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
+        this.syntheticSourceLicenseService = syntheticSourceLicenseService;
+    }
+
+    @Override
+    public Settings getAdditionalIndexSettings(
+        String indexName,
+        String dataStreamName,
+        boolean isTimeSeries,
+        Metadata metadata,
+        Instant resolvedAt,
+        Settings indexTemplateAndCreateRequestSettings,
+        List<CompressedXContent> combinedTemplateMappings
+    ) {
+        if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
+            && syntheticSourceLicenseService.fallbackToStoredSource()) {
+            LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
+            // TODO: handle falling back to stored source
+        }
+        return Settings.EMPTY;
+    }
+
+    boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
+        // TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
+        // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
+        var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
+        IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
+        return indexMode != null && indexMode.isSyntheticSourceEnabled();
+    }
+}

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicensedFeature;
+import org.elasticsearch.license.XPackLicenseState;
+
+/**
+ * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
+ */
+public final class SyntheticSourceLicenseService {
+
+    private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
+
+    /**
+     * A setting that determines whether source mode should always be stored source. Regardless of licence.
+     */
+    public static final Setting<Boolean> FALLBACK_SETTING = Setting.boolSetting(
+        "xpack.mapping.synthetic_source_fallback_to_stored_source",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    private static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary(
+        MAPPINGS_FEATURE_FAMILY,
+        "synthetic-source",
+        License.OperationMode.ENTERPRISE
+    );
+
+    private XPackLicenseState licenseState;
+    private volatile boolean syntheticSourceFallback;
+
+    public SyntheticSourceLicenseService(Settings settings) {
+        syntheticSourceFallback = FALLBACK_SETTING.get(settings);
+    }
+
+    /**
+     * @return whether synthetic source mode should fallback to stored source.
+     */
+    public boolean fallbackToStoredSource() {
+        if (syntheticSourceFallback) {
+            return true;
+        }
+
+        return SYNTHETIC_SOURCE_FEATURE.check(licenseState) == false;
+    }
+
+    void setSyntheticSourceFallback(boolean syntheticSourceFallback) {
+        this.syntheticSourceFallback = syntheticSourceFallback;
+    }
+
+    void setLicenseState(XPackLicenseState licenseState) {
+        this.licenseState = licenseState;
+    }
+}