Set xpack.security.enabled to true for all licenses

qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java

@@ -20,6 +20,8 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
 import static java.nio.file.StandardOpenOption.APPEND;
@@ -46,6 +48,8 @@ public static void cleanupFiles() {
 
     public void test10Install() throws Exception {
         install();
+        // Enable security for this test only where it is necessary, until we can enable it for all
+        ServerUtils.enableSecurityFeatures(installation);
     }
 
     public void test20Help() {
@@ -107,8 +111,24 @@ public void test40RunWithCert() throws Exception {
 
         Files.write(installation.config("elasticsearch.yml"), yaml, CREATE, APPEND);
 
-        assertWhileRunning(
-            () -> ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), null, null, installation.config("certs/ca/ca.crt"))
-        );
+        assertWhileRunning(() -> {
+            final String password = setElasticPassword();
+            assertNotNull(password);
+            ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), "elastic", password, installation.config("certs/ca/ca.crt"));
+        });
+    }
+
+    private String setElasticPassword() {
+        final Pattern userpassRegex = Pattern.compile("PASSWORD (\\w+) = ([^\\s]+)");
+        Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+        Matcher matcher = userpassRegex.matcher(result.stdout);
+        assertNotNull(matcher);
+        while (matcher.find()) {
+            if (matcher.group(1).equals("elastic")) {
+                return matcher.group(2);
+            }
+        }
+        return null;
     }
+
 }

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -80,6 +80,8 @@
  */
 public class DockerTests extends PackagingTestCase {
     private Path tempDir;
+    private static final String USERNAME = "elastic";
+    private static final String PASSWORD = "nothunter2";
 
     @BeforeClass
     public static void filterDistros() {
@@ -88,7 +90,10 @@ public static void filterDistros() {
 
     @Before
     public void setupTest() throws IOException {
-        installation = runContainer(distribution(), builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false")));
+        installation = runContainer(
+            distribution(),
+            builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+        );
         tempDir = createTempDir(DockerTests.class.getSimpleName());
     }
 
@@ -106,12 +111,18 @@ public void test010Install() {
     }
 
     /**
-     * Check that the /_xpack API endpoint's presence is correct for the type of distribution being tested.
+     * Check that security is enabled
      */
-    public void test011PresenceOfXpack() throws Exception {
+    public void test011SecurityEnabledStatus() throws Exception {
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        final int statusCode = ServerUtils.makeRequestAndGetStatus(Request.Get("http://localhost:9200"), USERNAME, "wrong_password", null);
+        assertThat(statusCode, equalTo(401));
+
+        // restart container with security disabled
+        runContainer(distribution(), builder().envVars(Map.of("xpack.security.enabled", "false")));
         waitForElasticsearch(installation);
-        final int statusCode = Request.Get("http://localhost:9200/_xpack").execute().returnResponse().getStatusLine().getStatusCode();
-        assertThat(statusCode, equalTo(200));
+        final int unauthStatusCode = ServerUtils.makeRequestAndGetStatus(Request.Get("http://localhost:9200"), null, null, null);
+        assertThat(unauthStatusCode, equalTo(200));
     }
 
     /**
@@ -147,7 +158,7 @@ public void test041AmazonCaCertsAreInTheKeystore() {
      * Check that when the keystore is created on startup, it is created with the correct permissions.
      */
     public void test042KeystorePermissionsAreCorrect() throws Exception {
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
         assertPermissionsAndOwnership(installation.config("elasticsearch.keystore"), "elasticsearch", "root", p660);
     }
@@ -157,11 +168,11 @@ public void test042KeystorePermissionsAreCorrect() throws Exception {
      * is minimally functional.
      */
     public void test050BasicApiTests() throws Exception {
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
         assertTrue(existsInContainer(installation.logs.resolve("gc.log")));
 
-        ServerUtils.runElasticsearchTests();
+        ServerUtils.runElasticsearchTests(USERNAME, PASSWORD);
     }
 
     /**
@@ -185,11 +196,24 @@ public void test070BindMountCustomPathConfAndJvmOptions() throws Exception {
 
         // Restart the container
         final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/usr/share/elasticsearch/config"));
-        runContainer(distribution(), builder().volumes(volumes).envVars(Map.of("ES_JAVA_OPTS", "-XX:-UseCompressedOops")));
+        runContainer(
+            distribution(),
+            builder().volumes(volumes)
+                .envVars(
+                    Map.of(
+                        "ES_JAVA_OPTS",
+                        "-XX:-UseCompressedOops",
+                        "ingest.geoip.downloader.enabled",
+                        "false",
+                        "ELASTIC_PASSWORD",
+                        PASSWORD
+                    )
+                )
+        );
 
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
-        final JsonNode nodes = getJson("/_nodes").get("nodes");
+        final JsonNode nodes = getJson("/_nodes", USERNAME, PASSWORD).get("nodes");
         final String nodeId = nodes.fieldNames().next();
 
         final int heapSize = nodes.at("/" + nodeId + "/jvm/mem/heap_init_in_bytes").intValue();
@@ -213,11 +237,14 @@ public void test071BindMountCustomPathWithDifferentUID() throws Exception {
             // Restart the container
             final Map<Path, Path> volumes = Map.of(tempEsDataDir.toAbsolutePath(), installation.data);
 
-            runContainer(distribution(), builder().volumes(volumes));
+            runContainer(
+                distribution(),
+                builder().volumes(volumes).envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+            );
 
-            waitForElasticsearch(installation);
+            waitForElasticsearch(installation, USERNAME, PASSWORD);
 
-            final JsonNode nodes = getJson("/_nodes");
+            final JsonNode nodes = getJson("/_nodes", USERNAME, PASSWORD);
 
             assertThat(nodes.at("/_nodes/total").intValue(), equalTo(1));
             assertThat(nodes.at("/_nodes/successful").intValue(), equalTo(1));
@@ -264,9 +291,14 @@ public void test072RunEsAsDifferentUserAndGroup() throws Exception {
         volumes.put(tempEsLogsDir.toAbsolutePath(), installation.logs);
 
         // Restart the container
-        runContainer(distribution(), builder().volumes(volumes).uid(501, 501));
+        runContainer(
+            distribution(),
+            builder().volumes(volumes)
+                .envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+                .uid(501, 501)
+        );
 
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
     }
 
     /**
@@ -275,9 +307,14 @@ public void test072RunEsAsDifferentUserAndGroup() throws Exception {
      */
     public void test073RunEsAsDifferentUserAndGroupWithoutBindMounting() throws Exception {
         // Restart the container
-        runContainer(distribution(), builder().uid(501, 501).extraArgs("--group-add 0"));
+        runContainer(
+            distribution(),
+            builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+                .uid(501, 501)
+                .extraArgs("--group-add 0")
+        );
 
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
     }
 
     /**
@@ -290,13 +327,7 @@ public void test080ConfigurePasswordThroughEnvironmentVariableFile() throws Exce
         // ELASTIC_PASSWORD_FILE
         Files.writeString(tempDir.resolve(passwordFilename), xpackPassword + "\n");
 
-        Map<String, String> envVars = Map.of(
-            "ELASTIC_PASSWORD_FILE",
-            "/run/secrets/" + passwordFilename,
-            // Enable security so that we can test that the password has been used
-            "xpack.security.enabled",
-            "true"
-        );
+        Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + passwordFilename);
 
         // File permissions need to be secured in order for the ES wrapper to accept
         // them for populating env var values
@@ -344,13 +375,7 @@ public void test081SymlinksAreFollowedWithEnvironmentVariableFiles() throws Exce
         // it won't resolve inside the container.
         Files.createSymbolicLink(tempDir.resolve(symlinkFilename), Path.of(passwordFilename));
 
-        Map<String, String> envVars = Map.of(
-            "ELASTIC_PASSWORD_FILE",
-            "/run/secrets/" + symlinkFilename,
-            // Enable security so that we can test that the password has been used
-            "xpack.security.enabled",
-            "true"
-        );
+        Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + symlinkFilename);
 
         // File permissions need to be secured in order for the ES wrapper to accept
         // them for populating env var values. The wrapper will resolve the symlink
@@ -436,13 +461,7 @@ public void test084SymlinkToFileWithInvalidPermissionsIsRejected() throws Except
         // it won't resolve inside the container.
         Files.createSymbolicLink(tempDir.resolve(symlinkFilename), Path.of(passwordFilename));
 
-        Map<String, String> envVars = Map.of(
-            "ELASTIC_PASSWORD_FILE",
-            "/run/secrets/" + symlinkFilename,
-            // Enable security so that we can test that the password has been used
-            "xpack.security.enabled",
-            "true"
-        );
+        Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + symlinkFilename);
 
         // Set invalid permissions on the file that the symlink targets
         Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p775);
@@ -469,10 +488,7 @@ public void test084SymlinkToFileWithInvalidPermissionsIsRejected() throws Except
      * `docker exec`, where the Docker image's entrypoint is not executed.
      */
     public void test085EnvironmentVariablesAreRespectedUnderDockerExec() throws Exception {
-        installation = runContainer(
-            distribution(),
-            builder().envVars(Map.of("xpack.security.enabled", "true", "ELASTIC_PASSWORD", "hunter2"))
-        );
+        installation = runContainer(distribution(), builder().envVars(Map.of("ELASTIC_PASSWORD", "hunter2")));
 
         // The tool below requires a keystore, so ensure that ES is fully initialised before proceeding.
         waitForElasticsearch("green", null, installation, "elastic", "hunter2");
@@ -672,7 +688,7 @@ public void test110OrgOpencontainersLabels() throws Exception {
      * Check that the container logs contain the expected content for Elasticsearch itself.
      */
     public void test120DockerLogsIncludeElasticsearchLogs() throws Exception {
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
         final Result containerLogs = getContainerLogs();
 
         assertThat("Container logs should contain full class names", containerLogs.stdout, containsString("org.elasticsearch.node.Node"));
@@ -683,9 +699,12 @@ public void test120DockerLogsIncludeElasticsearchLogs() throws Exception {
      * Check that it is possible to write logs to disk
      */
     public void test121CanUseStackLoggingConfig() throws Exception {
-        runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "file")));
+        runContainer(
+            distribution(),
+            builder().envVars(Map.of("ES_LOG_STYLE", "file", "ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+        );
 
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
         final Result containerLogs = getContainerLogs();
         final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
@@ -702,9 +721,12 @@ public void test121CanUseStackLoggingConfig() throws Exception {
      * Check that the default logging config can be explicitly selected.
      */
     public void test122CanUseDockerLoggingConfig() throws Exception {
-        runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "console")));
+        runContainer(
+            distribution(),
+            builder().envVars(Map.of("ES_LOG_STYLE", "console", "ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+        );
 
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
         final Result containerLogs = getContainerLogs();
         final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
@@ -726,14 +748,14 @@ public void test123CannotUseUnknownLoggingConfig() {
      * Check that it when configuring logging to write to disk, the container can be restarted.
      */
     public void test124CanRestartContainerWithStackLoggingConfig() throws Exception {
-        runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "file")));
+        runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "file", "ELASTIC_PASSWORD", PASSWORD)));
 
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
         restartContainer();
 
         // If something went wrong running Elasticsearch the second time, this will fail.
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
     }
 
     /**
@@ -769,9 +791,9 @@ public void test131InitProcessHasCorrectPID() {
      * Check that Elasticsearch reports per-node cgroup information.
      */
     public void test140CgroupOsStatsAreAvailable() throws Exception {
-        waitForElasticsearch(installation);
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
-        final JsonNode nodes = getJson("/_nodes/stats/os").get("nodes");
+        final JsonNode nodes = getJson("/_nodes/stats/os", USERNAME, PASSWORD).get("nodes");
 
         final String nodeId = nodes.fieldNames().next();
 
@@ -800,8 +822,13 @@ public void test150MachineDependentHeap() throws Exception {
         Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions));
 
         // Now run the container, being explicit about the available memory
-        runContainer(distribution(), builder().memory("942m").volumes(Map.of(jvmOptionsPath, containerJvmOptionsPath)));
-        waitForElasticsearch(installation);
+        runContainer(
+            distribution(),
+            builder().memory("942m")
+                .volumes(Map.of(jvmOptionsPath, containerJvmOptionsPath))
+                .envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
+        );
+        waitForElasticsearch(installation, USERNAME, PASSWORD);
 
         // Grab the container output and find the line where it print the JVM arguments. This will
         // let us see what the automatic heap sizing calculated.