Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set xpack.security.enabled to true for all licenses #72300

Merged
merged 56 commits into from
Aug 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
f04b68a
wip
jkakavas Apr 26, 2021
233e92b
Update remaining usages
jkakavas Apr 27, 2021
ff23d23
fix unused imports
jkakavas Apr 27, 2021
90c258e
remove unecessary tests
jkakavas Apr 27, 2021
c23e153
Fix a number of tests by explicitly disabling security so that they r…
jkakavas Apr 27, 2021
e4eecb6
Remove acknowledgments when switching to basic
jkakavas Apr 28, 2021
2de689e
fix enrich tests
jkakavas Apr 28, 2021
c6549d4
fix some packaging tests
jkakavas Apr 28, 2021
4c94272
spotless
jkakavas Apr 28, 2021
ade580e
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Apr 28, 2021
3bdf272
Fix archive and docker tests
jkakavas May 4, 2021
4099306
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas May 4, 2021
695332c
fix compilation error
jkakavas May 4, 2021
676ab57
remove redundant setting
jkakavas May 4, 2021
cde3cf1
unused import
jkakavas May 4, 2021
332d85a
adjust Docker tests
jkakavas May 4, 2021
6434324
Explicitly disable security for package tests
jkakavas May 4, 2021
605be3e
Disable security in TransformInternalIndexIT
jkakavas May 5, 2021
825bf4e
change how tests figure out if security is enabled
jkakavas May 5, 2021
33326e6
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas May 5, 2021
1a5ed61
Don't overwrite the whole file when enabling security
jkakavas May 6, 2021
3da7c42
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas May 6, 2021
5083db7
adjust new transport action
jkakavas May 6, 2021
b8b661a
unused import
jkakavas May 6, 2021
749d472
Fix CertGenCliTests
jkakavas May 6, 2021
93d5b5d
spotless
jkakavas May 6, 2021
43181a4
revert unintended change
jkakavas May 7, 2021
8ee411f
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Jun 8, 2021
12e087a
disable security in new ML tests
jkakavas Jun 9, 2021
42355f9
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Jul 8, 2021
479704d
add back checkFeature tests for licenses
jkakavas Jul 8, 2021
4ac7e40
address feedback
jkakavas Jul 13, 2021
a0f1e24
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Jul 13, 2021
539f300
explicitly disable security for AbstractMultiClustersTestCase
jkakavas Jul 13, 2021
7ff9ad1
Move log message indicating if security is enabled to Security
jkakavas Jul 13, 2021
1810af5
Disable security only when we can/need
jkakavas Jul 13, 2021
60854eb
fix test
jkakavas Jul 13, 2021
045ec1a
address feedback
jkakavas Jul 19, 2021
928798a
address feedback
jkakavas Aug 2, 2021
9b37b99
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Aug 2, 2021
67eace7
add tracking meta issue for enabling security in tests
jkakavas Aug 2, 2021
05b4636
More SecurityServerTransportInterceptor cleanup
jkakavas Aug 2, 2021
eafef85
spotless and a test fix
jkakavas Aug 2, 2021
b2f26a0
moar spotless violations
jkakavas Aug 2, 2021
364a984
Attempt to fix KeystoreManagementTests and other feedback
jkakavas Aug 2, 2021
5a008fa
remove redundant checks
jkakavas Aug 3, 2021
1c98a3c
add test for Security#onIndexModule
jkakavas Aug 3, 2021
7783563
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Aug 3, 2021
528bb2c
minor fixes
jkakavas Aug 3, 2021
36f4e5e
incorporate Albert's fixes for packaging tests
jkakavas Aug 3, 2021
d3b362c
change security check test
jkakavas Aug 4, 2021
9e33058
Merge remote-tracking branch 'origin/master' into security-enabled-de…
jkakavas Aug 4, 2021
5e4af06
Remove moar redundant checks for whether security is enabled in compo…
jkakavas Aug 4, 2021
a6bf19c
SecurityRestFilter is always installed, even with security disabled
jkakavas Aug 4, 2021
00758af
Re-enable QA test for enabling security on an existing cluster
jkakavas Aug 4, 2021
a12494d
add test for no realms
jkakavas Aug 4, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
import static java.nio.file.StandardOpenOption.APPEND;
Expand All @@ -46,6 +48,8 @@ public static void cleanupFiles() {

public void test10Install() throws Exception {
install();
// Enable security for this test only where it is necessary, until we can enable it for all
ServerUtils.enableSecurityFeatures(installation);
}

public void test20Help() {
Expand Down Expand Up @@ -107,8 +111,24 @@ public void test40RunWithCert() throws Exception {

Files.write(installation.config("elasticsearch.yml"), yaml, CREATE, APPEND);

assertWhileRunning(
() -> ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), null, null, installation.config("certs/ca/ca.crt"))
);
assertWhileRunning(() -> {
final String password = setElasticPassword();
assertNotNull(password);
ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), "elastic", password, installation.config("certs/ca/ca.crt"));
});
}

private String setElasticPassword() {
final Pattern userpassRegex = Pattern.compile("PASSWORD (\\w+) = ([^\\s]+)");
Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
Matcher matcher = userpassRegex.matcher(result.stdout);
assertNotNull(matcher);
while (matcher.find()) {
if (matcher.group(1).equals("elastic")) {
return matcher.group(2);
}
}
return null;
}

}
137 changes: 82 additions & 55 deletions qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@
*/
public class DockerTests extends PackagingTestCase {
private Path tempDir;
private static final String USERNAME = "elastic";
private static final String PASSWORD = "nothunter2";

@BeforeClass
public static void filterDistros() {
Expand All @@ -88,7 +90,10 @@ public static void filterDistros() {

@Before
public void setupTest() throws IOException {
installation = runContainer(distribution(), builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false")));
installation = runContainer(
distribution(),
builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
jkakavas marked this conversation as resolved.
Show resolved Hide resolved
);
tempDir = createTempDir(DockerTests.class.getSimpleName());
}

Expand All @@ -106,12 +111,18 @@ public void test010Install() {
}

/**
* Check that the /_xpack API endpoint's presence is correct for the type of distribution being tested.
* Check that security is enabled
*/
public void test011PresenceOfXpack() throws Exception {
public void test011SecurityEnabledStatus() throws Exception {
waitForElasticsearch(installation, USERNAME, PASSWORD);
final int statusCode = ServerUtils.makeRequestAndGetStatus(Request.Get("http://localhost:9200"), USERNAME, "wrong_password", null);
assertThat(statusCode, equalTo(401));

// restart container with security disabled
runContainer(distribution(), builder().envVars(Map.of("xpack.security.enabled", "false")));
waitForElasticsearch(installation);
final int statusCode = Request.Get("http://localhost:9200/_xpack").execute().returnResponse().getStatusLine().getStatusCode();
assertThat(statusCode, equalTo(200));
final int unauthStatusCode = ServerUtils.makeRequestAndGetStatus(Request.Get("http://localhost:9200"), null, null, null);
assertThat(unauthStatusCode, equalTo(200));
}

/**
Expand Down Expand Up @@ -147,7 +158,7 @@ public void test041AmazonCaCertsAreInTheKeystore() {
* Check that when the keystore is created on startup, it is created with the correct permissions.
*/
public void test042KeystorePermissionsAreCorrect() throws Exception {
waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

assertPermissionsAndOwnership(installation.config("elasticsearch.keystore"), "elasticsearch", "root", p660);
}
Expand All @@ -157,11 +168,11 @@ public void test042KeystorePermissionsAreCorrect() throws Exception {
* is minimally functional.
*/
public void test050BasicApiTests() throws Exception {
waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

assertTrue(existsInContainer(installation.logs.resolve("gc.log")));

ServerUtils.runElasticsearchTests();
ServerUtils.runElasticsearchTests(USERNAME, PASSWORD);
}

/**
Expand All @@ -185,11 +196,24 @@ public void test070BindMountCustomPathConfAndJvmOptions() throws Exception {

// Restart the container
final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/usr/share/elasticsearch/config"));
runContainer(distribution(), builder().volumes(volumes).envVars(Map.of("ES_JAVA_OPTS", "-XX:-UseCompressedOops")));
runContainer(
distribution(),
builder().volumes(volumes)
.envVars(
Map.of(
"ES_JAVA_OPTS",
"-XX:-UseCompressedOops",
"ingest.geoip.downloader.enabled",
"false",
"ELASTIC_PASSWORD",
PASSWORD
)
)
);

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

final JsonNode nodes = getJson("/_nodes").get("nodes");
final JsonNode nodes = getJson("/_nodes", USERNAME, PASSWORD).get("nodes");
final String nodeId = nodes.fieldNames().next();

final int heapSize = nodes.at("/" + nodeId + "/jvm/mem/heap_init_in_bytes").intValue();
Expand All @@ -213,11 +237,14 @@ public void test071BindMountCustomPathWithDifferentUID() throws Exception {
// Restart the container
final Map<Path, Path> volumes = Map.of(tempEsDataDir.toAbsolutePath(), installation.data);

runContainer(distribution(), builder().volumes(volumes));
runContainer(
distribution(),
builder().volumes(volumes).envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
);

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

final JsonNode nodes = getJson("/_nodes");
final JsonNode nodes = getJson("/_nodes", USERNAME, PASSWORD);

assertThat(nodes.at("/_nodes/total").intValue(), equalTo(1));
assertThat(nodes.at("/_nodes/successful").intValue(), equalTo(1));
Expand Down Expand Up @@ -264,9 +291,14 @@ public void test072RunEsAsDifferentUserAndGroup() throws Exception {
volumes.put(tempEsLogsDir.toAbsolutePath(), installation.logs);

// Restart the container
runContainer(distribution(), builder().volumes(volumes).uid(501, 501));
runContainer(
distribution(),
builder().volumes(volumes)
.envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
.uid(501, 501)
);

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);
}

/**
Expand All @@ -275,9 +307,14 @@ public void test072RunEsAsDifferentUserAndGroup() throws Exception {
*/
public void test073RunEsAsDifferentUserAndGroupWithoutBindMounting() throws Exception {
// Restart the container
runContainer(distribution(), builder().uid(501, 501).extraArgs("--group-add 0"));
runContainer(
distribution(),
builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
.uid(501, 501)
.extraArgs("--group-add 0")
);

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);
}

/**
Expand All @@ -290,13 +327,7 @@ public void test080ConfigurePasswordThroughEnvironmentVariableFile() throws Exce
// ELASTIC_PASSWORD_FILE
Files.writeString(tempDir.resolve(passwordFilename), xpackPassword + "\n");

Map<String, String> envVars = Map.of(
"ELASTIC_PASSWORD_FILE",
"/run/secrets/" + passwordFilename,
// Enable security so that we can test that the password has been used
"xpack.security.enabled",
"true"
);
Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + passwordFilename);

// File permissions need to be secured in order for the ES wrapper to accept
// them for populating env var values
Expand Down Expand Up @@ -344,13 +375,7 @@ public void test081SymlinksAreFollowedWithEnvironmentVariableFiles() throws Exce
// it won't resolve inside the container.
Files.createSymbolicLink(tempDir.resolve(symlinkFilename), Path.of(passwordFilename));

Map<String, String> envVars = Map.of(
"ELASTIC_PASSWORD_FILE",
"/run/secrets/" + symlinkFilename,
// Enable security so that we can test that the password has been used
"xpack.security.enabled",
"true"
);
Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + symlinkFilename);

// File permissions need to be secured in order for the ES wrapper to accept
// them for populating env var values. The wrapper will resolve the symlink
Expand Down Expand Up @@ -436,13 +461,7 @@ public void test084SymlinkToFileWithInvalidPermissionsIsRejected() throws Except
// it won't resolve inside the container.
Files.createSymbolicLink(tempDir.resolve(symlinkFilename), Path.of(passwordFilename));

Map<String, String> envVars = Map.of(
"ELASTIC_PASSWORD_FILE",
"/run/secrets/" + symlinkFilename,
// Enable security so that we can test that the password has been used
"xpack.security.enabled",
"true"
);
Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + symlinkFilename);

// Set invalid permissions on the file that the symlink targets
Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p775);
Expand All @@ -469,10 +488,7 @@ public void test084SymlinkToFileWithInvalidPermissionsIsRejected() throws Except
* `docker exec`, where the Docker image's entrypoint is not executed.
*/
public void test085EnvironmentVariablesAreRespectedUnderDockerExec() throws Exception {
installation = runContainer(
distribution(),
builder().envVars(Map.of("xpack.security.enabled", "true", "ELASTIC_PASSWORD", "hunter2"))
);
installation = runContainer(distribution(), builder().envVars(Map.of("ELASTIC_PASSWORD", "hunter2")));

// The tool below requires a keystore, so ensure that ES is fully initialised before proceeding.
waitForElasticsearch("green", null, installation, "elastic", "hunter2");
Expand Down Expand Up @@ -672,7 +688,7 @@ public void test110OrgOpencontainersLabels() throws Exception {
* Check that the container logs contain the expected content for Elasticsearch itself.
*/
public void test120DockerLogsIncludeElasticsearchLogs() throws Exception {
waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);
final Result containerLogs = getContainerLogs();

assertThat("Container logs should contain full class names", containerLogs.stdout, containsString("org.elasticsearch.node.Node"));
Expand All @@ -683,9 +699,12 @@ public void test120DockerLogsIncludeElasticsearchLogs() throws Exception {
* Check that it is possible to write logs to disk
*/
public void test121CanUseStackLoggingConfig() throws Exception {
runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "file")));
runContainer(
distribution(),
builder().envVars(Map.of("ES_LOG_STYLE", "file", "ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
);

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

final Result containerLogs = getContainerLogs();
final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
Expand All @@ -702,9 +721,12 @@ public void test121CanUseStackLoggingConfig() throws Exception {
* Check that the default logging config can be explicitly selected.
*/
public void test122CanUseDockerLoggingConfig() throws Exception {
runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "console")));
runContainer(
distribution(),
builder().envVars(Map.of("ES_LOG_STYLE", "console", "ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
);

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

final Result containerLogs = getContainerLogs();
final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
Expand All @@ -726,14 +748,14 @@ public void test123CannotUseUnknownLoggingConfig() {
* Check that it when configuring logging to write to disk, the container can be restarted.
*/
public void test124CanRestartContainerWithStackLoggingConfig() throws Exception {
runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "file")));
runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "file", "ELASTIC_PASSWORD", PASSWORD)));

waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

restartContainer();

// If something went wrong running Elasticsearch the second time, this will fail.
waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);
}

/**
Expand Down Expand Up @@ -769,9 +791,9 @@ public void test131InitProcessHasCorrectPID() {
* Check that Elasticsearch reports per-node cgroup information.
*/
public void test140CgroupOsStatsAreAvailable() throws Exception {
waitForElasticsearch(installation);
waitForElasticsearch(installation, USERNAME, PASSWORD);

final JsonNode nodes = getJson("/_nodes/stats/os").get("nodes");
final JsonNode nodes = getJson("/_nodes/stats/os", USERNAME, PASSWORD).get("nodes");

final String nodeId = nodes.fieldNames().next();

Expand Down Expand Up @@ -800,8 +822,13 @@ public void test150MachineDependentHeap() throws Exception {
Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions));

// Now run the container, being explicit about the available memory
runContainer(distribution(), builder().memory("942m").volumes(Map.of(jvmOptionsPath, containerJvmOptionsPath)));
waitForElasticsearch(installation);
runContainer(
distribution(),
builder().memory("942m")
.volumes(Map.of(jvmOptionsPath, containerJvmOptionsPath))
.envVars(Map.of("ingest.geoip.downloader.enabled", "false", "ELASTIC_PASSWORD", PASSWORD))
);
waitForElasticsearch(installation, USERNAME, PASSWORD);

// Grab the container output and find the line where it print the JVM arguments. This will
// let us see what the automatic heap sizing calculated.
Expand Down
Loading