ensure FIPS-mode can be enabled by fixing existent and creating addit…

…ional tests.

Signed-off-by: Iwan Igonin <[email protected]>

client/rest/build.gradle

@@ -34,8 +34,8 @@ apply plugin: 'opensearch.build'
 apply plugin: 'opensearch.publish'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_1_8
-  sourceCompatibility = JavaVersion.VERSION_1_8
+  targetCompatibility = JavaVersion.VERSION_11
+  sourceCompatibility = JavaVersion.VERSION_11
 }
 
 base {
@@ -60,6 +60,7 @@ dependencies {
   api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
 
   testImplementation project(":client:test")
+  testImplementation project(':libs:opensearch-common')
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -71,6 +72,7 @@ dependencies {
   testImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   testImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
   testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  testImplementation "commons-io:commons-io:${versions.commonsio}"
 }
 
 tasks.named("dependencyLicenses").configure {
@@ -83,6 +85,7 @@ tasks.withType(CheckForbiddenApis).configureEach {
 }
 
 forbiddenPatterns {
+  exclude '**/*.bcfks'
   exclude '**/*.der'
 }
 

client/rest/src/main/java/org/opensearch/client/FipsEnabledThreadFactory.java

@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.client;
+
+import org.apache.hc.core5.concurrent.DefaultThreadFactory;
+import org.bouncycastle.crypto.CryptoServicesRegistrar;
+
+import java.util.concurrent.ThreadFactory;
+
+public class FipsEnabledThreadFactory implements ThreadFactory {
+
+    private final ThreadFactory defaultFactory;
+    private final boolean isFipsEnabled;
+
+    public FipsEnabledThreadFactory(String namePrefix, boolean isFipsEnabled) {
+        this.defaultFactory = new DefaultThreadFactory(namePrefix);
+        this.isFipsEnabled = isFipsEnabled;
+    }
+
+    @Override
+    public Thread newThread(final Runnable target) {
+        return defaultFactory.newThread(() -> {
+            if (isFipsEnabled) {
+                CryptoServicesRegistrar.setApprovedOnlyMode(true);
+            }
+            target.run();
+        });
+    }
+
+}

client/rest/src/main/java/org/opensearch/client/RestClientBuilder.java

@@ -48,6 +48,7 @@
 import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
 import org.apache.hc.core5.reactor.ssl.TlsDetails;
 import org.apache.hc.core5.util.Timeout;
+import org.bouncycastle.crypto.CryptoServicesRegistrar;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
@@ -334,6 +335,7 @@ public TlsDetails create(final SSLEngine sslEngine) {
             HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClientBuilder.create()
                 .setDefaultRequestConfig(requestConfigBuilder.build())
                 .setConnectionManager(connectionManager)
+                .setThreadFactory(new FipsEnabledThreadFactory("os-client-dispatcher", CryptoServicesRegistrar.isInApprovedOnlyMode()))
                 .setTargetAuthenticationStrategy(DefaultAuthenticationStrategy.INSTANCE)
                 .disableAutomaticRetries();
             if (httpClientConfigCallback != null) {

client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java

@@ -38,6 +38,9 @@
 import com.sun.net.httpserver.HttpsServer;
 
 import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.opensearch.common.crypto.KeyStoreFactory;
+import org.opensearch.common.crypto.KeyStoreType;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -50,19 +53,18 @@
 import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.security.AccessController;
-import java.security.KeyFactory;
 import java.security.KeyStore;
 import java.security.PrivilegedAction;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateFactory;
-import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.SecureRandom;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 
 /**
@@ -75,8 +77,11 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
     @BeforeClass
     public static void startHttpServer() throws Exception {
         httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
-        httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext()));
+        httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext(true)));
         httpsServer.createContext("/", new ResponseHandler());
+        var threadFactory = new FipsEnabledThreadFactory("test-httpserver-dispatch", inFipsJvm());
+        Executor executor = Executors.newFixedThreadPool(1, threadFactory);
+        httpsServer.setExecutor(executor);
         httpsServer.start();
     }
 
@@ -90,8 +95,22 @@ public void handle(HttpExchange httpExchange) throws IOException {
 
     @AfterClass
     public static void stopHttpServers() throws IOException {
-        httpsServer.stop(0);
-        httpsServer = null;
+        if (httpsServer != null) {
+            httpsServer.stop(0); // Stop the server
+            Executor executor = httpsServer.getExecutor();
+            if (executor instanceof ExecutorService) {
+                ((ExecutorService) executor).shutdown(); // Shutdown the executor
+                try {
+                    if (!((ExecutorService) executor).awaitTermination(15, TimeUnit.SECONDS)) {
+                        ((ExecutorService) executor).shutdownNow(); // Force shutdown if not terminated
+                    }
+                } catch (InterruptedException ex) {
+                    ((ExecutorService) executor).shutdownNow(); // Force shutdown on interruption
+                    Thread.currentThread().interrupt();
+                }
+            }
+            httpsServer = null;
+        }
     }
 
     public void testBuilderUsesDefaultSSLContext() throws Exception {
@@ -106,7 +125,7 @@ public void testBuilderUsesDefaultSSLContext() throws Exception {
                 }
             }
 
-            SSLContext.setDefault(getSslContext());
+            SSLContext.setDefault(getSslContext(false));
             try (RestClient client = buildRestClient()) {
                 Response response = client.performRequest(new Request("GET", "/"));
                 assertEquals(200, response.getStatusLine().getStatusCode());
@@ -121,34 +140,37 @@ private RestClient buildRestClient() {
         return RestClient.builder(new HttpHost("https", address.getHostString(), address.getPort())).build();
     }
 
-    private static SSLContext getSslContext() throws Exception {
-        SSLContext sslContext = SSLContext.getInstance(getProtocol());
+    private static SSLContext getSslContext(boolean server) throws Exception {
+        SSLContext sslContext;
+        char[] password = "password".toCharArray();
+        SecureRandom secureRandom = SecureRandom.getInstance("DEFAULT", "BCFIPS");
+
         try (
-            InputStream certFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test.crt");
-            InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore.jks")
+            InputStream trustStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore.bcfks");
+            InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/testks.bcfks")
         ) {
-            // Build a keystore of default type programmatically since we can't use JKS keystores to
-            // init a KeyManagerFactory in FIPS 140 JVMs.
-            KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
-            keyStore.load(null, "password".toCharArray());
-            CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
-            PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(
-                Files.readAllBytes(Paths.get(RestClientBuilderIntegTests.class.getResource("/test.der").toURI()))
-            );
-            KeyFactory keyFactory = KeyFactory.getInstance("RSA");
-            keyStore.setKeyEntry(
-                "mykey",
-                keyFactory.generatePrivate(privateKeySpec),
-                "password".toCharArray(),
-                new Certificate[] { certFactory.generateCertificate(certFile) }
-            );
+            KeyStore keyStore = KeyStoreFactory.getInstance(KeyStoreType.BCFKS);
+            keyStore.load(keyStoreFile, password);
             KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            kmf.init(keyStore, "password".toCharArray());
-            KeyStore trustStore = KeyStore.getInstance("JKS");
-            trustStore.load(keyStoreFile, "password".toCharArray());
+            kmf.init(keyStore, password);
+
+            KeyStore trustStore = KeyStoreFactory.getInstance(KeyStoreType.BCFKS);
+            trustStore.load(trustStoreFile, password);
             TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
             tmf.init(trustStore);
-            sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+
+            var sslContextBuilder = SSLContextBuilder.create()
+                .setProvider("BCJSSE")
+                .setProtocol(getProtocol())
+                .setSecureRandom(secureRandom);
+
+            if (server) {
+                sslContextBuilder.loadKeyMaterial(keyStore, password).build();
+            } else {
+                sslContextBuilder.loadTrustMaterial(trustStore, null);
+            }
+            sslContext = sslContextBuilder.build();
+
         }
         return sslContext;
     }

client/rest/src/test/java/org/opensearch/client/documentation/RestClientDocumentation.java

@@ -67,6 +67,7 @@
 import org.opensearch.client.RestClient;
 import org.opensearch.client.RestClientBuilder;
 import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback;
+import org.opensearch.common.crypto.KeyStoreFactory;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
@@ -84,6 +85,8 @@
 import java.util.Iterator;
 import java.util.concurrent.CountDownLatch;
 
+import static org.opensearch.common.crypto.KeyStoreType.PKCS_12;
+
 /**
  * This class is used to generate the Java low-level REST client documentation.
  * You need to wrap your code between two tags like:
@@ -418,7 +421,7 @@ public HttpAsyncClientBuilder customizeHttpClient(
             String keyStorePass = "";
             //tag::rest-client-config-encrypted-communication
             Path trustStorePath = Paths.get("/path/to/truststore.p12");
-            KeyStore truststore = KeyStore.getInstance("pkcs12");
+            KeyStore truststore = KeyStoreFactory.getInstance(PKCS_12);
             try (InputStream is = Files.newInputStream(trustStorePath)) {
                 truststore.load(is, keyStorePass.toCharArray());
             }
@@ -460,7 +463,7 @@ public TlsDetails create(final SSLEngine sslEngine) {
             try (InputStream is = Files.newInputStream(caCertificatePath)) {
                 trustedCa = factory.generateCertificate(is);
             }
-            KeyStore trustStore = KeyStore.getInstance("pkcs12");
+            KeyStore trustStore = KeyStoreFactory.getInstance(PKCS_12);
             trustStore.load(null, null);
             trustStore.setCertificateEntry("ca", trustedCa);
             SSLContextBuilder sslContextBuilder = SSLContexts.custom()
@@ -498,8 +501,8 @@ public TlsDetails create(final SSLEngine sslEngine) {
             //tag::rest-client-config-mutual-tls-authentication
             Path trustStorePath = Paths.get("/path/to/your/truststore.p12");
             Path keyStorePath = Paths.get("/path/to/your/keystore.p12");
-            KeyStore trustStore = KeyStore.getInstance("pkcs12");
-            KeyStore keyStore = KeyStore.getInstance("pkcs12");
+            KeyStore trustStore = KeyStoreFactory.getInstance(PKCS_12);
+            KeyStore keyStore = KeyStoreFactory.getInstance(PKCS_12);
             try (InputStream is = Files.newInputStream(trustStorePath)) {
                 trustStore.load(is, trustStorePass.toCharArray());
             }

client/rest/src/test/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumerTests.java

@@ -35,34 +35,34 @@ public void tearDown() {
     }
 
     public void testConsumerAllocatesBufferLimit() throws IOException {
-        consumer.consume((ByteBuffer) randomByteBufferOfLength(1000).flip());
+        consumer.consume(randomByteBufferOfLength(1000).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(1000));
     }
 
     public void testConsumerAllocatesEmptyBuffer() throws IOException {
-        consumer.consume((ByteBuffer) ByteBuffer.allocate(0).flip());
+        consumer.consume(ByteBuffer.allocate(0).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(0));
     }
 
     public void testConsumerExpandsBufferLimits() throws IOException {
-        consumer.consume((ByteBuffer) randomByteBufferOfLength(1000).flip());
-        consumer.consume((ByteBuffer) randomByteBufferOfLength(2000).flip());
-        consumer.consume((ByteBuffer) randomByteBufferOfLength(3000).flip());
+        consumer.consume(randomByteBufferOfLength(1000).flip());
+        consumer.consume(randomByteBufferOfLength(2000).flip());
+        consumer.consume(randomByteBufferOfLength(3000).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(6000));
     }
 
     public void testConsumerAllocatesLimit() throws IOException {
-        consumer.consume((ByteBuffer) randomByteBufferOfLength(BUFFER_LIMIT).flip());
+        consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(BUFFER_LIMIT));
     }
 
     public void testConsumerFailsToAllocateOverLimit() throws IOException {
-        assertThrows(ContentTooLongException.class, () -> consumer.consume((ByteBuffer) randomByteBufferOfLength(BUFFER_LIMIT + 1).flip()));
+        assertThrows(ContentTooLongException.class, () -> consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT + 1).flip()));
     }
 
     public void testConsumerFailsToExpandOverLimit() throws IOException {
-        consumer.consume((ByteBuffer) randomByteBufferOfLength(BUFFER_LIMIT).flip());
-        assertThrows(ContentTooLongException.class, () -> consumer.consume((ByteBuffer) randomByteBufferOfLength(1).flip()));
+        consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT).flip());
+        assertThrows(ContentTooLongException.class, () -> consumer.consume(randomByteBufferOfLength(1).flip()));
     }
 
     private static ByteBuffer randomByteBufferOfLength(int length) {

client/test/build.gradle

@@ -43,6 +43,12 @@ dependencies {
   api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   api "junit:junit:${versions.junit}"
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
+  api "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
+  api "org.bouncycastle:bcutil-fips:${versions.bouncycastle_util}"
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /bc.*/, to: 'bouncycastle'
 }
 
 tasks.named('forbiddenApisMain').configure {

client/test/src/main/java/org/opensearch/client/RestClientTestCase.java

@@ -45,6 +45,8 @@
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 import org.apache.hc.core5.http.Header;
+import org.bouncycastle.crypto.CryptoServicesRegistrar;
+import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -125,7 +127,14 @@ private static void addValueToListEntry(final Map<String, List<String>> map, fin
         values.add(value);
     }
 
+    @BeforeClass
+    public static void setFipsJvm() {
+        boolean isFipsEnabled = Boolean.parseBoolean(System.getProperty("tests.fips.enabled", "false"));
+        CryptoServicesRegistrar.setApprovedOnlyMode(isFipsEnabled);
+    }
+
     public static boolean inFipsJvm() {
-        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
+        return CryptoServicesRegistrar.isInApprovedOnlyMode();
     }
+
 }

distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java

@@ -71,6 +71,7 @@ public void setupEnv() throws IOException {
     }
 
     public void testLoadSecureSettings() throws Exception {
+        assumeFalse("Can't use empty password in a FIPS JVM", inFipsJvm());
         final Path configPath = env.configDir();
         final SecureString seed;
         try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {