make tests run without BC (not BCFIPS) libraries.

Signed-off-by: Iwan Igonin <[email protected]>
opensearch-project · Aug 7, 2024 · 84e7aa9 · 84e7aa9
1 parent 6016d5d
commit 84e7aa9
Show file tree

Hide file tree

Showing 54 changed files with 457 additions and 259 deletions.
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -123,7 +123,7 @@ dependencies {
   api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
   api "org.ajoberstar.grgit:grgit-core:5.2.1"
-  api "org.bouncycastle:bc-fips:1.0.2.5"
+  api "org.bouncycastle:bc-fips:${props.getProperty('bouncycastle_jce')}"
 
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java
@@ -164,6 +164,11 @@ public void execute(Task t) {
                 test.systemProperty("tests.seed", BuildParams.getTestSeed());
             }
 
+            test.systemProperty(
+                "java.security.properties",
+                project.getRootProject().getLayout().getProjectDirectory() + "/distribution/src/config/fips_java.security"
+            );
+
             // don't track these as inputs since they contain absolute paths and break cache relocatability
             File gradleHome = project.getGradle().getGradleUserHomeDir();
             String gradleVersion = project.getGradle().getGradleVersion();

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/info/BuildParams.java b/buildSrc/src/main/java/org/opensearch/gradle/info/BuildParams.java
@@ -52,7 +52,6 @@ public class BuildParams {
     private static JavaVersion gradleJavaVersion;
     private static JavaVersion runtimeJavaVersion;
     private static String runtimeJavaDetails;
-    @Deprecated
     private static Boolean inFipsJvm;
     private static String gitRevision;
     private static String gitOrigin;

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -54,7 +54,10 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.78
+bouncycastle_jce=1.0.2.4
+bouncycastle_tls=1.0.19
+bouncycastle_pkix=1.0.7
+bouncycastle_pg=1.0.7.1
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2

diff --git a/client/rest/build.gradle b/client/rest/build.gradle
@@ -50,6 +50,8 @@ dependencies {
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
+  api "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
+  api "org.bouncycastle:bctls-fips:${versions.bouncycastle_tls}"
 
   testImplementation project(":client:test")
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
@@ -65,6 +67,10 @@ dependencies {
   testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
 }
 
+tasks.named("dependencyLicenses").configure {
+  mapping from: /bc.*/, to: 'bouncycastle'
+}
+
 tasks.withType(CheckForbiddenApis).configureEach {
   //client does not depend on server, so only jdk and http signatures should be checked
   replaceSignatureFiles('jdk-signatures', 'http-signatures')
@@ -109,6 +115,20 @@ thirdPartyAudit.ignoreMissingClasses(
   'javax.servlet.ServletContextEvent',
   'javax.servlet.ServletContextListener'
 )
+thirdPartyAudit.ignoreViolations(
+  'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator',
+  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator$2'
+)
 
 tasks.withType(JavaCompile) {
   // Suppressing '[options] target value 8 is obsolete and will be removed in a future release'

diff --git a/client/rest/licenses/bc-fips-1.0.2.4.jar.sha1 b/client/rest/licenses/bc-fips-1.0.2.4.jar.sha1
@@ -0,0 +1 @@
+9008d04fc13da6455e6a792935b93b629757335d
diff --git a/client/rest/licenses/bctls-fips-1.0.19.jar.sha1 b/client/rest/licenses/bctls-fips-1.0.19.jar.sha1
@@ -0,0 +1 @@
+b15d650f6e2a9de08d5569e25a642b6a384dbfd2
diff --git a/client/rest/licenses/bouncycastle-LICENSE.txt b/client/rest/licenses/bouncycastle-LICENSE.txt
@@ -0,0 +1,14 @@
+Copyright (c) 2000 - 2023 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/client/rest/licenses/bouncycastle-NOTICE.txt b/client/rest/licenses/bouncycastle-NOTICE.txt
@@ -0,0 +1 @@
+
diff --git a/client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java
@@ -43,7 +43,7 @@
 
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLException;
 import javax.net.ssl.TrustManagerFactory;
 
 import java.io.IOException;
@@ -95,15 +95,14 @@ public static void stopHttpServers() throws IOException {
     }
 
     public void testBuilderUsesDefaultSSLContext() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             try (RestClient client = buildRestClient()) {
                 try {
                     client.performRequest(new Request("GET", "/"));
-                    fail("connection should have been rejected due to SSL handshake");
+                    fail("connection should have been rejected due to SSL failure");
                 } catch (Exception e) {
-                    assertThat(e, instanceOf(SSLHandshakeException.class));
+                    assertThat(e.getCause(), instanceOf(SSLException.class));
                 }
             }
 

diff --git a/distribution/src/config/fips_java.security b/distribution/src/config/fips_java.security
@@ -22,9 +22,9 @@ ssl.KeyManagerFactory.algorithm=PKIX
 ssl.TrustManagerFactory.algorithm=PKIX
 networkaddress.cache.negative.ttl=10
 krb5.kdc.bad.policy = tryLast
-jdk.certpath.disabledAlgorithms=MD2, MD5, SHA1, jdkCA&usageTLSServer, RSA keySize < 1024, DSA keySize < 1024, EC keySize < 224
-jdk.jar.disabledAlgorithms=MD2, MD5, RSA keySize < 1024, DSA keySize < 1024
-jdk.tls.disabledAlgorithms=SSLv3, TLSv1, TLSv1.1, RC4, MD5withRSA, DH keySize < 1024, EC keySize < 224, DES40_CBC, RC4_40, 3DES_EDE_CBC
+jdk.certpath.disabledAlgorithms=MD2, MD5, SHA1, jdkCA&usageTLSServer, RSA keySize < 2048, DSA keySize < 2048, EC keySize < 224
+jdk.jar.disabledAlgorithms=MD2, MD5, RSA keySize < 2048, DSA keySize < 2048
+jdk.tls.disabledAlgorithms=SSLv3, TLSv1, TLSv1.1, RC4, MD5withRSA, DH keySize < 2048, EC keySize < 224, DES40_CBC, RC4_40, 3DES_EDE_CBC
 jdk.tls.legacyAlgorithms= \
         K_NULL, C_NULL, M_NULL, \
         DH_anon, ECDH_anon, \

diff --git a/...eystore-cli/src/test/java/org/opensearch/common/settings/AddFileKeyStoreCommandTests.java b/...eystore-cli/src/test/java/org/opensearch/common/settings/AddFileKeyStoreCommandTests.java
@@ -210,17 +210,13 @@ public void testIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongkeystorepassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo", file.toString()));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        if (inFipsJvm()) {
-            assertThat(
-                e.getMessage(),
-                anyOf(
-                    containsString("Provided keystore password was incorrect"),
-                    containsString("Keystore has been corrupted or tampered with")
-                )
-            );
-        } else {
-            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
-        }
+        assertThat(
+            e.getMessage(),
+            anyOf(
+                containsString("Provided keystore password was incorrect"),
+                containsString("Keystore has been corrupted or tampered with")
+            )
+        );
     }
 
     public void testAddToUnprotectedKeystore() throws Exception {

diff --git a/...store-cli/src/test/java/org/opensearch/common/settings/AddStringKeyStoreCommandTests.java b/...store-cli/src/test/java/org/opensearch/common/settings/AddStringKeyStoreCommandTests.java
@@ -71,17 +71,13 @@ public void testInvalidPassphrease() throws Exception {
         terminal.addSecretInput("thewrongpassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo2"));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        if (inFipsJvm()) {
-            assertThat(
-                e.getMessage(),
-                anyOf(
-                    containsString("Provided keystore password was incorrect"),
-                    containsString("Keystore has been corrupted or tampered with")
-                )
-            );
-        } else {
-            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
-        }
+        assertThat(
+            e.getMessage(),
+            anyOf(
+                containsString("Provided keystore password was incorrect"),
+                containsString("Keystore has been corrupted or tampered with")
+            )
+        );
 
     }
 

diff --git a/...-cli/src/test/java/org/opensearch/common/settings/ChangeKeyStorePasswordCommandTests.java b/...-cli/src/test/java/org/opensearch/common/settings/ChangeKeyStorePasswordCommandTests.java
@@ -104,16 +104,12 @@ public void testChangeKeyStorePasswordWrongExistingPassword() throws Exception {
         // We'll only be prompted once (for the old password)
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        if (inFipsJvm()) {
-            assertThat(
-                e.getMessage(),
-                anyOf(
-                    containsString("Provided keystore password was incorrect"),
-                    containsString("Keystore has been corrupted or tampered with")
-                )
-            );
-        } else {
-            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
-        }
+        assertThat(
+            e.getMessage(),
+            anyOf(
+                containsString("Provided keystore password was incorrect"),
+                containsString("Keystore has been corrupted or tampered with")
+            )
+        );
     }
 }
diff --git a/...tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java b/...tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java
@@ -132,17 +132,13 @@ public void testDecryptKeyStoreWithWrongPassword() throws Exception {
             SecurityException.class,
             () -> loadedKeystore.decrypt(new char[] { 'i', 'n', 'v', 'a', 'l', 'i', 'd' })
         );
-        if (inFipsJvm()) {
-            assertThat(
-                exception.getMessage(),
-                anyOf(
-                    containsString("Provided keystore password was incorrect"),
-                    containsString("Keystore has been corrupted or tampered with")
-                )
-            );
-        } else {
-            assertThat(exception.getMessage(), containsString("Provided keystore password was incorrect"));
-        }
+        assertThat(
+            exception.getMessage(),
+            anyOf(
+                containsString("Provided keystore password was incorrect"),
+                containsString("Keystore has been corrupted or tampered with")
+            )
+        );
     }
 
     public void testCannotReadStringFromClosedKeystore() throws Exception {
@@ -373,8 +369,8 @@ public void testBackcompatV1() throws Exception {
             output.writeString("PKCS12");
             output.writeString("PBE");
 
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
-            KeyStore keystore = KeyStore.getInstance("PKCS12");
+            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE", "SunJCE");
+            KeyStore keystore = KeyStore.getInstance("PKCS12", "SUN");
             keystore.load(null, null);
             SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
             KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
@@ -414,8 +410,8 @@ public void testBackcompatV2() throws Exception {
             output.writeString("file_setting");
             output.writeString("FILE");
 
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
-            KeyStore keystore = KeyStore.getInstance("PKCS12");
+            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE", "SunJCE");
+            KeyStore keystore = KeyStore.getInstance("PKCS12", "SUN");
             keystore.load(null, null);
             SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
             KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);

diff --git a/...s/keystore-cli/src/test/java/org/opensearch/common/settings/ListKeyStoreCommandTests.java b/...s/keystore-cli/src/test/java/org/opensearch/common/settings/ListKeyStoreCommandTests.java
@@ -90,17 +90,13 @@ public void testListWithIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongkeystorepassword");
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        if (inFipsJvm()) {
-            assertThat(
-                e.getMessage(),
-                anyOf(
-                    containsString("Provided keystore password was incorrect"),
-                    containsString("Keystore has been corrupted or tampered with")
-                )
-            );
-        } else {
-            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
-        }
+        assertThat(
+            e.getMessage(),
+            anyOf(
+                containsString("Provided keystore password was incorrect"),
+                containsString("Keystore has been corrupted or tampered with")
+            )
+        );
     }
 
     public void testListWithUnprotectedKeystore() throws Exception {

diff --git a/...e-cli/src/test/java/org/opensearch/common/settings/RemoveSettingKeyStoreCommandTests.java b/...e-cli/src/test/java/org/opensearch/common/settings/RemoveSettingKeyStoreCommandTests.java
@@ -107,18 +107,13 @@ public void testRemoveWithIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongpassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo"));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        if (inFipsJvm()) {
-            assertThat(
-                e.getMessage(),
-                anyOf(
-                    containsString("Provided keystore password was incorrect"),
-                    containsString("Keystore has been corrupted or tampered with")
-                )
-            );
-        } else {
-            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
-        }
-
+        assertThat(
+            e.getMessage(),
+            anyOf(
+                containsString("Provided keystore password was incorrect"),
+                containsString("Keystore has been corrupted or tampered with")
+            )
+        );
     }
 
     public void testRemoveFromUnprotectedKeystore() throws Exception {

diff --git a/...bution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/...bution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java
@@ -87,8 +87,8 @@ static List<String> systemJvmOptions(final Path config) {
     }
 
     private static String loadJavaSecurityProperties(final Path config) {
-        var securityFile = config.resolve("fips_java.security").toFile();
-        return "-Djava.security.properties=" + securityFile.getAbsolutePath();
+        var securityFile = config.resolve("fips_java.security");
+        return "-Djava.security.properties=" + securityFile.toAbsolutePath();
     }
 
     private static String allowSecurityManagerOption() {

diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
@@ -37,8 +37,7 @@ base {
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
-  api "org.bouncycastle:bcpg-fips:1.0.7.1"
-  api "org.bouncycastle:bc-fips:1.0.2.5"
+  api "org.bouncycastle:bcpg-fips:${versions.bouncycastle_pg}"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.3.0'
   testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
@@ -58,33 +57,6 @@ test {
   jvmArgs += [ "-Djava.security.egd=file:/dev/urandom" ]
 }
 
-/*
- * these two classes intentionally use the following JDK internal APIs in order to offer the necessary
- * functionality
- *
- * sun.security.internal.spec.TlsKeyMaterialParameterSpec
- * sun.security.internal.spec.TlsKeyMaterialSpec
- * sun.security.internal.spec.TlsMasterSecretParameterSpec
- * sun.security.internal.spec.TlsPrfParameterSpec
- * sun.security.internal.spec.TlsRsaPremasterSecretParameterSpec
- * sun.security.provider.SecureRandom
- *
- */
-thirdPartyAudit.ignoreViolations(
-  'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator',
-  'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSExtendedMasterSecretGenerator$2'
-)
-
 thirdPartyAudit.ignoreMissingClasses(
   'org.brotli.dec.BrotliInputStream',
   'org.objectweb.asm.AnnotationVisitor',

diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.5.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.5.jar.sha1
diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle
@@ -36,8 +36,8 @@ dependencies {
   api project(':libs:opensearch-common')
 
   // bouncyCastle
-  implementation 'org.bouncycastle:bcpkix-fips:1.0.7'
-  compileOnly 'org.bouncycastle:bc-fips:1.0.2.5'
+  implementation "org.bouncycastle:bcpkix-fips:${versions.bouncycastle_pkix}"
+  compileOnly "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
 
   testImplementation(project(":test:framework")) {
     exclude group: 'org.opensearch', module: 'opensearch-ssl-config'

diff --git a/libs/ssl-config/licenses/bc-fips-1.0.2.5.jar.sha1 b/libs/ssl-config/licenses/bc-fips-1.0.2.5.jar.sha1