openssl: rsa: use openssl3 limitations for public exponent

In OpenSSL 3, the RSA public exponent (e) can be the following:
- 3, for legacy reasons, when in non FIPS-mode
- any odd integer greater than or equal to 65537

Let's update our code -- and tests -- to match these expectations.

lib/openssl/rsa.c

@@ -21,6 +21,35 @@
 
 #include <string.h>
 
+/* The following functions are from OpenSSL 3 code base:
+ * - bn_is_three()
+ * - check_public_exponent() is ossl_rsa_check_public_exponent(), with
+ *   a minor change -- no FIPS check for allowing RSA_3. */
+static int bn_is_three(const BIGNUM *bn)
+{
+    BIGNUM *num = BN_dup(bn);
+    int ret = (num != NULL && BN_sub_word(num, 3) && BN_is_zero(num));
+
+    BN_free(num);
+    return ret;
+}
+
+/* Check exponent is odd, and has a bitlen ranging from [17..256]
+ * In practice, it allows odd integers greater than or equal to 65537. 3 is
+ * also allowed, for legacy purposes. */
+static int check_public_exponent(const BIGNUM* e)
+{
+    int bitlen;
+
+    /* In OpenSSL 3, RSA_3 is allowed in non-FIPS mode only, for
+     * legacy purposes. */
+    if (bn_is_three(e)) {
+        return 1;
+    }
+    bitlen = BN_num_bits(e);
+    return (BN_is_odd(e) && bitlen > 16 && bitlen < 257);
+}
+
 static RSA *
 mkrsa(const json_t *jwk)
 {
@@ -59,6 +88,10 @@ mkrsa(const json_t *jwk)
         break;
     }
 
+    if (!check_public_exponent(bn)) {
+        return NULL;
+    }
+
     key = RSA_new();
     if (!key)
         return NULL;

tests/jose-jwk-gen

@@ -19,8 +19,11 @@ jose jwk gen -i '{ "kty": "EC", "crv": "P-384" }'
 jose jwk gen -i '{ "kty": "EC", "crv": "P-521" }'
 
 jose jwk gen -i '{ "kty": "RSA", "bits": 3072 }'
-jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 257 }'
-jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": "AQE" }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 257 }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 65536 }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 65537 }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": "AQE" }'   # 257.
+jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": "AQAB"}'     # 65537.
 
 jose jwk gen -i '{ "kty": "oct", "bytes": 32 }'