From 62649e3a1a7c84b6825698dcc010774fd4165f8e Mon Sep 17 00:00:00 2001
From: Sergio Correia <scorreia@redhat.com>
Date: Wed, 28 Apr 2021 07:56:01 -0300
Subject: [PATCH] openssl: rsa: use openssl3 limitations for public exponent

In OpenSSL 3, the RSA public exponent (e) can be the following:
- 3, for legacy reasons, when in non FIPS-mode
- any odd integer greater than or equal to 65537

Let's update our code -- and tests -- to match these expectations.
---
 lib/openssl/rsa.c  | 33 +++++++++++++++++++++++++++++++++
 tests/jose-jwk-gen |  7 +++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/lib/openssl/rsa.c b/lib/openssl/rsa.c
index 645070ab..2688e483 100644
--- a/lib/openssl/rsa.c
+++ b/lib/openssl/rsa.c
@@ -21,6 +21,35 @@
 
 #include <string.h>
 
+/* The following functions are from OpenSSL 3 code base:
+ * - bn_is_three()
+ * - check_public_exponent() is ossl_rsa_check_public_exponent(), with
+ *   a minor change -- no FIPS check for allowing RSA_3. */
+static int bn_is_three(const BIGNUM *bn)
+{
+    BIGNUM *num = BN_dup(bn);
+    int ret = (num != NULL && BN_sub_word(num, 3) && BN_is_zero(num));
+
+    BN_free(num);
+    return ret;
+}
+
+/* Check exponent is odd, and has a bitlen ranging from [17..256]
+ * In practice, it allows odd integers greater than or equal to 65537. 3 is
+ * also allowed, for legacy purposes. */
+static int check_public_exponent(const BIGNUM* e)
+{
+    int bitlen;
+
+    /* In OpenSSL 3, RSA_3 is allowed in non-FIPS mode only, for
+     * legacy purposes. */
+    if (bn_is_three(e)) {
+        return 1;
+    }
+    bitlen = BN_num_bits(e);
+    return (BN_is_odd(e) && bitlen > 16 && bitlen < 257);
+}
+
 static RSA *
 mkrsa(const json_t *jwk)
 {
@@ -59,6 +88,10 @@ mkrsa(const json_t *jwk)
         break;
     }
 
+    if (!check_public_exponent(bn)) {
+        return NULL;
+    }
+
     key = RSA_new();
     if (!key)
         return NULL;
diff --git a/tests/jose-jwk-gen b/tests/jose-jwk-gen
index efbdefde..1cf6b1db 100755
--- a/tests/jose-jwk-gen
+++ b/tests/jose-jwk-gen
@@ -19,8 +19,11 @@ jose jwk gen -i '{ "kty": "EC", "crv": "P-384" }'
 jose jwk gen -i '{ "kty": "EC", "crv": "P-521" }'
 
 jose jwk gen -i '{ "kty": "RSA", "bits": 3072 }'
-jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 257 }'
-jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": "AQE" }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 257 }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 65536 }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": 65537 }'
+! jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": "AQE" }'   # 257.
+jose jwk gen -i '{ "kty": "RSA", "bits": 3072, "e": "AQAB"}'     # 65537.
 
 jose jwk gen -i '{ "kty": "oct", "bytes": 32 }'