crypto: fail early if passphrase is too long

This causes OpenSSL to fail early if the decryption passphrase is too
long, and produces a somewhat helpful error message.

PR-URL: nodejs#27010
Refs: nodejs#25208
Reviewed-By: Ruben Bridgewater <[email protected]>
Reviewed-By: Sam Roberts <[email protected]>

doc/api/crypto.md

@@ -1826,6 +1826,9 @@ Creates and returns a new key object containing a private key. If `key` is a
 string or `Buffer`, `format` is assumed to be `'pem'`; otherwise, `key`
 must be an object with the properties described above.
 
+If the private key is encrypted, a `passphrase` must be specified. The length
+of the passphrase is limited to 1024 bytes.
+
 ### crypto.createPublicKey(key)
 <!-- YAML
 added: v11.6.0

src/node_crypto.cc

@@ -189,7 +189,8 @@ static int PasswordCallback(char* buf, int size, int rwflag, void* u) {
   if (passphrase != nullptr) {
     size_t buflen = static_cast<size_t>(size);
     size_t len = strlen(passphrase);
-    len = len > buflen ? buflen : len;
+    if (buflen < len)
+      return -1;
     memcpy(buf, passphrase, len);
     return len;
   }

test/parallel/test-crypto-key-objects.js

@@ -230,6 +230,27 @@ const privateDsa = fixtures.readKey('dsa_private_encrypted_1025.pem',
     message: 'Passphrase required for encrypted key'
   });
 
+  // Reading an encrypted key with a passphrase that exceeds OpenSSL's buffer
+  // size limit should fail with an appropriate error code.
+  common.expectsError(() => createPrivateKey({
+    key: privateDsa,
+    format: 'pem',
+    passphrase: Buffer.alloc(1025, 'a')
+  }), {
+    code: 'ERR_OSSL_PEM_BAD_PASSWORD_READ',
+    type: Error
+  });
+
+  // The buffer has a size of 1024 bytes, so this passphrase should be permitted
+  // (but will fail decryption).
+  common.expectsError(() => createPrivateKey({
+    key: privateDsa,
+    format: 'pem',
+    passphrase: Buffer.alloc(1024, 'a')
+  }), {
+    message: /bad decrypt/
+  });
+
   const publicKey = createPublicKey(publicDsa);
   assert.strictEqual(publicKey.type, 'public');
   assert.strictEqual(publicKey.asymmetricKeyType, 'dsa');