Test some obscure modes of EVP sign/verify with RSA keys

When the digest is unset, padding may be either RSA_PADDING_NONE or
RSA_PADDING_PKCS1.

If RSA_PADDING_NONE, this becomes raw RSA public and private key
operations, with signature verify comparing the "digest" against the
output of the raw public key operation.

If RSA_PADDING_PKCS1, this treats the "digest" as the raw DigestInfo
structure.

Test both of these, so we don't break them as we move code around. In
doing so, this revealed that verify in these modes, when the "digest"
doesn't match, forgot to add to the error queue. Fix that up.

Bug: 42290606
Change-Id: I3412a633124a12bda6dfebc08896f616b2d268aa
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/75228
Reviewed-by: Bob Beck <bbe@google.com>
Auto-Submit: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

crypto/evp/evp_tests.txt

@@ -1551,6 +1551,62 @@ Input = 2d207a73432a8fb4c03051b3f73b28a61764098dfa34c47a20995f8115aa6816679b557e
 Output = eaf1a73a1b0c4609537de69cd9228bbcfb9a8ca8c6c3efaf056fe4a7f4634ed00b7c39ec6922d7b8ea2c04ebac
 
 
+# RSA with no padding implements raw public and private transforms. This is not
+# a real signature scheme, but might be used to construct one.
+
+Sign = RSA-2048
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+
+Verify = RSA-2048
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+
+Verify = RSA-2048
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+Error = BAD_SIGNATURE
+
+Sign = RSA-2048
+RSAPadding = None
+Input = "Too short"
+Error = DATA_TOO_SMALL
+
+Verify = RSA-2048
+RSAPadding = None
+Input = "Too short"
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+Error = BAD_SIGNATURE
+
+Sign = RSA-2048
+Digest = SHA256
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002
+Error = INVALID_PADDING_MODE
+
+
+# RSASSA-PKCS1-v1_5 with no digest. This is not actually a defined mode in
+# PKCS #1, but OpenSSL's API interprets this to mean accepting an arbitrary
+# string instead of the serialized DigestInfo. This is not a real signature
+# scheme, but might be used to construct one.
+
+Sign = RSA-2048
+Input = 01234567
+Output = 9925a048eb9e225819dea311be206ea1287ad90bdf77218bb1348829ef7072e7b4a772b6cb1df1480d4ede4140feb84e03c4c99c851f91cf293ed3259a6ef8f567130164465a39e83306b69a22b74ec26adca77da8e1cc625c3721269b410819298f1c6399eb8c68bcc8df16ab149eae0959947a5918b4fd49396af80b557124712f27f760cc69d09d6d1db0883decf05ee072e98f15b153bd74e2ec98a00ea514a4c9c2a98a0813cb5b53381e030779612e3de972f8029cf363bec7098ecda568eee6371ecb10f6448e707e08b47f557a9f7e87024a2a1249d1f8dc540b35a79f9e6c0188cbc406cc434cefd83538c3cffb7663f9e46a7787a6a630c5a1c675
+
+Verify = RSA-2048
+Input = 01234567
+Output = 9925a048eb9e225819dea311be206ea1287ad90bdf77218bb1348829ef7072e7b4a772b6cb1df1480d4ede4140feb84e03c4c99c851f91cf293ed3259a6ef8f567130164465a39e83306b69a22b74ec26adca77da8e1cc625c3721269b410819298f1c6399eb8c68bcc8df16ab149eae0959947a5918b4fd49396af80b557124712f27f760cc69d09d6d1db0883decf05ee072e98f15b153bd74e2ec98a00ea514a4c9c2a98a0813cb5b53381e030779612e3de972f8029cf363bec7098ecda568eee6371ecb10f6448e707e08b47f557a9f7e87024a2a1249d1f8dc540b35a79f9e6c0188cbc406cc434cefd83538c3cffb7663f9e46a7787a6a630c5a1c675
+
+Verify = RSA-2048
+Input = 01234568
+Output = 9925a048eb9e225819dea311be206ea1287ad90bdf77218bb1348829ef7072e7b4a772b6cb1df1480d4ede4140feb84e03c4c99c851f91cf293ed3259a6ef8f567130164465a39e83306b69a22b74ec26adca77da8e1cc625c3721269b410819298f1c6399eb8c68bcc8df16ab149eae0959947a5918b4fd49396af80b557124712f27f760cc69d09d6d1db0883decf05ee072e98f15b153bd74e2ec98a00ea514a4c9c2a98a0813cb5b53381e030779612e3de972f8029cf363bec7098ecda568eee6371ecb10f6448e707e08b47f557a9f7e87024a2a1249d1f8dc540b35a79f9e6c0188cbc406cc434cefd83538c3cffb7663f9e46a7787a6a630c5a1c675
+Error = BAD_SIGNATURE
+
+
 # Single-shot signing tests.
 
 SignMessage = RSA-2048

crypto/evp/p_rsa.cc

@@ -184,8 +184,11 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t siglen,
   const size_t key_len = EVP_PKEY_size(ctx->pkey);
   if (!setup_tbuf(rctx, ctx) ||
       !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
-                      rctx->pad_mode) ||
-      rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
+                      rctx->pad_mode)) {
+    return 0;
+  }
+  if (rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_SIGNATURE);
     return 0;
   }