fix(crypto): hash input for RSASSA-PKCS1-v1_5 before signing (#11314)

cli/tests/unit/webcrypto_test.ts

@@ -56,3 +56,36 @@ unitTest(async function testSignECDSA() {
 
   assert(signature);
 });
+
+// https://github.com/denoland/deno/issues/11313
+unitTest(async function testSignRSASSAKey() {
+  const subtle = window.crypto.subtle;
+  assert(subtle);
+
+  const keyPair = await subtle.generateKey(
+    {
+      name: "RSASSA-PKCS1-v1_5",
+      modulusLength: 2048,
+      publicExponent: new Uint8Array([1, 0, 1]),
+      hash: "SHA-256",
+    },
+    true,
+    ["sign", "verify"],
+  );
+
+  assert(keyPair.privateKey);
+  assert(keyPair.publicKey);
+  assertEquals(keyPair.privateKey.extractable, true);
+  assert(keyPair.privateKey.usages.includes("sign"));
+
+  const encoder = new TextEncoder();
+  const encoded = encoder.encode("Hello, World!");
+
+  const signature = await window.crypto.subtle.sign(
+    { name: "RSASSA-PKCS1-v1_5" },
+    keyPair.privateKey,
+    encoded,
+  );
+
+  assert(signature);
+});

extensions/crypto/lib.rs

@@ -246,25 +246,53 @@ pub async fn op_crypto_sign_key(
   let signature = match algorithm {
     Algorithm::RsassaPkcs1v15 => {
       let private_key = RSAPrivateKey::from_pkcs8(&*args.key.data)?;
-      let padding = match args
+      let (padding, hashed) = match args
         .hash
         .ok_or_else(|| type_error("Missing argument hash".to_string()))?
       {
-        CryptoHash::Sha1 => PaddingScheme::PKCS1v15Sign {
+        CryptoHash::Sha1 => {
-          hash: Some(rsa::hash::Hash::SHA1),
+          let mut hasher = Sha1::new();
-        },
+          hasher.update(&data);
-        CryptoHash::Sha256 => PaddingScheme::PKCS1v15Sign {
+          (
-          hash: Some(rsa::hash::Hash::SHA2_256),
+            PaddingScheme::PKCS1v15Sign {
-        },
+              hash: Some(rsa::hash::Hash::SHA1),
-        CryptoHash::Sha384 => PaddingScheme::PKCS1v15Sign {
+            },
-          hash: Some(rsa::hash::Hash::SHA2_384),
+            hasher.finalize()[..].to_vec(),
-        },
+          )
-        CryptoHash::Sha512 => PaddingScheme::PKCS1v15Sign {
+        }
-          hash: Some(rsa::hash::Hash::SHA2_512),
+        CryptoHash::Sha256 => {
-        },
+          let mut hasher = Sha256::new();
+          hasher.update(&data);
+          (
+            PaddingScheme::PKCS1v15Sign {
+              hash: Some(rsa::hash::Hash::SHA2_256),
+            },
+            hasher.finalize()[..].to_vec(),
+          )
+        }
+        CryptoHash::Sha384 => {
+          let mut hasher = Sha384::new();
+          hasher.update(&data);
+          (
+            PaddingScheme::PKCS1v15Sign {
+              hash: Some(rsa::hash::Hash::SHA2_384),
+            },
+            hasher.finalize()[..].to_vec(),
+          )
+        }
+        CryptoHash::Sha512 => {
+          let mut hasher = Sha512::new();
+          hasher.update(&data);
+          (
+            PaddingScheme::PKCS1v15Sign {
+              hash: Some(rsa::hash::Hash::SHA2_512),
+            },
+            hasher.finalize()[..].to_vec(),
+          )
+        }
       };
 
-      private_key.sign(padding, &data)?
+      private_key.sign(padding, &hashed)?
     }
     Algorithm::RsaPss => {
       let private_key = RSAPrivateKey::from_pkcs8(&*args.key.data)?;