LibCrypto: Do not trim leading zeros in export_data by default

This fixes the issue with the exported data having a leading zero,
causing RSA::encrypt to trim the block down, and ruining the encryption.

Fixes #2691 :^)

5 files changed, 26 insertions, 18 deletions

diff --git a/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp b/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp
index 005d148731..fbfd372ff9 100644
--- a/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp
+++ b/Libraries/LibCrypto/BigInt/SignedBigInteger.cpp
@@ -36,11 +36,15 @@ SignedBigInteger SignedBigInteger::import_data(const u8* ptr, size_t length)
     return { move(unsigned_data), sign };
 }
 
-size_t SignedBigInteger::export_data(Bytes data) const
+size_t SignedBigInteger::export_data(Bytes data, bool remove_leading_zeros) const
 {
+    // FIXME: Support this:
+    //        m <0XX> -> m <XX> (if remove_leading_zeros)
+    ASSERT(!remove_leading_zeros);
+
     data[0] = m_sign;
     auto bytes_view = data.slice(1, data.size() - 1);
-    return m_unsigned_data.export_data(bytes_view) + 1;
+    return m_unsigned_data.export_data(bytes_view, remove_leading_zeros) + 1;
 }
 
 SignedBigInteger SignedBigInteger::from_base10(StringView str)
diff --git a/Libraries/LibCrypto/BigInt/SignedBigInteger.h b/Libraries/LibCrypto/BigInt/SignedBigInteger.h
index 81521fcff4..b2ff1dd118 100644
--- a/Libraries/LibCrypto/BigInt/SignedBigInteger.h
+++ b/Libraries/LibCrypto/BigInt/SignedBigInteger.h
@@ -67,7 +67,7 @@ public:
     static SignedBigInteger import_data(const AK::StringView& data) { return import_data((const u8*)data.characters_without_null_termination(), data.length()); }
     static SignedBigInteger import_data(const u8* ptr, size_t length);
 
-    size_t export_data(Bytes) const;
+    size_t export_data(Bytes, bool remove_leading_zeros = false) const;
 
     static SignedBigInteger from_base10(StringView str);
     String to_base10() const;
diff --git a/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp b/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp
index 64a603b48a..23ab7d2a8d 100644
--- a/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp
+++ b/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp
@@ -55,21 +55,23 @@ UnsignedBigInteger UnsignedBigInteger::create_invalid()
     return invalid;
 }
 
-size_t UnsignedBigInteger::export_data(Bytes data) const
+size_t UnsignedBigInteger::export_data(Bytes data, bool remove_leading_zeros) const
 {
     size_t word_count = trimmed_length();
     size_t out = 0;
     if (word_count > 0) {
         ssize_t leading_zeros = -1;
-        u32 word = m_words[word_count - 1];
-        for (size_t i = 0; i < sizeof(u32); i++) {
-            u8 byte = (u8)(word >> ((sizeof(u32) - i - 1) * 8));
-            data[out++] = byte;
-            if (leading_zeros < 0 && byte != 0)
-                leading_zeros = (int)i;
+        if (remove_leading_zeros) {
+            u32 word = m_words[word_count - 1];
+            for (size_t i = 0; i < sizeof(u32); i++) {
+                u8 byte = (u8)(word >> ((sizeof(u32) - i - 1) * 8));
+                data[out++] = byte;
+                if (leading_zeros < 0 && byte != 0)
+                    leading_zeros = (int)i;
+            }
         }
-        for (size_t i = word_count - 1; i > 0; i--) {
-            word = m_words[i - 1];
+        for (size_t i = word_count - (remove_leading_zeros ? 1 : 0); i > 0; i--) {
+            auto word = m_words[i - 1];
             data[out++] = (u8)(word >> 24);
             data[out++] = (u8)(word >> 16);
             data[out++] = (u8)(word >> 8);
diff --git a/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h b/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h
index 544fc68b2c..752343f4fb 100644
--- a/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h
+++ b/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h
@@ -59,7 +59,7 @@ public:
         return UnsignedBigInteger(ptr, length);
     }
 
-    size_t export_data(Bytes) const;
+    size_t export_data(Bytes, bool remove_leading_zeros = false) const;
 
     static UnsignedBigInteger from_base10(const String& str);
     String to_base10() const;
diff --git a/Libraries/LibCrypto/PK/RSA.cpp b/Libraries/LibCrypto/PK/RSA.cpp
index 1c984c60c2..c359a7c3b6 100644
--- a/Libraries/LibCrypto/PK/RSA.cpp
+++ b/Libraries/LibCrypto/PK/RSA.cpp
@@ -126,9 +126,11 @@ void RSA::encrypt(const ByteBuffer& in, ByteBuffer& out)
     }
     auto exp = NumberTheory::ModularPower(in_integer, m_public_key.public_exponent(), m_public_key.modulus());
     auto size = exp.export_data(out.span());
-    // FIXME: We should probably not do this...
-    if (size != out.size())
-        out = out.slice(out.size() - size, size);
+    auto outsize = out.size();
+    if (size != outsize) {
+        dbg() << "POSSIBLE RSA BUG!!! Size mismatch: " << outsize << " requested but " << size << " bytes generated";
+        out = out.slice(outsize - size, size);
+    }
 }
 
 void RSA::decrypt(const ByteBuffer& in, ByteBuffer& out)
@@ -252,8 +254,8 @@ void RSA_PKCS1_EME::encrypt(const ByteBuffer& in, ByteBuffer& out)
     // since arc4random can create zeros (shocking!)
     // we have to go through and un-zero the zeros
     for (size_t i = 0; i < ps_length; ++i)
-        if (!ps[i])
-            ps[i] = 0xfe;
+        while (!ps[i])
+            AK::fill_with_random(ps + i, 1);
 
     u8 paddings[] { 0x00, 0x02 };