LibCrypto: Avoid overly big allocs in intermediate ModularPower results

If we don't limit the sizes of the intermediate results, they will grow
indefinitely, causing each iteration to take longer and longer (in both
memcpy time, and algorithm runtime).
While calculating the trimmed length is fairly expensive, it's a small
cost to pay for uniform iteration times.

2 files changed, 10 insertions, 1 deletions

diff --git a/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp b/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp
index 0e08a76d9d..9ee6622009 100644
--- a/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp
+++ b/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.cpp
@@ -156,6 +156,13 @@ size_t UnsignedBigInteger::trimmed_length() const
     return m_cached_trimmed_length.value();
 }
 
+void UnsignedBigInteger::clamp_to_trimmed_length()
+{
+    auto length = trimmed_length();
+    if (m_words.size() > length)
+        m_words.resize(length);
+}
+
 FLATTEN UnsignedBigInteger UnsignedBigInteger::plus(const UnsignedBigInteger& other) const
 {
     UnsignedBigInteger result;
@@ -578,7 +585,7 @@ FLATTEN void UnsignedBigInteger::shift_left_without_allocation(
 
         // output += (carry_word << temp_result.length())
         // FIXME : Using temp_plus this way to transform carry_word into a bigint is not
-        // efficient nor pretty. Maybe we should have an "add_with_shift" method ?
+        //         efficient nor pretty. Maybe we should have an "add_with_shift" method ?
         temp_plus.set_to_0();
         temp_plus.m_words.append(carry_word);
         shift_left_by_n_words(temp_plus, temp_result.length(), temp_result);
diff --git a/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h b/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h
index 51c1d9f776..b2805e685c 100644
--- a/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h
+++ b/Userland/Libraries/LibCrypto/BigInt/UnsignedBigInteger.h
@@ -81,6 +81,8 @@ public:
     // The "trimmed length" is the number of words after trimming leading zeroed words
     size_t trimmed_length() const;
 
+    void clamp_to_trimmed_length();
+
     UnsignedBigInteger plus(const UnsignedBigInteger& other) const;
     UnsignedBigInteger minus(const UnsignedBigInteger& other) const;
     UnsignedBigInteger bitwise_or(const UnsignedBigInteger& other) const;