Fix RSA modulus, output and scratch bounds (N-146)

CHANGELOG.md

@@ -5,6 +5,7 @@ Add new entries under `Unreleased`; published release sections are history.
 
 ## Unreleased
 
+- RSA keys over 4096 bits are refused (N-146).
 - TLS rejects certificates with empty version, public-key or signature fields,
   and correctly parses certificates containing unique identifiers.
 - TLS rejects HelloRetryRequest cookies and ClientHello messages that do not

src/tls/ami_tls_crypto.c

@@ -446,6 +446,20 @@ UINT    i;
  */
 #define AMI_TLS_POWM_SCRATCH_LIMBS  2704u
 
+/*
+ * The modulus that scratch is sized for, in bits.  The vendored init's ceiling
+ * is NX_CRYPTO_MAX_RSA_MODULUS_SIZE, which a build may raise; ours may not
+ * follow it, because the area above is fixed (N-146).
+ */
+#define AMI_TLS_RSA_MAX_MODULUS_BITS  4096u
+
+/* m_len counts HN_UBASE limbs, 32 bits each here; window 1 is the least the
+   powm accepts.  The cap and the area are one decision. */
+_Static_assert(AMI_TLS_POWM_SCRATCH_LIMBS >=
+               C68K_POWM_SCRATCH_LIMBS(AMI_TLS_RSA_MAX_MODULUS_BITS / 32u, 1u),
+               "AMI_TLS_RSA_MAX_MODULUS_BITS needs more powm scratch than "
+               "AMI_TLS_POWM_SCRATCH_LIMBS provides");
+
 typedef struct AMI_CRYPTO_RSA_STRUCT
 {
     NX_CRYPTO_RSA   ami_rsa_base;
@@ -470,14 +484,41 @@ static UINT ami_rsa_exponentiate(const UCHAR *exponent, UINT exponent_length,
                                  const UCHAR *q, UINT q_length,
                                  const UCHAR *input, UINT input_length,
                                  UCHAR *output,
-                                 USHORT *scratch_buf_ptr,
+                                 USHORT *scratch_buf_ptr, UINT scratch_buf_length,
                                  HN_UBASE *powm_scratch, UINT powm_scratch_limbs)
 {
 
 HN_UBASE               *scratch;
 UINT                    mod_length;
+UINT                    status;
+ULONG                   scratch_needed;
 NX_CRYPTO_HUGE_NUMBER   modulus_hn, exponent_hn, input_hn, output_hn, p_hn, q_hn;
 
+    /* As the vendored operation now does (N-146): the carve's need against
+       the scratch, in bytes, before any of it is touched; scratch_buf_length
+       counts USHORTs.  The numbers' own bounds are setup()'s, whose status
+       is returned. */
+    if (modulus_length == 0)
+    {
+        return(NX_CRYPTO_SIZE_ERROR);
+    }
+    /* Rounded to whole limbs as the carve is; the vendored operation's
+       comment has the arithmetic. */
+    {
+        ULONG rounded = ((ULONG)modulus_length + HN_SIZE_ROUND) & ~(ULONG)HN_SIZE_ROUND;
+
+        if ((p != NX_CRYPTO_NULL) && (q != NX_CRYPTO_NULL))
+            scratch_needed = ((modulus_length % (2u * (HN_SIZE_ROUND + 1u))) == 0)
+                             ? ((10UL * modulus_length) + 24UL)
+                             : ((10UL * rounded) + 40UL);
+        else
+            scratch_needed = (7UL * rounded) + 8UL;
+    }
+    if (((ULONG)scratch_buf_length * sizeof(USHORT)) < scratch_needed)
+    {
+        return(NX_CRYPTO_SIZE_ERROR);
+    }
+
 
     scratch =  (HN_UBASE *)scratch_buf_ptr;
 
@@ -486,17 +527,28 @@ NX_CRYPTO_HUGE_NUMBER   modulus_hn, exponent_hn, input_hn, output_hn, p_hn, q_hn
     NX_CRYPTO_HUGE_NUMBER_INITIALIZE(&exponent_hn, scratch, modulus_length);
     NX_CRYPTO_HUGE_NUMBER_INITIALIZE(&output_hn,   scratch, modulus_length << 1);
 
-    _nx_crypto_huge_number_setup(&exponent_hn, exponent, exponent_length);
-    _nx_crypto_huge_number_setup(&input_hn,    input,    input_length);
-    _nx_crypto_huge_number_setup(&modulus_hn,  modulus,  modulus_length);
+    status = _nx_crypto_huge_number_setup(&exponent_hn, exponent, exponent_length);
+    if (status == NX_CRYPTO_SUCCESS)
+        status = _nx_crypto_huge_number_setup(&input_hn, input, input_length);
+    if (status == NX_CRYPTO_SUCCESS)
+        status = _nx_crypto_huge_number_setup(&modulus_hn, modulus, modulus_length);
+    if (status != NX_CRYPTO_SUCCESS)
+    {
+        return(status);
+    }
 
     if ((p != NX_CRYPTO_NULL) && (q != NX_CRYPTO_NULL))
     {
         NX_CRYPTO_HUGE_NUMBER_INITIALIZE(&p_hn, scratch, modulus_length >> 1);
         NX_CRYPTO_HUGE_NUMBER_INITIALIZE(&q_hn, scratch, modulus_length >> 1);
 
-        _nx_crypto_huge_number_setup(&p_hn, p, p_length);
-        _nx_crypto_huge_number_setup(&q_hn, q, q_length);
+        status = _nx_crypto_huge_number_setup(&p_hn, p, p_length);
+        if (status == NX_CRYPTO_SUCCESS)
+            status = _nx_crypto_huge_number_setup(&q_hn, q, q_length);
+        if (status != NX_CRYPTO_SUCCESS)
+        {
+            return(status);
+        }
 
         if (ami_arithmetic == AMI_TLS_ARITH_REFERENCE)
         {
@@ -527,9 +579,7 @@ NX_CRYPTO_HUGE_NUMBER   modulus_hn, exponent_hn, input_hn, output_hn, p_hn, q_hn
         }
     }
 
-    _nx_crypto_huge_number_extract(&output_hn, output, modulus_length, &mod_length);
-
-    return(NX_CRYPTO_SUCCESS);
+    return(_nx_crypto_huge_number_extract(&output_hn, output, modulus_length, &mod_length));
 }
 
 static UINT ami_crypto_method_rsa_init(struct NX_CRYPTO_METHOD_STRUCT *method,
@@ -550,6 +600,14 @@ static UINT ami_crypto_method_rsa_init(struct NX_CRYPTO_METHOD_STRUCT *method,
         return(NX_CRYPTO_PTR_ERROR);
     }
 
+    /* The accelerated path's own capacity, whatever the vendored ceiling is
+       set to; refused before the context is touched (N-146). */
+    if ((key_size_in_bits == 0) || ((key_size_in_bits & 7u) != 0) ||
+        (key_size_in_bits > AMI_TLS_RSA_MAX_MODULUS_BITS))
+    {
+        return(NX_CRYPTO_UNSUPPORTED_KEY_SIZE);
+    }
+
     return(_nx_crypto_method_rsa_init(method, key, key_size_in_bits, handle,
                                       crypto_metadata, crypto_metadata_size));
 }
@@ -640,7 +698,10 @@ ULONG                       elapsed;
         return(NX_CRYPTO_PTR_ERROR);
     }
 
-    if (output_length_in_byte < (ULONG)(key_size_in_bits >> 3))
+    /* The result is the modulus's length; key_size_in_bits is the
+       exponent's, 24 bits for 65537, so it bounded nothing (N-146). */
+    if ((base -> nx_crypto_rsa_modulus_length == 0) ||
+        (output_length_in_byte < (ULONG)(base -> nx_crypto_rsa_modulus_length)))
     {
         return(NX_CRYPTO_INVALID_BUFFER_SIZE);
     }
@@ -692,6 +753,7 @@ ULONG                       elapsed;
                                    input, (UINT)input_length_in_byte,
                                    output,
                                    base -> nx_crypto_rsa_scratch_buffer,
+                                   NX_CRYPTO_RSA_SCRATCH_BUFFER_SIZE,
                                    ctx -> ami_rsa_powm_scratch,
                                    AMI_TLS_POWM_SCRATCH_LIMBS);
 

tests/fuzz/CMakeLists.txt

@@ -593,6 +593,19 @@ if(CMAKE_SIZEOF_VOID_P EQUAL 4)
     target_link_libraries(tls_rsa_key_regression PRIVATE
         fuzz_tls_ami fuzz_tls_nx_secure fuzz_tls_nx_crypto fuzz_tls_netx
         fuzz_tls_ami fuzz_tls_nx_secure fuzz_tls_nx_crypto fuzz_tls_netx)
+    # N-146: both RSA methods refuse a modulus over NX_CRYPTO_MAX_RSA_MODULUS_SIZE,
+    # check the output against the modulus, and honour the scratch length.
+    add_executable(tls_rsa_bounds
+        tls_rsa_bounds.c
+        fuzz_nxstub.c
+        fuzz_txstub.c)
+    target_include_directories(tls_rsa_bounds PRIVATE ${_fuzz_tls_crypto_inc})
+    target_compile_definitions(tls_rsa_bounds PRIVATE ${_fuzz_tls_def})
+    target_compile_options(tls_rsa_bounds PRIVATE -UAMINETXDUO_IPV6 ${_fuzz_san} -Wall -Wextra)
+    target_link_options(tls_rsa_bounds PRIVATE ${_fuzz_san})
+    target_link_libraries(tls_rsa_bounds PRIVATE
+        fuzz_tls_ami fuzz_tls_nx_secure fuzz_tls_nx_crypto fuzz_tls_netx
+        fuzz_tls_ami fuzz_tls_nx_secure fuzz_tls_nx_crypto fuzz_tls_netx)
 
 endif()
 
@@ -716,6 +729,7 @@ if(CMAKE_SIZEOF_VOID_P EQUAL 4)
              COMMAND fuzz_tls_crypto -s)
 
     add_test(NAME tls_rsa_key_regression COMMAND tls_rsa_key_regression)
+    add_test(NAME tls_rsa_bounds COMMAND tls_rsa_bounds)
 
     # EIGHT STREAMS, NOT ONE, and the same number of mutations between them.
     # Four was the first attempt and left this at 8.14 s of the ten-second