Protect against combinatorial explosion

lib/tre-compile.c

@@ -1335,6 +1335,8 @@ tre_compute_npfl(tre_mem_t mem, tre_stack_t *stack, tre_ast_node_t *tree,
 		    /* Back references: nullable = false, firstpos = {i},
 		       lastpos = {i}. */
 		    node->nullable = 0;
+		    if (*nextpos == TRE_MAX_POS)
+		      return REG_ESPACE;
 		    lit->position = (*nextpos)++;
 		    node->firstpos = tre_set_one(mem, lit->position, 0,
 					     TRE_CHAR_MAX, 0, NULL, -1);
@@ -1363,6 +1365,8 @@ tre_compute_npfl(tre_mem_t mem, tre_stack_t *stack, tre_ast_node_t *tree,
 		    /* Literal at position i: nullable = false, firstpos = {i},
 		       lastpos = {i}. */
 		    node->nullable = 0;
+		    if (*nextpos == TRE_MAX_POS)
+		      return REG_ESPACE;
 		    lit->position = (*nextpos)++;
 		    node->firstpos =
 		      tre_set_one(mem, lit->position, lit->code_min,
@@ -2008,6 +2012,10 @@ tre_compile(regex_t *preg, const tre_char_t *regex, size_t n, int cflags)
   for (i = 0; i < numpos; i++)
     {
       offs[i] = add;
+      /* Note that counts[i] cannot exceed TRE_MAX_POS which is orders of
+	 magnitude smaller than TRE_MAX_TRANS */
+      if (add >= TRE_MAX_TRANS - counts[i] - 1)
+	  ERROR_EXIT(REG_ESPACE);
       add += counts[i] + 1;
       counts[i] = 0;
     }

lib/tre-internal.h

@@ -25,9 +25,11 @@
 
 #include "tre/tre.h"
 
-#define TRE_MAX_RE	65536
+#define TRE_MAX_RE	(1<<16)
 #define TRE_MAX_STRING	INT_MAX
-#define TRE_MAX_STACK	1048576
+#define TRE_MAX_STACK	(1<<20)
+#define TRE_MAX_POS	(1<<20)
+#define TRE_MAX_TRANS	(1<<24)
 
 #ifdef TRE_DEBUG
 #include <stdio.h>

tests/test-limits.c

@@ -33,8 +33,12 @@ static void notok(void) { fputc('-', stderr); ntests++; }
 static void done(void) { fputc('\n', stderr); exit(nok == ntests ? 0 : 1); }
 #define check(expr) do { ((expr) ? ok() : notok()); } while (0)
 
-int
-main(void)
+/*
+ * These tests exercise TRE_MAX_RE / TRE_MAX_STRING and allocate large
+ * amounts of memory.  They will most likely not run on a 32-bit system.
+ */
+static void
+limits(void)
 {
   regmatch_t pm[9];
   regex_t preg;
@@ -137,5 +141,41 @@ main(void)
     }
     free(buf);
   }
+}
+
+/*
+ * These tests exercise TRE_MAX_POS / TRE_MAX_TRANS and don't interact
+ * well with the various memory tricks which retest plays.
+ */
+static void
+extras(void)
+{
+  static const struct {
+    const char *regex;
+    int result;
+  } testcases[] = {
+    { "(a*){13}", 0 },
+    { "((a*){13}){13}", 0 },
+    { "(((a*){13}){13}){13}", 0 },
+    { "((((a*){13}){13}){13}){13}", REG_ESPACE },
+  };
+  regex_t preg;
+  int error;
+
+  fputc('X', stderr);
+  for (unsigned int i = 0; i < sizeof(testcases) / sizeof(testcases[0]); i++)
+    {
+      error = regcomp(&preg, testcases[i].regex, REG_EXTENDED);
+      check(error == testcases[i].result);
+      if (error == 0)
+	regfree(&preg);
+    }
+}
+
+int
+main(void)
+{
+  limits();
+  extras();
   done();
 }