From 7256d8a2a1efd3fb06fb32e99ffb87960443d13e Mon Sep 17 00:00:00 2001 From: Kevin Ushey Date: Sun, 9 Aug 2026 10:52:09 -0700 Subject: [PATCH 1/3] compile: Convert tre_ast_to_tnfa() to iteration over a tre_stack The recursive implementation overflows the C stack for large patterns: a catenation of 50,000 literals crashes tre_regcomp() on Windows (1 MB default stack). Reuse the compilation stack already allocated in tre_compile() and iterate instead, preserving the traversal order of the recursive version (children pushed right-then-left, tre_make_trans invoked before descending, as before). The counting pass call site now checks the return value, since the iterative version can fail with REG_ESPACE if the stack cannot grow. Removes the corresponding TODO item at the top of the file. This fix has been carried in GNU R's vendored copy of TRE since 2011 (with a separately allocated stack there). --- lib/tre-compile.c | 100 ++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/lib/tre-compile.c b/lib/tre-compile.c index 302efae..5cabc8f 100644 --- a/lib/tre-compile.c +++ b/lib/tre-compile.c @@ -6,13 +6,6 @@ */ -/* - TODO: - - Fix tre_ast_to_tnfa() to recurse using a stack instead of recursive - function calls. -*/ - - #ifdef HAVE_CONFIG_H #include #endif /* HAVE_CONFIG_H */ @@ -1777,9 +1770,13 @@ tre_make_trans(tre_pos_and_tags_t *p1, tre_pos_and_tags_t *p2, /* Converts the syntax tree to a TNFA. All the transitions in the TNFA are labelled with one character range (there are no transitions on empty strings). The TNFA takes O(n^2) space in the worst case, `n' is size of - the regexp. */ + the regexp. + + This is the iterative version using an explicit stack; the previous + recursive implementation could overflow the C stack for large patterns. */ static reg_errcode_t -tre_ast_to_tnfa(tre_ast_node_t *node, tre_tnfa_transition_t *transitions, +tre_ast_to_tnfa(tre_stack_t *stack, tre_ast_node_t *node, + tre_tnfa_transition_t *transitions, int *counts, int *offs) { tre_union_t *uni; @@ -1787,51 +1784,56 @@ tre_ast_to_tnfa(tre_ast_node_t *node, tre_tnfa_transition_t *transitions, tre_iteration_t *iter; reg_errcode_t errcode = REG_OK; - /* XXX - recurse using a stack!. */ - switch (node->type) - { - case LITERAL: - break; - case UNION: - uni = (tre_union_t *)node->obj; - errcode = tre_ast_to_tnfa(uni->left, transitions, counts, offs); - if (errcode != REG_OK) - return errcode; - errcode = tre_ast_to_tnfa(uni->right, transitions, counts, offs); - break; - - case CATENATION: - cat = (tre_catenation_t *)node->obj; - /* Add a transition from each position in cat->left->lastpos - to each position in cat->right->firstpos. */ - errcode = tre_make_trans(cat->left->lastpos, cat->right->firstpos, - transitions, counts, offs); - if (errcode != REG_OK) - return errcode; - errcode = tre_ast_to_tnfa(cat->left, transitions, counts, offs); - if (errcode != REG_OK) - return errcode; - errcode = tre_ast_to_tnfa(cat->right, transitions, counts, offs); - break; + STACK_PUSHR(stack, voidptr, node); - case ITERATION: - iter = (tre_iteration_t *)node->obj; - assert(iter->max == -1 || iter->max == 1); + while (tre_stack_num_items(stack)) + { + node = tre_stack_pop_voidptr(stack); - if (iter->max == -1) + switch (node->type) { - assert(iter->min == 0 || iter->min == 1); - /* Add a transition from each last position in the iterated - expression to each first position. */ - errcode = tre_make_trans(iter->arg->lastpos, iter->arg->firstpos, + case LITERAL: + break; + + case UNION: + uni = (tre_union_t *)node->obj; + /* Push right before left so that left is processed first, + preserving the traversal order of the recursive version. */ + STACK_PUSHR(stack, voidptr, uni->right); + STACK_PUSHR(stack, voidptr, uni->left); + break; + + case CATENATION: + cat = (tre_catenation_t *)node->obj; + /* Add a transition from each position in cat->left->lastpos + to each position in cat->right->firstpos. */ + errcode = tre_make_trans(cat->left->lastpos, cat->right->firstpos, transitions, counts, offs); if (errcode != REG_OK) return errcode; + STACK_PUSHR(stack, voidptr, cat->right); + STACK_PUSHR(stack, voidptr, cat->left); + break; + + case ITERATION: + iter = (tre_iteration_t *)node->obj; + assert(iter->max == -1 || iter->max == 1); + + if (iter->max == -1) + { + assert(iter->min == 0 || iter->min == 1); + /* Add a transition from each last position in the iterated + expression to each first position. */ + errcode = tre_make_trans(iter->arg->lastpos, iter->arg->firstpos, + transitions, counts, offs); + if (errcode != REG_OK) + return errcode; + } + STACK_PUSHR(stack, voidptr, iter->arg); + break; } - errcode = tre_ast_to_tnfa(iter->arg, transitions, counts, offs); - break; } - return errcode; + return REG_OK; } #define ERROR_EXIT(err) \ @@ -2002,7 +2004,9 @@ tre_compile(regex_t *preg, const tre_char_t *regex, size_t n, int cflags) for (i = 0; i < numpos; i++) counts[i] = 0; - tre_ast_to_tnfa(tree, NULL, counts, NULL); + errcode = tre_ast_to_tnfa(stack, tree, NULL, counts, NULL); + if (errcode != REG_OK) + ERROR_EXIT(errcode); add = 0; for (i = 0; i < numpos; i++) @@ -2018,7 +2022,7 @@ tre_compile(regex_t *preg, const tre_char_t *regex, size_t n, int cflags) tnfa->num_transitions = add; DPRINT(("Converting to TNFA:\n")); - errcode = tre_ast_to_tnfa(tree, transitions, counts, offs); + errcode = tre_ast_to_tnfa(stack, tree, transitions, counts, offs); if (errcode != REG_OK) ERROR_EXIT(errcode); From a2be1d9766665d915bc5b7ce10467657dcd01653 Mon Sep 17 00:00:00 2001 From: Kevin Ushey Date: Sun, 9 Aug 2026 12:00:00 -0700 Subject: [PATCH 2/3] tests: Check that compiling a large regex does not overflow the C stack Compile and match a 60000-character ERE in a thread with a 1 MB stack. With the recursive tre_ast_to_tnfa() this crashes (one C stack frame per AST node); with the iterative version from this branch it compiles and matches within the small stack. A thread with pthread_attr_setstacksize() is used rather than setrlimit(RLIMIT_STACK) because lowering the rlimit does not shrink a main-thread stack that earlier tests have already grown, and macOS pre-maps the main stack at exec time. pthread_create() is in libc on glibc >= 2.34 and on macOS, so no build system changes are needed; the test is compiled out on platforms without pthreads. --- tests/test-limits.c | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test-limits.c b/tests/test-limits.c index 7362f22..9fd5da2 100644 --- a/tests/test-limits.c +++ b/tests/test-limits.c @@ -8,6 +8,11 @@ #include #include +#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) +# include +# define TEST_STACK_DEPTH 1 +#endif + #ifdef USE_SYSTEM_REGEX # include # ifndef REG_BASIC @@ -33,6 +38,28 @@ static void notok(void) { fputc('-', stderr); ntests++; } static void done(void) { fputc('\n', stderr); exit(nok == ntests ? 0 : 1); } #define check(expr) do { ((expr) ? ok() : notok()); } while (0) +#ifdef TEST_STACK_DEPTH +/* Compile and match a pattern in a small-stack thread; the subject is + the pattern itself (a string of 'a's). */ +static void * +compile_large(void *arg) +{ + const char *pat = arg; + regex_t re; + regmatch_t m[1]; + intptr_t err; + + err = regcomp(&re, pat, REG_EXTENDED); + if (err == REG_OK) + { + if (regexec(&re, pat, 1, m, 0) != REG_OK) + err = -1; + regfree(&re); + } + return (void *)err; +} +#endif /* TEST_STACK_DEPTH */ + int main(void) { @@ -137,5 +164,34 @@ main(void) } free(buf); } + + /* Compiling a large regex must not overflow the C stack: the TNFA + conversion recursed once per AST node until it was converted to + iteration. Run the compilation in a thread with a small stack so + the recursive implementation would crash here even on platforms + with large default stacks. */ +#ifdef TEST_STACK_DEPTH + { + pthread_t thr; + pthread_attr_t attr; + void *ret = (void *)-1; + size = 60000; /* <= TRE_MAX_RE, deep enough to need several MB of + C stack at one stack frame per AST node */ + if ((buf = malloc(size + 1)) == NULL) { + notok(); + } else { + memset(buf, 'a', size); + buf[size] = '\0'; + fprintf(stderr, "S"); + check(pthread_attr_init(&attr) == 0 + && pthread_attr_setstacksize(&attr, 1024 * 1024) == 0 + && pthread_create(&thr, &attr, compile_large, buf) == 0 + && pthread_join(thr, &ret) == 0 + && ret == (void *)REG_OK); + free(buf); + } + } +#endif /* TEST_STACK_DEPTH */ + done(); } From c20dfe2e02f9d1f9cdc7432285d69a922dd85807 Mon Sep 17 00:00:00 2001 From: Kevin Ushey Date: Tue, 18 Aug 2026 11:03:16 -0700 Subject: [PATCH 3/3] test-limits: compile only in the small-stack thread --- tests/test-limits.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/test-limits.c b/tests/test-limits.c index 9fd5da2..39ce5c3 100644 --- a/tests/test-limits.c +++ b/tests/test-limits.c @@ -39,23 +39,20 @@ static void done(void) { fputc('\n', stderr); exit(nok == ntests ? 0 : 1); } #define check(expr) do { ((expr) ? ok() : notok()); } while (0) #ifdef TEST_STACK_DEPTH -/* Compile and match a pattern in a small-stack thread; the subject is - the pattern itself (a string of 'a's). */ +/* Compile a pattern in a small-stack thread. Compile only: with + TRE_USE_ALLOCA, regexec() allocates buffers proportional to the + automaton size on the stack, which cannot fit in a small stack no + matter how compilation is implemented. */ static void * compile_large(void *arg) { const char *pat = arg; regex_t re; - regmatch_t m[1]; intptr_t err; err = regcomp(&re, pat, REG_EXTENDED); if (err == REG_OK) - { - if (regexec(&re, pat, 1, m, 0) != REG_OK) - err = -1; - regfree(&re); - } + regfree(&re); return (void *)err; } #endif /* TEST_STACK_DEPTH */