Fix excessive backtracking in regex engine by introducing backtracking limit

The regex engine was prone to excessive backtracking, leading to
timeouts and infinite loops, particularly with patterns involving
nested quantifiers.

This commit introduces a backtracking counter and a limit of 1000
backtracking steps. When this limit is exceeded, the regex engine
aborts to prevent excessive backtracking.

Author: renatahodovan

libregexp.c

@@ -1929,6 +1929,7 @@ typedef struct {
     /* 0 = 8 bit chars, 1 = 16 bit chars, 2 = 16 bit chars, UTF-16 */
     int cbuf_type;
     int capture_count;
+    int backtrack_count;
     int stack_size_max;
     BOOL multi_line;
     BOOL ignore_case;
@@ -1995,6 +1996,10 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture,
 
     for(;;) {
         //        printf("top=%p: pc=%d\n", th_list.top, (int)(pc - (bc_buf + RE_HEADER_LEN)));
+        if (++s->backtrack_count > 1000) {
+            return -1; // backtracking limit exceeded
+        }
+
         opcode = *pc++;
         switch(opcode) {
         case REOP_match:
@@ -2401,6 +2406,7 @@ int lre_exec(uint8_t **capture,
     s->ignore_case = (re_flags & LRE_FLAG_IGNORECASE) != 0;
     s->is_unicode = (re_flags & LRE_FLAG_UNICODE) != 0;
     s->capture_count = bc_buf[RE_HEADER_CAPTURE_COUNT];
+    s->backtrack_count = 0;
     s->stack_size_max = bc_buf[RE_HEADER_STACK_SIZE];
     s->cbuf = cbuf;
     s->cbuf_end = cbuf + (clen << cbuf_type);