Do not assume "SET" as top level when preceded with "CHARACTER" (#150)

* Do not assume "SET" as top level when preceded with "CHARACTER"

* test PHP 8.4

Author: mvorisek

.github/workflows/continuous-integration.yml

@@ -13,6 +13,6 @@ jobs:
     name: "PHPUnit"
     uses: "doctrine/.github/.github/workflows/[email protected]"
     with:
-      php-versions: '["8.1", "8.2", "8.3"]'
+      php-versions: '["8.1", "8.2", "8.3", "8.4"]'
     secrets:
       CODECOV_TOKEN: "${{ secrets.CODECOV_TOKEN }}"

src/Tokenizer.php

@@ -868,7 +868,7 @@ private function makeTokenizeRegexes(): array
             Token::TOKEN_TYPE_BOUNDARY => $regexBoundaries,
             // A reserved word cannot be preceded by a '.'
             // this makes it so in "mytable.from", "from" is not considered a reserved word
-            Token::TOKEN_TYPE_RESERVED_TOPLEVEL => '(?<!\.)' . $regexReservedToplevel . '(?=$|\s|' . $regexBoundaries . ')',
+            Token::TOKEN_TYPE_RESERVED_TOPLEVEL => '(?<!\.|\sCHARACTER\s(?=SET\s))' . $regexReservedToplevel . '(?=$|\s|' . $regexBoundaries . ')',
             Token::TOKEN_TYPE_RESERVED_NEWLINE => '(?<!\.)' . $regexReservedNewline . '(?=$|\s|' . $regexBoundaries . ')',
             Token::TOKEN_TYPE_RESERVED => '(?<!\.)' . $regexReserved . '(?=$|\s|' . $regexBoundaries . ')'
                 // A function must be succeeded by '('

tests/clihighlight.txt

@@ -1201,3 +1201,7 @@ MY_NON_TOP_LEVEL_KEYWORD_FX_3();
   t
 WHERE
   t.start = t.end
+---
+CREATE TABLE t (
+  c VARCHAR(20)
+) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB

tests/compress.txt

@@ -109,3 +109,5 @@ BEGIN FOR i IN 1..5 LOOP DBMS_OUTPUT.PUT_LINE(i); END LOOP; END;
 SELECT a FROM test STRAIGHT_JOIN test2 ON test.id = test2.id
 ---
 SELECT t.id, t.start, t.end, t.end AS e2, t.limit, t.begin, t.case, t.when, t.then, t.else FROM t WHERE t.start = t.end
+---
+CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB

tests/format-highlight.html

@@ -1201,3 +1201,7 @@
   <span style="color: #333;">t</span>
 <span style="font-weight:bold;">WHERE</span>
   <span style="color: #333;">t</span><span >.</span><span style="color: #333;">start</span> <span >=</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">end</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">CREATE</span> <span style="font-weight:bold;">TABLE</span> <span style="color: #333;">t</span> (
+  <span style="color: #333;">c</span> <span style="font-weight:bold;">VARCHAR</span>(<span style="color: green;">20</span>)
+) <span style="font-weight:bold;">DEFAULT</span> <span style="font-weight:bold;">CHARACTER</span> <span style="font-weight:bold;">SET</span> <span style="color: #333;">utf8mb4</span> <span style="font-weight:bold;">ENGINE</span> <span >=</span> <span style="color: #333;">InnoDB</span></pre>

tests/format.txt

@@ -1199,3 +1199,7 @@ FROM
   t
 WHERE
   t.start = t.end
+---
+CREATE TABLE t (
+  c VARCHAR(20)
+) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB

tests/highlight.html

@@ -423,3 +423,5 @@
   <span style="color: #333;">t</span><span >.</span><span style="color: #333;">else</span>
 <span style="font-weight:bold;">FROM</span> <span style="color: #333;">t</span>
 <span style="font-weight:bold;">WHERE</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">start</span> <span >=</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">end</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">CREATE</span> <span style="font-weight:bold;">TABLE</span> <span style="color: #333;">t</span> (<span style="color: #333;">c</span> <span style="font-weight:bold;">VARCHAR</span>(<span style="color: green;">20</span>)) <span style="font-weight:bold;">DEFAULT</span> <span style="font-weight:bold;">CHARACTER</span> <span style="font-weight:bold;">SET</span> <span style="color: #333;">utf8mb4</span> <span style="font-weight:bold;">ENGINE</span> <span >=</span> <span style="color: #333;">InnoDB</span></pre>

tests/sql.sql

@@ -423,3 +423,5 @@ SELECT
   t.else
 FROM t
 WHERE t.start = t.end
+---
+CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB