Merge pull request #72 from ndm2/common-table-expressions-and-window-functions

Add support for common table expressions and window functions

Author: greg0ire

src/Tokenizer.php

@@ -70,6 +70,7 @@ final class Tokenizer
         'CONVERT',
         'CREATE',
         'CROSS',
+        'CURRENT ROW',
         'CURRENT_TIMESTAMP',
         'DATABASE',
         'DATABASES',
@@ -108,19 +109,22 @@ final class Tokenizer
         'FAST',
         'FIELDS',
         'FILE',
+        'FILTER',
         'FIRST',
         'FIXED',
         'FLUSH',
         'FOR',
         'FORCE',
+        'FOLLOWING',
         'FOREIGN',
         'FULL',
         'FULLTEXT',
         'FUNCTION',
         'GLOBAL',
         'GRANT',
         'GRANTS',
-        'GROUP_CONCAT',
+        'GROUP',
+        'GROUPS',
         'HEAP',
         'HIGH_PRIORITY',
         'HOSTS',
@@ -180,6 +184,7 @@ final class Tokenizer
         'MYISAM',
         'NAMES',
         'NATURAL',
+        'NO OTHERS',
         'NOT',
         'NOW()',
         'NULL',
@@ -192,12 +197,14 @@ final class Tokenizer
         'ON UPDATE',
         'ON DELETE',
         'OUTFILE',
+        'OVER',
         'PACK_KEYS',
         'PAGE',
         'PARTIAL',
         'PARTITION',
         'PARTITIONS',
         'PASSWORD',
+        'PRECEDING',
         'PRIMARY',
         'PRIVILEGES',
         'PROCEDURE',
@@ -213,6 +220,7 @@ final class Tokenizer
         'READ',
         'READ_ONLY',
         'READ_WRITE',
+        'RECURSIVE',
         'REFERENCES',
         'REGEXP',
         'RELOAD',
@@ -277,13 +285,15 @@ final class Tokenizer
         'TEMPORARY',
         'TERMINATED',
         'THEN',
+        'TIES',
         'TO',
         'TRAILING',
         'TRANSACTIONAL',
         'TRUE',
         'TRUNCATE',
         'TYPE',
         'TYPES',
+        'UNBOUNDED',
         'UNCOMMITTED',
         'UNIQUE',
         'UNLOCK',
@@ -307,6 +317,7 @@ final class Tokenizer
      * @var string[]
      */
     private $reservedToplevel = [
+        'WITH',
         'SELECT',
         'FROM',
         'WHERE',
@@ -327,6 +338,11 @@ final class Tokenizer
         'UNION',
         'EXCEPT',
         'INTERSECT',
+        'PARTITION BY',
+        'ROWS',
+        'RANGE',
+        'GROUPS',
+        'WINDOW',
     ];
 
     /** @var string[] */
@@ -341,6 +357,7 @@ final class Tokenizer
         'XOR',
         'OR',
         'AND',
+        'EXCLUDE',
     ];
 
     /** @var string[] */
@@ -351,6 +368,7 @@ final class Tokenizer
         'ADDTIME',
         'AES_DECRYPT',
         'AES_ENCRYPT',
+        'APPROX_COUNT_DISTINCT',
         'AREA',
         'ASBINARY',
         'ASCII',
@@ -380,6 +398,7 @@ final class Tokenizer
         'CHARACTER_LENGTH',
         'CHARSET',
         'CHAR_LENGTH',
+        'CHECKSUM_AGG',
         'COALESCE',
         'COERCIBILITY',
         'COLLATION',
@@ -395,8 +414,10 @@ final class Tokenizer
         'COS',
         'COT',
         'COUNT',
+        'COUNT_BIG',
         'CRC32',
         'CROSSES',
+        'CUME_DIST',
         'CURDATE',
         'CURRENT_DATE',
         'CURRENT_TIME',
@@ -418,6 +439,7 @@ final class Tokenizer
         'DECODE',
         'DEFAULT',
         'DEGREES',
+        'DENSE_RANK',
         'DES_DECRYPT',
         'DES_ENCRYPT',
         'DIFFERENCE',
@@ -437,6 +459,7 @@ final class Tokenizer
         'EXTRACTVALUE',
         'FIELD',
         'FIND_IN_SET',
+        'FIRST_VALUE',
         'FLOOR',
         'FORMAT',
         'FOUND_ROWS',
@@ -457,6 +480,8 @@ final class Tokenizer
         'GET_LOCK',
         'GLENGTH',
         'GREATEST',
+        'GROUPING',
+        'GROUPING_ID',
         'GROUP_CONCAT',
         'GROUP_UNIQUE_USERS',
         'HEX',
@@ -478,9 +503,12 @@ final class Tokenizer
         'ISSIMPLE',
         'IS_FREE_LOCK',
         'IS_USED_LOCK',
+        'LAG',
         'LAST_DAY',
         'LAST_INSERT_ID',
+        'LAST_VALUE',
         'LCASE',
+        'LEAD',
         'LEAST',
         'LEFT',
         'LENGTH',
@@ -489,6 +517,7 @@ final class Tokenizer
         'LINESTRING',
         'LINESTRINGFROMTEXT',
         'LINESTRINGFROMWKB',
+        'LISTAGG',
         'LN',
         'LOAD_FILE',
         'LOCALTIME',
@@ -536,6 +565,8 @@ final class Tokenizer
         'MULTIPOLYGONFROMTEXT',
         'MULTIPOLYGONFROMWKB',
         'NAME_CONST',
+        'NTH_VALUE',
+        'NTILE',
         'NULLIF',
         'NUMGEOMETRIES',
         'NUMINTERIORRINGS',
@@ -546,6 +577,9 @@ final class Tokenizer
         'ORD',
         'OVERLAPS',
         'PASSWORD',
+        'PERCENT_RANK',
+        'PERCENTILE_CONT',
+        'PERCENTILE_DISC',
         'PERIOD_ADD',
         'PERIOD_DIFF',
         'PI',
@@ -566,6 +600,7 @@ final class Tokenizer
         'QUOTE',
         'RADIANS',
         'RAND',
+        'RANK',
         'RELATED',
         'RELEASE_LOCK',
         'REPEAT',
@@ -574,6 +609,7 @@ final class Tokenizer
         'RIGHT',
         'ROUND',
         'ROW_COUNT',
+        'ROW_NUMBER',
         'RPAD',
         'RTRIM',
         'SCHEMA',
@@ -591,9 +627,12 @@ final class Tokenizer
         'SRID',
         'STARTPOINT',
         'STD',
+        'STDEV',
+        'STDEVP',
         'STDDEV',
         'STDDEV_POP',
         'STDDEV_SAMP',
+        'STRING_AGG',
         'STRCMP',
         'STR_TO_DATE',
         'SUBDATE',
@@ -630,7 +669,9 @@ final class Tokenizer
         'UTC_TIME',
         'UTC_TIMESTAMP',
         'UUID',
+        'VAR',
         'VARIANCE',
+        'VARP',
         'VAR_POP',
         'VAR_SAMP',
         'VERSION',

tests/clihighlight.html

@@ -824,3 +824,154 @@
 FROM
   b
   LEFT OUTER JOIN c on (d = f);
+---
+WITH
+  cte AS (
+    SELECT
+      a,
+      b
+    FROM
+      table
+  ),
+  RECURSIVE fibonacci (n, fib_n, next_fib_n) AS (
+    SELECT
+      1,
+      0,
+      1
+    UNION ALL
+    SELECT
+      n + 1,
+      next_fib_n,
+      fib_n + next_fib_n
+    FROM
+      fibonacci
+    WHERE
+      n < 10
+  )
+SELECT
+  *
+FROM
+  fibonacci;
+---
+WITH
+  cte1 AS (
+    SELECT
+      a,
+      b
+    FROM
+      table1
+  ),
+  cte2 AS (
+    SELECT
+      c,
+      d
+    FROM
+      table2
+  )
+SELECT
+  b,
+  d
+FROM
+  cte1
+  JOIN cte2
+WHERE
+  cte1.a = cte2.c;
+---
+SELECT
+  a,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE NO OTHERS
+  ) AS no_others,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE CURRENT ROW
+  ) AS current_row,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE GROUP
+  ) AS grp,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE TIES
+  ) AS tie,
+  GROUP_CONCAT(b, '.') FILTER (
+    WHERE
+      c != 'two'
+  ) OVER (
+    ORDER BY
+      a
+  ) AS filtered,
+  CONVERT(
+    VARCHAR(20),
+    AVG(SalesYTD) OVER (
+      PARTITION BY
+        TerritoryID
+      ORDER BY
+        DATEPART(yy, ModifiedDate)
+    ),
+    1
+  ) AS MovingAvg,
+  AVG(starting_salary) OVER w2 AVG,
+  MIN(starting_salary) OVER w2 MIN_STARTING_SALARY,
+  MAX(starting_salary) OVER (
+    w1
+    ORDER BY
+      hire_date
+  ),
+  LISTAGG(arg, ',') OVER (
+    PARTITION BY
+      part
+    ORDER BY
+      ord
+    ROWS
+      BETWEEN 1 PRECEDING
+      AND 1 FOLLOWING
+  ) AS LISTAGG_ROWS,
+  LISTAGG(arg, ',') OVER (
+    PARTITION BY
+      part
+    ORDER BY
+      ord
+    RANGE
+      BETWEEN 1 PRECEDING
+      AND 1 FOLLOWING
+  ) AS LISTAGG_RANGE,
+  MIN(Revenue) OVER (
+    PARTITION BY
+      DepartmentID
+    ORDER BY
+      RevenueYear
+    ROWS
+      BETWEEN CURRENT ROW
+      AND UNBOUNDED FOLLOWING
+  ) AS MinRevenueBeyond
+FROM
+  t1
+WINDOW
+  w1 AS (
+    PARTITION BY
+      department,
+      division
+  ),
+  w2 AS (
+    w1
+    ORDER BY
+      hire_date
+  );