Add support for window functions

Author: ndm2

src/Tokenizer.php

@@ -70,6 +70,7 @@ final class Tokenizer
         'CONVERT',
         'CREATE',
         'CROSS',
+        'CURRENT ROW',
         'CURRENT_TIMESTAMP',
         'DATABASE',
         'DATABASES',
@@ -108,19 +109,22 @@ final class Tokenizer
         'FAST',
         'FIELDS',
         'FILE',
+        'FILTER',
         'FIRST',
         'FIXED',
         'FLUSH',
         'FOR',
         'FORCE',
+        'FOLLOWING',
         'FOREIGN',
         'FULL',
         'FULLTEXT',
         'FUNCTION',
         'GLOBAL',
         'GRANT',
         'GRANTS',
-        'GROUP_CONCAT',
+        'GROUP',
+        'GROUPS',
         'HEAP',
         'HIGH_PRIORITY',
         'HOSTS',
@@ -180,6 +184,7 @@ final class Tokenizer
         'MYISAM',
         'NAMES',
         'NATURAL',
+        'NO OTHERS',
         'NOT',
         'NOW()',
         'NULL',
@@ -192,12 +197,14 @@ final class Tokenizer
         'ON UPDATE',
         'ON DELETE',
         'OUTFILE',
+        'OVER',
         'PACK_KEYS',
         'PAGE',
         'PARTIAL',
         'PARTITION',
         'PARTITIONS',
         'PASSWORD',
+        'PRECEDING',
         'PRIMARY',
         'PRIVILEGES',
         'PROCEDURE',
@@ -278,13 +285,15 @@ final class Tokenizer
         'TEMPORARY',
         'TERMINATED',
         'THEN',
+        'TIES',
         'TO',
         'TRAILING',
         'TRANSACTIONAL',
         'TRUE',
         'TRUNCATE',
         'TYPE',
         'TYPES',
+        'UNBOUNDED',
         'UNCOMMITTED',
         'UNIQUE',
         'UNLOCK',
@@ -329,6 +338,11 @@ final class Tokenizer
         'UNION',
         'EXCEPT',
         'INTERSECT',
+        'PARTITION BY',
+        'ROWS',
+        'RANGE',
+        'GROUPS',
+        'WINDOW',
     ];
 
     /** @var string[] */
@@ -343,6 +357,7 @@ final class Tokenizer
         'XOR',
         'OR',
         'AND',
+        'EXCLUDE',
     ];
 
     /** @var string[] */
@@ -353,6 +368,7 @@ final class Tokenizer
         'ADDTIME',
         'AES_DECRYPT',
         'AES_ENCRYPT',
+        'APPROX_COUNT_DISTINCT',
         'AREA',
         'ASBINARY',
         'ASCII',
@@ -382,6 +398,7 @@ final class Tokenizer
         'CHARACTER_LENGTH',
         'CHARSET',
         'CHAR_LENGTH',
+        'CHECKSUM_AGG',
         'COALESCE',
         'COERCIBILITY',
         'COLLATION',
@@ -397,8 +414,10 @@ final class Tokenizer
         'COS',
         'COT',
         'COUNT',
+        'COUNT_BIG',
         'CRC32',
         'CROSSES',
+        'CUME_DIST',
         'CURDATE',
         'CURRENT_DATE',
         'CURRENT_TIME',
@@ -420,6 +439,7 @@ final class Tokenizer
         'DECODE',
         'DEFAULT',
         'DEGREES',
+        'DENSE_RANK',
         'DES_DECRYPT',
         'DES_ENCRYPT',
         'DIFFERENCE',
@@ -439,6 +459,7 @@ final class Tokenizer
         'EXTRACTVALUE',
         'FIELD',
         'FIND_IN_SET',
+        'FIRST_VALUE',
         'FLOOR',
         'FORMAT',
         'FOUND_ROWS',
@@ -459,6 +480,8 @@ final class Tokenizer
         'GET_LOCK',
         'GLENGTH',
         'GREATEST',
+        'GROUPING',
+        'GROUPING_ID',
         'GROUP_CONCAT',
         'GROUP_UNIQUE_USERS',
         'HEX',
@@ -480,9 +503,12 @@ final class Tokenizer
         'ISSIMPLE',
         'IS_FREE_LOCK',
         'IS_USED_LOCK',
+        'LAG',
         'LAST_DAY',
         'LAST_INSERT_ID',
+        'LAST_VALUE',
         'LCASE',
+        'LEAD',
         'LEAST',
         'LEFT',
         'LENGTH',
@@ -491,6 +517,7 @@ final class Tokenizer
         'LINESTRING',
         'LINESTRINGFROMTEXT',
         'LINESTRINGFROMWKB',
+        'LISTAGG',
         'LN',
         'LOAD_FILE',
         'LOCALTIME',
@@ -538,6 +565,8 @@ final class Tokenizer
         'MULTIPOLYGONFROMTEXT',
         'MULTIPOLYGONFROMWKB',
         'NAME_CONST',
+        'NTH_VALUE',
+        'NTILE',
         'NULLIF',
         'NUMGEOMETRIES',
         'NUMINTERIORRINGS',
@@ -548,6 +577,9 @@ final class Tokenizer
         'ORD',
         'OVERLAPS',
         'PASSWORD',
+        'PERCENT_RANK',
+        'PERCENTILE_CONT',
+        'PERCENTILE_DISC',
         'PERIOD_ADD',
         'PERIOD_DIFF',
         'PI',
@@ -568,6 +600,7 @@ final class Tokenizer
         'QUOTE',
         'RADIANS',
         'RAND',
+        'RANK',
         'RELATED',
         'RELEASE_LOCK',
         'REPEAT',
@@ -576,6 +609,7 @@ final class Tokenizer
         'RIGHT',
         'ROUND',
         'ROW_COUNT',
+        'ROW_NUMBER',
         'RPAD',
         'RTRIM',
         'SCHEMA',
@@ -593,9 +627,12 @@ final class Tokenizer
         'SRID',
         'STARTPOINT',
         'STD',
+        'STDEV',
+        'STDEVP',
         'STDDEV',
         'STDDEV_POP',
         'STDDEV_SAMP',
+        'STRING_AGG',
         'STRCMP',
         'STR_TO_DATE',
         'SUBDATE',
@@ -632,7 +669,9 @@ final class Tokenizer
         'UTC_TIME',
         'UTC_TIMESTAMP',
         'UUID',
+        'VAR',
         'VARIANCE',
+        'VARP',
         'VAR_POP',
         'VAR_SAMP',
         'VERSION',

tests/clihighlight.html

@@ -876,3 +876,102 @@
   JOIN cte2
 WHERE
   cte1.a = cte2.c;
+---
+SELECT
+  a,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE NO OTHERS
+  ) AS no_others,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE CURRENT ROW
+  ) AS current_row,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE GROUP
+  ) AS grp,
+  GROUP_CONCAT(b, '.') OVER (
+    ORDER BY
+      c
+    GROUPS
+      BETWEEN UNBOUNDED PRECEDING
+      AND CURRENT ROW
+      EXCLUDE TIES
+  ) AS tie,
+  GROUP_CONCAT(b, '.') FILTER (
+    WHERE
+      c != 'two'
+  ) OVER (
+    ORDER BY
+      a
+  ) AS filtered,
+  CONVERT(
+    VARCHAR(20),
+    AVG(SalesYTD) OVER (
+      PARTITION BY
+        TerritoryID
+      ORDER BY
+        DATEPART(yy, ModifiedDate)
+    ),
+    1
+  ) AS MovingAvg,
+  AVG(starting_salary) OVER w2 AVG,
+  MIN(starting_salary) OVER w2 MIN_STARTING_SALARY,
+  MAX(starting_salary) OVER (
+    w1
+    ORDER BY
+      hire_date
+  ),
+  LISTAGG(arg, ',') OVER (
+    PARTITION BY
+      part
+    ORDER BY
+      ord
+    ROWS
+      BETWEEN 1 PRECEDING
+      AND 1 FOLLOWING
+  ) AS LISTAGG_ROWS,
+  LISTAGG(arg, ',') OVER (
+    PARTITION BY
+      part
+    ORDER BY
+      ord
+    RANGE
+      BETWEEN 1 PRECEDING
+      AND 1 FOLLOWING
+  ) AS LISTAGG_RANGE,
+  MIN(Revenue) OVER (
+    PARTITION BY
+      DepartmentID
+    ORDER BY
+      RevenueYear
+    ROWS
+      BETWEEN CURRENT ROW
+      AND UNBOUNDED FOLLOWING
+  ) AS MinRevenueBeyond
+FROM
+  t1
+WINDOW
+  w1 AS (
+    PARTITION BY
+      department,
+      division
+  ),
+  w2 AS (
+    w1
+    ORDER BY
+      hire_date
+  );

tests/compress.html

@@ -89,3 +89,5 @@
 WITH cte AS (SELECT a, b FROM table), RECURSIVE fibonacci (n, fib_n, next_fib_n) AS ( SELECT 1, 0, 1 UNION ALL SELECT n + 1, next_fib_n, fib_n + next_fib_n FROM fibonacci WHERE n < 10 ) SELECT * FROM fibonacci;
 ---
 WITH cte1 AS (SELECT a, b FROM table1), cte2 AS (SELECT c, d FROM table2) SELECT b, d FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;
+---
+SELECT a, GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS no_others, GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW) AS current_row, GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP) AS grp, GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES) AS tie, GROUP_CONCAT(b, '.') FILTER (WHERE c != 'two') OVER (ORDER BY a) AS filtered, CONVERT(VARCHAR(20), AVG(SalesYTD) OVER (PARTITION BY TerritoryID ORDER BY DATEPART(yy, ModifiedDate)), 1) AS MovingAvg, AVG(starting_salary) OVER w2 AVG, MIN(starting_salary) OVER w2 MIN_STARTING_SALARY, MAX(starting_salary) OVER (w1 ORDER BY hire_date), LISTAGG(arg, ',') OVER (PARTITION BY part ORDER BY ord ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS LISTAGG_ROWS, LISTAGG(arg, ',') OVER (PARTITION BY part ORDER BY ord RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS LISTAGG_RANGE, MIN(Revenue) OVER (PARTITION BY DepartmentID ORDER BY RevenueYear ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS MinRevenueBeyond FROM t1 WINDOW w1 AS (PARTITION BY department, division), w2 AS (w1 ORDER BY hire_date);