Update tokenizer to use Moo instead

.babelrc

@@ -1,4 +1,8 @@
 {
   "presets": ["@babel/preset-env", "@babel/preset-typescript"],
-  "plugins": ["@babel/plugin-proposal-class-properties", "add-module-exports"]
+  "plugins": [
+    "@babel/plugin-proposal-class-properties",
+    "add-module-exports",
+    ["babel-plugin-inline-import", { "extensions": [".sql"] }]
+  ]
 }

.eslintrc

@@ -28,6 +28,7 @@
     "no-useless-concat": "off",
     "no-restricted-syntax": "off",
     "no-constant-condition": "off",
+    "no-unused-expressions": "warn",
     "prefer-template": "off",
     "default-case": "off",
     "prettier/prettier": ["error"],
@@ -36,7 +37,7 @@
     "@typescript-eslint/indent": "off",
     "@typescript-eslint/lines-between-class-members": "off",
     "@typescript-eslint/naming-convention": "error",
-    "@typescript-eslint/no-unused-vars": "error",
+    "@typescript-eslint/no-unused-vars": "warn",
     "@typescript-eslint/quotes": [
       "error",
       "single",

package.json

@@ -98,7 +98,8 @@
     "url": "https://github.com/zeroturnaround/sql-formatter/issues"
   },
   "dependencies": {
-    "argparse": "^2.0.1"
+    "argparse": "^2.0.1",
+    "moo": "^0.5.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.10.4",
@@ -109,12 +110,14 @@
     "@jest/globals": "^28.0.2",
     "@types/babel__core": "^7.1.15",
     "@types/jest": "^27.5.0",
+    "@types/moo": "^0.5.5",
     "@typescript-eslint/eslint-plugin": "^5.21.0",
     "@typescript-eslint/parser": "^5.21.0",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^28.0.2",
     "babel-loader": "^8.1.0",
     "babel-plugin-add-module-exports": "^1.0.2",
+    "babel-plugin-inline-import": "^3.0.0",
     "dedent-js": "^1.0.1",
     "eslint": "^8.14.0",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -131,7 +134,7 @@
     "rimraf": "^3.0.2",
     "ts-jest": "^28.0.0",
     "ts-loader": "^9.2.6",
-    "typescript": "^4.3.5",
+    "typescript": "^4.7.2",
     "webpack": "^5.64.1",
     "webpack-cli": "^4.9.1",
     "webpack-merge": "^5.8.0"
@@ -144,6 +147,9 @@
     "collectCoverage": true,
     "collectCoverageFrom": [
       "src/**/*.ts"
-    ]
+    ],
+    "moduleNameMapper": {
+      "src/(.*)": "<rootDir>/src/$1"
+    }
   }
 }

src/core/AliasAs.ts

@@ -1,5 +1,6 @@
-import { AliasMode } from '../types';
-import { isCommand, isToken, Token, TokenType } from './token';
+import type { AliasMode } from 'src/types';
+
+import { isCommand, isToken, type Token, TokenType } from './token';
 
 export interface TokenStream {
   isWithinSelect(): boolean;

src/core/AsTokenFactory.ts

@@ -1,5 +1,6 @@
-import { KeywordCase } from '../types';
-import { isToken, Token, TokenType } from './token';
+import type { KeywordCase } from 'src/types';
+
+import { isToken, type Token, TokenType } from './token';
 
 export default class AsTokenFactory {
   private detectedCase: KeywordCase;

src/core/Formatter.ts

@@ -1,12 +1,12 @@
+import type { FormatOptions } from 'src/types';
 import Params from './Params';
 import Tokenizer from './Tokenizer';
-import { FormatOptions } from '../types';
 import formatCommaPositions from './formatCommaPositions';
 import formatAliasPositions from './formatAliasPositions';
 import AsTokenFactory from './AsTokenFactory';
-import Parser, { Statement } from './Parser';
+import Parser, { type Statement } from './Parser';
 import StatementFormatter from './StatementFormatter';
-import { Token } from './token';
+import { type Token } from './token';
 import { indentString } from './config';
 
 /** Main formatter class that produces a final output string from list of tokens */

src/core/Indentation.ts

@@ -1,4 +1,4 @@
-import { last } from '../utils';
+import { last } from 'src/utils';
 
 const INDENT_TYPE_TOP_LEVEL = 'top-level';
 const INDENT_TYPE_BLOCK_LEVEL = 'block-level';

src/core/InlineBlock.ts

@@ -1,4 +1,4 @@
-import { isToken, Token, TokenType } from './token';
+import { isToken, type Token, TokenType } from './token';
 
 /**
  * Bookkeeper for inline blocks.

src/core/Params.ts

@@ -1,4 +1,4 @@
-import type { Token } from './token';
+import { type Token } from './token';
 
 export type ParamItems = { [k: string]: string };
 

src/core/Parser.ts

@@ -1,4 +1,4 @@
-import { EOF_TOKEN, Token, TokenType } from './token';
+import { EOF_TOKEN, type Token, TokenType } from './token';
 /* eslint-disable no-cond-assign */
 
 export type Statement = {

src/core/StatementFormatter.ts

@@ -1,13 +1,14 @@
+import type { FormatOptions } from 'src/types';
+import { equalizeWhitespace } from 'src/utils';
+
 import Indentation from './Indentation';
 import InlineBlock from './InlineBlock';
 import Params from './Params';
-import { equalizeWhitespace } from '../utils';
-import { isReserved, isCommand, isToken, Token, TokenType, EOF_TOKEN } from './token';
-import { FormatOptions } from '../types';
+import { isReserved, isCommand, isToken, type Token, TokenType, EOF_TOKEN } from './token';
 import toTabularFormat from './tabularStyle';
 import AliasAs from './AliasAs';
 import AsTokenFactory from './AsTokenFactory';
-import { Statement } from './Parser';
+import { type Statement } from './Parser';
 import { indentString, isTabularStyle } from './config';
 import WhitespaceBuilder, { WS } from './WhitespaceBuilder';
 

src/core/Tokenizer.ts

@@ -1,6 +1,7 @@
+import { equalizeWhitespace, escapeRegExp, id } from 'src/utils';
+
 import * as regexFactory from './regexFactory';
-import { equalizeWhitespace, escapeRegExp, id } from '../utils';
-import { Token, TokenType } from './token'; // convert to partial type import in TS 4.5
+import { type Token, TokenType } from './token';
 
 export const WHITESPACE_REGEX = /^(\s+)/u;
 const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing
@@ -28,7 +29,7 @@ interface TokenizerOptions {
 
 /** Converts SQL language string into a token stream */
 export default class Tokenizer {
-  REGEX_MAP: { [tokenType in TokenType]: RegExp };
+  REGEX_MAP: Partial<{ [tokenType in TokenType]: RegExp }>;
 
   INDEXED_PLACEHOLDER_REGEX?: RegExp;
   IDENT_NAMED_PLACEHOLDER_REGEX?: RegExp;
@@ -165,7 +166,7 @@ export default class Tokenizer {
       this.getTokenOnFirstMatch({
         input,
         type: tokenType,
-        regex: this.REGEX_MAP[tokenType],
+        regex: this.REGEX_MAP[tokenType]!,
         transform: id,
       });
 
@@ -253,7 +254,7 @@ export default class Tokenizer {
         this.getTokenOnFirstMatch({
           input,
           type: tokenType,
-          regex: this.REGEX_MAP[tokenType],
+          regex: this.REGEX_MAP[tokenType]!,
           transform: toCanonicalKeyword,
         }),
       undefined

src/core/WhitespaceBuilder.ts

@@ -1,4 +1,5 @@
-import { last } from '../utils';
+import { last } from 'src/utils';
+
 import Indentation from './Indentation';
 
 /** Whitespace modifiers to be used with add() method */

src/core/config.ts

@@ -1,4 +1,4 @@
-import { FormatOptions } from '../types';
+import type { FormatOptions } from 'src/types';
 
 // Utility functions for config options
 

src/core/formatAliasPositions.ts

@@ -1,4 +1,4 @@
-import { maxLength } from '../utils';
+import { maxLength } from 'src/utils';
 
 /**
  * Handles select alias placement - tabulates if enabled

src/core/formatCommaPositions.ts

@@ -1,5 +1,6 @@
-import { CommaPosition } from '../types';
-import { maxLength } from '../utils';
+import type { CommaPosition } from 'src/types';
+import { maxLength } from 'src/utils';
+
 import { WHITESPACE_REGEX } from './Tokenizer';
 
 /**

src/core/regexFactory.ts

@@ -1,4 +1,4 @@
-import { escapeRegExp, isEmpty, sortByLengthDesc } from '../utils';
+import { escapeRegExp, isEmpty, sortByLengthDesc } from 'src/utils';
 
 /**
  * Builds a RegExp containing all operators for a SQL dialect

src/core/tabularStyle.ts

@@ -1,4 +1,4 @@
-import { IndentStyle } from '../types';
+import type { IndentStyle } from 'src/types';
 
 /**
  * When tabular style enabled,

src/core/token.ts

@@ -11,6 +11,7 @@ export enum TokenType {
   RESERVED_CASE_START = 'RESERVED_CASE_START',
   RESERVED_CASE_END = 'RESERVED_CASE_END',
   OPERATOR = 'OPERATOR',
+  COMMA = 'COMMA',
   BLOCK_START = 'BLOCK_START',
   BLOCK_END = 'BLOCK_END',
   LINE_COMMENT = 'LINE_COMMENT',

src/languages/bigquery.formatter.ts

@@ -1,8 +1,8 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import type { StringPatternType } from '../core/regexFactory';
-import { EOF_TOKEN, Token } from '../core/token';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { EOF_TOKEN, type Token } from 'src/core/token';
+import { dedupe } from 'src/utils';
 
 /**
  * Priority 5 (last)
@@ -835,8 +835,8 @@ export default class BigQueryFormatter extends Formatter {
       reservedBinaryCommands,
       reservedDependentClauses,
       reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []),
-        ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []),
+        ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []),
+        ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []),
       ]),
       stringTypes: BigQueryFormatter.stringTypes,
       indexedPlaceholderTypes: ['?'],

src/languages/db2.formatter.ts

@@ -1,7 +1,7 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import type { StringPatternType } from '../core/regexFactory';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { dedupe } from 'src/utils';
 
 /**
  * Priority 5 (last)
@@ -868,8 +868,8 @@ export default class Db2Formatter extends Formatter {
       reservedBinaryCommands,
       reservedDependentClauses,
       reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []),
-        ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []),
+        ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []),
+        ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []),
       ]),
       stringTypes: Db2Formatter.stringTypes,
       indexedPlaceholderTypes: ['?'],

src/languages/hive.formatter.ts

@@ -1,7 +1,7 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import type { StringPatternType } from '../core/regexFactory';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { dedupe } from 'src/utils';
 
 /**
  * Priority 5 (last)
@@ -623,8 +623,8 @@ export default class HiveFormatter extends Formatter {
       reservedBinaryCommands,
       reservedDependentClauses,
       reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []),
-        ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []),
+        ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []),
+        ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []),
       ]),
       stringTypes: HiveFormatter.stringTypes,
       indexedPlaceholderTypes: ['?'],

src/languages/mariadb.formatter.ts

@@ -1,8 +1,8 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import { EOF_TOKEN, isToken, Token, TokenType } from '../core/token';
-import type { StringPatternType } from '../core/regexFactory';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { dedupe } from 'src/utils';
 
 /**
  * Priority 5 (last)
@@ -1165,7 +1165,7 @@ export default class MariaDbFormatter extends Formatter {
       reservedBinaryCommands,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]),
+      reservedKeywords: dedupe(reservedKeywords.concat(reservedFunctions)),
       stringTypes: MariaDbFormatter.stringTypes,
       indexedPlaceholderTypes: ['?'],
       lineCommentTypes: ['--', '#'],

src/languages/mysql.formatter.ts

@@ -1,8 +1,8 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import { EOF_TOKEN, isToken, Token, TokenType } from '../core/token';
-import type { StringPatternType } from '../core/regexFactory';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { dedupe } from 'src/utils';
 
 // TODO: split this into object with function categories
 /**
@@ -1328,7 +1328,7 @@ export default class MySqlFormatter extends Formatter {
       reservedBinaryCommands,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]),
+      reservedKeywords: dedupe(reservedKeywords.concat(reservedFunctions)),
       stringTypes: MySqlFormatter.stringTypes,
       indexedPlaceholderTypes: ['?'],
       lineCommentTypes: ['--', '#'],

src/languages/n1ql.formatter.ts

@@ -1,7 +1,7 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import type { StringPatternType } from '../core/regexFactory';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { dedupe } from 'src/utils';
 
 // TODO: split this into object with function categories
 /**
@@ -521,7 +521,7 @@ export default class N1qlFormatter extends Formatter {
       reservedBinaryCommands,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]),
+      reservedKeywords: dedupe(reservedKeywords.concat(reservedFunctions)),
       stringTypes: N1qlFormatter.stringTypes,
       blockStart: ['(', '[', '{'],
       blockEnd: [')', ']', '}'],

src/languages/plsql.formatter.ts

@@ -1,8 +1,8 @@
-import Formatter from '../core/Formatter';
-import Tokenizer from '../core/Tokenizer';
-import { EOF_TOKEN, isReserved, isToken, Token, TokenType } from '../core/token'; // convert to partial type import in TS 4.5
-import type { StringPatternType } from '../core/regexFactory';
-import { dedupe } from '../utils';
+import Formatter from 'src/core/Formatter';
+import Tokenizer from 'src/core/Tokenizer';
+import { EOF_TOKEN, isReserved, isToken, type Token, TokenType } from 'src/core/token';
+import { type StringPatternType } from 'src/core/regexFactory';
+import { dedupe } from 'src/utils';
 
 /**
  * Priority 5 (last)