PROT-5778: support identifiers that begin with digits

The lexer dispatched any digit-leading token to lexNumber, which
consumed the digit run and returned a Number without checking what
followed. An identifier like "1_x_coffee_buyer" therefore lexed as
Number(1) followed by Ident(_x_coffee_buyer), which then failed to
parse ("Unexpected token in statement").

lexNumber now hands off to lexIdent when the token is followed by
identifier characters, so the whole thing lexes as a single Ident.
The handoff is guarded on the token containing a digit: the dispatcher
also routes a bare sign here, so "-foo" keeps its previous behavior.

JSON property names are arbitrary customer data rather than
programmer-chosen symbols, so leading digits are legitimate. This
mirrors the equivalent fix in the Go implementation
(segmentio/fql#86) -- both now lex "friends.1.last" identically.

Author: MichaelGHSeg

src/lexer.test.ts

@@ -130,6 +130,41 @@ test('Lexer passes Number fixtures', () => {
   ])
 })
 
+test('Lexer passes digit-first ident fixtures', () => {
+  testFixtures([
+    // Identifiers may begin with digits: JSON property names are arbitrary
+    // customer data, not programmer-chosen symbols. These used to lex as a
+    // Number followed by a separate Ident, which then failed to parse.
+    fix('1val', [t.Ident('1val'), t.EOS()], false),
+    fix('123audience', [t.Ident('123audience'), t.EOS()], false),
+    fix('1_x_coffee_buyer', [t.Ident('1_x_coffee_buyer'), t.EOS()], false),
+    fix('1-x-coffee-buyer', [t.Ident('1-x-coffee-buyer'), t.EOS()], false),
+
+    // ...including as a segment of a dotted path
+    fix(
+      'properties.123audience',
+      [t.Ident('properties'), t.Dot(), t.Ident('123audience'), t.EOS()],
+      false
+    ),
+    fix(
+      'properties.1_x_coffee_buyer = true',
+      [
+        t.Ident('properties'),
+        t.Dot(),
+        t.Ident('1_x_coffee_buyer'),
+        t.Operator('='),
+        t.Ident('true'),
+        t.EOS()
+      ],
+      false
+    ),
+
+    // A purely numeric token is still a Number, not an Ident
+    fix('123', [t.Number('123'), t.EOS()], false),
+    fix('123 456', [t.Number('123'), t.Number('456'), t.EOS()], false)
+  ])
+})
+
 test('Lexer passes Strings fixtures', () => {
   testFixtures([
     // Strings

src/lexer.ts

@@ -176,7 +176,21 @@ export class Lexer {
       comingUp = this.peek()
     }
 
-    return t.Number(previous + str)
+    const token = previous + str
+
+    // What looked like a number is actually an identifier that begins with
+    // digits (e.g. "123audience", "1_x_coffee_buyer"). Hand the consumed
+    // digits to lexIdent as its prefix rather than emitting a Number and
+    // letting the rest of the identifier lex as a separate token.
+    //
+    // Guarded on the token containing a digit: the dispatcher also routes a
+    // bare sign here, so this keeps "-foo" behaving as it did instead of
+    // turning it into an identifier.
+    if (isIdent(this.peek()) && /[0-9]/.test(token)) {
+      return this.lexIdent(token)
+    }
+
+    return t.Number(token)
   }
 
   private lexOperatorOrConditional(previous: string): Token {