fixes #34; Remove number and string literal suffixes (#36)

* fixes #34; Remove number and string literal suffixes

* support nifreader

* supports nifbuilder

* remove echo

* handle comments

* document NumberWithSuffix

* fixes documentation

* IntLit, FloatLit don't need suffix

* fixes gear2 bridge

* supports nifc

* supports SufC

---------

Co-authored-by: araq <[email protected]>

doc/nif-spec.md

@@ -194,14 +194,15 @@ Grammar:
 
 ```
 Digit ::= [0-9]
-NumberSuffix ::= [a-z]+ [0-9a-z]*  # suffixes can only contain lowercase letters
 FloatingPointPart ::= ('.' Digit+ ('E' '-'? Digit+)? ) | 'E' '-'? Digit+
-Number ::= ('+' | '-') Digit+ FloatingPointPart? NumberSuffix?
+Number ::= ('+' | '-') Digit+ (FloatingPointPart | 'u')?
 ```
 
-Numbers must start with a plus or a minus and only their decimal notation is supported. Numbers can have
-a suffix that has to start with a lowercase letter. For example Nim's `0xff'i32` would become `256i32x`.
-(The `x` encodes the fact that the number was originally written in hex.)
+Numbers must start with a plus or a minus and only their decimal notation is supported.
+For example Nim's `0xff` would become `256`.
+
+Unsigned numbers always have a `u` suffix. Floating point numbers must contain a dot or `E`.
+Every other number is interpreted as a signed integer.
 
 Note that numbers that do not start with a plus nor a minus are interpreted as "line information". See
 the corresponding section for more details.
@@ -225,9 +226,8 @@ Char literals are enclosed in single quotes. The only supported escape sequence
 Grammar:
 
 ```
-StringSuffix ::= Identifier
 EscapedData ::= (VisibleChar | Escape | Whitespace)*
-StringLiteral ::= '"' EscapedData '"' StringSuffix?
+StringLiteral ::= '"' EscapedData '"'
 ```
 
 String literals are enclosed in double quotes. The only supported escape sequence is `\xx`.
@@ -243,14 +243,6 @@ For example:
 
 Produces: `"This is a single \n  literal string"`.
 
-A string literal can have a suffix that is usually ignored but can be used to store the
-original format of the string. For example, Nim supports "raw string literals" and "triple
-string literals". These could be modelled as `R` and `T` suffixes:
-
-```nif
-  "This was a triple quoted Nim string"T
-```
-
 
 <div style="page-break-after: always;"></div>
 

src/gear2/bridge.nim

@@ -149,7 +149,7 @@ proc toNif*(n, parent: PNode; c: var WContext) =
     c.b.addIntLit n.intVal, "i64"
   of nkUIntLit:
     relLineInfo(n, parent, c, true)
-    c.b.addUIntLit cast[BiggestUInt](n.intVal), "u"
+    c.b.addUIntLit cast[BiggestUInt](n.intVal)
   of nkUInt8Lit:
     relLineInfo(n, parent, c, true)
     c.b.addUIntLit cast[BiggestUInt](n.intVal), "u8"

src/lib/nifbuilder.nim

@@ -120,7 +120,7 @@ proc addSymbolDef*(b: var Builder; s: string) =
     else:
       b.put c
 
-proc addStrLit*(b: var Builder; s: string; suffix = "") =
+proc addStrLit*(b: var Builder; s: string) =
   addSep b
   b.put '"'
   for c in s:
@@ -129,7 +129,6 @@ proc addStrLit*(b: var Builder; s: string; suffix = "") =
     else:
       b.put c
   b.put '"'
-  b.put suffix
 
 proc addEmpty*(b: var Builder; count = 1) =
   addSep b
@@ -145,20 +144,19 @@ proc addCharLit*(b: var Builder; c: char) =
     b.put c
   b.put '\''
 
-proc addIntLit*(b: var Builder; i: BiggestInt; suffix = "") =
+proc addIntLit*(b: var Builder; i: BiggestInt) =
   addSep b
   if i >= 0:
     b.buf.add '+'
   b.put $i
-  b.put suffix
 
-proc addUIntLit*(b: var Builder; u: BiggestUInt; suffix = "") =
+proc addUIntLit*(b: var Builder; u: BiggestUInt) =
   addSep b
   b.buf.add '+'
   b.put $u
-  b.put suffix
+  b.buf.add 'u'
 
-proc addFloatLit*(b: var Builder; f: BiggestFloat; suffix = "") =
+proc addFloatLit*(b: var Builder; f: BiggestFloat) =
   addSep b
   let myLen = b.buf.len
   drainPending b
@@ -170,7 +168,6 @@ proc addFloatLit*(b: var Builder; f: BiggestFloat; suffix = "") =
   if b.mode == UsesFile:
     b.f.write b.buf
     b.buf.setLen 0
-  b.put suffix
 
 proc addLine(s: var string; x: int32) =
   if x < 0:
@@ -237,6 +234,16 @@ template withTree*(b: var Builder; kind: string; body: untyped) =
   body
   endTree b
 
+proc addUIntLit*(b: var Builder; u: BiggestUInt; suffix: string) =
+  withTree(b, "suf"):
+    addUIntLit(b, u)
+    addStrLit(b, suffix)
+
+proc addStrLit*(b: var Builder; s: string; suffix: string) =
+  withTree(b, "suf"):
+    addStrLit(b, s)
+    addStrLit(b, suffix)
+
 proc addHeader*(b: var Builder; vendor = "", dialect = "") =
   b.put "(.nif24)\n"
   if vendor.len > 0:

src/lib/nifreader.nim

@@ -233,13 +233,6 @@ proc handleNumber(r: var Reader; result: var Token) =
         while p < eof and ^p in Digits:
           inc p
           inc result.s.len
-      if p < eof and ^p in NumberSuffixChars:
-        result.suffix.p = p
-        if ^p == 'u': result.tk = UIntLit
-        elif ^p == 'f': result.tk = FloatLit
-        while p < eof and ^p in NumberSuffixChars:
-          inc p
-          inc result.suffix.len
 
 proc handleLineInfo(r: var Reader; result: var Token) =
   useCpuRegisters:
@@ -318,6 +311,7 @@ proc next*(r: var Reader): Token =
         let repl = r.ksubs.getOrDefault(result.s)
         if repl[0] != UnknownToken:
           result.s = repl[1]
+
     of ')':
       result.tk = ParRi
       result.s.p = r.p
@@ -345,13 +339,6 @@ proc next*(r: var Reader): Token =
             inc r.line
           inc result.s.len
           inc p
-
-        if p < eof and ^p in StringSuffixChars:
-          result.suffix.p = p
-          while true:
-            inc p
-            inc result.suffix.len
-            if p == eof or ^p notin StringSuffixChars: break
     of '\'':
       inc r.p
       result.s.p = r.p

src/nifc/codegen.nim

@@ -9,7 +9,7 @@
 
 # We produce C code as a list of tokens.
 
-import std / [assertions, syncio, tables, sets, intsets, formatfloat]
+import std / [assertions, syncio, tables, sets, intsets, formatfloat, strutils]
 from std / strutils import parseBiggestInt, parseBiggestUInt, parseInt
 from std / os import changeFileExt
 

src/nifc/genexprs.nim

@@ -185,5 +185,13 @@ proc genx(c: var GeneratedCode; t: Tree; n: NodePos) =
   of LtC: cmpop " < "
   of CastC: typedUnOp ""
   of ConvC: typedUnOp ""
+  of SufC:
+    let (value, suffix) = sons2(t, n)
+    case t[value].kind
+    of StrLit:
+      c.add makeCString(c.m.lits.strings[t[value].litId])
+    else:
+      assert c.m.lits.strings[t[suffix].litId].startsWith('u')
+      genUIntLit c, t[value].litId
   else:
     genLvalue c, t, n

src/nifc/nifc_model.nim

@@ -6,7 +6,7 @@
 
 ## Parse NIF into a packed tree representation.
 
-import std / [hashes, tables]
+import std / [hashes, tables, strutils]
 import "../lib" / [bitabs, lineinfos, stringviews, packedtrees, nifreader, keymatcher,
   nifbuilder]
 
@@ -112,6 +112,7 @@ type
     ImpC = "imp"
     NodeclC = "nodecl"
     InclC = "incl"
+    SufC = "suf"
 
 const
   CallingConventions* = {CdeclC..MemberC}
@@ -168,7 +169,6 @@ proc parse*(r: var Reader; dest: var PackedTree[NifcKind]; m: var Module; parent
       while true:
         let progress = parse(r, d[], m, currentInfo)
         if not progress: break
-
   of UnknownToken:
     copyInto dest, Err, currentInfo:
       dest.addAtom StrLit, m.lits.strings.getOrIncl(decodeStr t), currentInfo

src/nifler/bridge.nim

@@ -105,7 +105,6 @@ type
     section: string
 
 proc absLineInfo(i: TLineInfo; em: var Emitter; c: var TranslationContext) =
-  em.addRaw "@"
   em.addLine int32 i.col
   em.addRaw ","
   em.addLine int32 i.line
@@ -122,25 +121,19 @@ proc relLineInfo(n, parent: PNode; em: var Emitter; c: var TranslationContext;
   let colDiff = int32(i.col) - int32(p.col)
   var seps = 0
   if colDiff != 0:
-    em.addRaw "@"
     em.addLine colDiff
     seps = 1
   let lineDiff = int32(i.line) - int32(p.line)
   if lineDiff != 0:
-    if seps != 1:
-      em.addRaw "@"
     seps = 2
     em.addRaw ","
     em.addLine lineDiff
   if i.fileIndex != p.fileIndex:
     case seps
     of 0:
-      em.addRaw "@,,"
-    of 1:
       em.addRaw ",,"
-    of 2:
+    else:
       em.addRaw ","
-    else: discard
     inc seps
     em.addIdent toFullPath(c.conf, i.fileIndex)
   if seps > 0 and emitSpace:
@@ -182,7 +175,7 @@ proc toNif*(n, parent: PNode; em: var Emitter; c: var TranslationContext) =
     em.addIntLit n.intVal, "i64"
   of nkUIntLit:
     relLineInfo(n, parent, em, c, true)
-    em.addUIntLit cast[BiggestUInt](n.intVal), "u"
+    em.addUIntLit cast[BiggestUInt](n.intVal)
   of nkUInt8Lit:
     relLineInfo(n, parent, em, c, true)
     em.addUIntLit cast[BiggestUInt](n.intVal), "u8"

src/nifler/emitter.nim

@@ -45,24 +45,6 @@ proc addIdent*(em: var Emitter; s: string) =
       r.add c
   emit em, r, r.len
 
-proc addStrLit*(em: var Emitter; s, suffix: string) =
-  var r = "\""
-  var l = em.lineLen + 1
-  var lastPart = 1
-  var afterNewline = false
-  for c in s:
-    if c in ControlChars:
-      r.escape c
-      inc l, 3
-      inc lastPart, 3
-    else:
-      r.add c
-      inc l, 1
-      inc lastPart, 1
-    afterNewline = false
-  r.add "\""
-  em.emit r, lastPart
-  em.emit suffix, suffix.len
 
 
 type
@@ -126,6 +108,45 @@ proc addEmpty*(em: var Emitter; count = 1) =
   for i in 1..count:
     em.emit ".", 1
 
+template addSuffixLit(em: var Emitter, suffix: string, body: typed) =
+  let suffixLit = '"' & suffix & '"'
+  var a = prepare(em, "suf")
+  em.addSep a
+  body
+  em.addSep a
+  em.emit suffixLit, suffixLit.len
+  em.patch(a)
+
+template addSuffixLitDispatch(em: var Emitter, suffix: string, body: typed) =
+  if suffix.len > 0:
+    addSuffixLit(em, suffix):
+      body
+  else:
+    body
+
+
+proc addStrLitImpl(em: var Emitter; s: string) =
+  var r = "\""
+  var l = em.lineLen + 1
+  var lastPart = 1
+  var afterNewline = false
+  for c in s:
+    if c in ControlChars:
+      r.escape c
+      inc l, 3
+      inc lastPart, 3
+    else:
+      r.add c
+      inc l, 1
+      inc lastPart, 1
+    afterNewline = false
+  r.add "\""
+  em.emit r, lastPart
+
+proc addStrLit*(em: var Emitter; s, suffix: string) =
+  addSuffixLitDispatch(em, suffix):
+    em.addStrLitImpl s
+
 proc addCharLit*(em: var Emitter; c: char) =
   em.output.add '\''
   if c.needsEscape:
@@ -141,35 +162,52 @@ template upateLen(body) =
   body
   inc em.lineLen, em.output.len - oldLen
 
-proc addIntLit*(em: var Emitter; i: BiggestInt; suffix = "") =
+proc addIntLit*(em: var Emitter; i: BiggestInt) =
   upateLen:
+    if i >= 0: em.output.add '+'
     em.output.addInt i
-    em.output.add suffix
+
+proc addIntLit*(em: var Emitter; u: BiggestInt; suffix: string) =
+  addSuffixLitDispatch(em, suffix):
+    addIntLit(em, u)
 
 proc addLine*(em: var Emitter; i: int32) =
   upateLen:
-    em.output.addInt i
+    if i < 0'i32:
+      em.output.add '~'
+      em.output.addInt(-i)
+    else:
+      em.output.addInt i
 
-proc addUIntLit*(em: var Emitter; u: BiggestUInt; suffix = "") =
+proc addUIntLit*(em: var Emitter; u: BiggestUInt) =
   upateLen:
+    em.output.add '+'
     em.output.add $u
-    em.output.add suffix
+    em.output.add 'u'
+
+proc addUIntLit*(em: var Emitter; u: BiggestUInt; suffix: string) =
+  addSuffixLitDispatch(em, suffix):
+    addUIntLit(em, u)
 
-proc addFloatLit*(em: var Emitter; f: BiggestFloat; suffix = "") =
+proc addFloatLit*(em: var Emitter; f: BiggestFloat) =
   let myLen = em.output.len
   upateLen:
+    if f >= 0.0: em.output.add '+'
     em.output.addFloat f
     for i in myLen ..< em.output.len:
       if em.output[i] == 'e': em.output[i] = 'E'
-    em.output.add suffix
+
+proc addFloatLit*(em: var Emitter; f: BiggestFloat; suffix: string) =
+  addSuffixLitDispatch(em, suffix):
+    addFloatLit(em, f)
 
 when isMainModule:
   var em = Emitter()
   var a = prepare(em, "proc")
   em.addSep a
-  em.addStrLit "#(escaped?)\n"
+  em.addStrLit "#(escaped?)\n", ""
   em.addSep a
-  em.addStrLit "more here"
+  em.addStrLit "more here", ""
   em.patch(a)
 
   echo em.output