API extensions (#40)

* API extensions

* typo

* typo

* cleanups

* removed dead code

src/lib/generictrees.nim

@@ -40,14 +40,6 @@ proc `==`*(a, b: TagId): bool {.borrow.}
 proc addAtom*[L](dest: var PackedTree[NifKind]; kind: NifKind; lit: L; info: PackedLineInfo) =
   packedtrees.addAtom dest, kind, uint32(lit), info
 
-template withSuffix(body) =
-  if t.suffix.len > 0:
-    copyInto(dest, Suffixed, currentInfo):
-      body
-      dest.addAtom StrLit, lits.strings.getOrInclFromView(t.suffix), currentInfo
-  else:
-    body
-
 const
   LastTag = 255'u32
 
@@ -95,17 +87,13 @@ proc parse*(r: var Reader; dest: var PackedTree[NifKind]; lits: var Literals;
   of CharLit:
     dest.addAtom CharLit, uint32 decodeChar(t), currentInfo
   of StringLit:
-    withSuffix:
-      dest.addAtom StrLit, lits.strings.getOrIncl(decodeStr t), currentInfo
+    dest.addAtom StrLit, lits.strings.getOrIncl(decodeStr t), currentInfo
   of IntLit:
-    withSuffix:
-      dest.addAtom IntLit, lits.integers.getOrIncl(decodeInt t), currentInfo
+    dest.addAtom IntLit, lits.integers.getOrIncl(decodeInt t), currentInfo
   of UIntLit:
-    withSuffix:
-      dest.addAtom UIntLit, lits.uintegers.getOrIncl(decodeUInt t), currentInfo
+    dest.addAtom UIntLit, lits.uintegers.getOrIncl(decodeUInt t), currentInfo
   of FloatLit:
-    withSuffix:
-      dest.addAtom FloatLit, lits.floats.getOrIncl(decodeFloat t), currentInfo
+    dest.addAtom FloatLit, lits.floats.getOrIncl(decodeFloat t), currentInfo
 
 type
   Tree* = PackedTree[NifKind]

src/lib/nifcursors.nim

@@ -16,10 +16,15 @@ type
 proc fromBuffer*(x: openArray[PackedToken]): Cursor {.inline.} =
   Cursor(p: addr(x[0]), rem: x.len)
 
+proc setSpan*(c: var Cursor; beyondLast: Cursor) {.inline.} =
+  c.rem = cast[int](beyondLast.p) - cast[int](c.p)
+
 proc load*(c: Cursor): PackedToken {.inline.} = c.p[]
 
 proc kind*(c: Cursor): TokenKind {.inline.} = c.load.kind
 
+proc info*(c: Cursor): PackedLineInfo {.inline.} = c.load.info
+
 proc litId*(c: Cursor): StrId {.inline.} = nifstreams.litId(c.load)
 proc intId*(c: Cursor): IntId {.inline.} = nifstreams.intId(c.load)
 proc uintId*(c: Cursor): UIntId {.inline.} = nifstreams.uintId(c.load)
@@ -83,6 +88,10 @@ proc add*(b: var TokenBuf; item: PackedToken) {.inline.} =
 
 proc len*(b: TokenBuf): int {.inline.} = b.len
 
+proc `[]`*(b: TokenBuf; i: int): PackedToken {.inline.} =
+  assert i >= 0 and i < b.len
+  result = b.data[i]
+
 proc add*(result: var TokenBuf; c: Cursor) =
   assert c.kind != ParRi, "cursor at end?"
   if c.kind != ParLe:

src/lib/nifindexes.nim

@@ -10,9 +10,9 @@ import std / [os]
 import bitabs, lineinfos, nifreader, nifstreams, nifcursors
 
 const
-  PublicT = TagId(2)
-  PrivateT = TagId(3)
-  KvT = TagId(4)
+  PublicT = TagId(3)
+  PrivateT = TagId(4)
+  KvT = TagId(5)
 
 proc createIndex*(infile: string) =
   registerTag "public", PublicT

src/lib/nifreader.nim

@@ -12,9 +12,7 @@ import stringviews
 const
   ControlChars = {'(', ')', '[', ']', '{', '}', '~', '#', '\'', '"', ':'}
   ControlCharsOrWhite = ControlChars + {' ', '\n', '\t', '\r'}
-  HexChars* = {'0'..'9', 'A'..'F'} # lowercase letters are not in the NIF spec!
-  StringSuffixChars = {'A'..'Z', 'a'..'z', '_', '0'..'9'}
-  NumberSuffixChars = {'a'..'z', '_', '0'..'9'}
+  HexChars = {'0'..'9', 'A'..'F'} # lowercase letters are not in the NIF spec!
   Digits = {'0'..'9'}
 
 type
@@ -36,7 +34,6 @@ type
     kind*: uint16   # for clients to fill in ("known node kinds")
     s*: StringView
     pos*: FilePos
-    suffix*: StringView
     filename*: StringView
 
   MetaInfo* = object

src/lib/nifstreams.nim

@@ -71,12 +71,16 @@ proc hash*(x: TagId): Hash {.borrow.}
 
 const
   Suffixed* = TagId(1)
+  ErrT* = TagId(2)
 
 proc createLiterals*(): Literals =
   result = default(Literals)
   let t = result.tags.getOrIncl("suf")
   assert t == Suffixed
 
+  let t2 = result.tags.getOrIncl("err")
+  assert t2 == ErrT
+
 var pool* = createLiterals()
 
 proc registerTag*(tag: string; expected: TagId) =
@@ -95,14 +99,6 @@ template copyIntoUnchecked*(dest: var seq[PackedToken]; tag: string; info: Packe
   body
   dest.addToken ParRi, 0'u32, info
 
-template withSuffix(body) =
-  if t.suffix.len > 0:
-    copyInto(dest, Suffixed, currentInfo):
-      body
-      dest.addToken StringLit, pool.strings.getOrInclFromView(t.suffix), currentInfo
-  else:
-    body
-
 type
   Stream* = object
     r*: Reader
@@ -204,17 +200,13 @@ proc parse*(r: var Reader; dest: var seq[PackedToken];
   of CharLit:
     dest.addToken CharLit, uint32 decodeChar(t), currentInfo
   of StringLit:
-    withSuffix:
-      dest.addToken StringLit, pool.strings.getOrIncl(decodeStr t), currentInfo
+    dest.addToken StringLit, pool.strings.getOrIncl(decodeStr t), currentInfo
   of IntLit:
-    withSuffix:
-      dest.addToken IntLit, pool.integers.getOrIncl(decodeInt t), currentInfo
+    dest.addToken IntLit, pool.integers.getOrIncl(decodeInt t), currentInfo
   of UIntLit:
-    withSuffix:
-      dest.addToken UIntLit, pool.uintegers.getOrIncl(decodeUInt t), currentInfo
+    dest.addToken UIntLit, pool.uintegers.getOrIncl(decodeUInt t), currentInfo
   of FloatLit:
-    withSuffix:
-      dest.addToken FloatLit, pool.floats.getOrIncl(decodeFloat t), currentInfo
+    dest.addToken FloatLit, pool.floats.getOrIncl(decodeFloat t), currentInfo
 
 proc litId*(n: PackedToken): StrId {.inline.} =
   assert n.kind in {Ident, StringLit}