WIP: nifindexes.nim refactorings (#558)

* WIP: nifindexes.nim refactorings * progress
nim-lang · Feb 14, 2025 · 9bcd020 · 9bcd020
1 parent 48bb5e1
commit 9bcd020
Show file tree

Hide file tree

Showing 4 changed files with 112 additions and 86 deletions.
diff --git a/doc/tags.md b/doc/tags.md
@@ -26,7 +26,7 @@
 | `(bracket X*)`         | NimonyExpr, NiflerKind | untyped array constructor |
 | `(curly X*)`           | NimonyExpr, NiflerKind | untyped set constructor |
 | `(curlyat X X)`        | NimonyExpr, NiflerKind | curly expression `a{i}` |
-| `(kv Y X)`             | NimonyOther, NifcOther, NiflerKind | key-value pair |
+| `(kv Y X)`             | NimonyOther, NifcOther, NiflerKind, NifIndex | key-value pair |
 | `(vv X X)`             | NimonyOther, NiflerKind | value-value pair (used for explicitly named arguments in function calls) | 
 | `(add T X X)`          | NifcExpr, NimonyExpr | |
 | `(sub T X X)`          | NifcExpr, NimonyExpr | |
@@ -49,27 +49,27 @@
 | `(cmd X X*)`             | NimonyStmt, NimonyExpr, NiflerKind | command operation |
 | `(range X X)`          | NifcOther, NimonyOther | `(range a b)` construct |
 | `(ranges (range ...)*)` | NifcOther, NimonyOther, NiflerKind | |
-| `(gvar D E P T X)`; `(gvar D P T X)` | NifcStmt, NimonyStmt, NimonySym, NifcSym | global variable declaration |
-| `(tvar D E P T X)`; `(tvar D P T X)` | NifcStmt, NimonyStmt, NimonySym, NifcSym | thread local variable declaration |
-| `(var D E P T X)`; `(var D P T X)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NiflerKind | variable declaration |
+| `(gvar D E P T X)`; `(gvar D P T X)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NifIndex | global variable declaration |
+| `(tvar D E P T X)`; `(tvar D P T X)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NifIndex | thread local variable declaration |
+| `(var D E P T X)`; `(var D P T X)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NiflerKind, NifIndex | variable declaration |
 | `(param D E P T X)`; `(param D P T)` | NifcOther, NimonyOther, NimonySym, NifcSym, NiflerKind | parameter declaration |
-| `(const D E P T X)`; `(const D P T)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NiflerKind | const variable declaration |
+| `(const D E P T X)`; `(const D P T)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NiflerKind, NifIndex | const variable declaration |
 | `(result D E P T X)` | NimonySym, NimonyStmt | result variable declaration |
-| `(glet D E P T X)` | NimonyStmt, NimonySym | global let variable declaration |
-| `(tlet D E P T X)` | NimonyStmt, NimonySym | thread local let variable declaration |
-| `(let D E P T X)` | NimonySym, NimonyStmt, NiflerKind | let variable declaration |
-| `(cursor D E P T X)` | NimonySym, NimonyStmt | cursor variable declaration |
+| `(glet D E P T X)` | NimonyStmt, NimonySym, NifIndex | global let variable declaration |
+| `(tlet D E P T X)` | NimonyStmt, NimonySym, NifIndex | thread local let variable declaration |
+| `(let D E P T X)` | NimonySym, NimonyStmt, NiflerKind, NifIndex | let variable declaration |
+| `(cursor D E P T X)` | NimonySym, NimonyStmt, NifIndex | cursor variable declaration |
 | `(typevar D E P T X)` | NimonySym, NifcOther, NimonyOther, NiflerKind | type variable declaration |
 | `(efld D E P T X)`; `(efld D X)` | NimonySym, NifcSym, NifcOther, NimonyOther, NiflerKind | enum field declaration |
 | `(fld D E P T X)`; `(fld D P T)` | NifcOther, NimonyOther, NimonySym, NifcSym, NiflerKind | field declaration |
-| `(proc D ...)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NiflerKind | proc declaration |
-| `(func D ...)` | NimonyStmt, NimonySym, NiflerKind | function declaration |
-| `(iterator D ...)` | NimonyStmt, NimonySym, NimonyType, NiflerKind | iterator declaration |
-| `(converter D ...)` | NimonyStmt, NimonySym, NiflerKind | converter declaration |
-| `(method D ...)` | NimonyStmt, NimonySym, NiflerKind | method declaration |
-| `(macro D ...)` | NimonyStmt, NimonySym, NiflerKind | macro declaration |
-| `(template D ...)` | NimonyStmt, NimonySym, NiflerKind | template declaration |
-| `(type D ...)` | NifcStmt, NimonyStmt, NimonySym, NiflerKind | type declaration |
+| `(proc D ...)` | NifcStmt, NimonyStmt, NimonySym, NifcSym, NiflerKind, NifIndex | proc declaration |
+| `(func D ...)` | NimonyStmt, NimonySym, NiflerKind, NifIndex | function declaration |
+| `(iterator D ...)` | NimonyStmt, NimonySym, NimonyType, NiflerKind, NifIndex | iterator declaration |
+| `(converter D ...)` | NimonyStmt, NimonySym, NiflerKind, NifIndex | converter declaration |
+| `(method D ...)` | NimonyStmt, NimonySym, NiflerKind, NifIndex | method declaration |
+| `(macro D ...)` | NimonyStmt, NimonySym, NiflerKind, NifIndex | macro declaration |
+| `(template D ...)` | NimonyStmt, NimonySym, NiflerKind, NifIndex | template declaration |
+| `(type D ...)` | NifcStmt, NimonyStmt, NimonySym, NiflerKind, NifIndex | type declaration |
 | `(block .D X)` | NimonyStmt, NimonySym, NiflerKind | block declaration |
 | `(module)` | NimonySym | module declaration |
 | `(cchoice X X*)` | NimonyExpr, NimonySym | closed choice |
@@ -121,7 +121,7 @@
 | `(noconv)` | CallConv | no explicit calling convention |
 | `(member)`  | CallConv | `member` calling convention |
 | `(nimcall)` | CallConv | `nimcall` calling convention |
-| `(inline)` | NifcPragma, NimonyPragma | `inline` proc annotation |
+| `(inline)` | NifcPragma, NimonyPragma, NifIndex | `inline` proc annotation |
 | `(noinline)` | NifcPragma, NimonyPragma | `noinline` proc annotation |
 | `(attr STR)` | NifcPragma | general attribute annoation |
 | `(varargs)` | NifcPragma, NimonyPragma, NimonyType | `varargs` proc annotation |
@@ -204,7 +204,7 @@
 | `(noinit)` | NimonyPragma | `noinit` pragma |
 | `(requires X)` | NimonyPragma | `requires` pragma |
 | `(ensures X)` | NimonyPragma | `ensures` pragma |
-| `(build X)` | NimonyPragma | `build` pragma |
+| `(build X)`; `(build STR STR STR)` | NimonyPragma, NifIndex | `build` pragma |
 | `(string)` | NimonyPragma | `string` pragma |
 | `(quoted X+)` | NimonyExpr, NiflerKind | name in backticks |
 | `(hderef X)` | NimonyExpr | hidden pointer deref operation |
@@ -260,3 +260,11 @@
 | `(using (params...)+)` | NimonyStmt, NiflerKind | `using` statement |
 | `(asm X+)` | NimonyStmt, NiflerKind | `asm` statement |
 | `(defer X)` | NimonyStmt, NiflerKind | `defer` statement |
+| `(index (public ...) (private ...) (hooks ...) (converter ...) (build ...))` | NifIndex | index section |
+| `(public (kv Y INTLIT*)` | NifIndex | public section |
+| `(private (kv Y INTLIT*))` | NifIndex | private section |
+| `(cloner Y Y)` | NifIndex | cloner hook section |
+| `(tracer Y Y)` | NifIndex | tracer hook section |
+| `(disarmer Y Y)` | NifIndex | disarmer hook section |
+| `(mover Y Y)` | NifIndex | mover hook section |
+| `(dtor Y Y)` | NifIndex | destructor hook section |
diff --git a/src/lib/nifindexes.nim b/src/lib/nifindexes.nim
@@ -11,6 +11,13 @@ import bitabs, lineinfos, nifreader, nifstreams, nifcursors, nifchecksums
 
 #import std / [sha1]
 import "$nim"/dist/checksums/src/checksums/sha1
+import ".." / models / nifindex_tags
+
+proc entryKind(tag: TagId): NifIndex =
+  if rawTagIsNifIndex(tag.uint32):
+    result = cast[NifIndex](tag)
+  else:
+    result = NoIndexTag
 
 proc registerTag(tag: string): TagId = pool.tags.getOrIncl(tag)
 
@@ -31,24 +38,14 @@ proc isExported(n: Cursor): bool =
 proc processForChecksum(dest: var Sha1State; content: var TokenBuf) =
   var n = beginRead(content)
   var nested = 0
-  let letT = registerTag("let")
-  let varT = registerTag("var")
-  let cursorT = registerTag("cursor")
-  let constT = registerTag("const")
-  let typeT = registerTag("type")
-  let procT = registerTag("proc")
-  let templateT = registerTag("template")
-  let funcT = registerTag("func")
-  let macroT = registerTag("macro")
-  let converterT = registerTag("converter")
-  let methodT = registerTag("method")
-  let iteratorT = registerTag("iterator")
-  let inlineT = registerTag("inline")
+  let inlineT = TagId(InlineIdx)
   while true:
     case n.kind
     of ParLe:
       var foundInline = false
-      if n.tagId == letT or n.tagId == varT or n.tagId == cursorT or n.tagId == constT or n.tagId == typeT:
+      let k = entryKind(n.tagId)
+      case k
+      of LetIdx, VarIdx, CursorIdx, ConstIdx, TypeIdx, GletIdx, TletIdx, GvarIdx, TvarIdx:
         inc n # tag
         if isExported(n):
           updateLoop(dest, n, inlineT, foundInline) # SymbolDef
@@ -57,7 +54,7 @@ proc processForChecksum(dest: var Sha1State; content: var TokenBuf) =
           updateLoop(dest, n, inlineT, foundInline) # type
           updateLoop(dest, n, inlineT, foundInline) # value
         skipToEnd n
-      elif n.tagId == templateT or n.tagId == macroT or n.tagId == iteratorT:
+      of TemplateIdx, MacroIdx, IteratorIdx:
         # these always have inline semantics
         inc n # tag
         if isExported(n):
@@ -71,7 +68,7 @@ proc processForChecksum(dest: var Sha1State; content: var TokenBuf) =
           updateLoop(dest, n, inlineT, foundInline) # effects
           updateLoop(dest, n, inlineT, foundInline) # body
         skipToEnd n
-      elif n.tagId == procT or n.tagId == funcT or n.tagId == methodT or n.tagId == converterT:
+      of ProcIdx, FuncIdx, MethodIdx, ConverterIdx:
         inc n # tag
         if isExported(n):
           var dummy = false
@@ -88,7 +85,8 @@ proc processForChecksum(dest: var Sha1State; content: var TokenBuf) =
           else:
             skip n
         skipToEnd n
-      else:
+      of NoIndexTag, InlineIdx, KvIdx, BuildIdx, IndexIdx, PublicIdx, PrivateIdx,
+         ClonerIdx, TracerIdx, DisarmerIdx, MoverIdx, DtorIdx:
         inc n
         inc nested
     of ParRi:
@@ -99,35 +97,32 @@ proc processForChecksum(dest: var Sha1State; content: var TokenBuf) =
     else:
       inc n
 
-type IndexSections* = object
-  hooks*: Table[string, seq[(SymId, SymId)]]
-  converters*: seq[(SymId, SymId)]
-  toBuild*: TokenBuf
+type
+  IndexSections* = object
+    hooks*: Table[string, seq[(SymId, SymId)]]
+    converters*: seq[(SymId, SymId)]
+    toBuild*: TokenBuf
 
 proc getSection(tag: TagId; values: seq[(SymId, SymId)]; symToOffsetMap: Table[SymId, int]): TokenBuf =
-  let KvT = registerTag "kv"
-
-  result = default(TokenBuf)
+  result = createTokenBuf(30)
   result.addParLe tag
 
   for value in values:
     let (key, sym) = value
     let offset = symToOffsetMap[sym]
-    result.buildTree KvT, NoLineInfo:
+    result.buildTree TagId(KvIdx), NoLineInfo:
       result.add symToken(key, NoLineInfo)
       result.add intToken(pool.integers.getOrIncl(offset), NoLineInfo)
 
   result.addParRi()
 
 proc getSymbolSection(tag: TagId; values: seq[(SymId, SymId)]): TokenBuf =
-  let KvT = registerTag "kv"
-
-  result = default(TokenBuf)
+  result = createTokenBuf(30)
   result.addParLe tag
 
   for value in values:
     let (key, sym) = value
-    result.buildTree KvT, NoLineInfo:
+    result.buildTree TagId(KvIdx), NoLineInfo:
       if key == SymId(0):
         result.add dotToken(NoLineInfo)
       else:
@@ -137,10 +132,6 @@ proc getSymbolSection(tag: TagId; values: seq[(SymId, SymId)]): TokenBuf =
   result.addParRi()
 
 proc createIndex*(infile: string; buildChecksum: bool; sections: IndexSections) =
-  let PublicT = registerTag "public"
-  let PrivateT = registerTag "private"
-  let KvT = registerTag "kv"
-
   let indexName = changeFileExt(infile, ".idx.nif")
 
   var s = nifstreams.open(infile)
@@ -149,10 +140,10 @@ proc createIndex*(infile: string; buildChecksum: bool; sections: IndexSections)
   var previousPublicTarget = 0
   var previousPrivateTarget = 0
 
-  var public = default(TokenBuf)
-  var private = default(TokenBuf)
-  public.addParLe PublicT
-  private.addParLe PrivateT
+  var public = createTokenBuf(30)
+  var private = createTokenBuf(30)
+  public.addParLe TagId(PublicIdx)
+  private.addParLe TagId(PrivateIdx)
   var buf = createTokenBuf(100)
   var symToOffsetMap = initTable[SymId, int]()
 
@@ -178,7 +169,7 @@ proc createIndex*(infile: string; buildChecksum: bool; sections: IndexSections)
         symToOffsetMap[sym] = target
         let diff = if isPublic: target - previousPublicTarget
                   else: target - previousPrivateTarget
-        dest[].buildTree KvT, info:
+        dest[].buildTree TagId(KvIdx), info:
           dest[].add symToken(sym, NoLineInfo)
           dest[].add intToken(pool.integers.getOrIncl(diff), NoLineInfo)
         if isPublic:
@@ -204,8 +195,7 @@ proc createIndex*(infile: string; buildChecksum: bool; sections: IndexSections)
     content.add "\n"
 
   if sections.converters.len != 0:
-    let ConverterT = registerTag "converter"
-    let converterSectionBuf = getSymbolSection(ConverterT, sections.converters)
+    let converterSectionBuf = getSymbolSection(TagId(ConverterIdx), sections.converters)
 
     content.add toString(converterSectionBuf)
     content.add "\n"
@@ -243,15 +233,14 @@ type
     toBuild*: seq[(string, string, string)]
 
 proc readSection(s: var Stream; tab: var Table[string, NifIndexEntry]; useAbsoluteOffset = false) =
-  let KvT = registerTag "kv"
   var previousOffset = 0
   var t = next(s)
   var nested = 1
   while t.kind != EofToken:
     let info = t.info
     if t.kind == ParLe:
       inc nested
-      if t.tagId == KvT:
+      if t.tagId == TagId(KvIdx):
         t = next(s)
         var key: string
         if t.kind == Symbol:
@@ -286,13 +275,12 @@ proc readSection(s: var Stream; tab: var Table[string, NifIndexEntry]; useAbsolu
       #t = next(s)
 
 proc readSymbolSection(s: var Stream; tab: var Table[string, string]) =
-  let KvT = registerTag "kv"
   var t = next(s)
   var nested = 1
   while t.kind != EofToken:
     if t.kind == ParLe:
       inc nested
-      if t.tagId == KvT:
+      if t.tagId == TagId(KvIdx):
         t = next(s)
         var key: string
         if t.kind == Symbol:
@@ -334,45 +322,31 @@ proc readIndex*(indexName: string): NifIndex =
   let res = processDirectives(s.r)
   assert res == Success
 
-  let PublicT = registerTag "public"
-  let PrivateT = registerTag "private"
-  let IndexT = registerTag "index"
-
-  let ClonerT = registerTag "cloner"
-  let TracerT = registerTag "tracer"
-  let DisarmerT = registerTag "disarmer"
-  let MoverT = registerTag "mover"
-  let DtorT = registerTag "dtor"
-
-  let hookSet = toHashSet([ClonerT, TracerT, DisarmerT, MoverT, DtorT])
-
-  let ConverterT = registerTag "converter"
-
   result = default(NifIndex)
   var t = next(s)
-  if t.tag == IndexT:
+  if t.tag == TagId(IndexIdx):
     t = next(s)
-    if t.tag == PublicT:
+    if t.tag == TagId(PublicIdx):
       readSection s, result.public
     else:
       assert false, "'public' expected"
     t = next(s)
-    if t.tag == PrivateT:
+    if t.tag == TagId(PrivateIdx):
       readSection s, result.private
     else:
       assert false, "'private' expected"
     t = next(s)
-    while t.tag in hookSet:
+    # XXX Dup is missing here!
+    while t.tag.entryKind in {DtorIdx, DisarmerIdx, TracerIdx, ClonerIdx, MoverIdx}:
       let tagName = pool.tags[t.tag]
       result.hooks[tagName] = initTable[string, NifIndexEntry]()
       readSection(s, result.hooks[tagName])
       t = next(s)
-    if t.tag == ConverterT:
+    if t.tag == TagId(ConverterIdx):
       readSymbolSection(s, result.converters)
       t = next(s)
 
-    let BuildT = registerTag "build"
-    if t.tag == BuildT:
+    if t.tag == TagId(BuildIdx):
       t = next(s)
       while t.kind != EofToken and t.kind != ParRi:
         # tup

diff --git a/src/models/nifindex_tags.nim b/src/models/nifindex_tags.nim
@@ -4,7 +4,35 @@
 type
   NifIndex* = enum
     NoIndexTag
+    KvIdx = (27, "kv")  ## key-value pair
+    GvarIdx = (50, "gvar")  ## global variable declaration
+    TvarIdx = (51, "tvar")  ## thread local variable declaration
+    VarIdx = (52, "var")  ## variable declaration
+    ConstIdx = (54, "const")  ## const variable declaration
+    GletIdx = (56, "glet")  ## global let variable declaration
+    TletIdx = (57, "tlet")  ## thread local let variable declaration
+    LetIdx = (58, "let")  ## let variable declaration
+    CursorIdx = (59, "cursor")  ## cursor variable declaration
+    ProcIdx = (63, "proc")  ## proc declaration
+    FuncIdx = (64, "func")  ## function declaration
+    IteratorIdx = (65, "iterator")  ## iterator declaration
+    ConverterIdx = (66, "converter")  ## converter declaration
+    MethodIdx = (67, "method")  ## method declaration
+    MacroIdx = (68, "macro")  ## macro declaration
+    TemplateIdx = (69, "template")  ## template declaration
+    TypeIdx = (70, "type")  ## type declaration
+    InlineIdx = (122, "inline")  ## `inline` proc annotation
+    BuildIdx = (205, "build")  ## `build` pragma
+    IndexIdx = (261, "index")  ## index section
+    PublicIdx = (262, "public")  ## public section
+    PrivateIdx = (263, "private")  ## private section
+    ClonerIdx = (264, "cloner")  ## cloner hook section
+    TracerIdx = (265, "tracer")  ## tracer hook section
+    DisarmerIdx = (266, "disarmer")  ## disarmer hook section
+    MoverIdx = (267, "mover")  ## mover hook section
+    DtorIdx = (268, "dtor")  ## destructor hook section
 
 proc rawTagIsNifIndex*(raw: uint32): bool {.inline.} =
-  raw >= -1'u32 and raw <= -1'u32
+  let r = raw - 27'u32
+  r <= 255'u32 and r.uint8 in {0'u8, 23'u8, 24'u8, 25'u8, 27'u8, 29'u8, 30'u8, 31'u8, 32'u8, 36'u8, 37'u8, 38'u8, 39'u8, 40'u8, 41'u8, 42'u8, 43'u8, 95'u8, 178'u8, 234'u8, 235'u8, 236'u8, 237'u8, 238'u8, 239'u8, 240'u8, 241'u8}
 
diff --git a/src/models/tags.nim b/src/models/tags.nim
@@ -261,7 +261,15 @@ const
     ("mixin", 257),
     ("using", 258),
     ("asm", 259),
-    ("defer", 260)
+    ("defer", 260),
+    ("index", 261),
+    ("public", 262),
+    ("private", 263),
+    ("cloner", 264),
+    ("tracer", 265),
+    ("disarmer", 266),
+    ("mover", 267),
+    ("dtor", 268)
   ]
 const
   ErrTagId* = 1
@@ -524,3 +532,11 @@ const
   UsingTagId* = 258
   AsmTagId* = 259
   DeferTagId* = 260
+  IndexTagId* = 261
+  PublicTagId* = 262
+  PrivateTagId* = 263
+  ClonerTagId* = 264
+  TracerTagId* = 265
+  DisarmerTagId* = 266
+  MoverTagId* = 267
+  DtorTagId* = 268