Initial version

Author: toyray

README.md

@@ -1 +1,101 @@
-# go_moduledata_parser
+# go_moduledata_parser
+
+Personal project to parse and extract function and type metadata from Go binaries
+as JSON. Since the parser output is JSON, this allows integration with different tools.
+
+Currently, only generation of an IDC for annotating an IDA dissassembly is supported. This IDC
+can be applied to the disassembly to rename functions and types.
+
+[[_TOC_]]
+
+# Example Usage
+
+```bash
+pip3 install pyelftools pefile
+python3 parser.py win64s.exe > win64s.json
+```
+
+A sample JSON can be viewed [here](integrations/ida/sample.json)
+
+## IDA integration
+```
+python3 integrations/ida/generate_go_idc.py win64s.json annotate_win64s.idc
+```
+
+To use the IDC (tested on IDA Freeware only!)
+* **File > Load File > Parse C header file ...** *(Ctrl+F9)* and select `go_32.h` or `go_64.h` depending on the bitness of the binary to import the necessary structs
+* **View > Open subviews > Local types** *(Shift+F1)*, select all types (Ctrl+A) and right click **Synchronize to IDB**
+* **File > Script File ...** *(Alt+F7)* and select the IDC file
+
+### Before
+
+![before](imgs/before.png)
+
+### After
+
+Types and functions are annotated.
+
+![after](imgs/after.png)
+
+# Limitations
+
+* Assumes that only one moduledata struct in use, Go binaries can contain more than one
+* Assumes that there is only one text section
+* Assumes that architecture is little-endian
+* Assumes that binary is built with later Go versions (currently 1.15), moduledata struct is not the same for binaries built with earlier versions
+* Only Windows and Linux (both x86/x64) supported (same architectures and OSes supported by IDA Freeware)
+* Code is not very Pythonic.:sweat_smile:
+
+# Project Organization
+
+## root
+
+All the Python code needed for parsing the moduledata and related structures
+within a Golang binary. Start from `parser.py`
+
+## go_files
+
+* [custom.go](go_files/custom.go) contains the sample Go code to test the parsing
+and annotation
+* [build.sh](go_files/build.sh) builds the Go code into stripped, unstripped versions for x86/x64 Windows and Linux
+* Prebuilt binaries. Stripped binaries are suffixed with **s**.
+
+## integrations
+
+Currently only IDA is supported. 
+
+### ida
+
+* [generate_go_idc.py](integrations/ida/generate_go_idc.py) generates a IDC script from JSON for types and functions (useful for stripped binaries)
+* [generate_go_idc_types.py](integration/ida/generate_go_idc_types.py) generates a IDC script from JSON for types only (useful for unstripped binaries so that you don't rename what IDA already generated for you)
+* [go_32.h](integrations/ida/go_32.h) contains the 32 bit version of the Golang structs
+* [go_64.h](integrations/ida/go_64.h) contains the 64 bit version of the Golang structs
+* [go_structs.h](integrations/ida/go_64.h) contains the bitness-independent
+  structs for selected Golang types. Primitive data types are defined in `go_32.h` and `go_64.h`
+* [sample](integrations/ida/sample) contains the generated JSON and IDC for a stripped Windows x64 binary
+
+# Background
+
+This started out as an attempt to convert the structs in Golang's source code (mainly in runtime/type.go, runtime/symtab.go, runtime/runtime2.go, reflect/type.go and reflect/value.go) into a header file usable by IDA and then manually identify the types from the disassembly and apply each struct by hand.
+
+A good way to learn this stuff was to write a simple Go program using the different features of the language (see [custom.go](go_files/custom.go)) and then building with and without stripping the symbols (see [build.sh](go_files/build.sh)).
+
+By comparing the disassembly of both binaries side-by-side, it is possible to manually identify the main functions, recognize types. etc.
+
+After spending enough time manually annotating a stripped binary, I decided to automate most of these tasks and hence this project :smiley:
+
+Some of the excellent articles and tools that I referred to during this project are listed below.
+
+# References
+
+## Reverse Engineering Articles
+
+* https://lekstu.ga/tags/go/
+* https://x0r19x91.gitlab.io/categories/golang/
+
+## Other Good :thumbsup: Tools
+
+* https://go-re.tk/redress/ (Windows and Linux, r2 integration)
+* https://github.com/alexander-hanel/gopep (Windows only, good notes and references as well)
+* https://github.com/getCUJO/ThreatIntel/tree/master/Scripts/Ghidra (Ghidra integration)
+

go_files/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o "$1win64s.exe"
+GOOS=windows GOARCH=amd64 go build -o "$1win64.exe"
+GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o "$1win32s.exe"
+GOOS=windows GOARCH=386 go build -o "$1win32.exe"
+GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o "$1lnx64s"
+GOOS=linux GOARCH=amd64 go build -o "$1lnx64"
+GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o "$1lnx32s"
+GOOS=linux GOARCH=386 go build -o "$1lnx32"
+

go_files/custom.go

@@ -0,0 +1,50 @@
+package main
+
+import "fmt"
+
+type Updater interface {
+	UpdateQty(qty int, rounds int) int
+}
+
+type custom struct {
+	name  string
+	qty   int
+	legit bool
+	toys  map[int]string
+}
+
+func (c *custom) UpdateQty(qty int, rounds int) int {
+	for i := 0; i < rounds; i++ {
+		c.qty = c.qty + qty
+	}
+	return c.qty
+}
+
+func (c custom) PrintName(greeting string) {
+	if len(c.name) != 0 {
+		fmt.Printf("%s %s ", greeting, c.name)
+	}
+}
+
+func main() {
+	c := &custom{
+		name:  "Wheee",
+		qty:   -33,
+		legit: false,
+		toys:  map[int]string{1: "thiss", 2: "thatt"},
+	}
+
+	fmt.Printf("%s\n", c.name)
+	fmt.Printf("%d\n", c.qty)
+	fmt.Printf("%t\n", c.legit)
+	fmt.Printf("%#v\n", c.toys)
+
+	c.PrintName("Hello World!")
+	c.UpdateQty(111, 3)
+
+	var u Updater
+	u = c
+	u.UpdateQty(11, 8)
+
+	fmt.Printf("%v\n", c)
+}

go_files/lnx32

@@ -0,0 +1,70 @@
+from struct import *
+
+class Parser:
+
+    def __init__(self, md_parser):
+        self.md_parser = md_parser
+
+    def parse_ftabs(self):
+        mdp = self.md_parser
+        mdp.f.seek(mdp.moduledata_raw, 0)
+
+        FTABS_OFFSET = 3 * mdp.ptr_size
+        FTABS_SIZE = 3 * mdp.ptr_size
+
+        mdp.f.seek(FTABS_OFFSET, 1)
+        ftabs = mdp.f.read(FTABS_SIZE)
+
+        ftabs_va, ftabs_len, _ = unpack('<' + 3 * mdp.ptr_type, ftabs)
+        ftabs_raw = mdp.va2raw(ftabs_va)
+
+        FTAB_SIZE = mdp.ptr_size * 2
+
+        funcs = {}
+
+        for i in range(0, ftabs_len - 1, 1):
+            mdp.f.seek(ftabs_raw + (i * FTAB_SIZE), 0)
+            ftab = mdp.f.read(FTAB_SIZE)
+            func_addr, _func_offset = unpack("<" + 2 * mdp.ptr_type, ftab)
+            _func_raw =  mdp.pclntab_off( _func_offset)
+            func = self._parse_func(_func_raw)
+            funcs[hex(func_addr)] = func
+        return funcs
+
+    def _parse_func(self, _func_raw):
+        # https://golang.org/src/runtime/runtime2.go
+        # type _func struct {
+	#     entry   uintptr // start pc
+	#     nameoff int32   // function name
+        #
+	#     args        int32  // in/out args size
+	#     deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
+        #
+	#     pcsp      int32
+	#     pcfile    int32
+	#     pcln      int32
+	#     npcdata   int32
+	#     funcID    funcID  // set for certain special runtime functions
+	#     _         [2]int8 // unused
+	#     nfuncdata uint8   // must be last
+        # }
+
+        mdp = self.md_parser
+        mdp.f.seek(_func_raw)
+
+        # _func is bigger than this, but we only want to read the offset to the
+        # function name for now
+        FUNC_SIZE = mdp.ptr_size + 4
+        _func = mdp.f.read(mdp.ptr_size + 4)
+        _func_va, _func_name_offset = unpack("<" + mdp.ptr_type + "L", _func)
+        _func_name_raw =  mdp.pclntab_off(_func_name_offset)
+
+        mdp.f.seek(_func_name_raw)
+
+        # TODO: Not sure if there's a max length for function names
+        data = mdp.f.read(512)
+        for i in range(0, len(data), 1):
+            if data[i] == 0:
+                break
+        _func_name = data[:i].decode("Utf-8")
+        return { "name": _func_name }

go_files/lnx32s

@@ -0,0 +1,62 @@
+from struct import *
+from go_type import Kind
+
+class Parser:
+
+    def __init__(self, md_parser):
+        self.md_parser = md_parser
+
+    def parse_itablinks(self, types):
+        mdp = self.md_parser
+        mdp.f.seek(mdp.moduledata_raw, 0)
+
+        ITABLINKS_OFFSET = 33 * mdp.ptr_size
+        ITABLINKS_SIZE = 3 * mdp.ptr_size
+
+        mdp.f.seek(ITABLINKS_OFFSET, 1)
+        itablinks_data = mdp.f.read(ITABLINKS_SIZE)
+
+        itablinks_va, itablinks_len, _ = unpack("<" + 3 * mdp.ptr_type, itablinks_data)
+        itablinks_raw = mdp.va2raw(itablinks_va)
+
+        ITABLINK_SIZE =  mdp.ptr_size
+        ITAB_SIZE = 3 * mdp.ptr_size + 8
+
+        itablinks = {}
+        itabs = {}
+
+        for i in range(0, itablinks_len, 1):
+            itl_raw = itablinks_raw + (i * ITABLINK_SIZE)
+            itl_va = itablinks_va + (i * ITABLINK_SIZE)
+            itablinks[hex(itl_va)] = { "name": "moduledata_itablink." + str(i+1) }
+
+            mdp.f.seek(itl_raw, 0)
+
+            itl_data = mdp.f.read(ITABLINK_SIZE)
+            it_va = unpack("<" + mdp.ptr_type, itl_data)[0]
+            it_raw = mdp.va2raw(it_va)
+
+            mdp.f.seek(it_raw)
+            itab_data = mdp.f.read(ITAB_SIZE)
+            itab_inter_va, itab_type_va , _, _ , itab_func_ptrs = unpack("<" +
+                    2 * mdp.ptr_type + "LL" + mdp.ptr_type, itab_data)
+
+            itab_inter = types.get(hex(itab_inter_va))
+            itab_type = types.get(hex(itab_type_va))
+
+            if itab_inter is None:
+                itab_inter_name = "undefined_interface_" + hex(itab_inter_va)
+            else:
+                itab_inter_name = itab_inter["name"]
+
+            if itab_type is None:
+                itab_type_name = "undefined_type_" + hex(itab_type_va)
+            else:
+                itab_type_name = itab_type["name"]
+
+            itabs[hex(it_va)] = {
+                    "interface_name": itab_inter_name,
+                    "type_name": itab_type_name
+            }
+
+        return itablinks, itabs