Introduce symdb (#767)

* Increase parquet writer PageBufferSize

* reduce by 2 page buffer size

* Introduce symdb

* Add chunk format description

* Add chunk format description

* Improve naming

* Implement stack trace appender

* Limit chunk by number of nodes

* Stacktrace ID is uint32

* Add in-memory stacktrace resolver

* Add writer

* Add writer

* Fix stacktrace resolver

* Single pass write

* Index file refactoring

* Fixes, improvements, notes

* Ignore empty stacktraces

* Fix chunk boundary check

* Fix tests

* Store chunk headers sorted

* Make chunk index explicit

* Add file reader

* Use group varint encoding

* Refine stacktrace tree

* Stacktrace tree race condition elimination

* Remove unused stacktracesResolve.do

* Better nil coalescence in stack trace appender

* Format imports

* Use the new symDB package  (#770)

* Ingest stacktraces in the new symdb

* Setup read in memory read path

* Fix up a comment placement

* Start setting up the read path

* Update to uint32

* Introduce stacktrace partition (#775)

* Introduce stacktrace partition

This determines the partition of a particular profile, by looking first
at its metadata:

* If there is a `Filename` on the main mapping use its
  filepath.Base(Filename)
* Failing that take the externally supplied `service_name`
* Fallback to `unknown`

Take the underlying string value and hash.

* After a chat with cyril we decided to not longer mod and use the hash
straight away.

We don't wanted to risk the collisions of two very big stacktrace
applications.

* Remove reconstructMeta from singleBlockQuerier

* support multiple versions of stacktraces resolver

* Integrate v2 reader for stacktraces in block reader

* Fixes tests

* Rewrite locations Ids

* Rewrite test for counting uniq stacktraces

* lint and fmt

* Fixes more tests

* Fixes leftover from todo

---------

Co-authored-by: Christian Simon <[email protected]>

* Use prefixed bucket for symbols

* Initialize locationsIdsByStacktraceID

* Initialize locationsIdsByStacktraceID for pprof as well

* Fix chunk headers sort

* Inline node alloc

* Mapping filename extraction

* Tidy go.mod

* Fix TestHeadIngestStacktraces

* Use symdb.DefaultDirName

* Sort mappings on write

* Make column iterator to respect the context

* Fix unexpected EOF on stacktrace chunk unmarshal

* Fix symbols upload

* Fix symbols upload

* Release fetched data

* 3MB Page Buffer Size

* Sort stacktraces IDs as expected by the resolver

---------

Co-authored-by: Cyril Tovena <[email protected]>
Co-authored-by: Christian Simon <[email protected]>

Author: 3 people

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/bufbuild/connect-grpchealth-go v1.0.0
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/dennwc/varint v1.0.0
+	github.com/dgryski/go-groupvarint v0.0.0-20211110171037-f5865eef0641
 	github.com/drone/envsubst v1.0.3
 	github.com/dustin/go-humanize v1.0.0
 	github.com/felixge/fgprof v0.9.4-0.20221116204635-ececf7638e93

go.sum

@@ -571,6 +571,8 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa
 github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-groupvarint v0.0.0-20211110171037-f5865eef0641 h1:z0nAHq7J9u9XvP/iyxka0dG6vGobRH5dAz38APXRCKo=
+github.com/dgryski/go-groupvarint v0.0.0-20211110171037-f5865eef0641/go.mod h1:MlkUQveSLEDbIgq2r1e++tSf0zfzU9mQpa9Qkczl+9Y=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/digitalocean/godo v1.98.0 h1:potyC1eD0N9n5/P4/WmJuKgg+OGYZOBWEW+/aKTX6QQ=

pkg/model/profile.go

@@ -1,13 +1,16 @@
 package model
 
 import (
+	"path/filepath"
 	"strings"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/gogo/status"
 	"github.com/google/pprof/profile"
 	"github.com/prometheus/prometheus/model/labels"
 	"google.golang.org/grpc/codes"
 
+	profilev1 "github.com/grafana/phlare/api/gen/proto/go/google/v1"
 	ingestv1 "github.com/grafana/phlare/api/gen/proto/go/ingester/v1"
 	typesv1 "github.com/grafana/phlare/api/gen/proto/go/types/v1"
 )
@@ -61,3 +64,43 @@ func SetProfileMetadata(p *profile.Profile, ty *typesv1.ProfileType) {
 		p.Period = 1
 	}
 }
+
+func StacktracePartitionFromProfile(lbls []Labels, p *profilev1.Profile) uint64 {
+	return xxhash.Sum64String(stacktracePartitionKeyFromProfile(lbls, p))
+}
+
+func stacktracePartitionKeyFromProfile(lbls []Labels, p *profilev1.Profile) string {
+	// take the first mapping (which is the main binary's file basename)
+	if len(p.Mapping) > 0 {
+		if filenameID := p.Mapping[0].Filename; filenameID > 0 {
+			if filename := extractMappingFilename(p.StringTable[filenameID]); filename != "" {
+				return filename
+			}
+		}
+	}
+
+	// failing that look through the labels for the ServiceName
+	if len(lbls) > 0 {
+		for _, lbl := range lbls[0] {
+			if lbl.Name == LabelNameServiceName {
+				return lbl.Value
+			}
+		}
+	}
+
+	return "unknown"
+}
+
+func extractMappingFilename(filename string) string {
+	// See github.com/google/pprof/profile/profile.go
+	// It's unlikely that the main binary mapping is one of them.
+	if filename == "" ||
+		strings.HasPrefix(filename, "[") ||
+		strings.HasPrefix(filename, "linux-vdso") ||
+		strings.HasPrefix(filename, "/dev/dri/") {
+		return ""
+	}
+	// Like filepath.ToSlash but doesn't rely on OS.
+	n := strings.ReplaceAll(filename, `\`, `/`)
+	return strings.TrimSpace(filepath.Base(filepath.Clean(n)))
+}

pkg/model/profile_test.go

@@ -0,0 +1,36 @@
+package model
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_filename_extraction(t *testing.T) {
+	assert.Equal(t, "app", extractMappingFilename(`app`))
+	assert.Equal(t, "app", extractMappingFilename(`./app`))
+	assert.Equal(t, "app", extractMappingFilename(`/usr/bin/app`))
+	assert.Equal(t, "app", extractMappingFilename(`../../../app`))
+	assert.Equal(t, "app", extractMappingFilename(`/usr/bin/app\`))
+	assert.Equal(t, "app", extractMappingFilename(`/usr/bin/app\\`))
+	assert.Equal(t, "my awesome app", extractMappingFilename(`/usr/bin/my awesome app`))
+	assert.Equal(t, "app", extractMappingFilename(`/usr/bin/my\ awesome\ app`))
+
+	assert.Equal(t, "app.exe", extractMappingFilename(`C:\\app.exe`))
+	assert.Equal(t, "app.exe", extractMappingFilename(`C:\\./app.exe`))
+	assert.Equal(t, "app.exe", extractMappingFilename(`./app.exe`))
+	assert.Equal(t, "app.exe", extractMappingFilename(`./../app.exe`))
+	assert.Equal(t, "app.exe", extractMappingFilename(`C:\\build\app.exe`))
+	assert.Equal(t, "My App.exe", extractMappingFilename(`C:\\build\My App.exe`))
+	assert.Equal(t, "Not My App.exe", extractMappingFilename(`C:\\build\Not My App.exe`))
+	assert.Equal(t, "app.exe", extractMappingFilename(`\\app.exe`))
+	assert.Equal(t, "app.exe", extractMappingFilename(`\\build\app.exe`))
+
+	assert.Equal(t, "bin", extractMappingFilename(`/usr/bin/`))
+	assert.Equal(t, "build", extractMappingFilename(`\\build\`))
+
+	assert.Equal(t, "", extractMappingFilename(""))
+	assert.Equal(t, "", extractMappingFilename(`[vdso]`))
+	assert.Equal(t, "", extractMappingFilename(`[vsyscall]`))
+	assert.Equal(t, "not a path actually", extractMappingFilename(`not a path actually`))
+}

pkg/phlaredb/block/metadata.go

@@ -38,6 +38,7 @@ type MetaVersion int
 const (
 	// Version1 is a enumeration of Phlare section of TSDB meta supported by Phlare.
 	MetaVersion1 = MetaVersion(1)
+	MetaVersion2 = MetaVersion(2)
 )
 
 type BlockStats struct {
@@ -130,6 +131,7 @@ func NewMeta() *Meta {
 		MinTime: math.MaxInt64,
 		MaxTime: 0,
 		Labels:  make(map[string]string),
+		Version: MetaVersion2,
 	}
 }
 
@@ -143,7 +145,10 @@ func MetaFromDir(dir string) (*Meta, int64, error) {
 	if err := json.Unmarshal(b, &m); err != nil {
 		return nil, 0, err
 	}
-	if m.Version != MetaVersion1 {
+	switch m.Version {
+	case MetaVersion1:
+	case MetaVersion2:
+	default:
 		return nil, 0, errors.Errorf("unexpected meta file version %d", m.Version)
 	}
 
@@ -174,8 +179,6 @@ func (meta *Meta) WriteTo(w io.Writer) (int64, error) {
 }
 
 func (meta *Meta) WriteToFile(logger log.Logger, dir string) (int64, error) {
-	meta.Version = MetaVersion1
-
 	// Make any changes to the file appear atomic.
 	path := filepath.Join(dir, MetaFilename)
 	tmp := path + ".tmp"
@@ -249,7 +252,10 @@ func Read(rc io.ReadCloser) (_ *Meta, err error) {
 		return nil, err
 	}
 
-	if m.Version != MetaVersion1 {
+	switch m.Version {
+	case MetaVersion1:
+	case MetaVersion2:
+	default:
 		return nil, errors.Errorf("unexpected meta file version %d", m.Version)
 	}