DNS: Account for systrap dispatcher in maxSysmsgThreads.

pkg/sentry/platform/systrap/metrics.go

@@ -15,6 +15,7 @@
 package systrap
 
 import (
+	"runtime"
 	"time"
 
 	"gvisor.dev/gvisor/pkg/atomicbitops"
@@ -259,14 +260,15 @@ var (
 	// only thing that matters here is whether the Sentry handles syscall faster
 	// than the overhead of scheduling another stub thread.
 	//
-	// It is set after maxSysmsgThreads is initialized.
+	// Stub threads across all active subprocs are taken into account for
+	// this value.
 	fastPathContextLimit = uint32(0)
 )
 
 // controlFastPath is used to spawn a goroutine when creating the Systrap
 // platform.
 func controlFastPath() {
-	fastPathContextLimit = uint32(maxSysmsgThreads * 2)
+	fastPathContextLimit = uint32(runtime.GOMAXPROCS(0) * 2)
 
 	for {
 		time.Sleep(recordingPeriod)
@@ -297,6 +299,10 @@ func (s *fastPathState) stubFastPath() bool {
 // enableSentryFP is a wrapper to unconditionally enable sentry FP and increment
 // a debug metric.
 func (s *fastPathState) enableSentryFP() {
+	// Account for the ~1 core the dispatcher will consume processing the
+	// sentry fastpath.
+	maxSysmsgThreads.Add(^uint32(0))
+
 	s.sentryFastPathEnabled.Store(true)
 	numTimesSentryFastPathEnabled.Increment()
 }
@@ -311,6 +317,9 @@ func (s *fastPathState) disableSentryFP() bool {
 	if s.consecutiveSentryFPFailures < numConsecutiveFailsToDisableFP {
 		return false
 	}
+	// Dispatcher will no longer consume 1 core.
+	maxSysmsgThreads.Add(1)
+
 	s.consecutiveSentryFPFailures = 0
 	s.sentryFastPathEnabled.Store(false)
 	numTimesSentryFastPathDisabled.Increment()
@@ -367,8 +376,9 @@ func (s *fastPathState) shouldDisableSentryFP(stubMedian, sentryMedian cpuTicks)
 	if sentryMedian < sentryBaseline {
 		// Assume the number of productive stubs is the core count on the
 		// system, not counting the 1 core taken by the dispatcher for
-		// the fast path.
-		n := cpuTicks(maxSysmsgThreads - 1)
+		// the fastpath. maxSysmsgThreads is already adjusted based
+		// whether sentry fastpath is on or off.
+		n := cpuTicks(maxSysmsgThreads.Load())
 		// If the sentry fastpath is causing the stub latency to be
 		// higher than normal, the point at which it's considered to be
 		// too high is when the time saved via the sentry fastpath is

pkg/sentry/platform/systrap/subprocess.go

@@ -105,11 +105,11 @@ type requestStub struct {
 // maxSysmsgThreads is the maximum number of sysmsg threads that a subprocess
 // can create. It is based on GOMAXPROCS and set once, so it must be set after
 // GOMAXPROCS has been adjusted (see loader.go:Args.NumCPU).
-var maxSysmsgThreads = 0
+var maxSysmsgThreads atomicbitops.Uint32
 
 // maxChildThreads is the max number of all child system threads that a
 // subprocess can create, including sysmsg threads.
-var maxChildThreads = 0
+var maxChildThreads uint32 = 0
 
 const (
 	// maxGuestContexts specifies the maximum number of task contexts that a
@@ -169,7 +169,7 @@ type subprocess struct {
 	// numSysmsgThreads counts the number of active sysmsg threads; we use a
 	// counter instead of using len(sysmsgThreads) because we need to synchronize
 	// how many threads get created _before_ the creation happens.
-	numSysmsgThreads int
+	numSysmsgThreads uint32
 
 	// contextQueue is a queue of all contexts that are ready to switch back to
 	// user mode.
@@ -766,15 +766,15 @@ func (t *thread) NotifyInterrupt() {
 
 func (s *subprocess) incAwakeContexts() {
 	nr := atomic.AddUint32(&s.contextQueue.numAwakeContexts, 1)
-	if nr > uint32(maxSysmsgThreads) {
+	if nr > maxSysmsgThreads.Load() {
 		return
 	}
 	fastpath.nrMaxAwakeStubThreads.Add(1)
 }
 
 func (s *subprocess) decAwakeContexts() {
 	nr := atomic.AddUint32(&s.contextQueue.numAwakeContexts, ^uint32(0))
-	if nr >= uint32(maxSysmsgThreads) {
+	if nr >= maxSysmsgThreads.Load() {
 		return
 	}
 	fastpath.nrMaxAwakeStubThreads.Add(^uint32(0))
@@ -948,7 +948,7 @@ func (s *subprocess) kickSysmsgThread() bool {
 	}
 	numTimesStubKicked.Increment()
 	atomic.AddUint32(&s.contextQueue.numThreadsToWakeup, 1)
-	if s.numSysmsgThreads < maxSysmsgThreads && s.numSysmsgThreads < int(nrThreads) {
+	if s.numSysmsgThreads < maxSysmsgThreads.Load() && s.numSysmsgThreads < nrThreads {
 		s.numSysmsgThreads++
 		s.sysmsgThreadsMu.Unlock()
 		if err := s.createSysmsgThread(); err != nil {

pkg/sentry/platform/systrap/systrap.go

@@ -242,13 +242,13 @@ func (*Systrap) MinUserAddress() hostarch.Addr {
 
 // New returns a new seccomp-based implementation of the platform interface.
 func New() (*Systrap, error) {
-	if maxSysmsgThreads == 0 {
+	if maxSysmsgThreads.Load() == 0 {
 		// CPUID information has been initialized at this point.
 		archState.Init()
 		// GOMAXPROCS has been set at this point.
-		maxSysmsgThreads = runtime.GOMAXPROCS(0)
+		maxSysmsgThreads.Store(uint32(runtime.GOMAXPROCS(0)))
 		// Account for syscall thread.
-		maxChildThreads = maxSysmsgThreads + 1
+		maxChildThreads = maxSysmsgThreads.Load() + 1
 	}
 
 	mf, err := createMemoryFile()