buildkite: debug internal pipeline failures

.buildkite/hooks/post-command

@@ -1,5 +1,11 @@
 set -x
 
+ps axf
+ss -an
+uname -a
+docker info
+docker ps
+
 source .buildkite/hooks/libhook
 
 # Clear any downloaded credentials.

.buildkite/hooks/pre-command

@@ -4,6 +4,11 @@ source .buildkite/hooks/libhook
 
 clear_docker_containers
 
+uname -a
+ps axf
+ss -an
+sysctl -w net.ipv4.tcp_tw_reuse=1
+
 # Use a per-day bazel remote cache. As the cache object's TTL expires, they are
 # deleted on an ongoing basis. Such partial deletion can break the cache state.
 # Using per day cache will ensure that builds triggered on a certain day have
@@ -104,12 +109,12 @@ if [[ "${BUILDKITE_PIPELINE_INSTALL_RUNTIME:-}" == "true" ]]; then
     make sudo TARGETS=//runsc:runsc \
       ARGS="install --experimental=true --runtime=${RUNTIME} -- ${RUNTIME_ARGS:-}"
   fi
-  if [[ "$HAD_EXPERIMENTAL" != true ]]; then
+  if true; then
     # WARNING: We may be running in a container when this command executes.
     # This only makes sense if Docker's `live-restore` feature is enabled.
     echo 'Restarting Docker daemon with this new configuration:' >&2
     cat /etc/docker/daemon.json >&2
-    sudo systemctl restart docker
+    sudo chroot /proc/1/root systemctl restart docker
   else
     # If experimental-ness was already enabled, we don't need to restart, as the
     # only thing we modified is the list of runtimes, which can be reloaded with

runsc/cmd/do.go

@@ -49,6 +49,7 @@ type Do struct {
 	ip      string
 	quiet   bool
 	overlay bool
+	netns   string
 	uidMap  idMapSlice
 	gidMap  idMapSlice
 }
@@ -128,6 +129,7 @@ func (c *Do) SetFlags(f *flag.FlagSet) {
 	f.BoolVar(&c.overlay, "force-overlay", true, "use an overlay. WARNING: disabling gives the command write access to the host")
 	f.Var(&c.uidMap, "uid-map", "Add a user id mapping [ContainerID, HostID, Size]")
 	f.Var(&c.gidMap, "gid-map", "Add a group id mapping [ContainerID, HostID, Size]")
+	f.StringVar(&c.netns, "netns", "", "path to the pre-created network namespace")
 }
 
 // Execute implements subcommands.Command.Execute.
@@ -198,6 +200,12 @@ func (c *Do) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommand
 			conf.Network = config.NetworkHost
 		}
 
+	} else if c.netns != "" {
+		netns := specs.LinuxNamespace{
+			Type: specs.NetworkNamespace,
+			Path: c.netns,
+		}
+		addNamespace(spec, netns)
 	} else {
 		switch clean, err := c.setupNet(cid, spec); err {
 		case errNoDefaultInterface:

runsc/cmd/install.go

@@ -34,6 +34,7 @@ type Install struct {
 	ConfigFile     string
 	Runtime        string
 	Experimental   bool
+	UserlandProxy  bool
 	Clobber        bool
 	CgroupDriver   string
 	executablePath string
@@ -60,6 +61,7 @@ func (i *Install) SetFlags(fs *flag.FlagSet) {
 	fs.StringVar(&i.ConfigFile, "config_file", "/etc/docker/daemon.json", "path to Docker daemon config file")
 	fs.StringVar(&i.Runtime, "runtime", "runsc", "runtime name")
 	fs.BoolVar(&i.Experimental, "experimental", false, "enable/disable experimental features")
+	fs.BoolVar(&i.UserlandProxy, "userland-proxy", false, "enable/disable userland-proxy features")
 	fs.BoolVar(&i.Clobber, "clobber", true, "clobber existing runtime configuration")
 	fs.StringVar(&i.CgroupDriver, "cgroupdriver", "", "docker cgroup driver")
 }
@@ -158,6 +160,9 @@ func doInstallConfig(i *Install, rw configReaderWriter) error {
 	if i.Experimental {
 		c["experimental"] = true
 	}
+	if !i.UserlandProxy {
+		c["userland-proxy"] = false
+	}
 
 	re := regexp.MustCompile(`^native.cgroupdriver=`)
 	// Set the cgroupdriver if required.

runsc/cmd/install_test.go

@@ -65,7 +65,8 @@ var defaultInput = map[string]any{
 			runtimeArgs: []string{"super", "cool", "args"},
 		},
 	},
-	"exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"},
+	"exec-opts":      []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"},
+	"userland-proxy": false,
 }
 
 func TestInstall(t *testing.T) {
@@ -102,8 +103,9 @@ func TestInstall(t *testing.T) {
 						runtimeArgs: []string{"new", "cool", "args"},
 					},
 				},
-				"exec-opts":    []string{"some-cgroup-driver=something", "native.cgroupdriver=my_driver"},
-				"experimental": true,
+				"exec-opts":      []string{"some-cgroup-driver=something", "native.cgroupdriver=my_driver"},
+				"experimental":   true,
+				"userland-proxy": false,
 			},
 		},
 		{
@@ -132,8 +134,9 @@ func TestInstall(t *testing.T) {
 						runtimeArgs: []string{"super", "cool", "args"},
 					},
 				},
-				"exec-opts":    []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver", "native.cgroupdriver=my_driver"},
-				"experimental": true,
+				"exec-opts":      []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver", "native.cgroupdriver=my_driver"},
+				"experimental":   true,
+				"userland-proxy": false,
 			},
 		},
 		{
@@ -164,8 +167,9 @@ func TestInstall(t *testing.T) {
 						runtimeArgs: []string{"super", "cool", "args"},
 					},
 				},
-				"exec-opts":    []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"},
-				"experimental": true,
+				"exec-opts":      []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"},
+				"experimental":   true,
+				"userland-proxy": false,
 			},
 		},
 	} {
@@ -231,7 +235,8 @@ func TestUninstall(t *testing.T) {
 						runtimeArgs: []string{"super", "cool", "args"},
 					},
 				},
-				"exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"},
+				"userland-proxy": false,
+				"exec-opts":      []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"},
 			},
 		},
 		{