From 089e480313e0bd2e8176ab2315db3e538c9b47c2 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Wed, 26 Mar 2025 11:38:48 -0700 Subject: [PATCH] buildkite: debug internal pipeline failures PiperOrigin-RevId: 740845658 --- .buildkite/hooks/post-command | 6 ++++++ .buildkite/hooks/pre-command | 9 +++++++-- runsc/cmd/do.go | 8 ++++++++ runsc/cmd/install.go | 5 +++++ runsc/cmd/install_test.go | 21 +++++++++++++-------- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index 85c31572a2..e399214bd0 100644 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -1,5 +1,11 @@ set -x +ps axf +ss -an +uname -a +docker info +docker ps + source .buildkite/hooks/libhook # Clear any downloaded credentials. diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8dabdd6fd1..bd41b2e09e 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -4,6 +4,11 @@ source .buildkite/hooks/libhook clear_docker_containers +uname -a +ps axf +ss -an +sysctl -w net.ipv4.tcp_tw_reuse=1 + # Use a per-day bazel remote cache. As the cache object's TTL expires, they are # deleted on an ongoing basis. Such partial deletion can break the cache state. # Using per day cache will ensure that builds triggered on a certain day have @@ -104,12 +109,12 @@ if [[ "${BUILDKITE_PIPELINE_INSTALL_RUNTIME:-}" == "true" ]]; then make sudo TARGETS=//runsc:runsc \ ARGS="install --experimental=true --runtime=${RUNTIME} -- ${RUNTIME_ARGS:-}" fi - if [[ "$HAD_EXPERIMENTAL" != true ]]; then + if true; then # WARNING: We may be running in a container when this command executes. # This only makes sense if Docker's `live-restore` feature is enabled. echo 'Restarting Docker daemon with this new configuration:' >&2 cat /etc/docker/daemon.json >&2 - sudo systemctl restart docker + sudo chroot /proc/1/root systemctl restart docker else # If experimental-ness was already enabled, we don't need to restart, as the # only thing we modified is the list of runtimes, which can be reloaded with diff --git a/runsc/cmd/do.go b/runsc/cmd/do.go index 4d908dcd86..c511d2cf1c 100644 --- a/runsc/cmd/do.go +++ b/runsc/cmd/do.go @@ -49,6 +49,7 @@ type Do struct { ip string quiet bool overlay bool + netns string uidMap idMapSlice gidMap idMapSlice } @@ -128,6 +129,7 @@ func (c *Do) SetFlags(f *flag.FlagSet) { f.BoolVar(&c.overlay, "force-overlay", true, "use an overlay. WARNING: disabling gives the command write access to the host") f.Var(&c.uidMap, "uid-map", "Add a user id mapping [ContainerID, HostID, Size]") f.Var(&c.gidMap, "gid-map", "Add a group id mapping [ContainerID, HostID, Size]") + f.StringVar(&c.netns, "netns", "", "path to the pre-created network namespace") } // Execute implements subcommands.Command.Execute. @@ -198,6 +200,12 @@ func (c *Do) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommand conf.Network = config.NetworkHost } + } else if c.netns != "" { + netns := specs.LinuxNamespace{ + Type: specs.NetworkNamespace, + Path: c.netns, + } + addNamespace(spec, netns) } else { switch clean, err := c.setupNet(cid, spec); err { case errNoDefaultInterface: diff --git a/runsc/cmd/install.go b/runsc/cmd/install.go index 365728c7e8..95582d72cb 100644 --- a/runsc/cmd/install.go +++ b/runsc/cmd/install.go @@ -34,6 +34,7 @@ type Install struct { ConfigFile string Runtime string Experimental bool + UserlandProxy bool Clobber bool CgroupDriver string executablePath string @@ -60,6 +61,7 @@ func (i *Install) SetFlags(fs *flag.FlagSet) { fs.StringVar(&i.ConfigFile, "config_file", "/etc/docker/daemon.json", "path to Docker daemon config file") fs.StringVar(&i.Runtime, "runtime", "runsc", "runtime name") fs.BoolVar(&i.Experimental, "experimental", false, "enable/disable experimental features") + fs.BoolVar(&i.UserlandProxy, "userland-proxy", false, "enable/disable userland-proxy features") fs.BoolVar(&i.Clobber, "clobber", true, "clobber existing runtime configuration") fs.StringVar(&i.CgroupDriver, "cgroupdriver", "", "docker cgroup driver") } @@ -158,6 +160,9 @@ func doInstallConfig(i *Install, rw configReaderWriter) error { if i.Experimental { c["experimental"] = true } + if !i.UserlandProxy { + c["userland-proxy"] = false + } re := regexp.MustCompile(`^native.cgroupdriver=`) // Set the cgroupdriver if required. diff --git a/runsc/cmd/install_test.go b/runsc/cmd/install_test.go index 981274f159..3bbf10ea42 100644 --- a/runsc/cmd/install_test.go +++ b/runsc/cmd/install_test.go @@ -65,7 +65,8 @@ var defaultInput = map[string]any{ runtimeArgs: []string{"super", "cool", "args"}, }, }, - "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"}, + "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"}, + "userland-proxy": false, } func TestInstall(t *testing.T) { @@ -102,8 +103,9 @@ func TestInstall(t *testing.T) { runtimeArgs: []string{"new", "cool", "args"}, }, }, - "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=my_driver"}, - "experimental": true, + "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=my_driver"}, + "experimental": true, + "userland-proxy": false, }, }, { @@ -132,8 +134,9 @@ func TestInstall(t *testing.T) { runtimeArgs: []string{"super", "cool", "args"}, }, }, - "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver", "native.cgroupdriver=my_driver"}, - "experimental": true, + "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver", "native.cgroupdriver=my_driver"}, + "experimental": true, + "userland-proxy": false, }, }, { @@ -164,8 +167,9 @@ func TestInstall(t *testing.T) { runtimeArgs: []string{"super", "cool", "args"}, }, }, - "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"}, - "experimental": true, + "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"}, + "experimental": true, + "userland-proxy": false, }, }, } { @@ -231,7 +235,8 @@ func TestUninstall(t *testing.T) { runtimeArgs: []string{"super", "cool", "args"}, }, }, - "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"}, + "userland-proxy": false, + "exec-opts": []string{"some-cgroup-driver=something", "native.cgroupdriver=init_driver"}, }, }, {