Skip to content

Bug: #1224

Open
#1232
Open
Bug:#1224
#1232

Description

@tonsV2
tonsV2
opened on Dec 11, 2025

Describe the bug
The Kubernetes configuration file isn't retrieved. This stopped working all of a sudden.

To Reproduce

// SetupK8s creates a K8s container (using k3s).
func SetupK8s(t *testing.T) *K8sClient {
	t.Helper()

	container, err := gnomock.Start(
		k3s.Preset(
			k3s.WithVersion("v1.33.2-k3s1"),
			func(p *k3s.P) {
				p.K3sServerFlags = []string{"--debug"}
			},
		),
		gnomock.WithDebugMode(),
	)
	require.NoError(t, err, "failed to start k3s")
	t.Cleanup(func() { require.NoError(t, gnomock.Stop(container), "failed to stop k3s") })

	k8sConfig, err := k3s.Config(container)
	require.NoError(t, err, "failed to get k3s config from container")
	k8sClient, err := kubernetes.NewForConfig(k8sConfig)
	require.NoError(t, err, "failed to create k8s client")

	k3sConfigBytes, err := k3s.ConfigBytes(container)
	require.NoError(t, err, "failed to get k3s config from container as bytes")

	return &K8sClient{
		Client: k8sClient,
		Config: k3sConfigBytes,
	}
}

Expected behavior

The below line should retrieving the file from the endpoint http://127.0.0.1:32789/kubeconfig.yaml

k3sConfigBytes, err := k3s.ConfigBytes(container)

System (please complete the following information):

  • OS: Ubuntu 24.04
  • Version: v0.32.0
  • Docker version: 28.5.2

Additional context
The truncated error log can be see here

?   	github.com/dhis2-sre/im-manager/cmd/serve	[no test files]
ok  	github.com/dhis2-sre/im-manager/internal/errdef	(cached)
ok  	github.com/dhis2-sre/im-manager/internal/handler	(cached)
ok  	github.com/dhis2-sre/im-manager/internal/log	(cached)
ok  	github.com/dhis2-sre/im-manager/internal/middleware	(cached)
?   	github.com/dhis2-sre/im-manager/internal/server	[no test files]
ok  	github.com/dhis2-sre/im-manager/pkg/cluster	(cached)
panic: test timed out after 10m0s
	running tests:
		TestDatabaseHandler (10m0s)

goroutine 9367 [running]:
testing.(*M).startAlarm.func1()
	/snap/go/10984/src/testing/testing.go:2682 +0x605
created by time.goFunc
	/snap/go/10984/src/time/sleep.go:215 +0x45

goroutine 1 [chan receive, 10 minutes]:
testing.tRunner.func1()
	/snap/go/10984/src/testing/testing.go:1891 +0x9ad
testing.tRunner(0xc00058bdc0, 0xc00008dae0)
	/snap/go/10984/src/testing/testing.go:1940 +0x256
testing.runTests(0xc000640f90, {0xa9ab360, 0x1, 0x1}, {0x0?, 0x1e1d225?, 0xabc5be0?})
	/snap/go/10984/src/testing/testing.go:2475 +0x96d
testing.(*M).Run(0xc000764960)
	/snap/go/10984/src/testing/testing.go:2337 +0xed5
main.main()
	_testmain.go:47 +0x165

goroutine 44 [select]:
github.com/orlangure/gnomock.(*g).wait(0xc0007a2990, {0x84566c8, 0xc000400000}, 0xc000a58440, 0xc000ebe240)
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:299 +0x36c
github.com/orlangure/gnomock.newContainer(0xc0007a2990, {0xc000a7a1e0, 0x25}, 0xc000ebc780, 0xc000ebe240)
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:141 +0x5e5
github.com/orlangure/gnomock.StartCustom({0xc000a7a1e0, 0x25}, 0xc000ebc780, {0xc0007d33d0, 0x3, 0x3})
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:102 +0x626
github.com/orlangure/gnomock.Start({0x843e240, 0xc000778d80}, {0x0, 0x0, 0x1e30b09?})
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:200 +0x1f0
github.com/dhis2-sre/im-manager/pkg/inttest.SetupS3(0xc000250700, {0xc000a7a180, 0x26})
	/home/ubuntu/im-manager/pkg/inttest/s3.go:20 +0x2d9
github.com/dhis2-sre/im-manager/pkg/database_test.TestDatabaseHandler(0xc000250700)
	/home/ubuntu/im-manager/pkg/database/database_integration_test.go:38 +0x5e7
testing.tRunner(0xc000250700, 0x7d92e50)
	/snap/go/10984/src/testing/testing.go:1934 +0x21d
created by testing.(*T).Run in goroutine 1
	/snap/go/10984/src/testing/testing.go:1997 +0x9d3

goroutine 46 [select, 10 minutes]:
net/http.(*persistConn).readLoop(0xc00065fe60)
	/snap/go/10984/src/net/http/transport.go:2398 +0x14ec
created by net/http.(*Transport).dialConn in goroutine 45
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 47 [select, 10 minutes]:
net/http.(*persistConn).writeLoop(0xc00065fe60)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 45
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 53 [select, 9 minutes]:
net/http.(*persistConn).readLoop(0xc000ebe360)
	/snap/go/10984/src/net/http/transport.go:2398 +0x14ec
created by net/http.(*Transport).dialConn in goroutine 52
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 62 [select, 9 minutes]:
net/http.(*persistConn).roundTrip(0xc000ebe7e0, 0xc00054a1e0)
	/snap/go/10984/src/net/http/transport.go:2836 +0xd16
net/http.(*Transport).roundTrip(0xaa25500, 0xc0007ae140)
	/snap/go/10984/src/net/http/transport.go:696 +0x14dc
net/http.(*Transport).RoundTrip(0xaa25500, 0xc0007ae140)
	/snap/go/10984/src/net/http/roundtrip.go:33 +0x39
net/http.send(0xc0007ae140, {0x83fe9c0, 0xaa25500}, {0x8?, 0x8?, 0x0?})
	/snap/go/10984/src/net/http/client.go:259 +0x8cb
net/http.(*Client).send(0xabc4040, 0xc0007ae140, {0x0?, 0x279f5d2?, 0x0?})
	/snap/go/10984/src/net/http/client.go:180 +0x14d
net/http.(*Client).do(0xabc4040, 0xc0007ae140)
	/snap/go/10984/src/net/http/client.go:729 +0x13b9
net/http.(*Client).Do(...)
	/snap/go/10984/src/net/http/client.go:587
created by github.com/orlangure/gnomock/internal/cleaner.Notify in goroutine 68
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/internal/cleaner/cleaner.go:28 +0x2e8

goroutine 13 [select, 10 minutes]:
net/http.(*persistConn).readLoop(0xc000ad6360)
	/snap/go/10984/src/net/http/transport.go:2398 +0x14ec
created by net/http.(*Transport).dialConn in goroutine 12
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 14 [select, 10 minutes]:
net/http.(*persistConn).writeLoop(0xc000ad6360)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 12
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 51 [select, 10 minutes]:
net/http.(*persistConn).roundTrip(0xc000822120, 0xc00092e370)
	/snap/go/10984/src/net/http/transport.go:2836 +0xd16
net/http.(*Transport).roundTrip(0xaa25500, 0xc00095cdc0)
	/snap/go/10984/src/net/http/transport.go:696 +0x14dc
net/http.(*Transport).RoundTrip(0xaa25500, 0xc00095cdc0)
	/snap/go/10984/src/net/http/roundtrip.go:33 +0x39
net/http.send(0xc00095cdc0, {0x83fe9c0, 0xaa25500}, {0x8?, 0x8?, 0x0?})
	/snap/go/10984/src/net/http/client.go:259 +0x8cb
net/http.(*Client).send(0xabc4040, 0xc00095cdc0, {0x0?, 0x279f5d2?, 0x0?})
	/snap/go/10984/src/net/http/client.go:180 +0x14d
net/http.(*Client).do(0xabc4040, 0xc00095cdc0)
	/snap/go/10984/src/net/http/client.go:729 +0x13b9
net/http.(*Client).Do(...)
	/snap/go/10984/src/net/http/client.go:587
created by github.com/orlangure/gnomock/internal/cleaner.Notify in goroutine 25
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/internal/cleaner/cleaner.go:28 +0x2e8

goroutine 26 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x76b5eff36400, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc0002ca120, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0002ca100, {0xc00089d000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc0002ca100, {0xc00089d000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc000ad2040, {0xc00089d000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc000ad6360, {0xc00089d000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc000242720)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).ReadSlice(0xc000242720, 0xa)
	/snap/go/10984/src/bufio/bufio.go:380 +0x85
net/http/internal.readChunkLine(0xc000242720)
	/snap/go/10984/src/net/http/internal/chunked.go:156 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc0007e2de0)
	/snap/go/10984/src/net/http/internal/chunked.go:49 +0x5a
net/http/internal.(*chunkedReader).Read(0xc0007e2de0, {0xc000818000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/internal/chunked.go:125 +0x2ad
net/http.(*body).readLocked(0xc0007aa640, {0xc000818000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:845 +0xa4
net/http.(*body).Read(0xc0007aa640, {0xc000818000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:837 +0x17a
net/http.(*bodyEOFSignal).Read(0xc0007aa680, {0xc000818000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transport.go:3000 +0x135
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*wrappedBody).Read(0xc0007aa700, {0xc000818000, 0x8009, 0x8009})
	/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.61.0/transport.go:229 +0x6d
github.com/docker/docker/pkg/stdcopy.StdCopy({0x83fe040, 0xbca0020}, {0x83fe040, 0xbca0020}, {0x76b5a5322990, 0xc00038ec80})
	/home/ubuntu/go/pkg/mod/github.com/docker/docker@v28.2.2+incompatible/pkg/stdcopy/stdcopy.go:108 +0x55e
github.com/orlangure/gnomock.(*g).setupLogForwarding.copyf.func1()
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:156 +0x65
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x87
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 25
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x11d

goroutine 31 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x76b5eff36600, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc00022daa0, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00022da80, {0xc00077a000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc00022da80, {0xc00077a000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc000713f58, {0xc00077a000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc00065fe60, {0xc00077a000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc000a775c0)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).ReadSlice(0xc000a775c0, 0xa)
	/snap/go/10984/src/bufio/bufio.go:380 +0x85
net/http/internal.readChunkLine(0xc000a775c0)
	/snap/go/10984/src/net/http/internal/chunked.go:156 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc0007e3230)
	/snap/go/10984/src/net/http/internal/chunked.go:49 +0x5a
net/http/internal.(*chunkedReader).Read(0xc0007e3230, {0xc000838000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/internal/chunked.go:125 +0x2ad
net/http.(*body).readLocked(0xc0007aa880, {0xc000838000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:845 +0xa4
net/http.(*body).Read(0xc0007aa880, {0xc000838000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:837 +0x17a
net/http.(*bodyEOFSignal).Read(0xc0007aa8c0, {0xc000838000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transport.go:3000 +0x135
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*wrappedBody).Read(0xc0007aa940, {0xc000838000, 0x8009, 0x8009})
	/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.61.0/transport.go:229 +0x6d
github.com/docker/docker/pkg/stdcopy.StdCopy({0x83fe040, 0xbca0020}, {0x83fe040, 0xbca0020}, {0x76b5a5322990, 0xc00038ee40})
	/home/ubuntu/go/pkg/mod/github.com/docker/docker@v28.2.2+incompatible/pkg/stdcopy/stdcopy.go:108 +0x55e
github.com/orlangure/gnomock.(*g).setupLogForwarding.copyf.func1()
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:156 +0x65
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x87
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 44
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x11d

goroutine 29 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x76b5eff36200, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc0008260a0, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000826080, {0xc000836000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc000826080, {0xc000836000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc0007b60f0, {0xc000836000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc000822120, {0xc000836000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc000900960)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).Peek(0xc000900960, 0x1)
	/snap/go/10984/src/bufio/bufio.go:152 +0xc5
net/http.(*persistConn).readLoop(0xc000822120)
	/snap/go/10984/src/net/http/transport.go:2278 +0x32a
created by net/http.(*Transport).dialConn in goroutine 27
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 30 [select, 10 minutes]:
net/http.(*persistConn).writeLoop(0xc000822120)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 27
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 60 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x76b5eff35a00, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc000ec05a0, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ec0580, {0xc000ed9000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc000ec0580, {0xc000ed9000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc0007b66b0, {0xc000ed9000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc000ebe7e0, {0xc000ed9000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc0006913e0)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).Peek(0xc0006913e0, 0x1)
	/snap/go/10984/src/bufio/bufio.go:152 +0xc5
net/http.(*persistConn).readLoop(0xc000ebe7e0)
	/snap/go/10984/src/net/http/transport.go:2278 +0x32a
created by net/http.(*Transport).dialConn in goroutine 58
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 67 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc0007ff790, {0x8456690, 0xc0007eb310})
	/snap/go/10984/src/database/sql/sql.go:1261 +0xeb
created by database/sql.OpenDB in goroutine 44
	/snap/go/10984/src/database/sql/sql.go:841 +0x287

goroutine 54 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc000ebe360)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 52
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 56 [select, 9 minutes]:
net/http.(*persistConn).readLoop(0xc000ebe6c0)
	/snap/go/10984/src/net/http/transport.go:2398 +0x14ec
created by net/http.(*Transport).dialConn in goroutine 55
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 57 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc000ebe6c0)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 55
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 16 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x76b5eff35c00, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc000ec03a0, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ec0380, {0xc000ed0000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc000ec0380, {0xc000ed0000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc0007b6668, {0xc000ed0000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc000ebe6c0, {0xc000ed0000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc0006912c0)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).ReadSlice(0xc0006912c0, 0xa)
	/snap/go/10984/src/bufio/bufio.go:380 +0x85
net/http/internal.readChunkLine(0xc0006912c0)
	/snap/go/10984/src/net/http/internal/chunked.go:156 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc000ebd140)
	/snap/go/10984/src/net/http/internal/chunked.go:49 +0x5a
net/http/internal.(*chunkedReader).Read(0xc000ebd140, {0xc0010a6000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/internal/chunked.go:125 +0x2ad
net/http.(*body).readLocked(0xc0007791c0, {0xc0010a6000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:845 +0xa4
net/http.(*body).Read(0xc0007791c0, {0xc0010a6000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:837 +0x17a
net/http.(*bodyEOFSignal).Read(0xc000779200, {0xc0010a6000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transport.go:3000 +0x135
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*wrappedBody).Read(0xc000a589c0, {0xc0010a6000, 0x8009, 0x8009})
	/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.61.0/transport.go:229 +0x6d
github.com/docker/docker/pkg/stdcopy.StdCopy({0x83fe040, 0xbca0020}, {0x83fe040, 0xbca0020}, {0x76b5a5322990, 0xc0008e8f50})
	/home/ubuntu/go/pkg/mod/github.com/docker/docker@v28.2.2+incompatible/pkg/stdcopy/stdcopy.go:108 +0x55e
github.com/orlangure/gnomock.(*g).setupLogForwarding.copyf.func1()
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:156 +0x65
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x87
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 68
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x11d

goroutine 61 [select, 9 minutes]:
net/http.(*persistConn).writeLoop(0xc000ebe7e0)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 58
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 69 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x76b5eff35e00, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc000ec00a0, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ec0080, {0xc000ec6000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc000ec0080, {0xc000ec6000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc0007b6640, {0xc000ec6000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc000ebe360, {0xc000ec6000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc0006911a0)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).ReadSlice(0xc0006911a0, 0xa)
	/snap/go/10984/src/bufio/bufio.go:380 +0x85
net/http/internal.readChunkLine(0xc0006911a0)
	/snap/go/10984/src/net/http/internal/chunked.go:156 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc0010045d0)
	/snap/go/10984/src/net/http/internal/chunked.go:49 +0x5a
net/http/internal.(*chunkedReader).Read(0xc0010045d0, {0xc00101a000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/internal/chunked.go:125 +0x2ad
net/http.(*body).readLocked(0xc00091e4c0, {0xc00101a000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:845 +0xa4
net/http.(*body).Read(0xc00091e4c0, {0xc00101a000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:837 +0x17a
net/http.(*bodyEOFSignal).Read(0xc00091e500, {0xc00101a000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transport.go:3000 +0x135
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*wrappedBody).Read(0xc00091e580, {0xc00101a000, 0x8009, 0x8009})
	/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.61.0/transport.go:229 +0x6d
github.com/docker/docker/pkg/stdcopy.StdCopy({0x83fe040, 0xbca0020}, {0x83fe040, 0xbca0020}, {0x76b5a5322990, 0xc0010082d0})
	/home/ubuntu/go/pkg/mod/github.com/docker/docker@v28.2.2+incompatible/pkg/stdcopy/stdcopy.go:108 +0x55e
github.com/orlangure/gnomock.(*g).setupLogForwarding.copyf.func1()
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:156 +0x65
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x87
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 44
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x11d
FAIL	github.com/dhis2-sre/im-manager/pkg/database	600.112s
ok  	github.com/dhis2-sre/im-manager/pkg/event	(cached)
ok  	github.com/dhis2-sre/im-manager/pkg/group	(cached)
?   	github.com/dhis2-sre/im-manager/pkg/health	[no test files]
ok  	github.com/dhis2-sre/im-manager/pkg/inspector	(cached)
{"L":"INFO","T":"2025-12-11T05:01:03.852Z","M":"starting","id":"850415f5-1352-418d-b8d8-2faf945a6461","image":"docker.io/rancher/k3s:v1.33.2-k3s1","ports":{"default":{"protocol":"tcp","port":48443,"host_port":48443},"kubeconfig":{"protocol":"tcp","port":48480,"host_port":0}}}
{"L":"INFO","T":"2025-12-11T05:01:03.852Z","M":"using config","id":"850415f5-1352-418d-b8d8-2faf945a6461","image":"docker.io/rancher/k3s:v1.33.2-k3s1","ports":{"default":{"protocol":"tcp","port":48443,"host_port":48443},"kubeconfig":{"protocol":"tcp","port":48480,"host_port":0}},"config":{"timeout":600000000000,"env":["K3S_KUBECONFIG_OUTPUT=/var/gnomock/kubeconfig.yaml","K3S_KUBECONFIG_MODE=644"],"debug":true,"privileged":true,"container_name":"","cmd":null,"entrypoint":["/bin/sh","-c","mkdir -p /var/lib/rancher/k3s/server/manifests && echo \"eyJhcGlWZXJzaW9uIjoidjEiLCJraW5kIjoiUG9kIiwibWV0YWRhdGEiOnsibmFtZSI6Imt1YmVjb25maWctaHR0cGQiLCJuYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSJ9LCJzcGVjIjp7ImNvbnRhaW5lcnMiOlt7ImNvbW1hbmQiOlsiaHR0cGQiLCItZiIsIi12IiwiLXAiLCI0ODQ4MCJdLCJpbWFnZSI6ImRvY2tlci5pby9saWJyYXJ5L2J1c3lib3g6bGF0ZXN0IiwibmFtZSI6IndlYiIsInBvcnRzIjpbeyJjb250YWluZXJQb3J0Ijo0ODQ4MCwibmFtZSI6Imh0dHAiLCJwcm90b2NvbCI6IlRDUCJ9XSwidm9sdW1lTW91bnRzIjpbeyJtb3VudFBhdGgiOiIvdmFyL2dub21vY2svIiwibmFtZSI6Imt1YmVjb25maWctZGlyIn1dLCJ3b3JraW5nRGlyIjoiL3Zhci9nbm9tb2NrLyJ9XSwiaG9zdE5ldHdvcmsiOnRydWUsInZvbHVtZXMiOlt7Imhvc3RQYXRoIjp7InBhdGgiOiIvdmFyL2dub21vY2svIiwidHlwZSI6IkRpcmVjdG9yeSJ9LCJuYW1lIjoia3ViZWNvbmZpZy1kaXIifV19fQ==\" | base64 -d > \"/var/lib/rancher/k3s/server/manifests/kubeconfig-httpd.json\" && /bin/k3s server --https-listen-port 48443 --debug"],"host_mounts":null,"disable_cleanup":false,"use_local_images_first":false,"custom_named_ports":null,"auth":"","extraHosts":null,"reuse":false,"customImage":"","user":""}}
{"L":"INFO","T":"2025-12-11T05:01:03.852Z","M":"connecting to docker engine","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:03.852Z","M":"connected to docker engine","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:03.852Z","M":"starting container","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:03.852Z","M":"pulling image","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:08.811Z","M":"image pulled","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:09.017Z","M":"waiting for container network","id":"850415f5-1352-418d-b8d8-2faf945a6461","container":"66e950984815341a498c1aa58ece8e40f0717bace451c0311a24e8e30a35b94d"}
{"L":"INFO","T":"2025-12-11T05:01:09.271Z","M":"waiting for port allocation","id":"850415f5-1352-418d-b8d8-2faf945a6461","container":"66e950984815341a498c1aa58ece8e40f0717bace451c0311a24e8e30a35b94d"}
{"L":"INFO","T":"2025-12-11T05:01:09.271Z","M":"container started","id":"850415f5-1352-418d-b8d8-2faf945a6461","container":{"id":"66e950984815341a498c1aa58ece8e40f0717bace451c0311a24e8e30a35b94d","host":"127.0.0.1","ports":{"default":{"protocol":"tcp","port":48443,"host_port":0},"kubeconfig":{"protocol":"tcp","port":32789,"host_port":0}}}}
{"L":"INFO","T":"2025-12-11T05:01:09.271Z","M":"starting container logs forwarder","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:09.271Z","M":"container logs forwarder ready","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:09.271Z","M":"waiting for healthcheck to pass","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:09.525Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:49202->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:09.774Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": EOF","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:10.022Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:49214->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:01:10.272Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": EOF","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
...
{"L":"INFO","T":"2025-12-11T05:11:01.773Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:47966->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:02.023Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:47968->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:02.273Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:47978->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:02.523Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": EOF","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:02.772Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": EOF","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:03.023Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:48008->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:03.273Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:48014->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:03.524Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:48028->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
{"L":"INFO","T":"2025-12-11T05:11:03.773Z","M":"healthcheck failed: failed to get kubeconfig: can't get kubeconfig bytes: kubeconfig unavailable: Get \"http://127.0.0.1:32789/kubeconfig.yaml\": read tcp 127.0.0.1:48044->127.0.0.1:32789: read: connection reset by peer","id":"850415f5-1352-418d-b8d8-2faf945a6461"}
panic: test timed out after 10m0s
	running tests:
		TestInstanceHandler (10m0s)

goroutine 9579 [running]:
testing.(*M).startAlarm.func1()
	/snap/go/10984/src/testing/testing.go:2682 +0x605
created by time.goFunc
	/snap/go/10984/src/time/sleep.go:215 +0x45

goroutine 1 [chan receive, 10 minutes]:
testing.(*T).Run(0xc000612fc0, {0x80049c5, 0x13}, 0x81ad400)
	/snap/go/10984/src/testing/testing.go:2005 +0x9fe
testing.runTests.func1(0xc000612fc0)
	/snap/go/10984/src/testing/testing.go:2477 +0x86
testing.tRunner(0xc000612fc0, 0xc00008dae0)
	/snap/go/10984/src/testing/testing.go:1934 +0x21d
testing.runTests(0xc0006d9368, {0xafb4180, 0x3, 0x3}, {0x1?, 0x4?, 0xb160ae0?})
	/snap/go/10984/src/testing/testing.go:2475 +0x96d
testing.(*M).Run(0xc000812a00)
	/snap/go/10984/src/testing/testing.go:2337 +0xed5
main.main()
	_testmain.go:51 +0x165

goroutine 30 [select]:
github.com/orlangure/gnomock.(*g).wait(0xc00088e1e0, {0x88bdb40, 0xc00042a310}, 0xc00089cf00, 0xc00089e120)
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:299 +0x36c
github.com/orlangure/gnomock.newContainer(0xc00088e1e0, {0xc00088c060, 0x22}, 0xc0008843f0, 0xc00089e120)
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:141 +0x5e5
github.com/orlangure/gnomock.StartCustom({0xc00088c060, 0x22}, 0xc0008843f0, {0xc0008843c0, 0x6, 0x6})
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:102 +0x626
github.com/orlangure/gnomock.Start({0x88a8fa0, 0xc00089c200}, {0xc000825c48, 0x1, 0x25?})
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:200 +0x1f0
github.com/dhis2-sre/im-manager/pkg/inttest.SetupK8s(0xc000203dc0)
	/home/ubuntu/im-manager/pkg/inttest/k8s.go:24 +0x1af
github.com/dhis2-sre/im-manager/pkg/instance_test.TestInstanceHandler(0xc000203dc0)
	/home/ubuntu/im-manager/pkg/instance/instance_integration_test.go:39 +0x3f
testing.tRunner(0xc000203dc0, 0x81ad400)
	/snap/go/10984/src/testing/testing.go:1934 +0x21d
created by testing.(*T).Run in goroutine 1
	/snap/go/10984/src/testing/testing.go:1997 +0x9d3

goroutine 32 [select, 10 minutes]:
net/http.(*persistConn).readLoop(0xc00089e240)
	/snap/go/10984/src/net/http/transport.go:2398 +0x14ec
created by net/http.(*Transport).dialConn in goroutine 31
	/snap/go/10984/src/net/http/transport.go:1947 +0x2d52

goroutine 33 [select, 10 minutes]:
net/http.(*persistConn).writeLoop(0xc00089e240)
	/snap/go/10984/src/net/http/transport.go:2600 +0x1b6
created by net/http.(*Transport).dialConn in goroutine 31
	/snap/go/10984/src/net/http/transport.go:1948 +0x2ddd

goroutine 83 [IO wait]:
internal/poll.runtime_pollWait(0x71c2645c9000, 0x72)
	/snap/go/10984/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc0008b2220, 0x72, 0x0)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
	/snap/go/10984/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0008b2200, {0xc0008e1000, 0x1000, 0x1000})
	/snap/go/10984/src/internal/poll/fd_unix.go:165 +0x453
net.(*netFD).Read(0xc0008b2200, {0xc0008e1000, 0x1000, 0x1000})
	/snap/go/10984/src/net/fd_posix.go:68 +0x4b
net.(*conn).Read(0xc00019c258, {0xc0008e1000, 0x1000, 0x1000})
	/snap/go/10984/src/net/net.go:196 +0xad
net/http.(*persistConn).Read(0xc00089e240, {0xc0008e1000, 0x1000, 0x1000})
	/snap/go/10984/src/net/http/transport.go:2125 +0xf5
bufio.(*Reader).fill(0xc0005541e0)
	/snap/go/10984/src/bufio/bufio.go:113 +0x2a4
bufio.(*Reader).ReadSlice(0xc0005541e0, 0xa)
	/snap/go/10984/src/bufio/bufio.go:380 +0x85
net/http/internal.readChunkLine(0xc0005541e0)
	/snap/go/10984/src/net/http/internal/chunked.go:156 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc0009676e0)
	/snap/go/10984/src/net/http/internal/chunked.go:49 +0x5a
net/http/internal.(*chunkedReader).Read(0xc0009676e0, {0xc000982000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/internal/chunked.go:125 +0x2ad
net/http.(*body).readLocked(0xc00089d000, {0xc000982000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:845 +0xa4
net/http.(*body).Read(0xc00089d000, {0xc000982000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transfer.go:837 +0x17a
net/http.(*bodyEOFSignal).Read(0xc00089d040, {0xc000982000, 0x8009, 0x8009})
	/snap/go/10984/src/net/http/transport.go:3000 +0x135
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*wrappedBody).Read(0xc00089d0c0, {0xc000982000, 0x8009, 0x8009})
	/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.61.0/transport.go:229 +0x6d
github.com/docker/docker/pkg/stdcopy.StdCopy({0x8864400, 0xc23b9a0}, {0x8864400, 0xc23b9a0}, {0x71c21c17be90, 0xc0004c8200})
	/home/ubuntu/go/pkg/mod/github.com/docker/docker@v28.2.2+incompatible/pkg/stdcopy/stdcopy.go:108 +0x55e
github.com/orlangure/gnomock.(*g).setupLogForwarding.copyf.func1()
	/home/ubuntu/go/pkg/mod/github.com/orlangure/gnomock@v0.32.0/gnomock.go:156 +0x65
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x87
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 30
	/home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x11d
FAIL	github.com/dhis2-sre/im-manager/pkg/instance	600.124s
ok  	github.com/dhis2-sre/im-manager/pkg/integration	(cached)
?   	github.com/dhis2-sre/im-manager/pkg/inttest	[no test files]
ok  	github.com/dhis2-sre/im-manager/pkg/model	(cached)
ok  	github.com/dhis2-sre/im-manager/pkg/stack	(cached)
?   	github.com/dhis2-sre/im-manager/pkg/storage	[no test files]
?   	github.com/dhis2-sre/im-manager/pkg/token	[no test files]
ok  	github.com/dhis2-sre/im-manager/pkg/token/helper	(cached)
ok  	github.com/dhis2-sre/im-manager/pkg/user	(cached)
FAIL

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions