Resolve selinux installation issues

.github/workflows/ci.yaml

@@ -910,6 +910,7 @@ jobs:
         test:
           - TestResetAndReinstallAirgap
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeSelinux
           - TestSingleNodeAirgapUpgradeConfigValues
           - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestMultiNodeAirgapUpgrade

.github/workflows/release-prod.yaml

@@ -569,6 +569,7 @@ jobs:
         test:
           - TestResetAndReinstallAirgap
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeSelinux
           - TestSingleNodeAirgapUpgradeConfigValues
           - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestMultiNodeAirgapUpgrade

e2e/cluster/cmx/cluster.go

@@ -205,7 +205,15 @@ func copyScriptsToNode(node Node) error {
 }
 
 func getSSHEndpoint(nodeID string) (string, error) {
-	output, err := exec.Command("replicated", "vm", "ssh-endpoint", nodeID).CombinedOutput()
+	args := []string{
+		"vm",
+		"ssh-endpoint",
+		nodeID,
+	}
+	if user := os.Getenv("CMX_SSH_USERNAME"); user != "" {
+		args = append(args, "--username", user)
+	}
+	output, err := exec.Command("replicated", args...).CombinedOutput()
 	if err != nil {
 		return "", fmt.Errorf("%v: %s", err, string(output))
 	}
@@ -335,7 +343,7 @@ func runCommandOnNode(node Node, line []string, envs ...map[string]string) (stri
 			line = append([]string{fmt.Sprintf("%s=%s", k, v)}, line...)
 		}
 	}
-	line = append([]string{"sudo"}, line...)
+	line = append([]string{"sudo", "PATH=$PATH:/usr/local/bin"}, line...)
 
 	cmd := exec.Command("ssh", append(sshArgs(), node.sshEndpoint, strings.Join(line, " "))...)
 	var stdout, stderr bytes.Buffer
@@ -363,7 +371,7 @@ func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (st
 
 func (c *Cluster) SetupPlaywright(envs ...map[string]string) error {
 	c.t.Logf("%s: bypassing kurl-proxy", time.Now().Format(time.RFC3339))
-	_, stderr, err := c.RunCommandOnNode(0, []string{"bypass-kurl-proxy.sh"}, envs...)
+	_, stderr, err := c.RunCommandOnNode(0, []string{"/usr/local/bin/bypass-kurl-proxy.sh"}, envs...)
 	if err != nil {
 		return fmt.Errorf("bypass kurl-proxy: %v: %s", err, string(stderr))
 	}
@@ -401,7 +409,7 @@ func (c *Cluster) generateSupportBundle(envs ...map[string]string) {
 		go func(i int, wg *sync.WaitGroup) {
 			defer wg.Done()
 			c.t.Logf("%s: generating host support bundle from node %d", time.Now().Format(time.RFC3339), i)
-			if stdout, stderr, err := c.RunCommandOnNode(i, []string{"collect-support-bundle-host.sh"}, envs...); err != nil {
+			if stdout, stderr, err := c.RunCommandOnNode(i, []string{"/usr/local/bin/collect-support-bundle-host.sh"}, envs...); err != nil {
 				c.t.Logf("stdout: %s", stdout)
 				c.t.Logf("stderr: %s", stderr)
 				c.t.Logf("fail to generate support bundle from node %d: %v", i, err)
@@ -419,7 +427,7 @@ func (c *Cluster) generateSupportBundle(envs ...map[string]string) {
 	}
 
 	c.t.Logf("%s: generating cluster support bundle from node %d", time.Now().Format(time.RFC3339), c.supportBundleNodeIndex)
-	if stdout, stderr, err := c.RunCommandOnNode(c.supportBundleNodeIndex, []string{"collect-support-bundle-cluster.sh"}, envs...); err != nil {
+	if stdout, stderr, err := c.RunCommandOnNode(c.supportBundleNodeIndex, []string{"/usr/local/bin/collect-support-bundle-cluster.sh"}, envs...); err != nil {
 		c.t.Logf("stdout: %s", stdout)
 		c.t.Logf("stderr: %s", stderr)
 		c.t.Logf("fail to generate cluster support bundle from node %d: %v", c.supportBundleNodeIndex, err)

e2e/install_test.go

@@ -694,6 +694,95 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestSingleNodeAirgapUpgradeSelinux(t *testing.T) {
+	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
+	tc := cmx.NewCluster(&cmx.ClusterInput{
+		T:            t,
+		Nodes:        1,
+		Distribution: "almalinux",
+		Version:      "8",
+	})
+	defer tc.Cleanup()
+
+	t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339))
+	initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA"))
+	runInParallel(t,
+		func(t *testing.T) error {
+			return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID)
+		}, func(t *testing.T) error {
+			return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID)
+		},
+	)
+
+	t.Logf("%s: airgapping cluster", time.Now().Format(time.RFC3339))
+	if err := tc.Airgap(); err != nil {
+		t.Fatalf("failed to airgap cluster: %v", err)
+	}
+
+	t.Logf("%s: creating /.autorelabel file for SELinux relabeling", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"touch", "/.autorelabel"}); err != nil {
+		t.Fatalf("fail to create /.autorelabel file on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	t.Logf("%s: rebooting VM for SELinux relabeling", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"reboot"}); err != nil {
+		t.Fatalf("fail to reboot node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	t.Logf("%s: waiting for node to reboot", time.Now().Format(time.RFC3339))
+	tc.WaitForReboot()
+
+	t.Logf("%s: setting selinux to Enforcing mode", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"setenforce 1"}); err != nil {
+		t.Fatalf("fail to set selinux to Enforcing mode %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339))
+	line := []string{"/usr/local/bin/airgap-prepare.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to prepare airgap files on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	installSingleNodeWithOptions(t, tc, installOptions{
+		isAirgap:                true,
+		version:                 initialVersion,
+		localArtifactMirrorPort: "50001", // choose an alternate lam port
+	})
+
+	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
+	line = []string{"/usr/local/bin/check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()}
+	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	checkNodeJoinCommand(t, tc, 0)
+
+	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
+	line = []string{"/usr/local/bin/airgap-update.sh"}
+	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to run airgap update: %v", err)
+	}
+
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr)
+	}
+
+	checkPostUpgradeState(t, tc)
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 	t.Parallel()
 

e2e/shared.go

@@ -76,9 +76,9 @@ func installSingleNodeWithOptions(t *testing.T, tc cluster.Cluster, opts install
 	line := []string{}
 
 	if opts.isAirgap {
-		line = append(line, "single-node-airgap-install.sh")
+		line = append(line, "/usr/local/bin/single-node-airgap-install.sh")
 	} else {
-		line = append(line, "single-node-install.sh")
+		line = append(line, "/usr/local/bin/single-node-install.sh")
 		// the cli/ui option is currently only applicable for online installs
 		if opts.viaCLI {
 			line = append(line, "cli")
@@ -136,7 +136,7 @@ func checkInstallationState(t *testing.T, tc cluster.Cluster) {
 }
 
 func checkInstallationStateWithOptions(t *testing.T, tc cluster.Cluster, opts installationStateOptions) {
-	line := []string{"check-installation-state.sh"}
+	line := []string{"/usr/local/bin/check-installation-state.sh"}
 	if opts.version != "" {
 		line = append(line, opts.version)
 	} else {
@@ -233,23 +233,23 @@ func joinWorkerNodeWithOptions(t *testing.T, tc cluster.Cluster, node int, opts
 
 func waitForNodes(t *testing.T, tc cluster.Cluster, nodes int, envs map[string]string, args ...string) {
 	t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339))
-	stdout, stderr, err := tc.RunCommandOnNode(0, append([]string{"wait-for-ready-nodes.sh", fmt.Sprintf("%d", nodes)}, args...), envs)
+	stdout, stderr, err := tc.RunCommandOnNode(0, append([]string{"/usr/local/bin/wait-for-ready-nodes.sh", fmt.Sprintf("%d", nodes)}, args...), envs)
 	if err != nil {
 		t.Fatalf("fail to wait for ready nodes: %v: %s: %s", err, stdout, stderr)
 	}
 }
 
 func checkWorkerProfile(t *testing.T, tc cluster.Cluster, node int) {
 	t.Logf("checking worker profile on node %d", node)
-	line := []string{"check-worker-profile.sh"}
+	line := []string{"/usr/local/bin/check-worker-profile.sh"}
 	if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil {
 		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", node, err, stdout, stderr)
 	}
 }
 
 func checkNodeJoinCommand(t *testing.T, tc cluster.Cluster, node int) {
 	t.Logf("node join command generation on node %d", node)
-	line := []string{"check-node-join-command.sh"}
+	line := []string{"/usr/local/bin/check-node-join-command.sh"}
 	if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil {
 		t.Fatalf("fail to check if node join command is generated successfully on node %d: %v: %s: %s", node, err, stdout, stderr)
 	}
@@ -261,7 +261,7 @@ func downloadECRelease(t *testing.T, tc cluster.Cluster, node int) {
 
 func downloadECReleaseWithOptions(t *testing.T, tc cluster.Cluster, node int, opts downloadECReleaseOptions) {
 	t.Logf("%s: downloading embedded cluster release on node %d", time.Now().Format(time.RFC3339), node)
-	line := []string{"vandoor-prepare.sh"}
+	line := []string{"/usr/local/bin/vandoor-prepare.sh"}
 
 	if opts.version != "" {
 		line = append(line, opts.version)
@@ -292,7 +292,7 @@ func resetInstallationWithOptions(t *testing.T, tc cluster.Cluster, node int, op
 
 func resetInstallationWithError(t *testing.T, tc cluster.Cluster, node int, opts resetInstallationOptions) (string, string, error) {
 	t.Logf("%s: resetting the installation on node %d", time.Now().Format(time.RFC3339), node)
-	line := []string{"reset-installation.sh"}
+	line := []string{"/usr/local/bin/reset-installation.sh"}
 	if opts.force {
 		line = append(line, "--force")
 	}
@@ -304,7 +304,7 @@ func checkPostUpgradeState(t *testing.T, tc cluster.Cluster) {
 }
 
 func checkPostUpgradeStateWithOptions(t *testing.T, tc cluster.Cluster, opts postUpgradeStateOptions) {
-	line := []string{"check-postupgrade-state.sh"}
+	line := []string{"/usr/local/bin/check-postupgrade-state.sh"}
 
 	if opts.k8sVersion != "" {
 		line = append(line, opts.k8sVersion)

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/replicatedhq/embedded-cluster/kinds v0.0.0
 	github.com/replicatedhq/embedded-cluster/utils v0.0.0
 	github.com/replicatedhq/kotskinds v0.0.0-20250411153224-089dbeb7ba2a
-	github.com/replicatedhq/troubleshoot v0.121.0
+	github.com/replicatedhq/troubleshoot v0.121.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
@@ -285,7 +285,7 @@ require (
 	go.opentelemetry.io/otel v1.37.0 // indirect
 	go.opentelemetry.io/otel/metric v1.37.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
 	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect

go.sum

@@ -1453,8 +1453,8 @@ github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRl
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/replicatedhq/kotskinds v0.0.0-20250411153224-089dbeb7ba2a h1:aNZ7qcuEmPGIUIIfxF7c0sdKR2+zL2vc5r2V8j8a49I=
 github.com/replicatedhq/kotskinds v0.0.0-20250411153224-089dbeb7ba2a/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
-github.com/replicatedhq/troubleshoot v0.121.0 h1:Sa3gRAcTfwq+RvwzSX4YNunLeGPxGRFkXCC1S3ZnRqA=
-github.com/replicatedhq/troubleshoot v0.121.0/go.mod h1:9wTqSv515XRV6m5xmS3xP9oNXlRW23QL+z3ffcZGt90=
+github.com/replicatedhq/troubleshoot v0.121.1 h1:3KBKXeznMymrtrvViOWn71Zy3Tt7tLmj+kUJNr0qz2U=
+github.com/replicatedhq/troubleshoot v0.121.1/go.mod h1:bqtzIYtqavduEnwmJsVV74ds5mOrjyInKJDAQ9vcU6k=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -1669,8 +1669,8 @@ go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdD
 go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
 go.opentelemetry.io/otel/sdk/log v0.8.0 h1:zg7GUYXqxk1jnGF/dTdLPrK06xJdrXgqgFLnI4Crxvs=
 go.opentelemetry.io/otel/sdk/log v0.8.0/go.mod h1:50iXr0UVwQrYS45KbruFrEt4LvAdCaWWgIrsN3ZQggo=
-go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
-go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
+go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
 go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
 go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=

pkg-new/hostutils/initialize.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
@@ -39,6 +40,31 @@ func (h *HostUtils) ConfigureHost(ctx context.Context, rc runtimeconfig.RuntimeC
 		}
 	}
 
+	h.logger.Debugln("checking for semanage binary in $PATH")
+	if _, err := exec.LookPath("semanage"); err != nil {
+		h.logger.Debugln("semanage not found in $PATH")
+	} else {
+		// Set selinux fcontext for embedded-cluster binary directory to bin_t
+		out, err := exec.Command("semanage", "fcontext", "-a", "-s", "system_u", "-t", "bin_t", rc.EmbeddedClusterBinsSubDir()+"(/.*)?").CombinedOutput()
+		if err != nil {
+			h.logger.Debugf("unable to set contexts on binary directory: %v", err)
+			h.logger.Debugln(out)
+		}
+
+	}
+
+	h.logger.Debugln("checking for restorecon binary in $PATH")
+	if _, err := exec.LookPath("restorecon"); err != nil {
+		h.logger.Debugln("restorecon not found in $PATH")
+	} else {
+		// Relabel whole embedded-cluster data directory since it's created with unconfined_u
+		out, err := exec.Command("restorecon", "-RvF", rc.EmbeddedClusterHomeDirectory()).CombinedOutput()
+		if err != nil {
+			h.logger.Debugf("unable to run restorecon: %v", err)
+			h.logger.Debugln(out)
+		}
+	}
+
 	h.logger.Debugf("configuring sysctl")
 	if err := h.ConfigureSysctl(); err != nil {
 		h.logger.Debugf("unable to configure sysctl: %v", err)