diff --git a/.github/workflows/e2e-build-images.yaml b/.github/workflows/e2e-build-images.yaml index 371659eb7..2310e9090 100644 --- a/.github/workflows/e2e-build-images.yaml +++ b/.github/workflows/e2e-build-images.yaml @@ -114,7 +114,7 @@ jobs: - name: Build base VM if: steps.check-vm-template.outputs.image-version != '' run: | - go run ./e2e/cmd/build_base_image/01_prepare_base_vm --vm-image ${{ steps.check-vm-template.outputs.image-version }} --codename ${{ matrix.codename }} + go run ./e2e/cmd/build_base_image/01_prepare_base_vm --vm-image ${{ steps.check-vm-template.outputs.image-version }} --codename ${{ matrix.codename }} --keep - name: Create template version if: steps.check-vm-template.outputs.image-version != '' run: | diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 6790df271..ec2d1b455 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -118,7 +118,7 @@ jobs: cert: ${{ secrets.VPN_CERT }} key: ${{ secrets.VPN_KEY }} - name: Provision client VM - run: go run ./e2e/cmd/run_tests/01_provision_client + run: go run ./e2e/cmd/run_tests/01_provision_client --debug -k - name: Provision AD server run: go run ./e2e/cmd/run_tests/02_provision_ad - name: Recompile PAM module with coverage support diff --git a/e2e/cmd/build_base_image/01_prepare_base_vm/main.go b/e2e/cmd/build_base_image/01_prepare_base_vm/main.go index b33befb2c..57ef65901 100644 --- a/e2e/cmd/build_base_image/01_prepare_base_vm/main.go +++ b/e2e/cmd/build_base_image/01_prepare_base_vm/main.go @@ -23,6 +23,7 @@ var vmImage, codename, sshKey string var keep bool func main() { + log.SetLevel(log.DebugLevel) os.Exit(run()) } @@ -138,20 +139,30 @@ func action(ctx context.Context, cmd *command.Command) error { } defer client.Close() + // Edit SSH config for keepalive and reload it. + if _, err := client.Run(ctx, `echo 'ClientAliveInterval 30' | sudo tee -a /etc/ssh/sshd_config`); err != nil { + return fmt.Errorf("failed to update sshd_config") + } + if _, err := client.Run(ctx, `echo 'ClientAliveCountMax 30' | sudo tee -a /etc/ssh/sshd_config`); err != nil { + return fmt.Errorf("failed to update sshd_config") + } + if _, err := client.Run(ctx, `sudo sshd -t && sudo systemctl reload ssh`); err != nil { + return fmt.Errorf("failed to reload sshd") + } + // Install required dependencies - log.Infof("Installing eatmydata to speed up package installation...") - if _, err := client.Run(ctx, `echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io && \ -sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y eatmydata`); err != nil { - return fmt.Errorf("failed to set up eatmydata: %w", err) + if _, err := client.Run(ctx, `sudo apt-get update`); err != nil { + return fmt.Errorf("failed to update packages: %w", err) } log.Infof("Installing required packages on VM...") - if _, err := client.Run(ctx, `echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io && \ -sudo eatmydata apt-get update && sudo DEBIAN_FRONTEND=noninteractive eatmydata apt-get upgrade -y && \ -sudo DEBIAN_FRONTEND=noninteractive eatmydata apt-get install -y ubuntu-desktop realmd nfs-common cifs-utils && \ -sudo sync && \ -sudo rm -f /etc/dpkg/dpkg.cfg.d/force-unsafe-io -`); err != nil { + _, err = client.Run(ctx, `sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y`) + if err != nil { + return fmt.Errorf("failed to update packages: %w", err) + } + + _, err = client.Run(ctx, `sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-desktop-minimal realmd nfs-common cifs-utils`) + if err != nil { return fmt.Errorf("failed to install required packages: %w", err) } diff --git a/e2e/cmd/run_tests/01_provision_client/main.go b/e2e/cmd/run_tests/01_provision_client/main.go index a28de2a8d..fdfdd2384 100644 --- a/e2e/cmd/run_tests/01_provision_client/main.go +++ b/e2e/cmd/run_tests/01_provision_client/main.go @@ -176,14 +176,19 @@ func action(ctx context.Context, cmd *command.Command) error { } } - log.Infof("Upgrading packages...") - _, err = client.Run(ctx, "apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y upgrade") + log.Infof("Updating and upgrading packages...") + _, err = client.Run(ctx, "apt-get update") if err != nil { return fmt.Errorf("failed to update package list: %w", err) } + _, err = client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get --no-show-upgraded -q -y -o=Dpkg::Use-Pty=0 upgrade") + if err != nil { + return fmt.Errorf("failed to upgrade packages: %w", err) + } + log.Infof("Installing adsys package...") - _, err = client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install -y /debs/*.deb") + _, err = client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install /debs/*.deb -y") if err != nil { return fmt.Errorf("failed to install adsys package: %w", err) } @@ -191,7 +196,7 @@ func action(ctx context.Context, cmd *command.Command) error { // TODO: remove this once the packages installed below are MIRed and installed by default with adsys // Allow errors here on account on packages not being available on the tested Ubuntu version log.Infof("Installing universe packages required for some policy managers...") - if _, err := client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-proxy-manager python3-cepces"); err != nil { + if _, err := client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install -q -y -o=Dpkg::Use-Pty=0 ubuntu-proxy-manager python3-cepces"); err != nil { log.Warningf("Some packages failed to install: %v", err) } diff --git a/e2e/internal/remote/remote.go b/e2e/internal/remote/remote.go index 1efdc4182..e8ade71b2 100644 --- a/e2e/internal/remote/remote.go +++ b/e2e/internal/remote/remote.go @@ -3,13 +3,14 @@ package remote import ( - "bufio" + "bytes" "context" "errors" "fmt" + "io" + "net" "os" "path/filepath" - "strings" "sync" "time" @@ -62,29 +63,51 @@ func NewClient(host string, username string, secret string) (Client, error) { Timeout: 10 * time.Second, } - var client *ssh.Client - interval := 3 * time.Second retries := 10 + var connErr error + var sshClient Client for i := 1; i <= retries; i++ { + dialer := net.Dialer{ + KeepAlive: 30 * time.Second, + KeepAliveConfig: net.KeepAliveConfig{ + Enable: true, + Idle: 30 * time.Second, + Interval: 30 * time.Second, + Count: 60, + }, + } + + conn, err := dialer.Dial("tcp", host+":22") + if err != nil { + connErr = err + log.Warningf("Failed to connect to %q: %v (attempt %d/%d)", host, err, i, retries) + time.Sleep(interval) + continue + } + log.Debugf("Establishing SSH connection to %q (attempt %d/%d)", host, i, retries) - client, err = ssh.Dial("tcp", host+":22", config) - if err == nil { - break + sshCon, newChan, reqChan, err := ssh.NewClientConn(conn, host+":22", config) + if err != nil { + connErr = err + log.Warningf("Failed to connect to %q: %v (attempt %d/%d)", host, err, i, retries) + time.Sleep(interval) + continue + } + + sshClient = Client{ + client: ssh.NewClient(sshCon, newChan, reqChan), + config: config, + host: host, } - log.Warningf("Failed to connect to %q: %v (attempt %d/%d)", host, err, i, retries) - time.Sleep(interval) + break } - if err != nil { + if connErr != nil { return Client{}, fmt.Errorf("failed to connect to %q: %w", host, err) } - return Client{ - client: client, - config: config, - host: host, - }, nil + return sshClient, nil } // Close closes the SSH connection. @@ -114,45 +137,59 @@ func (c Client) Run(ctx context.Context, cmd string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("failed to create stderr pipe: %w", err) } + session.Stdin = nil log.Infof("Running command %q on remote host %q", cmd, c.client.RemoteAddr().String()) - - // Start the remote command - startTime := time.Now() - if err := session.Start(cmd); err != nil { - return nil, fmt.Errorf("failed to start command: %w", err) - } - // Create scanners to read stdout and stderr line by line - stdoutScanner := bufio.NewScanner(stdout) - stderrScanner := bufio.NewScanner(stderr) - var combinedOutput []string - var mu sync.Mutex + var stdoutBuff, stderrBuff bytes.Buffer var wg sync.WaitGroup // Use goroutines to read and print both stdout and stderr concurrently wg.Add(2) go func() { - for stdoutScanner.Scan() { - line := stdoutScanner.Text() - log.Debug("\t", line) - mu.Lock() - combinedOutput = append(combinedOutput, line) - mu.Unlock() + defer wg.Done() + log.Debug("Starting to read stdout") + written, err := io.Copy(&stdoutBuff, stdout) + if err != nil { + log.Warningf("Error when copying stdout: %v", err) } - wg.Done() + log.Debugf("Written %d bytes to stdout", written) }() go func() { - for stderrScanner.Scan() { - line := stderrScanner.Text() - log.Warning("\t", line) - mu.Lock() - combinedOutput = append(combinedOutput, line) - mu.Unlock() + defer wg.Done() + log.Debug("Starting to read stderr") + written, err := io.Copy(&stderrBuff, stderr) + if err != nil { + log.Warningf("Error when copying stderr: %v", err) } - wg.Done() + log.Debugf("Written %d bytes to stderr", written) }() + // Start keepalive goroutine + keepaliveDone := make(chan struct{}) + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + _, _, err := c.client.SendRequest("keepalive@openssh.com", true, nil) + if err != nil { + log.Warnf("Keepalive failed: %v", err) + return + } + case <-keepaliveDone: + return + } + } + }() + + // Start the remote command + startTime := time.Now() + if err := session.Start(cmd); err != nil { + return nil, fmt.Errorf("failed to start command: %w", err) + } + waitDone := make(chan error, 1) go func() { waitDone <- session.Wait() @@ -160,6 +197,11 @@ func (c Client) Run(ctx context.Context, cmd string) ([]byte, error) { select { case <-ctx.Done(): + close(keepaliveDone) + if err := session.Signal(ssh.SIGKILL); err != nil { + log.Warningf("Failed to stop the running session: %v", err) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { return nil, fmt.Errorf("command timed out after %s", commandTimeout) } @@ -167,16 +209,19 @@ func (c Client) Run(ctx context.Context, cmd string) ([]byte, error) { case err := <-waitDone: elapsedTime := time.Since(startTime) wg.Wait() // wait for scanners to finish - mu.Lock() - defer mu.Unlock() + close(keepaliveDone) - out := []byte(strings.Join(combinedOutput, "\n")) + out := []byte("STDOUT: " + stdoutBuff.String() + "\nSTDERR: " + stderrBuff.String()) + if err != nil && errors.Is(err, &ssh.ExitMissingError{}) { + log.Warningf("Command %q did not return any exit status: %v", cmd, err) + log.Warningf("Output: %s", out) + return nil, err + } if err != nil { log.Warningf("Command %q failed in %s", cmd, elapsedTime) return out, fmt.Errorf("command failed: %w", err) } log.Infof("Command %q finished in %s", cmd, elapsedTime) - return out, nil } }