ubuntu · denisonbarbosa · Jun 5, 2025
@@ -114,7 +114,7 @@ jobs:
       - name: Build base VM
         if: steps.check-vm-template.outputs.image-version != ''
         run: |
-          go run ./e2e/cmd/build_base_image/01_prepare_base_vm --vm-image ${{ steps.check-vm-template.outputs.image-version }} --codename ${{ matrix.codename }}
+          go run ./e2e/cmd/build_base_image/01_prepare_base_vm --vm-image ${{ steps.check-vm-template.outputs.image-version }} --codename ${{ matrix.codename }} --keep
       - name: Create template version
         if: steps.check-vm-template.outputs.image-version != ''
         run: |

@@ -118,7 +118,7 @@ jobs:
           cert: ${{ secrets.VPN_CERT }}
           key: ${{ secrets.VPN_KEY }}
       - name: Provision client VM
-        run: go run ./e2e/cmd/run_tests/01_provision_client
+        run: go run ./e2e/cmd/run_tests/01_provision_client --debug -k
       - name: Provision AD server
         run: go run ./e2e/cmd/run_tests/02_provision_ad
       - name: Recompile PAM module with coverage support

@@ -23,6 +23,7 @@ var vmImage, codename, sshKey string
 var keep bool
 
 func main() {
+	log.SetLevel(log.DebugLevel)
 	os.Exit(run())
 }
 
@@ -138,20 +139,30 @@ func action(ctx context.Context, cmd *command.Command) error {
 	}
 	defer client.Close()
 
+	// Edit SSH config for keepalive and reload it.
+	if _, err := client.Run(ctx, `echo 'ClientAliveInterval 30' | sudo tee -a /etc/ssh/sshd_config`); err != nil {
+		return fmt.Errorf("failed to update sshd_config")
+	}
+	if _, err := client.Run(ctx, `echo 'ClientAliveCountMax 30' | sudo tee -a /etc/ssh/sshd_config`); err != nil {
+		return fmt.Errorf("failed to update sshd_config")
+	}
+	if _, err := client.Run(ctx, `sudo sshd -t && sudo systemctl reload ssh`); err != nil {
+		return fmt.Errorf("failed to reload sshd")
+	}
+
 	// Install required dependencies
-	log.Infof("Installing eatmydata to speed up package installation...")
-	if _, err := client.Run(ctx, `echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io && \
-sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y eatmydata`); err != nil {
-		return fmt.Errorf("failed to set up eatmydata: %w", err)
+	if _, err := client.Run(ctx, `sudo apt-get update`); err != nil {
+		return fmt.Errorf("failed to update packages: %w", err)
 	}
 
 	log.Infof("Installing required packages on VM...")
-	if _, err := client.Run(ctx, `echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io && \
-sudo eatmydata apt-get update && sudo DEBIAN_FRONTEND=noninteractive eatmydata apt-get upgrade -y && \
-sudo DEBIAN_FRONTEND=noninteractive eatmydata apt-get install -y ubuntu-desktop realmd nfs-common cifs-utils && \
-sudo sync && \
-sudo rm -f /etc/dpkg/dpkg.cfg.d/force-unsafe-io
-`); err != nil {
+	_, err = client.Run(ctx, `sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y`)
+	if err != nil {
+		return fmt.Errorf("failed to update packages: %w", err)
+	}
+
+	_, err = client.Run(ctx, `sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-desktop-minimal realmd nfs-common cifs-utils`)
+	if err != nil {
 		return fmt.Errorf("failed to install required packages: %w", err)
 	}
 

@@ -176,22 +176,27 @@ func action(ctx context.Context, cmd *command.Command) error {
 		}
 	}
 
-	log.Infof("Upgrading packages...")
-	_, err = client.Run(ctx, "apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y upgrade")
+	log.Infof("Updating and upgrading packages...")
+	_, err = client.Run(ctx, "apt-get update")
 	if err != nil {
 		return fmt.Errorf("failed to update package list: %w", err)
 	}
 
+	_, err = client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get --no-show-upgraded -q -y -o=Dpkg::Use-Pty=0 upgrade")
+	if err != nil {
+		return fmt.Errorf("failed to upgrade packages: %w", err)
+	}
+
 	log.Infof("Installing adsys package...")
-	_, err = client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install -y /debs/*.deb")
+	_, err = client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install /debs/*.deb -y")
 	if err != nil {
 		return fmt.Errorf("failed to install adsys package: %w", err)
 	}
 
 	// TODO: remove this once the packages installed below are MIRed and installed by default with adsys
 	// Allow errors here on account on packages not being available on the tested Ubuntu version
 	log.Infof("Installing universe packages required for some policy managers...")
-	if _, err := client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-proxy-manager python3-cepces"); err != nil {
+	if _, err := client.Run(ctx, "DEBIAN_FRONTEND=noninteractive apt-get install -q -y -o=Dpkg::Use-Pty=0 ubuntu-proxy-manager python3-cepces"); err != nil {
 		log.Warningf("Some packages failed to install: %v", err)
 	}
 

@@ -3,13 +3,14 @@
 package remote
 
 import (
-	"bufio"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"net"
 	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
@@ -62,29 +63,51 @@ func NewClient(host string, username string, secret string) (Client, error) {
 		Timeout:         10 * time.Second,
 	}
 
-	var client *ssh.Client
-
 	interval := 3 * time.Second
 	retries := 10
 
+	var connErr error
+	var sshClient Client
 	for i := 1; i <= retries; i++ {
+		dialer := net.Dialer{
+			KeepAlive: 30 * time.Second,
+			KeepAliveConfig: net.KeepAliveConfig{
+				Enable:   true,
+				Idle:     30 * time.Second,
+				Interval: 30 * time.Second,
+				Count:    60,
+			},
+		}
+
+		conn, err := dialer.Dial("tcp", host+":22")
+		if err != nil {
+			connErr = err
+			log.Warningf("Failed to connect to %q: %v (attempt %d/%d)", host, err, i, retries)
+			time.Sleep(interval)
+			continue
+		}
+
 		log.Debugf("Establishing SSH connection to %q (attempt %d/%d)", host, i, retries)
-		client, err = ssh.Dial("tcp", host+":22", config)
-		if err == nil {
-			break
+		sshCon, newChan, reqChan, err := ssh.NewClientConn(conn, host+":22", config)
+		if err != nil {
+			connErr = err
+			log.Warningf("Failed to connect to %q: %v (attempt %d/%d)", host, err, i, retries)
+			time.Sleep(interval)
+			continue
+		}
+
+		sshClient = Client{
+			client: ssh.NewClient(sshCon, newChan, reqChan),
+			config: config,
+			host:   host,
 		}
-		log.Warningf("Failed to connect to %q: %v (attempt %d/%d)", host, err, i, retries)
-		time.Sleep(interval)
+		break
 	}
-	if err != nil {
+	if connErr != nil {
 		return Client{}, fmt.Errorf("failed to connect to %q: %w", host, err)
 	}
 
-	return Client{
-		client: client,
-		config: config,
-		host:   host,
-	}, nil
+	return sshClient, nil
 }
 
 // Close closes the SSH connection.
@@ -114,69 +137,91 @@ func (c Client) Run(ctx context.Context, cmd string) ([]byte, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create stderr pipe: %w", err)
 	}
+	session.Stdin = nil
 
 	log.Infof("Running command %q on remote host %q", cmd, c.client.RemoteAddr().String())
-
-	// Start the remote command
-	startTime := time.Now()
-	if err := session.Start(cmd); err != nil {
-		return nil, fmt.Errorf("failed to start command: %w", err)
-	}
-
 	// Create scanners to read stdout and stderr line by line
-	stdoutScanner := bufio.NewScanner(stdout)
-	stderrScanner := bufio.NewScanner(stderr)
-	var combinedOutput []string
-	var mu sync.Mutex
+	var stdoutBuff, stderrBuff bytes.Buffer
 	var wg sync.WaitGroup
 
 	// Use goroutines to read and print both stdout and stderr concurrently
 	wg.Add(2)
 	go func() {
-		for stdoutScanner.Scan() {
-			line := stdoutScanner.Text()
-			log.Debug("\t", line)
-			mu.Lock()
-			combinedOutput = append(combinedOutput, line)
-			mu.Unlock()
+		defer wg.Done()
+		log.Debug("Starting to read stdout")
+		written, err := io.Copy(&stdoutBuff, stdout)
+		if err != nil {
+			log.Warningf("Error when copying stdout: %v", err)
 		}
-		wg.Done()
+		log.Debugf("Written %d bytes to stdout", written)
 	}()
 	go func() {
-		for stderrScanner.Scan() {
-			line := stderrScanner.Text()
-			log.Warning("\t", line)
-			mu.Lock()
-			combinedOutput = append(combinedOutput, line)
-			mu.Unlock()
+		defer wg.Done()
+		log.Debug("Starting to read stderr")
+		written, err := io.Copy(&stderrBuff, stderr)
+		if err != nil {
+			log.Warningf("Error when copying stderr: %v", err)
 		}
-		wg.Done()
+		log.Debugf("Written %d bytes to stderr", written)
 	}()
 
+	// Start keepalive goroutine
+	keepaliveDone := make(chan struct{})
+	go func() {
+		ticker := time.NewTicker(30 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				_, _, err := c.client.SendRequest("[email protected]", true, nil)
+				if err != nil {
+					log.Warnf("Keepalive failed: %v", err)
+					return
+				}
+			case <-keepaliveDone:
+				return
+			}
+		}
+	}()
+
+	// Start the remote command
+	startTime := time.Now()
+	if err := session.Start(cmd); err != nil {
+		return nil, fmt.Errorf("failed to start command: %w", err)
+	}
+
 	waitDone := make(chan error, 1)
 	go func() {
 		waitDone <- session.Wait()
 	}()
 
 	select {
 	case <-ctx.Done():
+		close(keepaliveDone)
+		if err := session.Signal(ssh.SIGKILL); err != nil {
+			log.Warningf("Failed to stop the running session: %v", err)
+		}
+
 		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 			return nil, fmt.Errorf("command timed out after %s", commandTimeout)
 		}
 		return nil, fmt.Errorf("command cancelled: %w", ctx.Err())
 	case err := <-waitDone:
 		elapsedTime := time.Since(startTime)
 		wg.Wait() // wait for scanners to finish
-		mu.Lock()
-		defer mu.Unlock()
+		close(keepaliveDone)
 
-		out := []byte(strings.Join(combinedOutput, "\n"))
+		out := []byte("STDOUT: " + stdoutBuff.String() + "\nSTDERR: " + stderrBuff.String())
+		if err != nil && errors.Is(err, &ssh.ExitMissingError{}) {
+			log.Warningf("Command %q did not return any exit status: %v", cmd, err)
+			log.Warningf("Output: %s", out)
+			return nil, err
+		}
 		if err != nil {
 			log.Warningf("Command %q failed in %s", cmd, elapsedTime)
 			return out, fmt.Errorf("command failed: %w", err)
 		}
 		log.Infof("Command %q finished in %s", cmd, elapsedTime)
-
 		return out, nil
 	}
 }