Skip to content

Commit

Permalink
Merge branch 'dev' into xinhl/kversion
Browse files Browse the repository at this point in the history
  • Loading branch information
@fseldow
fseldow authored Dec 3, 2024
2 parents 2cf3bb5 + a609fdb commit c3617cc
Show file tree
Hide file tree
Showing 10 changed files with 463 additions and 739 deletions.
39 changes: 22 additions & 17 deletions e2e/config/vhd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,52 +34,61 @@ type Gallery struct {
Name string
}

type OS string

var (
OSWindows OS = "windows"
OSUbuntu OS = "ubuntu"
OSMariner OS = "mariner"
OSAzureLinux OS = "azurelinux"
)

var (
VHDUbuntu1804Gen2Containerd = &Image{
Name: "1804gen2containerd",
OS: "ubuntu",
OS: OSUbuntu,
Arch: "amd64",
Distro: datamodel.AKSUbuntuContainerd1804Gen2,
Gallery: linuxGallery,
}
VHDUbuntu2204Gen2Arm64Containerd = &Image{
Name: "2204gen2arm64containerd",
OS: "ubuntu",
OS: OSUbuntu,
Arch: "arm64",
Distro: datamodel.AKSUbuntuArm64Containerd2204Gen2,
Gallery: linuxGallery,
}
VHDUbuntu2204Gen2Containerd = &Image{
Name: "2204gen2containerd",
OS: "ubuntu",
OS: OSUbuntu,
Arch: "amd64",
Distro: datamodel.AKSUbuntuContainerd2404Gen2,
Gallery: linuxGallery,
}
VHDAzureLinuxV2Gen2Arm64 = &Image{
Name: "AzureLinuxV2gen2arm64",
OS: "azurelinux",
OS: OSAzureLinux,
Arch: "arm64",
Distro: datamodel.AKSAzureLinuxV2Arm64Gen2,
Gallery: linuxGallery,
}
VHDAzureLinuxV2Gen2 = &Image{
Name: "AzureLinuxV2gen2",
OS: "azurelinux",
OS: OSAzureLinux,
Arch: "amd64",
Distro: datamodel.AKSAzureLinuxV2Gen2,
Gallery: linuxGallery,
}
VHDCBLMarinerV2Gen2Arm64 = &Image{
Name: "CBLMarinerV2gen2arm64",
OS: "mariner",
OS: OSMariner,
Arch: "arm64",
Distro: datamodel.AKSCBLMarinerV2Arm64Gen2,
Gallery: linuxGallery,
}
VHDCBLMarinerV2Gen2 = &Image{
Name: "CBLMarinerV2gen2",
OS: "mariner",
OS: OSMariner,
Arch: "amd64",
Distro: datamodel.AKSCBLMarinerV2Gen2,
Gallery: linuxGallery,
Expand All @@ -89,7 +98,7 @@ var (
VHDUbuntu2204Gen2ContainerdPrivateKubePkg = &Image{
// 2204Gen2 is a special image definition holding historical VHDs used by agentbaker e2e's.
Name: "2204Gen2",
OS: "ubuntu",
OS: OSUbuntu,
Arch: "amd64",
Version: "1.1704411049.2812",
Distro: datamodel.AKSUbuntuContainerd2404Gen2,
Expand All @@ -99,7 +108,7 @@ var (
// without kubelet, kubectl, credential-provider and wasm
VHDUbuntu2204Gen2ContainerdAirgappedK8sNotCached = &Image{
Name: "2204Gen2",
OS: "ubuntu",
OS: OSUbuntu,
Arch: "amd64",
Version: "1.1725612526.29638",
Distro: datamodel.AKSUbuntuContainerd2404Gen2,
Expand All @@ -126,7 +135,7 @@ var (

VHDWindows2022ContainerdGen2 = &Image{
Name: "windows-2022-containerd-gen2",
OS: "windows",
OS: OSWindows,
Arch: "amd64",
Distro: datamodel.AKSWindows2022ContainerdGen2,
Latest: true,
Expand All @@ -135,7 +144,7 @@ var (

VHDWindows23H2 = &Image{
Name: "windows-23H2",
OS: "windows",
OS: OSWindows,
Arch: "amd64",
Distro: datamodel.AKSWindows23H2,
Latest: true,
Expand All @@ -144,7 +153,7 @@ var (

VHDWindows23H2Gen2 = &Image{
Name: "windows-23H2-gen2",
OS: "windows",
OS: OSWindows,
Arch: "amd64",
Distro: datamodel.AKSWindows23H2Gen2,
Latest: true,
Expand All @@ -158,7 +167,7 @@ type Image struct {
Arch string
Distro datamodel.Distro
Name string
OS string
OS OS
Version string
Gallery *Gallery
Latest bool // a hack to get the latest version of the image for windows, currently windows images are not tagged
Expand Down Expand Up @@ -191,10 +200,6 @@ func (i *Image) VHDResourceID(ctx context.Context, t *testing.T) (VHDResourceID,
return i.vhd, i.vhdErr
}

func (i *Image) Windows() bool {
return i.OS == "windows"
}

// VHDResourceID represents a resource ID pointing to a VHD in Azure. This could be theoretically
// be the resource ID of a managed image or SIG image version, though for now this will always be a SIG image version.
type VHDResourceID string
Expand Down
6 changes: 2 additions & 4 deletions e2e/exec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,9 @@ func getBootstrapToken(ctx context.Context, t *testing.T, kube *Kubeclient) stri
return fmt.Sprintf("%s.%s", id, token)
}

func execOnVM(ctx context.Context, kube *Kubeclient, vmPrivateIP, jumpboxPodName, sshPrivateKey, command string, isShellBuiltIn bool) (*podExecResult, error) {
func execOnVM(ctx context.Context, kube *Kubeclient, vmPrivateIP, jumpboxPodName, sshPrivateKey, command string) (*podExecResult, error) {
sshCommand := fmt.Sprintf(sshCommandTemplate, sshPrivateKey, strings.ReplaceAll(vmPrivateIP, ".", ""), vmPrivateIP)
if !isShellBuiltIn {
sshCommand = sshCommand + " sudo"
}
sshCommand = sshCommand + " sudo"
commandToExecute := fmt.Sprintf("%s %s", sshCommand, command)

execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, commandToExecute)
Expand Down
97 changes: 66 additions & 31 deletions e2e/scenario_helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,58 @@ func RunScenario(t *testing.T, s *Scenario) {
t.Parallel()
ctx := newTestCtx(t)
maybeSkipScenario(ctx, t, s)
s.PrepareRuntime(ctx)
createAndValidateVM(ctx, s)
cluster, err := s.Config.Cluster(ctx, s.T)
require.NoError(s.T, err)
s.Runtime = &ScenarioRuntime{
Cluster: cluster,
}
// use shorter timeout for faster feedback on test failures
ctx, cancel := context.WithTimeout(ctx, config.Config.TestTimeoutVMSS)
defer cancel()
prepareAKSNode(ctx, s)
validateVM(ctx, s)
}

func prepareAKSNode(ctx context.Context, s *Scenario) {
s.Runtime.VMSSName = generateVMSSName(s)
if (s.BootstrapConfigMutator == nil) == (s.AKSNodeConfigMutator == nil) {
s.T.Fatalf("exactly one of BootstrapConfigMutator or AKSNodeConfigMutator must be set")
}

nbc := getBaseNBC(s.Runtime.Cluster, s.VHD)
if s.VHD.OS == config.OSWindows {
nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL = windowsCSE(ctx, s.T)
}

if s.BootstrapConfigMutator != nil {
s.BootstrapConfigMutator(nbc)
s.Runtime.NBC = nbc
}
if s.AKSNodeConfigMutator != nil {
nodeconfig := nbcToAKSNodeConfigV1(nbc)
s.AKSNodeConfigMutator(nodeconfig)
s.Runtime.AKSNodeConfig = nodeconfig
}
var err error
s.Runtime.SSHKeyPrivate, s.Runtime.SSHKeyPublic, err = getNewRSAKeyPair()
require.NoError(s.T, err)
createVMSS(ctx, s)
err = getCustomScriptExtensionStatus(ctx, s)
require.NoError(s.T, err)
s.T.Logf("vmss %s creation succeeded", s.Runtime.VMSSName)

s.Runtime.KubeNodeName = waitUntilNodeReady(ctx, s.T, s.Runtime.Cluster.Kube, s.Runtime.VMSSName)
s.T.Logf("node %s is ready", s.Runtime.VMSSName)

s.Runtime.VMPrivateIP, err = getVMPrivateIPAddress(ctx, s)
require.NoError(s.T, err, "failed to get VM private IP address")
s.Runtime.HostPodName, err = getHostNetworkDebugPodName(ctx, s.Runtime.Cluster.Kube, s.T)
require.NoError(s.T, err, "failed to get host network debug pod name")
}

func maybeSkipScenario(ctx context.Context, t *testing.T, s *Scenario) {
s.Tags.Name = t.Name()
s.Tags.OS = s.VHD.OS
s.Tags.OS = string(s.VHD.OS)
s.Tags.Arch = s.VHD.Arch
s.Tags.ImageName = s.VHD.Name
if config.Config.TagsToRun != "" {
Expand Down Expand Up @@ -121,42 +166,33 @@ func maybeSkipScenario(ctx context.Context, t *testing.T, s *Scenario) {
t.Fatalf("could not find image for %q: %s", t.Name(), err)
}
}
t.Logf("running scenario %q with vhd: %q, tags %+v", t.Name(), vhd, s.Tags)
t.Logf("running scenario vhd: %q, tags %+v", vhd, s.Tags)
}

func createAndValidateVM(ctx context.Context, s *Scenario) {
ctx, cancel := context.WithTimeout(ctx, config.Config.TestTimeoutVMSS)
defer cancel()
rid, _ := s.VHD.VHDResourceID(ctx, s.T)

s.T.Logf("running scenario %q with image %q in aks cluster %q", s.T.Name(), rid, *s.Runtime.Cluster.Model.ID)

createVMSS(ctx, s)

err := getCustomScriptExtensionStatus(ctx, s)
require.NoError(s.T, err)

s.T.Logf("vmss %s creation succeeded, proceeding with node readiness and pod checks...", s.Runtime.VMSSName)
nodeName := s.validateNodeHealth(ctx)
func validateVM(ctx context.Context, s *Scenario) {
ValidatePodRunning(ctx, s)

// test-specific validation
if s.Config.Validator != nil {
s.Config.Validator(ctx, s)
}
// skip when outbound type is block as the wasm will create pod from gcr, however, network isolated cluster scenario will block egress traffic of gcr.
// TODO(xinhl): add another way to validate
if s.Runtime.NBC != nil && s.Runtime.NBC.AgentPoolProfile.WorkloadRuntime == datamodel.WasmWasi && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeBlock && s.Runtime.NBC.OutboundType != datamodel.OutboundTypeNone {
validateWasm(ctx, s.T, s.Runtime.Cluster.Kube, nodeName)
ValidateWASM(ctx, s, s.Runtime.KubeNodeName)
}
if s.Runtime.AKSNodeConfig != nil && s.Runtime.AKSNodeConfig.WorkloadRuntime == aksnodeconfigv1.WorkloadRuntime_WORKLOAD_RUNTIME_WASM_WASI {
validateWasm(ctx, s.T, s.Runtime.Cluster.Kube, nodeName)
ValidateWASM(ctx, s, s.Runtime.KubeNodeName)
}

s.T.Logf("node %s is ready, proceeding with validation commands...", s.Runtime.VMSSName)

vmPrivateIP, err := getVMPrivateIPAddress(ctx, s)
switch s.VHD.OS {
case config.OSWindows:
// TODO: validate something
default:
ValidateCommonLinux(ctx, s)

require.NoError(s.T, err, "get vm private IP %v", s.Runtime.VMSSName)
err = runLiveVMValidators(ctx, s.T, s.Runtime.VMSSName, vmPrivateIP, string(s.Runtime.SSHKeyPrivate), s)
require.NoError(s.T, err)

s.T.Logf("node %s bootstrapping succeeded!", s.Runtime.VMSSName)
}
s.T.Log("validation succeeded")
}

func getExpectedPackageVersions(packageName, distro, release string) []string {
Expand Down Expand Up @@ -197,7 +233,7 @@ func getCustomScriptExtensionStatus(ctx context.Context, s *Scenario) error {
}
for _, extension := range instanceViewResp.Extensions {
for _, status := range extension.Statuses {
if s.VHD.Windows() {
if s.VHD.OS == config.OSWindows {
if status.Code == nil || !strings.EqualFold(*status.Code, "ProvisioningState/succeeded") {
return fmt.Errorf("failed to get CSE output, error: %s", *status.Message)
}
Expand All @@ -209,9 +245,8 @@ func getCustomScriptExtensionStatus(ctx context.Context, s *Scenario) error {
return fmt.Errorf("Parse CSE message with error, error %w", err)
}
if resp.ExitCode != "0" {
return fmt.Errorf("vmssCSE %s, output=%s, error=%s", resp.ExitCode, resp.Output, resp.Error)
return fmt.Errorf("vmssCSE %s, output=%s, error=%s, cse output: %s", resp.ExitCode, resp.Output, resp.Error, *status.Message)
}
s.T.Logf("CSE completed successfully with exit code 0, cse output: %s", *status.Message)
return nil
}
}
Expand Down
11 changes: 6 additions & 5 deletions e2e/scenario_node_controller_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package e2e

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
Expand All @@ -24,7 +25,7 @@ import (
// the test results are unreliable, as there can be a version mismatch between the binary and the rest content of VHD image
// it's intended to be used for quick testing without rebuilding VHD images
// mostly executed locally
func Test_ubuntu2204AKSNodeController(t *testing.T) {
func Test_Ubuntu2204AKSNodeController(t *testing.T) {
ctx := newTestCtx(t)
if !config.Config.EnableAKSNodeControllerTest {
t.Skip("ENABLE_AKS_NODE_CONTROLLER_TEST is not set")
Expand Down Expand Up @@ -80,10 +81,10 @@ func Test_ubuntu2204AKSNodeController(t *testing.T) {
},
}
},
LiveVMValidators: []*LiveVMValidator{
mobyComponentVersionValidator("containerd", getExpectedPackageVersions("containerd", "ubuntu", "r2204")[0], "apt"),
mobyComponentVersionValidator("runc", getExpectedPackageVersions("runc", "ubuntu", "r2204")[0], "apt"),
FileHasContentsValidator("/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully"),
Validator: func(ctx context.Context, s *Scenario) {
ValidateInstalledPackageVersion(ctx, s, "moby-containerd", getExpectedPackageVersions("containerd", "ubuntu", "r2204")[0])
ValidateInstalledPackageVersion(ctx, s, "moby-runc", getExpectedPackageVersions("runc", "ubuntu", "r2204")[0])
ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully")
},
AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) {},
},
Expand Down
Loading

0 comments on commit c3617cc

Please sign in to comment.