Skip to content

Commit

Permalink
Support alternative partitions in e2e tests (#386)
Browse files Browse the repository at this point in the history 
*Issue #, if available:* N/A

*Description of changes:* Removes hard-coded `aws` partition, and uses
sts getCallerIdentity to retrieve an ARN which can be used to extract
the current partition. Additionally, replaces busybox from dockerhub
with busybox from ecr public.


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
  • Loading branch information
@muddyfish
muddyfish authored Feb 21, 2025
1 parent e519c4e commit 262f35b
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 27 deletions.
2 changes: 1 addition & 1 deletion tests/controller/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const s3CSIDriver = "s3.csi.aws.com"
const ebsCSIDriver = "ebs.csi.aws.com"

const defaultNamespace = "default"
const defaultContainerImage = "busybox:latest"
const defaultContainerImage = "public.ecr.aws/docker/library/busybox:stable-musl"

// Configuration values passed for `mppod.Config` while creating a controller to use in tests.
const mountpointNamespace = "mount-s3"
Expand Down
60 changes: 36 additions & 24 deletions tests/e2e-kubernetes/testsuites/credentials.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"
"time"

awsarn "github.com/aws/aws-sdk-go-v2/aws/arn"
"github.com/aws/aws-sdk-go-v2/aws/retry"
"github.com/aws/aws-sdk-go-v2/service/iam"
iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
Expand All @@ -33,9 +34,9 @@ import (
)

const (
iamPolicyS3FullAccess = "arn:aws:iam::aws:policy/AmazonS3FullAccess"
iamPolicyS3ReadOnlyAccess = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
iamPolicyS3NoAccess = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" // `AmazonEC2ReadOnlyAccess` gives no S3 access
iamPolicyS3FullAccess = "AmazonS3FullAccess"
iamPolicyS3ReadOnlyAccess = "AmazonS3ReadOnlyAccess"
iamPolicyS3NoAccess = "AmazonEC2ReadOnlyAccess" // `AmazonEC2ReadOnlyAccess` gives no S3 access
)

const (
Expand Down Expand Up @@ -294,13 +295,13 @@ func (t *s3CSICredentialsTestSuite) DefineTests(driver storageframework.TestDriv
var afterAllCleanup []func(context.Context) error

By("Pre-creating IAM roles for common policies")
for _, policyARN := range []string{
for _, policyName := range []string{
iamPolicyS3FullAccess,
iamPolicyS3ReadOnlyAccess,
iamPolicyS3NoAccess,
} {
role, removeRole := createRole(ctx, f, assumeRolePolicyDocument(ctx), policyARN)
policyRoleMapping[policyARN] = role
role, removeRole := createRole(ctx, f, assumeRolePolicyDocument(ctx), policyName)
policyRoleMapping[policyName] = role
afterAllCleanup = append(afterAllCleanup, removeRole)
}

Expand All @@ -317,11 +318,11 @@ func (t *s3CSICredentialsTestSuite) DefineTests(driver storageframework.TestDriv
cleanClusterWideResources(ctx)
})

updateCSIDriversServiceAccountRole := func(ctx context.Context, policyARN string) {
updateCSIDriversServiceAccountRole := func(ctx context.Context, policyName string) {
By("Updating CSI Driver's Service Account Role")
sa := csiDriverServiceAccount(ctx, f)

role, removeRole := createRole(ctx, f, assumeRoleWithWebIdentityPolicyDocument(ctx, oidcProvider, sa), policyARN)
role, removeRole := createRole(ctx, f, assumeRoleWithWebIdentityPolicyDocument(ctx, oidcProvider, sa), policyName)
deferCleanup(removeRole)

sa, restoreServiceAccountRole := overrideServiceAccountRole(ctx, f, sa, *role.Arn)
Expand All @@ -333,12 +334,12 @@ func (t *s3CSICredentialsTestSuite) DefineTests(driver storageframework.TestDriv
killCSIDriverPods(ctx, f)
}

updateDriverLevelKubernetesSecret := func(ctx context.Context, policyARN string) {
updateDriverLevelKubernetesSecret := func(ctx context.Context, policyName string) {
By("Updating Kubernetes Secret with temporary credentials")

role, ok := policyRoleMapping[policyARN]
role, ok := policyRoleMapping[policyName]
if !ok {
framework.Failf("Missing role mapping for policy %s", policyARN)
framework.Failf("Missing role mapping for policy %s", policyName)
}
assumeRoleOutput := assumeRole(ctx, f, *role.Arn)

Expand Down Expand Up @@ -441,23 +442,23 @@ func (t *s3CSICredentialsTestSuite) DefineTests(driver storageframework.TestDriv
}
})

assignPolicyToServiceAccount := func(ctx context.Context, sa *v1.ServiceAccount, policyARN string) *v1.ServiceAccount {
role, removeRole := createRole(ctx, f, assumeRoleWithWebIdentityPolicyDocument(ctx, oidcProvider, sa), policyARN)
assignPolicyToServiceAccount := func(ctx context.Context, sa *v1.ServiceAccount, policyName string) *v1.ServiceAccount {
role, removeRole := createRole(ctx, f, assumeRoleWithWebIdentityPolicyDocument(ctx, oidcProvider, sa), policyName)
deferCleanup(removeRole)

sa, _ = overrideServiceAccountRole(ctx, f, sa, *role.Arn)
waitUntilRoleIsAssumableWithWebIdentity(ctx, f, sa)
return sa
}

createServiceAccountWithPolicy := func(ctx context.Context, policyARN string) *v1.ServiceAccount {
createServiceAccountWithPolicy := func(ctx context.Context, policyName string) *v1.ServiceAccount {
sa, removeSA := createServiceAccount(ctx, f)
deferCleanup(removeSA)

return assignPolicyToServiceAccount(ctx, sa, policyARN)
return assignPolicyToServiceAccount(ctx, sa, policyName)
}

createPodWithServiceAccountAndPolicy := func(ctx context.Context, policyARN string, allowDelete bool, asNonRoot bool) (*v1.Pod, *v1.ServiceAccount) {
createPodWithServiceAccountAndPolicy := func(ctx context.Context, policyName string, allowDelete bool, asNonRoot bool) (*v1.Pod, *v1.ServiceAccount) {
By("Creating Pod with ServiceAccount")

mountOptions := []string{fmt.Sprintf("region %s", DefaultRegion)}
Expand All @@ -473,7 +474,7 @@ func (t *s3CSICredentialsTestSuite) DefineTests(driver storageframework.TestDriv
vol := createVolumeResourceWithMountOptions(enablePodLevelIdentity(ctx), l.config, pattern, mountOptions)
deferCleanup(vol.CleanupResource)

sa := createServiceAccountWithPolicy(ctx, policyARN)
sa := createServiceAccountWithPolicy(ctx, policyName)

var podModifiers []func(*v1.Pod)
podModifiers = append(podModifiers, func(pod *v1.Pod) {
Expand Down Expand Up @@ -653,15 +654,17 @@ func assumeRolePolicyDocument(ctx context.Context) string {
}

func assumeRoleWithWebIdentityPolicyDocument(ctx context.Context, oidcProvider string, sa *v1.ServiceAccount) string {
awsAccount := stsCallerIdentity(ctx).Account
identity := stsCallerIdentity(ctx)
awsAccount := identity.Account
partition := getARNPartition(*identity.Arn)

buf, err := json.Marshal(&jsonMap{
"Version": "2012-10-17",
"Statement": []jsonMap{
{
"Effect": "Allow",
"Principal": jsonMap{
"Federated": fmt.Sprintf("arn:aws:iam::%s:oidc-provider/%s", *awsAccount, oidcProvider),
"Federated": fmt.Sprintf("arn:%s:iam::%s:oidc-provider/%s", partition, *awsAccount, oidcProvider),
},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": jsonMap{
Expand All @@ -678,8 +681,15 @@ func assumeRoleWithWebIdentityPolicyDocument(ctx context.Context, oidcProvider s
return string(buf)
}

func createRole(ctx context.Context, f *framework.Framework, assumeRolePolicyDocument string, policyARNs ...string) (*iamtypes.Role, func(context.Context) error) {
func getARNPartition(arn string) string {
parsedArn, err := awsarn.Parse(arn)
framework.ExpectNoError(err)
return parsedArn.Partition
}

func createRole(ctx context.Context, f *framework.Framework, assumeRolePolicyDocument string, policyNames ...string) (*iamtypes.Role, func(context.Context) error) {
framework.Logf("Creating IAM role")
identity := stsCallerIdentity(ctx)

client := iam.NewFromConfig(awsConfig(ctx))

Expand All @@ -698,20 +708,22 @@ func createRole(ctx context.Context, f *framework.Framework, assumeRolePolicyDoc
return err
}

for _, p := range policyARNs {
for _, policyName := range policyNames {
policyArn := fmt.Sprintf("arn:%s:iam::aws:policy/%s", getARNPartition(*identity.Arn), policyName)
_, err := client.AttachRolePolicy(ctx, &iam.AttachRolePolicyInput{
RoleName: ptr.To(roleName),
PolicyArn: ptr.To(p),
PolicyArn: ptr.To(policyArn),
})
framework.ExpectNoError(err)
}

return role.Role, func(ctx context.Context) error {
var errs []error
for _, p := range policyARNs {
for _, policyName := range policyNames {
policyArn := fmt.Sprintf("arn:%s:iam::aws:policy/%s", getARNPartition(*identity.Arn), policyName)
_, err := client.DetachRolePolicy(ctx, &iam.DetachRolePolicyInput{
RoleName: ptr.To(roleName),
PolicyArn: ptr.To(p),
PolicyArn: ptr.To(policyArn),
})
errs = append(errs, err)
}
Expand Down
1 change: 0 additions & 1 deletion tests/e2e-kubernetes/testsuites/performance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@ func (t *s3CSIPerformanceTestSuite) DefineTests(driver storageframework.TestDriv
nodeSelector["kubernetes.io/hostname"] = nodeName
}
pod := e2epod.MakePod(f.Namespace.Name, nodeSelector, []*v1.PersistentVolumeClaim{resource.Pvc}, admissionapi.LevelBaseline, "")
pod.Spec.Containers[0].Image = "ubuntu:22.04"
var err error
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
framework.ExpectNoError(err)
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e-kubernetes/testsuites/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ func createServiceAccount(ctx context.Context, f *framework.Framework) (*v1.Serv
}

func awsConfig(ctx context.Context) aws.Config {
cfg, err := config.LoadDefaultConfig(ctx)
cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(DefaultRegion))
framework.ExpectNoError(err)
return cfg
}
Expand Down

0 comments on commit 262f35b

Please sign in to comment.