Use R2 for istio private prow (#5922)

* fix permissions

* Have istio-private use r2

Author: Stevenjin8

infra/gcp/istio-prow-private/iam.tf

@@ -63,7 +63,7 @@ resource "google_project_iam_member" "owners" {
 module "kubernetes_external_secrets_account" {
   source            = "../modules/workload-identity-service-account"
   project_id        = local.project_id
-  name              = "kubernetes-external-secrets-sa-private"
+  name              = "k8s-external-secrets-sa-priv"  # GCP SAs have a 30 character limit, so we need to abbreviate.
   description       = "Service account used by kubernetes-external-secrets operator on the private clusters."
   cluster_namespace = "default"
   secrets = [

prow/cluster/private/kubernetes-external-secrets_deployment.yaml

@@ -17,7 +17,7 @@ spec:
       labels:
         app.kubernetes.io/name: kubernetes-external-secrets
     spec:
-      serviceAccountName: kubernetes-external-secrets-sa-private
+      serviceAccountName: k8s-external-secrets-sa-priv
       containers:
         - name: kubernetes-external-secrets
           image: "ghcr.io/external-secrets/kubernetes-external-secrets:8.5.1"

prow/cluster/private/kubernetes-external-secrets_rbac.yaml

@@ -2,9 +2,9 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   annotations:
-    iam.gke.io/gcp-service-account: kubernetes-external-secrets-sa-private@istio-testing.iam.gserviceaccount.com
+    iam.gke.io/gcp-service-account: k8s-external-secrets-sa-priv@istio-prow-private.iam.gserviceaccount.com
   namespace: default
-  name: "kubernetes-external-secrets-sa-private"
+  name: "k8s-external-secrets-sa-priv"
 ---
 # Source: kubernetes-external-secrets/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -46,7 +46,7 @@ roleRef:
   kind: ClusterRole
   name: kubernetes-external-secrets
 subjects:
-  - name: kubernetes-external-secrets-sa-private
+  - name: k8s-external-secrets-sa-priv
     namespace: "default"
     kind: ServiceAccount
 ---
@@ -62,6 +62,6 @@ roleRef:
   kind: ClusterRole
   name: system:auth-delegator
 subjects:
-- name: kubernetes-external-secrets-sa-private
+- name: k8s-external-secrets-sa-priv
   namespace: "default"
   kind: ServiceAccount

prow/config.yaml

@@ -35,8 +35,9 @@ plank:
       gcs_credentials_secret: ""
       default_service_account_name: "prowjob-private"
       gcs_configuration:
-        bucket: "istio-prow-private"
+        bucket: "s3://istio-prow-private"
         path_strategy: "explicit"
+      s3_credentials_secret: "cf-r2-istio-prow-private-credentials"
       oauth_token_secret:
         name: github-oauth
         key: token