Leverage build scenario to figure out bucket and s3 path

Author: viveksinghggits

.evergreen-functions.yml

@@ -490,13 +490,6 @@ functions:
           - rh_pyxis
         binary: scripts/dev/run_python.sh scripts/preflight_images.py --image ${image_name} --submit "${preflight_submit}"
 
-#  build_multi_cluster_binary:
-#    - command: subprocess.exec
-#      type: setup
-#      params:
-#        working_dir: src/github.com/mongodb/mongodb-kubernetes
-#        binary: scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh
-
   install_macnotary:
     - command: shell.exec
       type: setup
@@ -543,7 +536,7 @@ functions:
           MACOS_NOTARY_KEY: ${macos_notary_keyid}
           MACOS_NOTARY_SECRET: ${macos_notary_secret}
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/evergreen/run_python.sh scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py
+        binary: scripts/dev/run_python.sh scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py
 
   build_and_push_appdb_database:
     - command: subprocess.exec

build_info.json

@@ -279,14 +279,14 @@
   "binaries": {
     "kubectl-mongodb": {
       "patch": {
-        "s3-store": "s3://kubectl-mongodb/dev",
+        "s3-store": "mongodb-kubernetes-dev",
         "platforms": [
           "linux/amd64"
         ]
       },
       "staging": {
-        "sign": true,
-        "s3-store": "s3://kubectl-mongodb/staging",
+        "sign": false,
+        "s3-store": "mongodb-kubernetes-staging",
         "platforms": [
           "darwin/amd64",
           "darwin/arm64",
@@ -296,7 +296,7 @@
       },
       "release": {
         "sign": true,
-        "s3-store": "s3://kubectl-mongodb/prod",
+        "s3-store": "mongodb-kubernetes-release",
         "platforms": [
           "darwin/amd64",
           "darwin/arm64",

scripts/release/kubectl-mongodb/kubectl_mac_notarize.sh

@@ -29,14 +29,6 @@ if [[ -f "./dist/kubectl-mongodb_darwin_amd64_v1/kubectl-mongodb" && -f "./dist/
 		-b com.mongodb.mongodb-kubectl-mongodb \
 		-o ./dist/kubectl-mongodb_macos_signed.zip
 
-  echo "printing output zip file content, unzip -l ./dist/kubectl-mongodb_macos_signed.zip"
-  unzip -l ./dist/kubectl-mongodb_macos_signed.zip
-  echo "done"
-
-  echo "running unzip -t ./dist/kubectl-mongodb_macos_signed.zip to verify that the op zip file of macnotary is correct"
-  unzip -t ./dist/kubectl-mongodb_macos_signed.zip
-  echo "done"
-
 	echo "replacing original files"
 	unzip -oj ./dist/kubectl-mongodb_macos_signed.zip dist/kubectl-mongodb_darwin_amd64_v1/kubectl-mongodb -d ./dist/kubectl-mongodb_darwin_amd64_v1/
 	unzip -oj ./dist/kubectl-mongodb_macos_signed.zip dist/kubectl-mongodb_darwin_arm64/kubectl-mongodb -d ./dist/kubectl-mongodb_darwin_arm64/

scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py

@@ -4,23 +4,23 @@
 
 import boto3
 from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError
-from mypy.types import ExtraAttrs
 
-# from lib.base_logger import logger
-
-DEV_S3_BUCKET_NAME = "mongodb-kubernetes-dev"
-STAGING_S3_BUCKET_NAME = "mongodb-kubernetes-staging"
-RELEASE_S3_BUCKET_NAME = "mongodb-kubernetes-release"
+from lib.base_logger import logger
+from scripts.release.build.build_info import (
+    load_build_info,
+)
+from scripts.release.build.build_scenario import (
+    BuildScenario,
+)
 
 AWS_REGION = "eu-north-1"
-S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = "kubectl-mongodb"
-
-COMMIT_SHA_ENV_VAR = "github_commit"
+KUBECTL_PLUGIN_BINARY_NAME = "kubectl-mongodb"
+S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = KUBECTL_PLUGIN_BINARY_NAME
 
 GORELEASER_DIST_DIR = "dist"
 
-# LOCAL_FILE_PATHis the full filename where tests image expects the kuebctl-mongodb binary to be available
-LOCAL_FILE_PATH = "docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_linux"
+# LOCAL_KUBECTL_PLUGIN_PATH the full filename where tests image expects the kuebctl-mongodb binary to be available
+LOCAL_KUBECTL_PLUGIN_PATH = "docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_linux"
 
 
 def run_goreleaser():
@@ -36,111 +36,102 @@ def run_goreleaser():
         exit_code = process.wait()
 
         if exit_code != 0:
-            print(f"GoReleaser command failed with exit code {exit_code}.")
+            logger.debug(f"GoReleaser command failed with exit code {exit_code}.")
             sys.exit(1)
 
-        print("GoReleaser build completed successfully!")
+        logger.info("GoReleaser build completed successfully!")
 
     except FileNotFoundError:
-        print("ERROR: 'goreleaser' command not found. Please ensure goreleaser is installed and in your system's PATH.")
+        logger.debug(
+            "ERROR: 'goreleaser' command not found. Please ensure goreleaser is installed and in your system's PATH."
+        )
         sys.exit(1)
     except Exception as e:
-        print(f"An unexpected error occurred while running `goreleaser build`: {e}")
+        logger.debug(f"An unexpected error occurred while running `goreleaser build`: {e}")
         sys.exit(1)
 
 
-def upload_artifacts_to_s3():
+# upload_artifacts_to_s3 uploads the artifacts that are generated by goreleaser to S3 bucket at a specific path.
+# The S3 bucket and version are figured out and passed to this function based on BuildScenario.
+def upload_artifacts_to_s3(s3_bucket: str, version: str):
     if not os.path.isdir(GORELEASER_DIST_DIR):
-        print(f"ERROR: GoReleaser dist directory '{GORELEASER_DIST_DIR}' not found.")
+        logger.info(f"ERROR: GoReleaser dist directory '{GORELEASER_DIST_DIR}' not found.")
         sys.exit(1)
 
     try:
         s3_client = boto3.client("s3", region_name=AWS_REGION)
     except (NoCredentialsError, PartialCredentialsError):
-        print("ERROR: Failed to create S3 client. AWS credentials not found.")
+        logger.debug("ERROR: Failed to create S3 client. AWS credentials not found.")
         sys.exit(1)
     except Exception as e:
-        print(f"An error occurred connecting to S3: {e}")
+        logger.debug(f"An error occurred connecting to S3: {e}")
         sys.exit(1)
 
     uploaded_files = 0
+    # iterate over all the files generated by goreleaser in the dist directory and upload them to S3
     for root, _, files in os.walk(GORELEASER_DIST_DIR):
         for filename in files:
             local_path = os.path.join(root, filename)
-            s3_key = s3_path(local_path)
-
-            print(f"Uploading artifact {local_path} to s3://{DEV_S3_BUCKET_NAME}/{s3_key}")
-
-            stat = os.stat(local_path)
-            permissions = str(oct(stat.st_mode)[-3:])
+            s3_key = s3_path(local_path, version)
 
+            logger.info(f"Uploading artifact {local_path} to s3://{s3_bucket}/{s3_key}")
             try:
-                s3_client.upload_file(
-                    local_path,
-                    DEV_S3_BUCKET_NAME,
-                    s3_key,
-                    ExtraArgs={
-                        "Metadata": {"posix-permissions": permissions},
-                    },
-                )
-                print(f"Successfully uploaded the artifact {filename}")
+                s3_client.upload_file(local_path, s3_bucket, s3_key)
+                logger.info(f"Successfully uploaded the artifact {filename}")
                 uploaded_files += 1
             except Exception as e:
-                print(f"ERROR: Failed to upload file {filename}: {e}")
+                logger.debug(f"ERROR: Failed to upload file {filename}: {e}")
 
     if uploaded_files > 0:
-        print(f"Successfully uploaded {uploaded_files} kubectl-mongodb plugin artifacts to S3.")
+        logger.info(f"Successfully uploaded {uploaded_files} kubectl-mongodb plugin artifacts to S3.")
 
 
-# s3_path returns the path where the artifacts should be uploaded to in S3 obect store.
+# s3_path returns the path where the artifacts should be uploaded to in S3 object store.
 # For dev workflows it's going to be `kubectl-mongodb/{evg-patch-id}/{goreleaser-artifact}`,
 # for staging workflows it would be `kubectl-mongodb/{commit-sha}/{goreleaser-artifact}`.
-def s3_path(local_path: str):
-    commit_sha = os.environ.get(COMMIT_SHA_ENV_VAR, "").strip()
-    if commit_sha == "":
-        print(
-            f"Error: The commit sha environment variable {COMMIT_SHA_ENV_VAR} is not set. It's required to form the S3 Path."
-        )
-        sys.exit(1)
-
-    return f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{commit_sha}/{local_path}"
+# The `version` string has the correct version (either patch id or commit sha), based on the BuildScenario.
+def s3_path(local_path: str, version: str):
+    return f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/{local_path}"
 
 
-def download_plugin_for_tests_image():
+# download_plugin_for_tests_image downloads just the linux amd64 version of the binary and places it
+# at the location LOCAL_KUBECTL_PLUGIN_PATH.
+def download_plugin_for_tests_image(build_scenario: BuildScenario, s3_bucket: str, version: str):
     try:
         s3_client = boto3.client("s3", region_name=AWS_REGION)
     except Exception as e:
-        print(f"An error occurred connecting to S3 to download kubectl plugin for tests image: {e}")
+        logger.debug(f"An error occurred connecting to S3 to download kubectl plugin for tests image: {e}")
         return
 
-    commit_sha = os.environ.get(COMMIT_SHA_ENV_VAR, "").strip()
-    if commit_sha == "":
-        print("Error: The commit sha environment variable is not set. It's required to form the S3 Path.")
-        sys.exit(1)
-
-    plugin_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{commit_sha}/dist/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb"
+    plugin_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb"
 
-    print(f"Downloading s3://{DEV_S3_BUCKET_NAME}/{plugin_path} to {LOCAL_FILE_PATH}")
+    logger.info(f"Downloading s3://{s3_bucket}/{plugin_path} to {LOCAL_KUBECTL_PLUGIN_PATH}")
     try:
-        s3_client.download_file(DEV_S3_BUCKET_NAME, plugin_path, LOCAL_FILE_PATH)
-        # change the file's permissions so that it can be executed
-        os.chmod(LOCAL_FILE_PATH, 0o755)
+        s3_client.download_file(s3_bucket, plugin_path, LOCAL_KUBECTL_PLUGIN_PATH)
+        # change the file's permissions to make file executable
+        os.chmod(LOCAL_KUBECTL_PLUGIN_PATH, 0o755)
 
-        print(f"Successfully downloaded artifact to {LOCAL_FILE_PATH}")
+        logger.info(f"Successfully downloaded artifact to {LOCAL_KUBECTL_PLUGIN_PATH}")
     except ClientError as e:
         if e.response["Error"]["Code"] == "404":
-            print(f"ERROR: Artifact not found at s3://{DEV_S3_BUCKET_NAME}/{plugin_path} ")
+            logger.debug(f"ERROR: Artifact not found at s3://{s3_bucket}/{plugin_path} ")
         else:
-            print(f"ERROR: Failed to download artifact. S3 Client Error: {e}")
+            logger.debug(f"ERROR: Failed to download artifact. S3 Client Error: {e}")
     except Exception as e:
-        print(f"An unexpected error occurred during download: {e}")
+        logger.debug(f"An unexpected error occurred during download: {e}")
 
 
 def main():
+    build_scenario = BuildScenario.infer_scenario_from_environment()
+    kubectl_plugin_build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY_NAME]
+
     run_goreleaser()
-    upload_artifacts_to_s3()
 
-    download_plugin_for_tests_image()
+    upload_artifacts_to_s3(kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version)
+
+    download_plugin_for_tests_image(
+        build_scenario, kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version
+    )
 
 
 if __name__ == "__main__":